From e1e8f5c9c56b89afdb0496960338d172ecf8f65f Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Wed, 16 Sep 2026 19:28:43 +0100 Subject: [PATCH 01/64] docs: developer_guides: modernize portal and add current code topics Reorganize the Developer Guides portal into five modern technical pillars: 1. Firmware Development (FW) 2. Kernel & Host Driver Development (Kernel) 3. Hardware & Platform Bringup (HW) 4. Testing, Simulation & Toolchains (SDK & Test) 5. Telemetry, Probes & Diagnostics (Debug) Key updates include: - Add Linux driver architecture guide (sound/soc/sof/, IPC3/IPC4, SRAM mailboxes, runtime PM). - Add ASoC machine drivers, ACPI/NHLT/DISCO, and DMI quirk guide. - Add hostless embedded firmware guide (sof_static_pipeline, static topologies, Teensy 4.1, ESP32-P4/C6, Zephyr shell). - Add hardware audio loopback verification guide (test_p4_loopback.py, test_c6_loopback.py, test_teensy_loopback.py). - Modernize unit tests guide for Zephyr Ztest & Twister runner. - Modernize rimage documentation for the modern Rust tool and TOML configs. - Modernize DSP traces for Zephyr logging, smex, and TCP probe server. - Modernize CMake build guide for Zephyr RTOS and West. - Add legacy notice to Topology 1.0 pointing to Topology 2.0. - Remove obsolete guides (lmdk_user_guide, setup_up_2_board, compile_wsl, fuzzing_in_docker). Signed-off-by: Liam Girdwood --- .../debugability/traces/index.rst | 140 +++++-- .../firmware/hostless_firmware.rst | 165 +++++++++ .../fuzzing/fuzzing_in_docker.rst | 62 ---- developer_guides/fuzzing/index.rst | 3 +- developer_guides/index.rst | 82 +++-- .../linux_driver/architecture.rst | 139 +++++++ developer_guides/linux_driver/index.rst | 5 +- .../linux_driver/machine_drivers_quirks.rst | 172 +++++++++ .../loadable_modules/lmdk_user_guide.rst | 25 -- developer_guides/rimage/index.rst | 95 ++++- .../setup_special_device/setup_up_2_board.rst | 112 ------ developer_guides/tech/cmake.rst | 272 +++++++------- developer_guides/tech/compile_wsl.rst | 104 ------ .../testing/hardware_loopback.rst | 122 ++++++ developer_guides/topology/topology.rst | 9 +- developer_guides/unit_tests.rst | 347 +++++++++++------- 16 files changed, 1201 insertions(+), 653 deletions(-) create mode 100644 developer_guides/firmware/hostless_firmware.rst delete mode 100644 developer_guides/fuzzing/fuzzing_in_docker.rst create mode 100644 developer_guides/linux_driver/architecture.rst create mode 100644 developer_guides/linux_driver/machine_drivers_quirks.rst delete mode 100644 developer_guides/loadable_modules/lmdk_user_guide.rst delete mode 100644 developer_guides/setup_special_device/setup_up_2_board.rst delete mode 100644 developer_guides/tech/compile_wsl.rst create mode 100644 developer_guides/testing/hardware_loopback.rst diff --git a/developer_guides/debugability/traces/index.rst b/developer_guides/debugability/traces/index.rst index 1df8bf6b..1a1dcb77 100644 --- a/developer_guides/debugability/traces/index.rst +++ b/developer_guides/debugability/traces/index.rst @@ -1,48 +1,124 @@ .. _dbg-traces: -Traces -###### +DSP Telemetry, Logging & Traces +############################### -A FW developer may log important events by adding ``trace_event(...)`` entries -to the source code. The data is collected in the internal buffer and -transmitted periodically to the host through the DMA. +Sound Open Firmware (SOF) features a high-performance, asynchronous logging and telemetry infrastructure. Because real-time audio processing operates on sub-millisecond deadlines, DSP logging cannot block on slow UART serial writes. Instead, SOF combines compile-time string dictionary extraction (**smex**), hardware DMA circular buffers, Zephyr RTOS structured logging, and network-accessible telemetry servers. -Building & Processing Traces -**************************** +Architecture Overview +********************* -During the compilation and linking, string literals and log entry metadata is -linked to the special *debug* sections that are extracted later to a special -*dictionary* file. This part is not loaded to the DSP and does not occupy the -DSP memory keeping both the memory footprint and the trace DMA payload small. +The logging system is split into compile-time extraction and runtime streaming: + +.. code-block:: text + + +-------------------------------------------------------------+ + | Build-Time Pipeline | + | | + | [ C Source (LOG_INF/DBG) ] ----> [ Compiler / Linker ] | + | | | + | v | + | [ ELF Binary ] | + | | | + | v | + | [ smex Extractor ] | + | | | + | v | + | [ Dictionary (.ldc) ] | + +-------------------------------------------------------------+ + + +-------------------------------------------------------------+ + | Runtime Streaming | + | | + | [ DSP Log Buffer ] ---> [ Trace DMA ] ---> [ Host SRAM / | + | debugfs trace ]| + | | | + | [ sof-logger ] <--- [ Dictionary (.ldc) ] <-------+ | + | | | + | v | + | [ Formatted Console Log / TCP Probe Server (port 9999) ] | + +-------------------------------------------------------------+ + +Zephyr Logging Integration +************************** + +Modern SOF leverages the native Zephyr logging subsystem (`zephyr/logging/log.h`). Modules register their log category and log level: + +.. code-block:: c + + #include + + /* Register module with default log level */ + LOG_MODULE_REGISTER(eq_fir, CONFIG_SOF_LOG_LEVEL); + + int eq_fir_process(struct comp_dev *dev) + { + LOG_DBG("eq_fir_process: dev %p, frame count %u", dev, dev->frames); + + if (dev->state != COMP_STATE_ACTIVE) { + LOG_WRN("eq_fir: processing called while not active"); + return -EINVAL; + } + + return 0; + } + +Standard Logging Macros: +======================== + +* ``LOG_ERR(...)``: Critical runtime errors and unexpected component failures. +* ``LOG_WRN(...)``: Recoverable issues or parameter warnings. +* ``LOG_INF(...)``: State transitions, stream creation, and hardware initialization milestones. +* ``LOG_DBG(...)``: Detailed per-buffer execution traces (disabled in release builds to save CPU cycles). + +Compile-Time String Extraction with Smex +**************************************** + +To minimize the firmware binary footprint and avoid transmitting text strings across DMA, SOF uses the **smex** (String Metadata Extractor) tool: + +1. String literals and format arguments in ``LOG_*`` calls are placed in a dedicated read-only section (``.static_log_entries``) of the ELF binary. +2. During the build, ``smex`` parses this section and extracts log metadata (file, line number, format string, argument types) into an external dictionary file (``.ldc``). +3. The DSP firmware binary only stores lightweight 32-bit log entry IDs and runtime argument values, keeping DSP memory usage and trace DMA payload sizes exceptionally small. .. graphviz:: images/build-traces.dot :caption: Traces - build process -Once the binary trace data is received by the host driver, it is accessible to -the trace decoder (logger) through the files located in the -_/sys/kernel/debug/sof/..._. The logger requires the *dictionary* file to -decode the trace data and "printf" them using format specified in the source -files. +Trace Collection and Streaming +****************************** -.. graphviz:: images/process-traces.dot - :caption: Traces - running & processing +Once the binary trace entries are written to internal SRAM buffers, they are flushed periodically to the host: + +Host Debugfs Trace Interface +============================ + +On Linux hosts with the SOF driver loaded, binary trace buffers are exposed via `debugfs`: + +.. code-block:: text -Enabling Traces -*************** + /sys/kernel/debug/sof/trace -When the traces are enabled by the driver, it stores the FW version information -received along with the *FW Ready* IPC message at the beginning of the local -trace files. It enables simple compatibility check between the trace data and -the *dictionary* file performed by the logger. +Real-Time TCP Probe Server (Port 9999) +====================================== -Note that the trace data may be collected on some machine and sent along with -the dictionary file to another person for investigation. It is important to be -able to verify the consistency of both by having the build version attached to -them. +On target development setups (DUTs), SOF integrates with a TCP probe server running on port **9999**. This service streams live DSP trace packets over the local network to remote debugging workstations: -.. uml:: images/trace-enable-flow.pu +.. code-block:: bash -Adding Traces -************* + # Connect to DUT trace server and stream live logs + nc 9999 | sof-logger -t -d build/sof-tgl.ldc -Refer to the *src/include/sof/trace.h*. +Using `sof-logger` +****************** + +The ``sof-logger`` host utility reads the binary trace stream, parses the dictionary file, and prints human-readable timestamps, module names, and formatted messages: + +.. code-block:: bash + + # Decode live trace stream from debugfs + sof-logger -t -d /path/to/sof-platform.ldc -l /sys/kernel/debug/sof/trace + + # Decode a saved binary trace file + sof-logger -d /path/to/sof-platform.ldc -i trace_dump.bin -o decoded_trace.txt + +.. graphviz:: images/process-traces.dot + :caption: Traces - running & processing diff --git a/developer_guides/firmware/hostless_firmware.rst b/developer_guides/firmware/hostless_firmware.rst new file mode 100644 index 00000000..815eb566 --- /dev/null +++ b/developer_guides/firmware/hostless_firmware.rst @@ -0,0 +1,165 @@ +.. _sof_hostless_firmware: + +Hostless Embedded Firmware +########################## + +While Sound Open Firmware (SOF) is widely deployed as a DSP coprocessor driven by an upstream Linux host kernel driver, SOF also supports **hostless embedded operation**. In hostless mode, the firmware boots autonomously under the Zephyr RTOS, establishes audio pipelines from compiled-in static topologies, and executes real-time audio signal processing without requiring an external host operating system or IPC mailbox connection. + +This architecture enables SOF deployment on standalone microcontrollers, dedicated audio processing bridges, smart speakers, and embedded IoT appliances. + +Hostless Architecture Overview +****************************** + +In a typical host-driven setup, an ALSA driver sends IPC messages to create pipelines, allocate buffers, bind components, and set mixer controls. In hostless mode, this initialization sequence is executed entirely inside the firmware using **Static Pipelines**: + +.. code-block:: text + + +-------------------------------------------------------------+ + | Zephyr RTOS | + | (Kernel Init, Board Bringup, Clock Gating, Device Tree) | + +-------------------------------------------------------------+ + | + v + +-------------------------------------------------------------+ + | SOF Initialization | + | (sof_init -> sys_comp_init) | + +-------------------------------------------------------------+ + | + v + +-------------------------------------------------------------+ + | sof_static_pipeline | + | - Static Topology Instantiation (ROM-compiled graph) | + | - Buffer Allocation & Component Binding | + | - Fixed Pipeline Scheduling (1ms / 10ms ticks) | + | - Static Kcontrol & Volume Defaults | + +-------------------------------------------------------------+ + | + +------------------------+------------------------+ + | | + v v + +---------------------------+ +---------------------------+ + | Physical Ingress (DAI) | | Physical Egress (DAI) | + | - I2S / TDM Receiver | | - I2S / TDM Transmitter | + | - PDM / DMIC Microphone | | - S/PDIF Transmitter | + +---------------------------+ +---------------------------+ + +Static Pipelines (`sof_static_pipeline`) +**************************************** + +Hostless platforms define their audio topology graph directly in C source files or compiled binary blobs rather than receiving `.tplg` files over IPC. + +Component Graph Definition +========================== + +A static pipeline defines the ingress DAI (e.g., I2S/PDM), processing modules (Volume, Equalizer, Mixer, SRC), and egress DAI: + +.. code-block:: c + + #include + #include + #include + + /* Define static audio pipeline elements */ + static struct comp_dev *dai_in; + static struct comp_dev *volume_comp; + static struct comp_dev *dai_out; + + int init_hostless_audio_pipeline(void) + { + struct processing_module *mod; + int ret; + + /* 1. Create pipeline scheduler */ + ret = sof_static_pipeline_create(PIPE_ID_PRIMARY, PRIORITY_MED); + if (ret < 0) + return ret; + + /* 2. Instantiate and link ingress DAI */ + dai_in = sof_static_comp_create(SOF_COMP_DAI, COMP_ID_DAI_IN); + volume_comp = sof_static_comp_create(SOF_COMP_VOLUME, COMP_ID_VOL); + dai_out = sof_static_comp_create(SOF_COMP_DAI, COMP_ID_DAI_OUT); + + /* 3. Bind audio pipeline routing */ + sof_static_pipeline_connect(dai_in, volume_comp); + sof_static_pipeline_connect(volume_comp, dai_out); + + /* 4. Complete and start pipeline */ + return sof_static_pipeline_start(PIPE_ID_PRIMARY); + } + +Static Kcontrols and Routing +============================ + +Because there is no ALSA user-space mixer to set initial volumes and switches, the static pipeline registers default gains and routing matrices: + +* **Initial Volume Levels**: Defaults to 0 dB unity gain or calibrated board defaults. +* **Mute/Unmute Logic**: Audio outputs start in a safe unmuted or soft-ramped state once clocks stabilize. +* **Fixed Sample Rates**: Ingress and egress DAI sample rates (typically 48 kHz, 16-bit or 32-bit PCM) are configured via Kconfig or Device Tree bindings. + +Supported Hostless Platforms +**************************** + +SOF supports hostless operation across several modern microcontroller architectures: + +Teensy 4.1 (NXP i.MX RT1062) +============================ + +* **Architecture**: ARM Cortex-M7 running at 600 MHz. +* **Features**: Hardware FPU, Audio PLL4 clock generation, S/PDIF transmitter, and multi-channel I2S/SAI interfaces. +* **Use Case**: High-precision audio bridge, S/PDIF loopback card, and real-time DSP filter prototyping. + +Espressif ESP32-P4 +================== + +* **Architecture**: Dual-core RISC-V (HP core) running at 400 MHz with hardware Single-Instruction Multiple-Data (SIMD) and FPU. +* **Features**: I2S, PDM microphone receiver, USB High-Speed Audio Class 2.0 (UAC2). +* **Use Case**: Hardware loopback testing (Pallas transmitter & Ceres receiver), smart microphone arrays, and voice capture frontends. + +Espressif ESP32-C6 +================== + +* **Architecture**: Single-core 32-bit RISC-V running at 160 MHz. +* **Features**: Compact low-power audio node, I2S transceiver, hardware crypto, 802.15.4 / Zigbee / Thread, and Wi-Fi 6. +* **Use Case**: Ultra-compact wireless audio sensors, tone generation, and hardware loopback validation. + +Interactive Zephyr Shell Diagnostics +************************************ + +When running hostless, developers interact with the firmware via the **Zephyr Shell** over a UART serial console or USB CDC ACM virtual COM port: + +.. code-block:: text + + uart:~$ sof + sof - Sound Open Firmware commands + Subcommands: + status : Print audio pipeline status + cap dump : Dump audio capture buffer samples + regs : Display peripheral register state + tone : Toggle onboard diagnostic tone generator + mode : Switch clock mode between primary and secondary + +Useful Diagnostic Commands +========================== + +* **Pipeline Status**: + + .. code-block:: text + + uart:~$ sof status + Pipeline 1: RUNNING, Period: 1000 us, Core: 0 + [Comp 1: DAI In] -> [Comp 2: Volume (0 dB)] -> [Comp 3: DAI Out] + +* **Tone Generation**: + + .. code-block:: text + + uart:~$ sof tone enable 1000 + Generating 1000 Hz sine wave on DAI Out... + +* **Buffer Verification**: + + .. code-block:: text + + uart:~$ sof cap dump --samples 16 + [00] 0x0000 0x0124 0x02a8 0x03fe 0x04f1 0x05a0 0x0602 0x05f8 + [08] 0x058e 0x04b1 0x0382 0x0210 0x0061 0xfe80 0xfcaa 0xfaf0 diff --git a/developer_guides/fuzzing/fuzzing_in_docker.rst b/developer_guides/fuzzing/fuzzing_in_docker.rst deleted file mode 100644 index 3ebf516e..00000000 --- a/developer_guides/fuzzing/fuzzing_in_docker.rst +++ /dev/null @@ -1,62 +0,0 @@ -.. _fuzzing-in-docker: - -Fuzzing in Docker -################# - -Instructions -************ - -#. Build a fuzzer in order to use it. Follow the instructions at Build SOF - with docker, :ref:`docker-topology-tools`. - -#. Enter the Docker container: - - :: - - #To be run from sof/ directory - ./scripts/docker-run.sh bash - - A container is created from the ``sof`` Docker image. We are - provided with a shell prompt. Let's call this Terminal #1. - -#. Connect to the container's shell from another terminal. - - To do this, you must first know the container ID. - - :: - - docker ps - - #Sample output - CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - 1c383e3c08ae sof "bash" 4 minutes ago Up 4 minutes objective_kilby - - The first column of the output gives you the container ID. - - To connect to the container's shell, do the following: - - :: - - docker exec -i -t container_id bash - - This opens a shell prompt. Let's call this Terminal #2. - -#. Run the QEMU DSP VM in Terminal #1 by following instructions from `Using - the QEMU DSP emulator `__. - -#. Run the sof-fuzzer built from Step 1. Run this in Terminal #2. - - When you see **FW boot complete** in Terminal #2, the setup is complete. - -Important notes -*************** - -#. The platform should be the same for the QEMU DSP VM and the fuzzer. - - Ex: If you run your QEMU DSP VM with the 'byt' platform, use the same platform when you run your fuzzer. - -#. Make sure that you pass your kernel using the '-k' flag in the QEMU DSP - VM. - -#. You must run the fuzzer and the QEMU DSP VM in the same container; - otherwise they can't communicate with each other! diff --git a/developer_guides/fuzzing/index.rst b/developer_guides/fuzzing/index.rst index 2967b46c..bf58d964 100644 --- a/developer_guides/fuzzing/index.rst +++ b/developer_guides/fuzzing/index.rst @@ -4,7 +4,6 @@ Fuzzing ####### .. toctree:: - :maxdepth: 2 + :maxdepth: 1 - fuzzing_in_docker testbench_afl_fuzzing \ No newline at end of file diff --git a/developer_guides/index.rst b/developer_guides/index.rst index b2a4af23..46eb115a 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -4,8 +4,22 @@ Developer Guides ################ -Firmware Development (FW) -************************* +Sound Open Firmware (SOF) provides comprehensive architectural specifications, developer runbooks, and implementation guides covering the entire audio stack: from low-level DSP firmware and Zephyr RTOS integration to mainline Linux kernel drivers, embedded microcontroller audio bridges, and automated verification suites. + +The developer documentation is organized into five core technical pillars: + +1. :ref:`fw_development_pillar` +2. :ref:`kernel_driver_pillar` +3. :ref:`hardware_bringup_pillar` +4. :ref:`testing_simulation_pillar` +5. :ref:`telemetry_diagnostics_pillar` + +--- + +.. _fw_development_pillar: + +1. Firmware Development (FW) +**************************** Guides and specifications for developing, compiling, and debugging DSP firmware components, Zephyr RTOS integration, audio processing algorithms, dynamic modules, and firmware image signing. @@ -59,29 +73,26 @@ Detailed filter design, coefficient generation, and tuning workflows: algorithms/src/sample_rate_conversion algorithms/tdfb/time_domain_fixed_beamformer -Firmware Packaging & Dynamic Modules -==================================== +Firmware Packaging, Modules & Hostless Mode +=========================================== + +Firmware image packaging, cryptographic signing, loadable modules, and standalone hostless embedded firmware: .. toctree:: :maxdepth: 1 rimage/index.rst firmware/llext_modules - loadable_modules/lmdk_user_guide - -DSP Telemetry, Probes & Debugging -================================= + firmware/hostless_firmware -.. toctree:: - :maxdepth: 1 +--- - debugability/index - uuid/index.rst +.. _kernel_driver_pillar: -Kernel & Host Driver Development (Kernel) -***************************************** +2. Kernel & Host Driver Development (Kernel) +******************************************** -Guides for Linux ASoC kernel driver developers, topology authors, virtualization environments, and host tuning utilities. +Guides for Linux ASoC kernel driver developers, machine drivers, DMI quirk authoring, topology configurations, virtualization environments, and host tuning utilities. .. toctree:: :maxdepth: 1 @@ -94,29 +105,52 @@ Guides for Linux ASoC kernel driver developers, topology authors, virtualization tuning/sof-ctl ktest/setup_ktest_environment -Hardware & Platform-Specific Guides (HW) -**************************************** +--- -Hardware integration, platform memory layouts, boot architectures, and bringup checklists across silicon vendors, legacy architectures, and embedded development boards. +.. _hardware_bringup_pillar: + +3. Hardware & Platform Bringup (HW) +*********************************** + +Hardware integration, platform memory layouts, boot architectures, and bringup checklists across silicon vendors and embedded development boards. .. toctree:: :maxdepth: 1 nxp/sof_imx_user_guide - setup_special_device/setup_up_2_board -Simulation, Testing & Toolchain (SDK) -************************************* +For embedded microcontroller audio bridges and hostless targets (Teensy 4.1, ESP32-P4, ESP32-C6), see :ref:`sof_hostless_firmware` and :ref:`sof_hardware_loopback_testing`. + +--- -Verification frameworks, host audio simulation, fuzzing, and compiler toolchains. +.. _testing_simulation_pillar: + +4. Testing, Simulation & Toolchains (SDK & Test) +************************************************ + +Unit testing with Zephyr Ztest and Twister runner, host audio pipeline simulation, automated hardware loopback verification, Zephyr CMake build flags, and fuzzing. .. toctree:: :maxdepth: 1 unit_tests - tech/cmake + testing/hardware_loopback testbench/index + tech/cmake xtrun/index fuzzing/index - tech/compile_wsl +--- + +.. _telemetry_diagnostics_pillar: + +5. DSP Telemetry, Logging & Diagnostics (Debug) +*********************************************** + +Real-time DSP trace streaming over network probes, Zephyr structured logging, compile-time string dictionary extraction (`smex`), `sof-logger`, interactive Zephyr shell, and kernel debug probes. + +.. toctree:: + :maxdepth: 1 + + debugability/index + uuid/index.rst diff --git a/developer_guides/linux_driver/architecture.rst b/developer_guides/linux_driver/architecture.rst new file mode 100644 index 00000000..df7b5461 --- /dev/null +++ b/developer_guides/linux_driver/architecture.rst @@ -0,0 +1,139 @@ +.. _sof_linux_driver_architecture: + +Linux Driver Architecture +######################### + +The Sound Open Firmware (SOF) Linux driver resides in the upstream Linux kernel under ``sound/soc/sof/``. As part of the Advanced Linux Sound Architecture (ALSA) System on Chip (ASoC) subsystem, the driver manages communication, topology loading, audio streaming, power management, and firmware lifecycle for digital signal processors (DSPs). + +Architectural Overview +********************** + +The SOF driver architecture is structured into layered abstractions that separate generic DSP audio logic from vendor-specific bus controllers and hardware interfaces: + +.. code-block:: text + + +-------------------------------------------------------------+ + | User Space | + | (ALSA Lib / TinyALSA / PipeWire / PulseAudio / UCM2) | + +-------------------------------------------------------------+ + | + v + +-------------------------------------------------------------+ + | ALSA / ASoC Core | + | (pcm, control, dapm, soc-topology) | + +-------------------------------------------------------------+ + | + v + +-------------------------------------------------------------+ + | SOF Core Subsystem | + | - Core DSP Management (core.c, pcm.c, control.c, loader.c) | + | - Topology Driver (topology.c) | + | - IPC Infrastructure (ipc.c, ipc3-*, ipc4-*) | + | - Runtime PM & Power States (pm.c) | + +-------------------------------------------------------------+ + | | + v v + +----------------------------------+ +----------------------+ + | Hardware DSP Drivers | | Bus Interconnects | + | - Intel (CAVS, ACE 1.x, ACE 3.x)| | - PCI / ACPI | + | - NXP (i.MX8, i.MX95, RT1062) | | - SoundWire | + | - AMD / Mediatek / Qualcomm | | - I2S / HD-Audio | + +----------------------------------+ +----------------------+ + | + v + +-------------------------------------------------------------+ + | Target DSP Hardware | + | (Execution Units, SRAM, L2 Cache, DMA Engines) | + +-------------------------------------------------------------+ + +Key Subsystems +************** + +1. Core DSP Lifecycle & Management +================================== + +The core layer (``sound/soc/sof/core.c``) orchestrates DSP probe, boot, firmware load, and shutdown: + +* **Probe & Discovery**: Identifies DSP hardware capabilities via PCI IDs, ACPI tables (e.g., NHLT), or Device Tree nodes. +* **Firmware Loader** (``loader.c``): Loads signed firmware images (generated by ``rimage``) using the standard Linux firmware subsystem (``request_firmware()``). +* **DSP Operations** (``struct snd_sof_dsp_ops``): An operations table implemented by each hardware backend (Intel, NXP, AMD, Mediatek) providing platform hooks for run, reset, power, memory read/write, and interrupt handling. + +2. IPC Subsystem: IPC3 vs IPC4 +============================== + +The SOF driver supports two major Inter-Processor Communication (IPC) protocols: + +* **IPC3 Protocol (Classic)**: + * Uses fixed header envelopes (``struct sof_ipc_cmd_hdr``). + * Relies on shared SRAM mailboxes for small commands and data payloads. + * Extensively used on CAVS platforms (Apollo Lake through Tiger Lake). +* **IPC4 Protocol (Modern Envelope)**: + * Uses bit-packed 64-bit message headers consisting of primary and extension registers. + * Designed for modern Intel ACE architectures (Meteor Lake, Arrow Lake, Panther Lake) and modular audio processing pipelines. + * Decouples control messaging from large payload data, providing gateway and module-specific configuration parameters. + +3. Mailbox and Interrupt Signaling +================================== + +Communication between host CPU and DSP relies on shared SRAM mailboxes and doorbell interrupts: + +* **Host-to-DSP (H2D)**: + 1. Host writes message headers into the shared inbound mailbox (inbox). + 2. Host asserts the H2D doorbell interrupt. + 3. DSP handles the interrupt, copies the message into local cache/SRAM, and clears the doorbell. + 4. DSP writes any response into the shared outbound mailbox (outbox) and asserts the D2H doorbell. +* **DSP-to-Host (D2H)**: + 1. DSP writes notifications (e.g., pipeline status, errors, trace position) into the outbox. + 2. DSP asserts the D2H doorbell interrupt. + 3. Host interrupt service routine (ISR) reads the outbox, dispatches the event to the appropriate handler, and acknowledges the interrupt. + +4. Topology Parsing & Component Creation +======================================== + +The SOF topology engine (``topology.c``) translates compiled ALSA topology binaries (``.tplg`` files generated by ``alsatplg``) into DSP runtime objects: + +* **Widgets**: Maps ALSA DAPM widgets to DSP components (Pipelines, Mixers, Volume, SRC, Equalizer, Copier). +* **Pipelines**: Instantiates scheduling pipelines with defined priorities, time periods (e.g., 1ms), and core affinities. +* **Routes & Buffers**: Establishes audio data links and allocates shared ring buffers between processing components. +* **Kcontrols**: Exposes mixer controls, volume sliders, and runtime configuration switches to ALSA user space. + +Power Management & Runtime PM +***************************** + +Audio DSPs require aggressive power management to achieve multi-day battery life on client devices: + +* **Active (D0)**: DSP cores are powered, streaming audio pipelines are active, and DMA engines are transferring data. +* **Autonomous Low-Power Idle (D0ix)**: Audio is streaming, but DSP cores dynamically clock-gate or enter low-power wait states between processing intervals. +* **Suspended (D3 / D3cold / D3hot)**: + * DSP memory contents are saved or powered off. + * When entering system suspend (S3 / S0ix), the driver tears down active streams, saves hardware contexts, and gates DSP power rails. + * On resume, the driver repowers the DSP, reloads firmware, restores kcontrol states, and restarts audio pipelines seamlessly. + +Platform DSP Operations Hook +**************************** + +Hardware drivers register their operations via ``snd_sof_dsp_ops``: + +.. code-block:: c + + struct snd_sof_dsp_ops { + /* DSP core boot and reset */ + int (*probe)(struct snd_sof_dev *sdev); + int (*remove)(struct snd_sof_dev *sdev); + int (*run)(struct snd_sof_dev *sdev); + int (*reset)(struct snd_sof_dev *sdev); + + /* Power management */ + int (*suspend)(struct snd_sof_dev *sdev, u32 target_state); + int (*resume)(struct snd_sof_dev *sdev); + int (*runtime_suspend)(struct snd_sof_dev *sdev); + int (*runtime_resume)(struct snd_sof_dev *sdev); + + /* IPC communication */ + int (*send_msg)(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg); + irqreturn_t (*irq_handler)(int irq, void *context); + irqreturn_t (*irq_thread)(int irq, void *context); + + /* Debug & Telemetry */ + void (*dbg_dump)(struct snd_sof_dev *sdev, u32 flags); + }; diff --git a/developer_guides/linux_driver/index.rst b/developer_guides/linux_driver/index.rst index c6534d70..e00e454c 100644 --- a/developer_guides/linux_driver/index.rst +++ b/developer_guides/linux_driver/index.rst @@ -3,8 +3,11 @@ SOF Linux Driver ################ +The Sound Open Firmware (SOF) Linux kernel driver subsystem provides upstream ALSA System on Chip (ASoC) support for digital signal processors (DSPs) across Intel, NXP, AMD, and Mediatek hardware architectures. + .. toctree:: :maxdepth: 1 + architecture + machine_drivers_quirks third_party/index - diff --git a/developer_guides/linux_driver/machine_drivers_quirks.rst b/developer_guides/linux_driver/machine_drivers_quirks.rst new file mode 100644 index 00000000..59138232 --- /dev/null +++ b/developer_guides/linux_driver/machine_drivers_quirks.rst @@ -0,0 +1,172 @@ +.. _sof_linux_machine_drivers_quirks: + +Machine Drivers & DMI Quirks +############################ + +In the ALSA System on Chip (ASoC) subsystem, the **Machine Driver** acts as the glue that binds together the platform DSP audio driver (SOF), the audio codec/amplifier drivers, and the physical audio interfaces (I2S, SoundWire, HD-Audio, DMIC). + +Because hardware manufacturers frequently design unique motherboard routing, clock schemes, and GPIO mappings, machine drivers rely on **DMI Quirk Tables** and **ACPI metadata** to configure sound cards accurately for each specific laptop or desktop platform. + +Role of the Machine Driver +************************** + +While the SOF core driver handles DSP firmware execution and IPC messaging, the machine driver specifies: + +* **DAI Links (Digital Audio Interfaces)**: Establishes connections between DSP PCM frontends and physical backend DAI controllers (e.g., SSP/I2S, SoundWire, SoundWire-Link, or PDM/DMIC). +* **Audio Routing & DAPM**: Defines physical audio paths between codec input/output pins, amplifiers, and external jacks (headphone, microphone, internal stereo speakers). +* **Jack Detection**: Configures GPIO interrupts and codec triggers to notify user space when headphones or headsets are plugged in. +* **Clock Configuration**: Programs system clocks (MCLK, BCLK, frame sync) and PLL dividers for audio codecs. + +ACPI Discovery & Hardware Tables +******************************** + +On x86 platforms, the Linux kernel relies on firmware tables provided by the platform BIOS / UEFI: + +1. **NHLT (Non-HD Audio Link Table)**: + * Describes physical audio endpoints connected to DSP interfaces (e.g., DMIC arrays, I2S codecs). + * Specifies audio formats, supported sample rates, channel configurations, and vendor-specific data. +2. **SoundWire DISCO (Device Information & Configuration Overrides)**: + * Provided via ACPI ``_DSD`` (Device Specific Data) properties. + * Identifies attached SoundWire target peripheral devices, link IDs, bus clock frequencies, and manufacturer device IDs. + +.. note:: + **OEM Firmware Shortcuts**: On systems designed primarily for Windows, Original Equipment Manufacturers (OEMs) and Original Design Manufacturers (ODMs) often cut development corners by hard-coding codec parameters directly into Windows driver packages while leaving ACPI tables incomplete or invalid. On Linux, where generic upstream drivers rely on compliant ACPI descriptors, incomplete tables prevent audio hardware initialization. Machine driver quirks provide the necessary overrides to support these devices. + +DMI Quirk Tables +**************** + +To support hardware with incomplete ACPI tables or proprietary audio routing, ASoC machine drivers maintain **DMI (Desktop Management Interface)** quirk tables. These tables match system BIOS strings (vendor, product name, motherboard model) and apply bitmasked hardware flags. + +Quirk Matching Example +====================== + +In `sound/soc/intel/boards/sof_rt5682.c`, the machine driver matches systems using ``struct dmi_system_id``: + +.. code-block:: c + + static const struct dmi_system_id sof_rt5682_quirk_table[] = { + { + .callback = sof_rt5682_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "0990"), + }, + .driver_data = (void *)(SOF_RT5682_MCLK_EN | + SOF_RT5682_SSP_CODEC(0) | + SOF_RT5682_NUM_HDMIDEV(3)), + }, + { + .callback = sof_rt5682_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Google"), + DMI_MATCH(DMI_PRODUCT_FAMILY, "Google_Volteer"), + }, + .driver_data = (void *)(SOF_RT5682_MCLK_EN | + SOF_RT5682_SSP_CODEC(0) | + SOF_RT5682_SSP_AMP(1) | + SOF_RT5682_NUM_HDMIDEV(4)), + }, + {} + }; + +Common Quirk Flags +================== + +Typical quirk flags specify: + +* ``SOF_SSP_CODEC(n)``: Specifies physical SSP/I2S port number connected to the primary codec. +* ``SOF_SSP_AMP(n)``: Specifies physical SSP/I2S port number connected to dedicated speaker amplifiers. +* ``SOF_RT5682_MCLK_EN``: Enables system clock (MCLK) output from DSP to codec. +* ``SOF_RT5682_MCLK_24MHZ``: Configures external oscillator frequency to 24 MHz instead of standard 19.2 MHz. +* ``SOF_BT_OFFLOAD_SSP(n)``: Designates SSP port for hardware Bluetooth audio offload. + +Step-by-Step: Adding a New DMI Quirk +************************************ + +When bringing up audio on a new laptop model where audio fails to initialize, follow this workflow: + +Step 1: Extract System DMI Strings +================================== + +Run ``dmidecode`` on the target device to obtain vendor, product name, and board information: + +.. code-block:: bash + + sudo dmidecode -t system + +Look for the following fields: + +.. code-block:: text + + Manufacturer: LENOVO + Product Name: 21AH002EUS + Family: ThinkPad T14 Gen 3 + +Step 2: Inspect Kernel Logs for Matching Failures +================================================= + +Review `dmesg` to identify which machine driver probed and whether fallback quirks were applied: + +.. code-block:: bash + + dmesg | grep -E "sof-|asoc|snd" + +Look for missing DAI link matches, failed codec clock initialization, or default fallback configurations. + +Step 3: Edit the Machine Driver +=============================== + +Locate the corresponding machine driver under ``sound/soc/intel/boards/`` (e.g., `sof_rt5682.c`, `sof_realtek_common.c`, or `sof_sdw.c` for SoundWire): + +1. Add a new entry to the `dmi_system_id` table: + +.. code-block:: c + + { + .callback = sof_rt5682_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21AH002EUS"), + }, + .driver_data = (void *)(SOF_RT5682_MCLK_EN | + SOF_RT5682_SSP_CODEC(0) | + SOF_RT5682_SSP_AMP(1)), + }, + +2. If custom GPIO or jack detection quirks are needed, update the board-specific initialization hooks. + +Step 4: Recompile and Install Modules +===================================== + +Rebuild the affected ASoC machine driver module: + +.. code-block:: bash + + make M=sound/soc/intel/boards modules + sudo make M=sound/soc/intel/boards modules_install + sudo depmod -a + +Step 5: Test and Verify +======================= + +Reload the audio drivers or reboot the system: + +.. code-block:: bash + + # Verify card detection + cat /proc/asound/cards + + # Inspect mixer controls + alsamixer -c 0 + + # Test playback + aplay -D plughw:0,0 test.wav + +ALSA Use Case Manager (UCM2) Integration +**************************************** + +Once the kernel machine driver binds the audio card and exposes ALSA mixer controls, user-space audio servers (PipeWire, PulseAudio) rely on **ALSA Use Case Manager (UCM2)** configuration profiles: + +* UCM profiles reside in `/usr/share/alsa/ucm2/`. +* Profiles map kernel mixer controls (e.g., `Speaker Switch`, `Headphone Volume`, `PGA Boost`) to standardized audio verbs (`HiFi`, `Record`, `VoiceCall`). +* For newly quirked platforms, ensure appropriate UCM device configurations exist to automatically manage routing, volume levels, and jack detection events. diff --git a/developer_guides/loadable_modules/lmdk_user_guide.rst b/developer_guides/loadable_modules/lmdk_user_guide.rst deleted file mode 100644 index b6085e01..00000000 --- a/developer_guides/loadable_modules/lmdk_user_guide.rst +++ /dev/null @@ -1,25 +0,0 @@ -.. _lmdk_user_guide: - -Loadable modules build guide using LMDK -####################################### - -What is LMDK -************ - -LMDK(Loadable Module Development Kit) is a standalone package required to build loadable module. It is independent from SOF FW but contains necessary data structures to interact with it. - -How to build -************ - -To build example loadable library execute: -.. code-block:: bash - - $ cd libraries/example - $ mkdir build - $ cd build - - $ cmake -DRIMAGE_COMMAND="/path/to/rimage" -DSIGNING_KEY="/path/to/signing/key.pem" .. - $ cmake --build . - -Here RIMAGE_COMMAND is path to rimage executable binary, SIGNING_KEY is path to -signing key for rimage. `LMDK ` diff --git a/developer_guides/rimage/index.rst b/developer_guides/rimage/index.rst index 2ac94ec7..1ac2c423 100644 --- a/developer_guides/rimage/index.rst +++ b/developer_guides/rimage/index.rst @@ -1,29 +1,96 @@ .. _rimage: -Rimage -###### +Rimage Firmware Image Creation & Signing +######################################## -Rimage is a DSP firmware image creation and signing tool used by -Sound Open Firmware (SOF) to generate binary image files. +**Rimage** is the official DSP firmware image packaging and cryptographic signing tool for Sound Open Firmware (SOF). Implemented in modern Rust (`thesofproject/rimage `_), rimage transforms compiled ELF executables into validated, hardware-loadable binary images (``.ri`` files) for Intel, NXP, AMD, and other DSP architectures. -Rimage contains a built-in generator for: +Rimage parses declarative target platform configuration files (TOML), verifies memory section alignments, packages dynamic modules and static manifests, and applies cryptographic digital signatures required by hardware boot ROMs. -#. Extended manifest - describes firmware metadata for drivers -#. CSE manifest -#. CSS manifest -#. ADSP manifest +Key Features & Capabilities +*************************** -For more details see: +* **Rust Architecture**: High-performance, memory-safe signing and image generation engine. +* **Declarative TOML Configuration**: Platform memory layouts, modules, and hardware parameters are defined in clean, human-readable TOML files under `config/`. +* **Hardware Manifest Generation**: + * **Extended Manifest**: Describes firmware versioning, compiler toolchains, and ABI metadata for the Linux host driver (`snd-sof`). + * **CSE Manifest**: Converged Security Engine descriptors for modern Intel platforms. + * **CSS Manifest**: Common Security Signature for Intel CAVS / ACE security coprocessors. + * **ADSP Manifest**: Audio DSP hardware memory segment descriptors. +* **Cryptographic Signing**: Supports RSA PKCS#1 v1.5 with SHA-256 and SHA-384, utilizing OpenSSL or pure Rust crypto backends. +* **IPC4 Multi-Module Packaging**: Bundles base firmware images with loadable library modules (LLEXT). .. toctree:: :maxdepth: 1 extended_manifest +TOML Platform Configuration +*************************** +Rimage relies on platform-specific TOML files to describe hardware memory mappings and signing requirements. For example, a target configuration defines memory segments, cache settings, and manifest types: -Build flow -========== +.. code-block:: toml -.. uml:: images/image_build_flow.pu - :caption: Image build generation + [platform] + name = "tgl" + arch = "xtensa" + + [manifest] + version = 4 + format = "cse" + + [[memory.regions]] + name = "iram" + vma = 0xa0000000 + size = 0x80000 + type = "code" + + [[memory.regions]] + name = "dram" + vma = 0xa0080000 + size = 0x60000 + type = "data" + +Command-Line Usage +****************** + +While rimage is typically invoked automatically by the Zephyr build system during `west build`, it can also be run standalone: + +.. code-block:: bash + + # Generate a signed Tiger Lake (TGL) image using test keys + rimage -k keys/otc_private.pem \ + -c config/tgl.toml \ + -o build/sof-tgl.ri \ + build/sof-tgl.elf + +Key Parameters: +=============== + +* ``-k, --key ``: Path to the RSA private key in PEM format used to sign the firmware image. +* ``-c, --config ``: Target platform TOML configuration file specifying memory layout and manifest rules. +* ``-o, --output ``: Target output path for the finalized binary image (`.ri`). +* ``-v, --verbose``: Enable verbose diagnostic output for inspecting section headers and offsets. + +Signing Keys and Production Workflow +************************************ + +1. **Development & Community Test Keys**: + SOF repositories include public test keys (e.g., `keys/otc_private.pem`) suitable for engineering samples and pre-production development hardware. +2. **Production OEM Keys**: + For commercial production devices, hardware boot ROMs enforce cryptographic verification against vendor fuses burned into the SoC. OEMs configure rimage to sign binaries with their secure hardware security modules (HSM) or offline private keys prior to distribution. + +Build System Integration +************************ + +In the modern Zephyr build workflow, rimage is integrated into the CMake toolchain as a post-build signing utility: + +.. code-block:: cmake + + # CMake hook automatically executed on successful elf link + add_custom_command( + TARGET sof_firmware POST_BUILD + COMMAND rimage -k ${RIMAGE_KEY} -c ${RIMAGE_CONFIG} -o ${BUILD_DIR}/sof.ri ${BUILD_DIR}/zephyr.elf + COMMENT "Packaging and signing SOF firmware with rimage" + ) diff --git a/developer_guides/setup_special_device/setup_up_2_board.rst b/developer_guides/setup_special_device/setup_up_2_board.rst deleted file mode 100644 index e4f36c6f..00000000 --- a/developer_guides/setup_special_device/setup_up_2_board.rst +++ /dev/null @@ -1,112 +0,0 @@ -.. _setup_up_2_board: - -Set up SOF on Up Squared board with Hifiberry DAC+ (STD) -######################################################## - -.. contents:: - :local: - :depth: 3 - -Prerequisites -************* - -Make sure you have the Standard version of Hifiberry DAC+. The Pro -version is not currently supported. - -Setup Instructions -****************** - -1. Flash BIOS version 4.0 onto the Up squared board. -====================================================== - -The BIOS main menu will show UP-APL01 R4.0. - -* Download the `BIOS `_. - -* If the current BIOS version is older than 1.8, please update to v1.8 - before flashing v4.0. - - .. note:: - - To check your BIOS version press - - 1) DELETE or - 2) F7 and select 'Enter Setup' - -* Press ENTER when prompted for password. - -* Use board `BIOS update `__ - instructions to flash the BIOS. - -2. Install Ubilinux or Ubuntu -============================= - -Press F7 and choose the Linux installation media as the boot device - -.. note:: - - Do not select UEFI. The built-in UEFI shell which will return you - to the BIOS menu. - -Use the `Ubilinux `__ installation -guide, if needed. - -3. Update kernel -================ - -Follow the :ref:`install-locally` guide. - -4. Firmware -=========== - -Build SOF firmware and copy ``sof-apl.ri`` into /lib/firmware/intel/sof - -5. Topology -=========== - -Copy test topology -``sof-apl-eq-pcm512x.tplg`` as -``sof-apl-pcm512x.tplg`` into /lib/firmware/intel/sof-tplg - -6. Add ACPI support for Hifiberry dac+ -====================================== - -Clone scripts from https://github.com/thesofproject/acpi-scripts - -.. code-block:: bash - - sudo ./install_hooks - sudo ./acpi-add Up2/PCM512X.asl - -Reboot and check if the status of the device is 15 - -.. code-block:: bash - - cat /sys/bus/acpi/devices/104C5122\:00/status - -7. Add sst drivers to blacklist-dsp.conf -======================================== - -Create blacklist-dsp.conf in /etc/modprobe.d/ if not exist - -:: - - blacklist snd\_soc\_sst\_acpi - blacklist snd\_soc\_sst\_dsp - blacklist snd\_soc\_sst\_firmware - blacklist snd\_soc\_sst\_ipc - blacklist snd\_soc\_sst\_match - blacklist snd\_soc\_skl - blacklist snd\_soc\_sst\_byt\_cht\_nocodec - blacklist snd\_intel\_sst\_acpi - blacklist snd\_intel\_sst\_core - blacklist snd\_hda\_intel - -8. Reboot -========= - -Make sure the green LED lights up on the Hifiberry. - -.. note:: - - If any problem has occured use ``dmesg | grep sof`` to track it. diff --git a/developer_guides/tech/cmake.rst b/developer_guides/tech/cmake.rst index 3edbbad3..af4ebccd 100644 --- a/developer_guides/tech/cmake.rst +++ b/developer_guides/tech/cmake.rst @@ -1,145 +1,127 @@ -.. _cmake: - -CMake Arguments -############### - -For firmware and unit tests only **TOOLCHAIN** and **ROOT_DIR** -arguments are mandatory. Other arguments are optional. - -For host build, only **BUILD_HOST** switch is needed. - -Firmware & Unit Tests -********************* - -Mandatory arguments for firmware and unit tests builds. - -TOOLCHAIN - Specifies toolchain to use, usually it's prefix to tools that - follow GCC naming convention. Toolchain should contain tools like: - - * -gcc - * -ar - * -objdump - * -objcopy - - There are more tools from GCC-like toolchains that may be used by build - system, but these are used in most cases. - For example toolchain *xtensa-apl-elf*, should have tools xtensa-apl-elf-gcc, - xtensa-apl-elf-ar, etc. - Toolchain has to be in PATH. - - .. code-block:: bash - - # Examples - cmake [...] -DTOOLCHAIN=xt [...] - cmake [...] -DTOOLCHAIN=xtensa-apl-elf [...] - cmake [...] -DTOOLCHAIN=xtensa-cnl-elf [...] - -ROOT_DIR - Path to directory with xtensa core's lib and include. - - .. code-block:: bash - - # Examples - cmake [...] -DROOT_DIR=$CONFIG_PATH/xtensa-elf [...] - cmake [...] -DROOT_DIR=/my-xtensa-newlib/xtensa-root/xtensa-apl-elf [...] - -Firmware -******** - -Optional arguments. Only for firmware. - -MEU_PATH - Path to directory with MEU tool. For example full path to MEU that will - be used, should be `$MEU_PATH/meu` or `$MEU_PATH/meu.exe`. - - .. code-block:: bash - - # Example - cmake [...] -DMEU_PATH=/path/to/meu/installation [...] - -MEU_PRIVATE_KEY - Path to file with key that will be used by meu. - - .. code-block:: bash - - # Example - cmake [...] -DMEU_PRIVATE_KEY=/path/to/meu/private-key.pem [...] - -MEU_OPENSSL - Default: /usr/bin/openssl - Path to OpenSSL binary used by MEU. Usually you should use it only - on Windows. - - .. code-block:: bash - - # Example - cmake [...] -DMEU_OPENSSL=C:/path/to/openssl.exe [...] - -FIRMWARE_NAME - Custom suffix for output binary. - - .. code-block:: bash - - # Example - cmake [...] -DFIRMWARE_NAME=custom [...] - -MEU_NO_SIGN - Flag that can be used to build unsigned FW binary, - that may be later used with MEU for signing. - - .. code-block:: bash - - # Example - cmake [...] -DMEU_NO_SIGN=ON [...] - -MEU_OFFSET - Default: determined by build-system, depends on MEU version. - Can be used to override MEU offset. - - .. code-block:: bash - - # Example - cmake [...] -DMEU_OFFSET=1344 [...] - -Unit Tests -********** - -Optional arguments. Only for unit tests. - -Read :ref:`unit_tests` first. - -BUILD_UNIT_TESTS - Default: OFF, if ON then builds unit tests. - - .. code-block:: bash - - # Example: build unit tests instead of firmware - cmake -DTOOLCHAIN=xt -DROOT_DIR=$CONFIG_PATH/xtensa-elf -DBUILD_UNIT_TESTS=ON [...] - -.. _cmocka-directory-label: - -CMOCKA_DIRECTORY - Path to directory with prebuilt Cmocka library. - Usually you shouldn't use it, because if this argument is not used, then - CMake will build Cmocka automatically for you in build directory. - Cmocka directory should contain include subdirectory with `cmocka.h` header - and lib subdirectory with `cmocka-static.a` library. - - .. code-block:: bash - - # Example - cmake [...] -DCMOCKA_DIRECTORY=/path/to/cmocka-install-apl [...] - -Host Testbench -************** - -Optional arguments. Only for host build. - -BUILD_HOST - Default: OFF, if ON then builds testbench for host. - - .. code-block:: bash - - # Example: build testbench instead of firmware - cmake -DBUILD_HOST=ON -DCMAKE_INSTALL_PREFIX=install [...] +.. _cmake: + +Zephyr CMake & Build Configuration +################################## + +Sound Open Firmware (SOF) builds as a native `Zephyr RTOS `_ application using CMake, Ninja, and the ``west`` meta-tool. This build architecture standardizes board definitions, Kconfig feature toggles, Device Tree hardware overlays, toolchain selection, and post-build binary signing across all supported DSP and microcontroller targets. + +Build System Architecture +************************* + +The SOF build pipeline executes through the following stages: + +.. code-block:: text + + +-------------------------------------------------------------+ + | west build | + | - Board Target (-b ) | + | - Kconfig Configurations (prj.conf + overlay-*.conf) | + | - Device Tree Overlays (*.overlay) | + +-------------------------------------------------------------+ + | + v + +-------------------------------------------------------------+ + | CMake & Ninja | + | - Toolchain Setup (LLVM / Zephyr SDK / Cadence XCC) | + | - C Compiler Flags (-DEXTRA_CFLAGS) | + | - Library & Module Linking (LLEXT, CMSIS, Xtensa HAL) | + +-------------------------------------------------------------+ + | + v + +-------------------------------------------------------------+ + | Firmware Artifacts | + | - zephyr.elf: Unstripped debug symbols & static logs | + | - smex: Extracts .ldc dictionary file | + | - rimage: Generates and cryptographically signs .ri image | + +-------------------------------------------------------------+ + +West Build Invocations +********************** + +Firmware compilation is invoked using `west build` from your SOF workspace: + +.. code-block:: bash + + # Build Tiger Lake (TGL) firmware using the LLVM toolchain + west build -b intel_adsp_cavs25 -d build-tgl app/ + + # Build Panther Lake (PTL / ACE 3.0) firmware + west build -b intel_ace30_ptl -d build-ptl app/ + + # Build Teensy 4.1 standalone hostless firmware + west build -b teensy41 -d build-teensy app/ + + # Build ESP32-P4 audio bridge firmware + west build -b esp32p4 -d build-esp32 app/ + +Common Build Options & CMake Flags +********************************** + +You can pass CMake flags to `west build` using the ``--`` delimiter: + +Toolchain Selection (``ZEPHYR_TOOLCHAIN_VARIANT``) +================================================== + +Specifies the active compiler backend: + +.. code-block:: bash + + # Use shared LLVM / Clang toolchain + export ZEPHYR_TOOLCHAIN_VARIANT=llvm + + # Use official Zephyr SDK cross-compilers + export ZEPHYR_TOOLCHAIN_VARIANT=zephyr + export ZEPHYR_SDK_INSTALL_DIR=/opt/zephyr-sdk + + # Use Cadence Xtensa XCC compiler (for proprietary DSP targets) + export ZEPHYR_TOOLCHAIN_VARIANT=xt-clang + +Kconfig Overlay Files (``FILE:EXTRA_CONF_FILE``) +================================================ + +Applies additional Kconfig fragments to enable specific features, debugging logs, or algorithm modules: + +.. code-block:: bash + + # Enable verbose DSP trace logging overlay + west build -b intel_adsp_cavs25 app/ -- -DFILE:EXTRA_CONF_FILE=overlay-debug.conf + + # Enable LLEXT dynamic module loading support + west build -b intel_ace15_mtlm app/ -- -DFILE:EXTRA_CONF_FILE=overlay-llext.conf + +Extra Compiler Flags (``EXTRA_CFLAGS``) +======================================= + +Injects custom C preprocessor defines or diagnostic flags into the build: + +.. code-block:: bash + + west build -b intel_adsp_cavs25 app/ -- -DEXTRA_CFLAGS="-Werror -DSOF_DEBUG_HOOKS=1" + +Interactive Kconfig Configuration (``menuconfig``) +************************************************** + +To inspect, search, and modify SOF firmware Kconfig options in an interactive terminal menu: + +.. code-block:: bash + + west build -t menuconfig + +From this interface, developers can toggle: + +* Supported audio components (Volume, Mixer, EQ, SRC, TDFB, TFLM). +* IPC version support (IPC3 vs IPC4). +* Log levels (`CONFIG_SOF_LOG_LEVEL_DBG`, `CONFIG_SOF_LOG_LEVEL_INF`). +* Hostless static pipeline presets. + +Host Testbench Compilation +************************** + +To build the host audio simulation testbench rather than DSP firmware: + +.. code-block:: bash + + cmake -B build-testbench -S tools/testbench \ + -DBUILD_TESTBENCH=ON \ + -DCMAKE_INSTALL_PREFIX=dist/ + cmake --build build-testbench -j$(nproc) diff --git a/developer_guides/tech/compile_wsl.rst b/developer_guides/tech/compile_wsl.rst deleted file mode 100644 index 43e3e134..00000000 --- a/developer_guides/tech/compile_wsl.rst +++ /dev/null @@ -1,104 +0,0 @@ -.. _compile_wsl: - -Build on Windows 10 with WSL -############################ - -Use these instructions to compile SOF on Windows using a 32-bit Linux -toolchain. - -Prerequisites -************* - -Download WSL (Windows Subsystem for Linux) from the Windows Store. -These instructions are written for Ubuntu WSL. Read how to install WSL at: https://docs.microsoft.com/en-us/windows/wsl/install-win10 - -Enable 32-bit binaries -********************** - -To use a 32-bit toolchain, enable 32-bit binaries by following these steps. - -#. Install QEMU that will run 32-bit binaries and then register it. - - .. code-block:: bash - - sudo apt update - sudo apt install qemu-user-static - sudo update-binfmts --install i386 /usr/bin/qemu-i386-static \ - --magic '\x7fELF\x01\x01\x01\x03\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x03\x00\x01\x00\x00\x00' \ - --mask '\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xf8\xff\xff\xff\xff\xff\xff\xff' - -#. Start the service that enables 32-bit support. Start it **every time** you want to enable 32-bit support; you can also add it to startup scripts if you want it to be enabled on every WSL launch. - - .. code-block:: bash - - sudo service binfmt-support start - -#. Add i386 arch for dpkg and install packages needed by most apps. - - .. code-block:: bash - - sudo dpkg --add-architecture i386 - sudo apt update - sudo apt install -y libc6:i386 libncurses5:i386 libstdc++6:i386 zlib1g:i386 zlib1g-dev:i386 - -Fix stat in 32-bit binaries -*************************** - -Many 32-bit apps cannot handle the 64-bit inodes of WSL filesystems. We can replace stat() with a function that partially supports 64-bit inodes by providing useful file properties. For example, even though it will return EOVERFLOW for file sizes >= 2^32, the struct with properties will contain some info. - -The following steps require GCC. Install it by entering: - -.. code-block:: bash - - sudo apt install gcc - -#. Build shared libs with the changed stat that will be used for preload. Download the source code with the modified stat here: https://raw.githubusercontent.com/jajanusz/sof-goodies/master/wsl_32bit_support/inode64.c - -#. Build it using the script below: - - .. code-block:: bash - - #!/bin/sh - - # info for ld - - cat > vers <| [Rx: SOF Audio Record] | + | |<--- I2S / S/PDIF ---| [Tx: SOF Audio Play] | + | [Rx: FFT & Analyzer] | | | + +--------------------------+ +--------------------------+ + | | + +-----------------------+------------------------+ + | + v + +-------------------------------+ + | Automated Test Controller | + | - test_p4_loopback.py | + | - test_c6_loopback.py | + | - test_teensy_loopback.py | + +-------------------------------+ + +Automated Test Suites +********************* + +SOF maintains automated Python test suites designed for continuous integration (CI) and pre-commit gate validation: + +ESP32-P4 Loopback (`test_p4_loopback.py`) +========================================= + +Validates I2S and PDM loopback using dual ESP32-P4 boards (Pallas as clock provider and Ceres as clock consumer): + +* **I2S Loopback Mode**: Tests 2-channel, 4-channel, and 8-channel TDM streams. +* **PDM Microphone Simulation**: Injects synthetic multi-frequency sine waves into the DUT's digital microphone (DMIC) inputs. +* **FFT & SNR Analysis**: Automatically computes Total Harmonic Distortion (THD), Signal-to-Noise Ratio (SNR), and phase alignment on captured buffers. + +ESP32-C6 Loopback (`test_c6_loopback.py`) +========================================= + +Validates lightweight I2S loopback on compact RISC-V nodes (e.g., Seeed Studio XIAO ESP32-C6, Waveshare ESP32-C6-Zero): + +* **Low-Power Verification**: Confirms audio frame synchronization and jitter margins under aggressive CPU frequency scaling. +* **Wireless Offload Verification**: Validates simultaneous audio streaming and IEEE 802.15.4 / Wi-Fi 6 radio activity. + +Teensy 4.1 Loopback (`test_teensy_loopback.py`) +=============================================== + +Validates high-bandwidth S/PDIF and multi-channel I2S on NXP i.MX RT1062: + +* **Clock Jitter Benchmarking**: Analyzes Audio PLL4 clock drift and PLL fractional dividers. +* **S/PDIF Protocol Compliance**: Verifies channel status bits, parity bits, and sample rate negotiation across 44.1 kHz, 48 kHz, 88.2 kHz, 96 kHz, and 192 kHz. + +Clock Role Configuration +************************ + +Audio bus testing requires evaluating both clock provider (transmitter generates BCLK and frame sync) and clock consumer (receiver synchronizes to external clocks) modes: + +* **Provider Mode**: The DSP or microcontroller drives the bit clock (BCLK) and word select (LRCLK / WCLK). +* **Consumer Mode**: The DSP listens to external clocks generated by the codec or audio bridge. + +On Linux DUTs, clock roles are toggled at runtime via ALSA mixer controls or through Topology 2.0 DAI definitions: + +.. code-block:: bash + + # Query current clock mode on target DUT + amixer -c 0 cget name='SSP0 Clock Role' + + # Set DUT as clock provider + amixer -c 0 cset name='SSP0 Clock Role' 'Provider' + +Rate and Format Verification Matrix +*********************************** + +Test suites systematically sweep the verification matrix to prevent format-conversion regressions: + +.. list-table:: + :widths: 20 25 25 30 + :header-rows: 1 + + * - Interface + - Sample Rates (kHz) + - Sample Formats + - Channel Configurations + * - **I2S / TDM** + - 16, 32, 44.1, 48, 96 + - S16_LE, S24_LE, S32_LE + - 2ch, 4ch TDM, 8ch TDM + * - **PDM / DMIC** + - 16, 48 + - S16_LE, S24_LE, S32_LE + - 1ch, 2ch, 4ch mic array + * - **S/PDIF** + - 44.1, 48, 88.2, 96, 192 + - S16_LE, S24_LE + - 2ch Stereo Linear PCM + +Executing Automated Loopback Tests +********************************** + +To run the complete automated test suite locally: + +.. code-block:: bash + + # Run ESP32-P4 I2S and PDM loopback validation + python3 scripts/test_p4_loopback.py --interface i2s --rate 48000 --channels 2 + python3 scripts/test_p4_loopback.py --interface pdm --rate 16000 --channels 4 + + # Run Teensy 4.1 S/PDIF validation + python3 scripts/test_teensy_loopback.py --interface spdif --rate 96000 diff --git a/developer_guides/topology/topology.rst b/developer_guides/topology/topology.rst index 3e2a23af..8bbaa3fd 100644 --- a/developer_guides/topology/topology.rst +++ b/developer_guides/topology/topology.rst @@ -1,7 +1,12 @@ .. _topology: -SOF Topology -############ +Legacy Topology 1.0 (M4-Based) +############################## + +.. note:: + **Legacy Topology Notice**: + This document describes the legacy M4 macro-based Topology 1.0 format. Modern SOF platforms standardize on ALSA Topology 2.0. For modern topology authoring, see :ref:`topology2`. + Topology defines the audio processing pipeline that is used by the firmware. In SOF, topologies are defined using M4_ macro language, diff --git a/developer_guides/unit_tests.rst b/developer_guides/unit_tests.rst index 355ff751..d312fb92 100644 --- a/developer_guides/unit_tests.rst +++ b/developer_guides/unit_tests.rst @@ -1,130 +1,217 @@ -.. _unit_tests: - -Unit Tests -########## - -Prerequisites -************* - -This guide assumes that you have the proper setup and that you know how to build firmware. If this is not correct, follow the instructions at :ref:`build_sof` first. - -`Cmocka `_ is fetched and built automatically. -For a successful compilation, it needs a toolchain thats supports C stdlib. - -Configuring for unit tests -************************** - -Unit tests are built from the same, top-level CMakeLists.txt as the -firmware but with different CMake flags: **-DBUILD_UNIT_TESTS=ON** and a -couple others. - -Building unit tests can be more complex than building the firmware -because for the firmware the script ``./xtensa-build-all.sh`` hides most -the CMake configuration. For unit tests you must find a working -combination of environment variables and CMake flags. Fortunately -``./xtensa-build-all.sh`` logs some of its magic that you can "steal" -and re-use to build unit tests. Like this: - -- Export ``XTENSA_TOOLS_ROOT`` as you normally do when building the - firmware. -- Build the firmware using ``./xtensa-build-all.sh`` and take note of the - following variables in the build log: ``PATH``, ``XTENSA_SYSTEM`` and - the ``-DROOT_DIR`` parameter. -- ``export`` the ``PATH`` and ``XTENSA_SYSTEM`` values found above. -- Run cmake with ``-DBUILD_UNIT_TESTS=ON``, the ``-DROOT_DIR`` parameter above, - ``-DINIT_CONFIG`` and a new build directory -- Build and run the tests with ``make test`` or ``ninja test``. - -.. note:: - - Use -DTOOLCHAIN=xt option. - - As of December 2021, -DTOOLCHAIN=xtensa--elf is not - supported. You can use a native toolchain, see below. - -If you get this double ``uintptr_t`` definition error: - -.. code-block:: bash - - [ 2%] Building C object test/cmocka/CMakeFiles/common_mock.dir/src/common_mocks.c.o - In file included from sof/test/cmocka/src/common_mocks.c:29: - sof/but/cmocka_git/src/cmocka_git/include/cmocka.h:132: - error: redefinition of typedef ‘uintptr_t’ - xcc/install/builds/RG-2017.8-linux/X4H3I16w2D48w3a_2017_8/xtensa-elf/include/stdint.h:252: - error: previous declaration of ‘uintptr_t’ was here - -... then append this to your cmake invocation: ``-DEXTRA_CFLAGS=-D_UINTPTR_T_DEFINED=1`` - -Additional unit tests options can be found in :ref:`cmake`. - -Example: Running tests for APL -============================== - -.. code-block:: bash - - mkdir build_ut && cd build_ut - cmake -DBUILD_UNIT_TESTS=ON -DTOOLCHAIN=xt -DINIT_CONFIG=apollolake_defconfig \ - -DROOT_DIR=/xcc/install/builds/RG-2017.8-linux/X4H3I16w2D48w3a_2017_8/xtensa-elf .. - make -j4 && ctest -j8 - -Compiling unit tests without a cross-compilation toolchain -========================================================== - -You can also compile and run unit tests with your native compiler: - -.. code-block:: bash - - rm -rf build_ut/ - cmake -B build_ut/ -DBUILD_UNIT_TESTS_HOST=yes \ - -DBUILD_UNIT_TESTS=ON -DINIT_CONFIG=something_defconfig - make -C build_ut/ -j8 && make -C build_ut/ test - -The ``scripts/run-cmocks.sh`` script does all that and can also run unit -tests with valgrind. - -Wrapping objects for unit tests -******************************* - -If you need to mock a symbol, define it in a unit test and include the .h file. There are two cases where this isn't possible: - -* Static functions in headers (those most probably are inline short functions - and don't have to be mocked). - -* Static functions that are in the same file as tested functionality and are - exceedingly large so they can't be tested as one functionality. - -Whatever the reason, mocking of those symbols can be done by using the --wrap linker functionality. To wrap the symbol follow these steps: - -#. Create mocked symbol named __wrap_symbol_name - -#. Pass instruction for the linker -Wl, --wrap=symbol_name during compilation. - -Now every symbol calls to symbol_name will call __wrap_symbol_name. - -Instructions can be passed to the linker in the SOF UT environment using -CFLAGS; however, they should be passed in separate variables in the makefile. - -Example: - -.. code-block:: cmake - - # some tests before ... - cmocka_test(pipeline_connect_upstream - pipeline_connect_upstream.c - ... - ) - target_link_libraries(pipeline_connect_upstream PRIVATE "-Wl,--wrap=symbol_name") - -Full information about wrapping can be found here: - -https://lwn.net/Articles/558106/ - -Notes -***** - -#. Use the **ctest -j** option while running tests that use xt-run - (to speed up tests significantly) by running multiple instances of the - xt-run simulator (it also speeds up the build if you have many unit tests). - -#. **ctest** only runs unit tests; to rebuild them, you have to explicitly - run **make**. +.. _unit_tests: + +Unit Testing with Zephyr Ztest & Twister +######################################## + +Sound Open Firmware (SOF) utilizes Zephyr's native **Ztest** testing framework and **Twister** test runner for unit testing and test-driven development (TDD). This modern testing architecture replaces legacy CMocka tests, seamlessly integrating SOF into the upstream Zephyr RTOS ecosystem. + +With Ztest and Twister, developers can compile and run firmware unit tests directly on the host machine using the **native_sim** target, verifying DSP processing algorithms, memory allocation, and pipeline lifecycle logic in milliseconds without requiring physical hardware or proprietary DSP toolchains. + +Architecture Overview +********************* + +Ztest unit tests execute in user space on the host development machine: + +.. code-block:: text + + +-------------------------------------------------------------+ + | Twister Test Runner | + | (Test Discovery, Parallel Execution, JUnit XML, Coverage) | + +-------------------------------------------------------------+ + | + v + +-------------------------------------------------------------+ + | native_sim Target | + | (Host x86_64 / Linux POSIX Sandbox) | + +-------------------------------------------------------------+ + | + v + +-------------------------------------------------------------+ + | Zephyr Ztest Suites | + | - Core Libs (math, lists, buffers, objpool) | + | - Audio Components (eq_fir, volume, mixer, tone, tflm) | + | - IPC Envelopes & Component Adapters | + +-------------------------------------------------------------+ + +Prerequisites & Environment Setup +********************************* + +Building and executing Ztest suites requires the Zephyr SDK, host build essentials, LLVM/Clang toolchain, and the ``west`` meta-tool. + +1. Install Host Dependencies +============================ + +.. code-block:: bash + + sudo apt-get update + sudo apt-get install -y clang llvm ninja-build device-tree-compiler \ + python3-pyelftools gcc-multilib g++-multilib + +2. Configure West Workspace +=========================== + +Ensure your SOF workspace is initialized with ``west``: + +.. code-block:: bash + + cd ~/work/sof + west init -l + west update --narrow --fetch-opt=--filter=tree:0 + +3. Set Toolchain Variant +======================== + +Configure Zephyr to use the LLVM/Clang compiler: + +.. code-block:: bash + + export ZEPHYR_TOOLCHAIN_VARIANT=llvm + +Running Unit Tests with Twister +******************************* + +The ``west twister`` command discovers, builds, and executes test suites across the repository. + +Executing All Unit Tests +======================== + +To execute all unit tests located under `sof/test/ztest/unit/` using the `native_sim` platform: + +.. code-block:: bash + + west twister --testsuite-root test/ztest/unit/ --platform native_sim \ + --verbose --inline-logs + +Twister outputs real-time test status to the terminal and records structured results, build logs, and reports in the `twister-out/` directory. + +Targeting Specific Test Suites +============================== + +To run a specific test suite or component (e.g., math or audio component tests): + +.. code-block:: bash + + # Run only math unit tests + west twister --testsuite-root test/ztest/unit/math/ --platform native_sim + + # Run matching a specific test scenario name + west twister --testsuite-root test/ztest/unit/ -s sof.unit.math --platform native_sim + +Generating Code Coverage Reports +================================ + +Twister integrates with `gcov` and `lcov` to calculate code coverage metrics: + +.. code-block:: bash + + west twister --testsuite-root test/ztest/unit/ --platform native_sim \ + --coverage -p native_sim + +Writing a Ztest Unit Test +************************* + +A typical SOF Ztest defines a test suite fixture (`setup`, `before`, `after`, `teardown`), initializes the mock SOF infrastructure (`sys_comp_init`), and validates component execution with assertions. + +Example: Testing an Audio Processing Component +=============================================== + +Below is an annotated example of a Ztest unit test for an audio filter component: + +.. code-block:: c + + // SPDX-License-Identifier: BSD-3-Clause + /* + * Copyright(c) 2026 Intel Corporation. + */ + + #include + #include + #include + #include + #include + #include + #include + + extern void sys_comp_module_eq_fir_interface_init(void); + + /* Suite setup fixture: runs once before all tests in this suite */ + static void *suite_setup(void) + { + struct sof *sof = sof_get(); + + /* Initialize SOF audio component framework */ + sys_comp_init(sof); + + if (!sof->ipc) { + sof->ipc = rzalloc(SOF_MEM_FLAG_COHERENT, sizeof(*sof->ipc)); + sof->ipc->comp_data = rzalloc(SOF_MEM_FLAG_COHERENT, 4096); + k_spinlock_init(&sof->ipc->lock); + list_init(&sof->ipc->msg_list); + list_init(&sof->ipc->comp_list); + } + + /* Register the component under test */ + sys_comp_module_eq_fir_interface_init(); + return NULL; + } + + /* Register the test suite with setup fixture */ + ZTEST_SUITE(sof_eq_fir_suite, NULL, suite_setup, NULL, NULL, NULL); + + /* Unit test case: verify component creation and parameter validation */ + ZTEST(sof_eq_fir_suite, test_eq_fir_create) + { + struct comp_dev *dev; + struct comp_ipc_config config = { + .id = 1, + .type = SOF_COMP_EQ_FIR, + .core = 0, + }; + + /* Test instantiation */ + dev = comp_new(&config); + zassert_not_null(dev, "Failed to create EQ FIR component"); + zassert_equal(dev->state, COMP_STATE_READY, "Component must initialize to READY state"); + + /* Free allocated component */ + comp_free(dev); + } + + /* Unit test case: verify processing with invalid channel configuration */ + ZTEST(sof_eq_fir_suite, test_eq_fir_invalid_channels) + { + struct comp_dev *dev; + struct comp_ipc_config config = { + .id = 2, + .type = SOF_COMP_EQ_FIR, + .core = 0, + }; + + dev = comp_new(&config); + zassert_not_null(dev, "Failed to create component"); + + /* Attempt to set invalid parameters */ + int ret = comp_set_attribute(dev, COMP_ATTR_CHANNELS, 0); + zassert_not_equal(ret, 0, "Zero channel count should return error"); + + comp_free(dev); + } + +Common Ztest Assertions +======================= + +Ztest provides robust macros that output descriptive failures when conditions are violated: + +* ``zassert_true(cond, msg)``: Asserts that a boolean condition is true. +* ``zassert_false(cond, msg)``: Asserts that a boolean condition is false. +* ``zassert_equal(a, b, msg)``: Asserts that two values are equal. +* ``zassert_not_equal(a, b, msg)``: Asserts that two values are not equal. +* ``zassert_not_null(ptr, msg)``: Asserts that a pointer is not ``NULL``. +* ``zassert_mem_equal(a, b, size, msg)``: Asserts that two memory buffers are bitwise identical. + +Deprecation Notice: Legacy CMocka +********************************* + +.. warning:: + **Legacy CMocka Deprecation**: + Prior versions of SOF used CMocka with custom build scripts (`scripts/run-cmocks.sh`). The CMocka framework has been deprecated and retired in favor of native Zephyr Ztest and Twister. All new unit tests must be authored using Ztest under `test/ztest/unit/`. From 5ff1abba793848b91100b0b3854b12337ad3b315 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Thu, 17 Sep 2026 14:54:46 +0100 Subject: [PATCH 02/64] docs: developer_guides: add high-level pipeline architecture guide Create comprehensive, high-level documentation explaining the SOF pipeline architecture without delving into low-level C code: - Explain pipeline containers, boundaries, and multi-core affinity. - Detail audio modules, sink/source pins, and topology models. - Contrast Low-Latency (LL) periodic timer scheduling against Data Processing (DP) asynchronous Zephyr RTOS threads. - Detail data movement through circular ring buffers and producer/consumer pointer mechanics. - Document the end-to-end construction, parameter propagation, streaming, and destruction lifecycle. - Detail the runtime state machine (INIT, READY, PRE_ACTIVE, ACTIVE, PAUSED, SUSPEND, XRUN_PAUSED) and trigger flows. - Explain audio XRUN detection and self-healing recovery. - Include 7 vector Graphviz diagrams illustrating each subject. - Cross-reference upstream src/audio/pipeline/README.md for implementation details and C struct definitions. - Update architectures/index.rst and developer_guides/index.rst. Signed-off-by: Liam Girdwood --- architectures/index.rst | 3 + .../firmware/pipeline_architecture.rst | 463 ++++++++++++++++++ developer_guides/index.rst | 9 +- 3 files changed, 471 insertions(+), 4 deletions(-) create mode 100644 developer_guides/firmware/pipeline_architecture.rst diff --git a/architectures/index.rst b/architectures/index.rst index 8c1a36d0..14698d0a 100644 --- a/architectures/index.rst +++ b/architectures/index.rst @@ -605,6 +605,9 @@ At the heart of the firmware is the audio processing pipeline framework: ctl_eq -> comp_eq [style=dashed, color="#d35400", label="IPC Set Data"]; } +.. seealso:: + For an in-depth architectural explanation of how pipelines, modules, Low-Latency (LL) and Data Processing (DP) scheduling domains, circular buffers, lifecycle management, and the runtime state machine operate, see the :ref:`pipeline_architecture` developer guide. + Topology 2 Architecture ======================= diff --git a/developer_guides/firmware/pipeline_architecture.rst b/developer_guides/firmware/pipeline_architecture.rst new file mode 100644 index 00000000..325bf082 --- /dev/null +++ b/developer_guides/firmware/pipeline_architecture.rst @@ -0,0 +1,463 @@ +.. _pipeline_architecture: + +Pipeline Architecture +##################### + +The **Pipeline Engine** is the core real-time audio scheduling and signal processing framework of Sound Open Firmware (SOF). It organizes audio processing components into connected execution graphs, coordinates data movement through circular ring buffers, enforces strict real-time deadlines, and provides an end-to-end operational state machine. + +This guide provides a high-level conceptual overview of how pipelines, modules, scheduling domains, and buffer queues function inside the DSP firmware without focusing on low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +1. What is an SOF Pipeline? +*************************** + +A **Pipeline** in SOF is a logical execution container that groups a collection of audio processing components and buffers into a single, cohesive scheduling entity. + +.. graphviz:: + :caption: Pipeline Containers, Inter-Pipeline Buffers, and Multi-Core Distribution + :align: center + + digraph pipeline_system { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_core0 { + label = "DSP Core 0 (Real-Time I/O Domain)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1b4f72"; + + subgraph cluster_pipe1 { + label = "Pipeline 1 (Low-Latency Host Ingest, Period: 1ms)"; + style = "filled,rounded"; + color = "#2471a3"; + fillcolor = "#d4e6f1"; + fontname = "Verdana-Bold"; + fontsize = 9; + + p1_host [label="Host Endpoint\n(DMA Reader)", fillcolor="#aed6f1"]; + p1_vol [label="Volume / Mute\n(Linear Gain)", fillcolor="#aed6f1"]; + p1_host -> p1_vol [label="Buffer"]; + } + + subgraph cluster_pipe3 { + label = "Pipeline 3 (Low-Latency DAI Egress, Period: 1ms)"; + style = "filled,rounded"; + color = "#16a085"; + fillcolor = "#d1f2eb"; + fontname = "Verdana-Bold"; + fontsize = 9; + + p3_vol [label="Main Volume\n(Soft Ramp)", fillcolor="#a3e4d7"]; + p3_dai [label="DAI Endpoint\n(I2S / SoundWire)", fillcolor="#a3e4d7"]; + p3_vol -> p3_dai [label="Buffer"]; + } + } + + subgraph cluster_core1 { + label = "DSP Core 1 (Heavy Compute / Offload Domain)"; + style = "filled,rounded"; + color = "#8e44ad"; + fillcolor = "#f4ecf7"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#512e5f"; + + subgraph cluster_pipe2 { + label = "Pipeline 2 (Data Processing Domain, Asynchronous Thread)"; + style = "filled,rounded"; + color = "#7d3c98"; + fillcolor = "#e8daef"; + fontname = "Verdana-Bold"; + fontsize = 9; + + p2_eq [label="Parametric EQ\n(10-Band Biquad)", fillcolor="#d7bde2"]; + p2_drc [label="Dynamic Range\nCompressor (DRC)", fillcolor="#d7bde2"]; + p2_aec [label="Echo Cancellation\n(AEC / NS)", fillcolor="#d7bde2"]; + + p2_eq -> p2_drc -> p2_aec [label="Buffer"]; + } + } + + /* Inter-pipeline connections */ + p1_vol -> p2_eq [color="#e67e22", penwidth=2, label="Inter-Pipeline\nShared Buffer"]; + p2_aec -> p3_vol [color="#e67e22", penwidth=2, label="Cross-Core\nShared Buffer"]; + } + +Why Separate Pipelines? +======================= + +Rather than placing all audio processing modules into one monolithic loop, SOF partitions graphs into distinct pipelines for three primary reasons: + +1. **Scheduling Boundaries**: Modules inside the same pipeline execute at the same scheduling period (e.g., 1ms low-latency intervals vs 10ms bulk processing). +2. **Core Affinity**: Different pipelines can be bound to separate DSP processor cores (e.g., Core 0 handles high-speed DMA transfers, while Core 1 handles intensive beamforming or AI inference). +3. **Power and Lifecycle Partitioning**: An input pipeline can remain active to capture microphone data while a playback pipeline is paused and powered down into a low-power sleep state. + +--- + +2. Audio Modules & Pin Interfaces +********************************* + +An **Audio Module** (or component) is the atomic building block of signal processing in SOF. Modules accept incoming audio frames on **Sink Pins** (inputs), process or transform the samples, and produce processed frames on **Source Pins** (outputs). + +.. graphviz:: + :caption: Anatomy of an SOF Audio Processing Module + :align: center + + digraph module_anatomy { + rankdir=LR; + nodesep=0.4; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + in_buf1 [label="Input Buffer 1\n(e.g., 48kHz Stereo)", fillcolor="#ebf5fb", shape=ellipse]; + in_buf2 [label="Input Buffer 2\n(e.g., Reference Audio)", fillcolor="#ebf5fb", shape=ellipse]; + + subgraph cluster_module { + label = "Audio Processing Module (Component)"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1e8449"; + + sink_pins [label="Sink Pins (Inputs)\n- Format negotiation\n- Minimum frame checks", fillcolor="#a9dfbf"]; + core_dsp [label="Internal DSP Kernel\n- SIMD / VFPU Math\n- In-place / Copy logic\n- Filter state history", fillcolor="#2ecc71", fontcolor="#ffffff", style="filled,bold"]; + src_pins [label="Source Pins (Outputs)\n- Buffer advance\n- Produced frame count", fillcolor="#a9dfbf"]; + + sink_pins -> core_dsp -> src_pins; + } + + out_buf [label="Output Buffer\n(Processed Audio)", fillcolor="#fef9e7", shape=ellipse]; + + ipc_ctl [label="Host Control Interface (IPC)\n- Volume level sliders\n- Equalizer coefficient blobs\n- Mute / Bypass switches", fillcolor="#fad7a0", shape=note]; + + in_buf1 -> sink_pins [label="Audio Data In"]; + in_buf2 -> sink_pins [label="Reference In"]; + src_pins -> out_buf [label="Audio Data Out"]; + ipc_ctl -> core_dsp [style=dashed, color="#d35400", label="Runtime Parameters"]; + } + +Module Topology Configurations +============================== + +Modules support different pin topologies depending on their functional role: + +* **Single-Input Single-Output (SISO)**: Standard processing filters such as Volume, Equalizer (EQ), Sample Rate Converter (SRC), and Dynamic Range Compressor (DRC). +* **Multi-Input Single-Output (MISO)**: Components that combine multiple audio streams into one, such as the Audio Mixer or Mixin/Mixout blocks. +* **Single-Input Multi-Output (SIMO)**: Components that split or replicate audio streams, such as the Demux, Channel Splitter, or Audio Copier. +* **Endpoints**: Components that bridge the DSP with external hardware: + * **Ingress Endpoints**: Host DMA readers (from host PC memory) and DAI receivers (from digital microphones or line-in). + * **Egress Endpoints**: Host DMA writers (to host PC memory for recording) and DAI transmitters (to speaker codecs or S/PDIF). + +--- + +3. Scheduling Domains: Low-Latency (LL) vs Data Processing (DP) +*************************************************************** + +Audio signal processing has diverse timing requirements. Simple volume adjustment must happen with sub-millisecond determinism to prevent hardware dropouts, whereas complex algorithms like Acoustic Echo Cancellation (AEC) or neural speech enhancement require flexible execution windows. + +To resolve these conflicting demands, SOF separates pipeline execution into two primary **Scheduling Domains**: + +.. list-table:: + :widths: 20 40 40 + :header-rows: 1 + + * - Characteristic + - Low-Latency (LL) Domain + - Data Processing (DP) Domain + * - **Trigger Source** + - Hardware timer tick (e.g., 1ms) or DMA completion interrupt. + - Asynchronous Zephyr RTOS thread notification when buffer data is available. + * - **Execution Model** + - Single cooperative task iterates through all modules in the graph synchronously. + - Independent preemptive or cooperative Zephyr RTOS thread with dedicated stack. + * - **Deadline Requirement** + - Hard real-time deadline; must complete within the 1ms timeslice. + - Soft real-time deadline; can buffer data across multiple milliseconds. + * - **Typical Modules** + - Host DMA Copier, Volume, Mixer, Tone Generator, DAI Copier. + - AEC, Beamforming (TDFB), Keyword Detect (WoV), TensorFlow Lite Micro (TFLM). + +Execution Comparison +==================== + +.. graphviz:: + :caption: LL Synchronous Periodic Walk vs DP Asynchronous Thread Processing + :align: center + + digraph scheduling_comparison { + rankdir=TB; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_ll { + label = "Low-Latency (LL) Domain Execution (Every 1ms Hardware Tick)"; + style = "filled,rounded"; + color = "#2471a3"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1b4f72"; + + hw_tick [label="Timer / DMA Interrupt\n(Ticks every 1000 µs)", shape=diamond, fillcolor="#aed6f1"]; + task_run [label="Pipeline Task Scheduled\n(Cooperative Execution)", fillcolor="#d4e6f1"]; + step_host [label="1. Read Host DMA Buffer\n(Fetch 48 frames)", fillcolor="#aed6f1"]; + step_vol [label="2. Apply Volume & Gain\n(In-place vector math)", fillcolor="#aed6f1"]; + step_dai [label="3. Write DAI Buffer\n(Transmit to Hardware)", fillcolor="#aed6f1"]; + task_done [label="Task Completes in < 150 µs\n(CPU enters low-power idle)", fillcolor="#abebc6"]; + + hw_tick -> task_run -> step_host -> step_vol -> step_dai -> task_done; + } + + subgraph cluster_dp { + label = "Data Processing (DP) Domain Execution (Autonomous Zephyr Thread)"; + style = "filled,rounded"; + color = "#7d3c98"; + fillcolor = "#f4ecf7"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#512e5f"; + + dp_wait [label="DP Thread Waiting on Semaphore\n(Thread Suspended, 0% CPU)", fillcolor="#e8daef"]; + dp_wake [label="Woken by LL Domain Buffer Write\n(160 frames available)", fillcolor="#d7bde2"]; + dp_proc [label="Execute Heavy Compute\n- Acoustic Echo Cancellation\n- Multi-channel Beamforming\n(Spans 4 to 8 ms across chunks)", fillcolor="#bb8fce"]; + dp_post [label="Push Processed Chunk into Output Buffer\nSignal Downstream Consumer", fillcolor="#d7bde2"]; + + dp_wait -> dp_wake -> dp_proc -> dp_post -> dp_wait [label="Loop"]; + } + + step_vol -> dp_wake [style=dashed, color="#e67e22", penwidth=2, label="Buffer threshold reached\n(Signals DP Thread)"]; + } + +--- + +4. Data Movement & Buffer Queues +******************************** + +Audio samples move through the pipeline via continuous **Circular Ring Buffers**. Rather than allocating dynamic memory packets on every audio tick, SOF pre-allocates cache-aligned circular memory pools during pipeline initialization. + +The Producer-Consumer Model +=========================== + +Every buffer connects an upstream **Producer** module to a downstream **Consumer** module: + +.. graphviz:: + :caption: Circular Buffer Producer-Consumer Queue Model + :align: center + + digraph circular_buffer { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + prod [label="Upstream Producer\n(e.g., Volume Module)\n\nWrites new audio samples\nAdvances Write Pointer", fillcolor="#a9dfbf", style="filled,bold"]; + + subgraph cluster_queue { + label = "Circular Ring Buffer in On-Chip SRAM (e.g., 1024 bytes)"; + style = "filled,rounded"; + color = "#d35400"; + fillcolor = "#fef5e7"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#a04000"; + + cell_free1 [label="Free Space", fillcolor="#ffffff", style=dotted]; + cell_data1 [label="Audio Data\nFrame 0-47", fillcolor="#fad7a0"]; + cell_data2 [label="Audio Data\nFrame 48-95", fillcolor="#fad7a0"]; + cell_free2 [label="Free Space", fillcolor="#ffffff", style=dotted]; + + cell_free1 -> cell_data1 [style=invis]; + cell_data1 -> cell_data2 [style=invis]; + cell_data2 -> cell_free2 [style=invis]; + } + + cons [label="Downstream Consumer\n(e.g., Equalizer Module)\n\nReads unread audio samples\nAdvances Read Pointer", fillcolor="#aed6f1", style="filled,bold"]; + + prod -> cell_data2 [color="#27ae60", penwidth=2, label="Write Pointer\n(write_ptr)"]; + cell_data1 -> cons [color="#2980b9", penwidth=2, label="Read Pointer\n(read_ptr)"]; + } + +Key Buffer Properties +===================== + +* **Occupancy Tracking**: The buffer tracks the number of unconsumed bytes currently stored (available to read) and the remaining free space (available to write). +* **Wrap-Around Handling**: When either the write or read pointer reaches the boundary of the allocated buffer, it wraps back to the starting memory address. +* **Cache Coherency & Memory Tiers**: + * For single-core pipelines, buffers reside in fast on-chip SRAM with zero-copy shared memory access. + * For cross-core pipelines, cache lines are invalidated and written back to ensure memory consistency across DSP cores. + +--- + +5. Pipeline Construction & Destruction Lifecycle +************************************************ + +Pipelines are created and destroyed dynamically by host drivers or instantiated statically during boot on hostless microcontrollers. The lifecycle consists of six distinct phases: + +.. graphviz:: + :caption: Pipeline Lifecycle: From Instantiation to Streaming and Destruction + :align: center + + digraph lifecycle { + rankdir=TB; + nodesep=0.25; + ranksep=0.35; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + step1 [label="1. Instantiation\n- Allocate pipeline tracking container\n- Assign unique Pipeline ID, priority, and core affinity", fillcolor="#d4e6f1"]; + step2 [label="2. Component Creation\n- Instantiate required modules (Host, Volume, EQ, DAI)\n- Configure default control parameters", fillcolor="#d4e6f1"]; + step3 [label="3. Graph Binding (Connect)\n- Allocate circular audio buffers between modules\n- Establish directional edges from source to sink", fillcolor="#d4e6f1"]; + step4 [label="4. Graph Validation (Complete)\n- Firmware walks entire graph from source to sink\n- Validates stream connections, rates, and formats\n- Pipeline transitions to READY state", fillcolor="#a9dfbf"]; + step5 [label="5. Parameter Propagation (Prepare)\n- Finalize PCM sample rate, channel maps, and bit depth\n- Initialize filter delay lines and clear buffers\n- Allocate task execution slots in scheduler", fillcolor="#a9dfbf"]; + step6 [label="6. Audio Streaming (Trigger Start)\n- Hardware timers / DMA interrupts begin firing\n- Pipeline actively processes audio frames", fillcolor="#2ecc71", fontcolor="#ffffff", style="filled,bold"]; + step7 [label="7. Teardown & Destruction (Free)\n- Stop trigger halts audio stream\n- Unschedule pipeline tasks\n- Detach and release ring buffers\n- Free module instances and pipeline container memory", fillcolor="#fadbd8"]; + + step1 -> step2 -> step3 -> step4 -> step5 -> step6 -> step7; + } + +1. **Instantiation**: The pipeline manager creates the container, registers a mailbox offset for host status notifications, and configures scheduling attributes. +2. **Component Creation**: Individual audio processing modules are instantiated in DSP memory. +3. **Graph Binding**: Audio buffers are attached between output pins and input pins, forming the Directed Acyclic Graph (DAG). +4. **Graph Validation**: The pipeline engine traverses the entire graph (`pipeline_complete`) to verify that all connections are valid, there are no unlinked endpoints, and no routing cycles exist. +5. **Parameter Propagation**: Stream parameters (e.g., 48 kHz, 32-bit float, stereo) propagate across all modules. Buffers calculate required period sizes, and DSP filters allocate scratch memory. +6. **Streaming**: The host issues a start trigger. The pipeline scheduler attaches to hardware interrupts and audio streaming commences. +7. **Teardown**: When the audio stream terminates, the host driver stops the pipeline, cancels scheduled tasks, flushes lingering audio samples, frees circular buffers, and reclaims heap memory. + +--- + +6. Pipeline & Module State Machine +********************************** + +Every pipeline and component operates according to a well-defined **State Machine**. Operational triggers command state transitions, which propagate down the graph from source to sink: + +.. graphviz:: + :caption: SOF Pipeline State Transition Diagram + :align: center + + digraph state_machine { + rankdir=TB; + nodesep=0.4; + ranksep=0.4; + node [shape=circle, style="filled", fontname="Verdana-Bold", fontsize=9, width=1.3, height=1.3, fixedsize=true]; + edge [fontname="Verdana", fontsize=8, color="#2c3e50"]; + + node [fillcolor="#eaeded"] INIT; + node [fillcolor="#d4e6f1"] READY; + node [fillcolor="#fef9e7"] PRE_ACTIVE; + node [fillcolor="#abebc6"] ACTIVE; + node [fillcolor="#fcf3cf"] PAUSED; + node [fillcolor="#ebdef0"] SUSPEND; + node [fillcolor="#fadbd8"] XRUN_PAUSED; + + /* State Transitions */ + INIT -> READY [label="pipeline_complete()\n(Graph validated)", color="#2980b9", fontcolor="#2980b9"]; + READY -> PRE_ACTIVE [label="TRIGGER_PRE_START\n(Clock ramp)", color="#27ae60", fontcolor="#27ae60"]; + PRE_ACTIVE -> ACTIVE [label="TRIGGER_START\n(Begin streaming)", color="#27ae60", fontcolor="#27ae60", penwidth=2]; + + ACTIVE -> PAUSED [label="TRIGGER_PAUSE\n(Host pause)", color="#f39c12", fontcolor="#b7950b"]; + PAUSED -> ACTIVE [label="TRIGGER_RELEASE\n(Host unpause)", color="#27ae60", fontcolor="#27ae60"]; + + ACTIVE -> SUSPEND [label="TRIGGER_SUSPEND\n(System sleep D3)", color="#8e44ad", fontcolor="#8e44ad"]; + SUSPEND -> ACTIVE [label="TRIGGER_RESUME\n(System wake D0)", color="#27ae60", fontcolor="#27ae60"]; + + ACTIVE -> READY [label="TRIGGER_STOP / RESET\n(Stream closed)", color="#c0392b", fontcolor="#c0392b"]; + PAUSED -> READY [label="TRIGGER_STOP\n(Stream aborted)", color="#c0392b", fontcolor="#c0392b"]; + + ACTIVE -> XRUN_PAUSED [label="TRIGGER_XRUN\n(Buffer under/overflow)", color="#e74c3c", fontcolor="#e74c3c", penwidth=2]; + XRUN_PAUSED -> READY [label="pipeline_xrun_recover()\n(Self-healing reset)", color="#2980b9", fontcolor="#2980b9"]; + } + +State Descriptions +================== + +* **INIT**: The pipeline is newly allocated; modules and buffers are being instantiated and bound. +* **READY**: The graph is completely constructed, validated, and initialized with stream parameters. It is idle and ready to stream. +* **PRE_ACTIVE**: An intermediate transition state where hardware clocks, PLLs, and power rails stabilize prior to sample delivery. +* **ACTIVE**: The pipeline is actively processing audio frames on every scheduling interval. +* **PAUSED**: Audio processing is halted upon host request, but sample history, filter coefficients, and buffer allocations are preserved for instant resume. +* **SUSPEND**: The DSP is entering a low-power system sleep state (S3 / S0ix / D3). Module states are preserved in retention memory or saved to host memory. +* **XRUN_PAUSED**: An audio buffer underrun or overrun has occurred; processing is temporarily suspended to prevent noise bursts or system panics while recovery executes. + +--- + +7. Error Handling & XRUN Self-Healing +************************************* + +In real-time audio systems, timing jitter can cause **XRUNs**: + +* **Underrun (Starvation)**: The consumer attempts to read audio data, but the buffer is empty because the producer has not produced samples in time. +* **Overrun (Overflow)**: The producer attempts to write audio data, but the buffer is full because the consumer has fallen behind. + +.. graphviz:: + :caption: Automated XRUN Detection and Self-Healing Recovery Workflow + :align: center + + digraph xrun_recovery { + rankdir=TB; + nodesep=0.3; + ranksep=0.35; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + detect [label="1. XRUN Detection\nHardware endpoint or buffer detects starvation/overflow\nFlags xrun_bytes counter", fillcolor="#fadbd8"]; + propagate [label="2. Immediate Pipeline Broadcast\nPipeline triggers COMP_TRIGGER_XRUN\nTransitions all components into XRUN_PAUSED state\nPrevents buffer corruption and audible pops", fillcolor="#f5b7b1"]; + signal [label="3. Host Notification\nUpdates stream position and registers error flag in IPC mailbox", fillcolor="#fad7a0"]; + self_heal [label="4. Self-Healing Recovery (pipeline_xrun_recover)\n- Resets downstream buffer read/write pointers\n- Re-runs pipeline_prepare() to synchronize clocks\n- Automatically issues internal COMP_TRIGGER_START", fillcolor="#a9dfbf", style="filled,bold"]; + resumed [label="5. Streaming Resumed\nAudio stream restarts seamlessly without crashing the sound card", fillcolor="#2ecc71", fontcolor="#ffffff", style="filled,bold"]; + + detect -> propagate -> signal -> self_heal -> resumed; + } + +Automated Self-Healing +====================== + +Rather than allowing an underrun to crash the audio subsystem, SOF features an automated **Self-Healing Recovery** mechanism: + +1. When a hardware DAI or DMA endpoint detects starvation, it immediately flags an XRUN. +2. The pipeline engine halts active processing, dropping components into `XRUN_PAUSED` to avoid rendering corrupted memory. +3. The scheduler checks the `xrun_bytes` flag. Unless explicitly disabled by firmware configuration, the pipeline: + * Flushes stale samples from affected ring buffers. + * Reinitializes read and write pointers to a safe initial offset. + * Re-prepares the pipeline components. + * Issues an internal `START` trigger to seamlessly resume streaming. + +--- + +8. Upstream Code Reference & Next Steps +*************************************** + +For developers seeking low-level C implementation details, data structures, and function prototypes: + +* **Upstream Pipeline Specification**: Consult the comprehensive code-level design guide in the main SOF repository at `thesofproject/sof: src/audio/pipeline/README.md `_. +* **Source Files**: + * `pipeline-graph.c`: Graph traversal, component binding, and route discovery. + * `pipeline-stream.c`: State machine triggers (`START`, `STOP`, `PAUSE`, `RESET`). + * `pipeline-params.c`: Parameter propagation, period configuration, and format negotiation. + * `pipeline-schedule.c`: Task scheduling, low-latency execution loops, and timer binding. + * `pipeline-xrun.c`: Overrun/underrun detection and self-healing recovery. + +Related Guides +============== + +* :ref:`topology2`: How pipelines and widgets are declared using ALSA Topology 2.0 configuration classes. +* :ref:`sof_hostless_firmware`: How to create static pipelines compiled into ROM for standalone microcontrollers. +* :ref:`llext_modules`: Building dynamic loadable modules (LLEXT) that integrate into SOF pipelines. +* :ref:`dbg-traces`: Monitoring pipeline execution and buffer positions in real time via TCP trace streaming. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 46eb115a..e5135e3b 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -31,7 +31,7 @@ The SOF firmware repository maintains detailed, up-to-date specifications for ea Core Infrastructure & Pipeline ------------------------------ -* `Pipeline Architecture `_ +* :ref:`pipeline_architecture` (High-level architecture; also see upstream `pipeline README `_) * `Audio Buffer Management `_ * `Scheduler `_ * `Module Framework `_ @@ -73,14 +73,15 @@ Detailed filter design, coefficient generation, and tuning workflows: algorithms/src/sample_rate_conversion algorithms/tdfb/time_domain_fixed_beamformer -Firmware Packaging, Modules & Hostless Mode -=========================================== +Pipeline Architecture, Packaging & Modules +========================================== -Firmware image packaging, cryptographic signing, loadable modules, and standalone hostless embedded firmware: +Core pipeline architecture, firmware image packaging, cryptographic signing, loadable modules, and standalone hostless embedded firmware: .. toctree:: :maxdepth: 1 + firmware/pipeline_architecture rimage/index.rst firmware/llext_modules firmware/hostless_firmware From d4a12da2a5923170ded9669bc148a253d6bfec71 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Thu, 17 Sep 2026 21:58:58 +0100 Subject: [PATCH 03/64] docs: developer_guides: add high-level module framework guide Create comprehensive, high-level documentation explaining the SOF Audio Processing Module Framework and Module Adapter without delving into low-level C code: - Document the Three-Tier architecture: Pipeline Schedulers, Module Adapter system layer, and Standardized Module Interface. - Detail the Module Adapter proxy and sandboxing container. - Explain the decoupled Source and Sink APIs (Get -> Manipulate -> Commit/Release pattern). - Detail pin topologies (SISO, MISO, SIMO) and dynamic graph binding. - Document the module runtime state machine (MODULE_DISABLED, MODULE_INITIALIZED, MODULE_IDLE, MODULE_PROCESSING). - Detail runtime parameter management (initialization blobs, large coefficient blobs, scalar controls via IPC). - Explain memory sandboxing (dp_heap_user, objpool) and automated leak protection (mod_free_all). - Include 7 vector Graphviz diagrams illustrating each concept. - Cross-reference upstream src/module/README.md and src/audio/module_adapter/README.md. - Update developer_guides/index.rst and pipeline_architecture.rst. Signed-off-by: Liam Girdwood --- data/sof_bin_releases.json | 22 +- .../firmware/module_framework.rst | 522 ++++++++++++++++++ .../firmware/pipeline_architecture.rst | 3 +- developer_guides/index.rst | 4 +- 4 files changed, 537 insertions(+), 14 deletions(-) create mode 100644 developer_guides/firmware/module_framework.rst diff --git a/data/sof_bin_releases.json b/data/sof_bin_releases.json index 95d46b51..da388e9f 100644 --- a/data/sof_bin_releases.json +++ b/data/sof_bin_releases.json @@ -1,4 +1,15 @@ [ + { + "tag_name": "v2026.09", + "name": "v2026.09", + "fw_version": "N/A", + "published_at": "2026-09-17", + "html_url": "https://github.com/thesofproject/sof-bin/releases/tag/v2026.09", + "asset_name": "sof-bin-2026.09.tar.gz", + "asset_url": "https://github.com/thesofproject/sof-bin/releases/download/v2026.09/sof-bin-2026.09.tar.gz", + "asset_size_mb": 16.7, + "prerelease": false + }, { "tag_name": "v2025.12.2", "name": "v2025.12.2", @@ -119,16 +130,5 @@ "asset_url": "https://github.com/thesofproject/sof-bin/releases/download/v2024.06/sof-bin-2024.06.tar.gz", "asset_size_mb": 9.4, "prerelease": false - }, - { - "tag_name": "v2024.03", - "name": "v2024.03", - "fw_version": "v2.9", - "published_at": "2024-04-02", - "html_url": "https://github.com/thesofproject/sof-bin/releases/tag/v2024.03", - "asset_name": "sof-bin-2024.03.tar.gz", - "asset_url": "https://github.com/thesofproject/sof-bin/releases/download/v2024.03/sof-bin-2024.03.tar.gz", - "asset_size_mb": 7.8, - "prerelease": false } ] \ No newline at end of file diff --git a/developer_guides/firmware/module_framework.rst b/developer_guides/firmware/module_framework.rst new file mode 100644 index 00000000..c1849798 --- /dev/null +++ b/developer_guides/firmware/module_framework.rst @@ -0,0 +1,522 @@ +.. _module_framework: + +Module Framework Architecture +############################# + +The **Audio Processing Module Framework** provides the standardized component interface and execution environment for all signal processing algorithms in Sound Open Firmware (SOF). By decoupling audio algorithms from low-level RTOS scheduling primitives, hardware platform drivers, and inter-processor communication (IPC) protocols, the module framework enables signal processing engineers to write portable, reusable audio processing blocks. + +This architecture supports both statically linked in-tree processing modules (Volume, Equalizers, Mixers, Sample Rate Converters) and dynamically loaded third-party proprietary libraries (via Zephyr LLEXT), ensuring strict memory sandboxing and automated leak protection. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +1. Architecture & Three-Tier Model +********************************** + +The SOF module architecture is organized into three distinct tiers: the **Standardized Module Interface**, the **Runtime Processing Module Instance**, and the **Module Adapter**: + +.. graphviz:: + :caption: Three-Tier Architecture: Pipeline Schedulers to Concrete Audio Modules + :align: center + + digraph module_architecture { + rankdir=TB; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_sched { + label = "Tier 1: SOF Core Pipeline Schedulers"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1b4f72"; + + ll_sched [label="Low-Latency (LL) Scheduler\n(1ms Hardware Timer / DMA Interrupts)", fillcolor="#aed6f1"]; + dp_sched [label="Data Processing (DP) Scheduler\n(Asynchronous Zephyr RTOS Threads)", fillcolor="#aed6f1"]; + } + + subgraph cluster_adapter { + label = "Tier 2: Module Adapter System Layer (Sandbox & Proxy)"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1e8449"; + + ma_proxy [label="Module Adapter Component Proxy\n(Masquerades as standard comp_dev)", fillcolor="#a9dfbf"]; + ma_ipc [label="IPC Parameter & Config Dispatcher\n(Decodes Set/Get Value and Set/Get Data)", fillcolor="#a9dfbf"]; + ma_mem [label="Memory Sandbox Manager\n(Component Heap & Object Pool Tracking)", fillcolor="#a9dfbf"]; + } + + subgraph cluster_interface { + label = "Tier 3: Standardized Module Framework & Processing APIs"; + style = "filled,rounded"; + color = "#8e44ad"; + fillcolor = "#f4ecf7"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#512e5f"; + + mod_ops [label="Standardized Operations\n(init, prepare, process, reset, free)", fillcolor="#d7bde2"]; + src_api [label="Source API (Inputs)\nsource_get_data / release", fillcolor="#d7bde2"]; + snk_api [label="Sink API (Outputs)\nsink_get_buffer / commit", fillcolor="#d7bde2"]; + } + + subgraph cluster_modules { + label = "Concrete Audio Processing Modules"; + style = "filled,rounded"; + color = "#d35400"; + fillcolor = "#fef5e7"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#a04000"; + + mod_vol [label="Volume / Mute\n(SIMD Vector Math)", fillcolor="#fad7a0"]; + mod_eq [label="Parametric EQ\n(FIR / IIR Filters)", fillcolor="#fad7a0"]; + mod_aec [label="Echo Cancellation\n(AEC / Beamformer)", fillcolor="#fad7a0"]; + mod_dyn [label="Loadable Dynamic Module\n(Zephyr LLEXT / Vendor IP)", fillcolor="#f5b041", style="filled,bold"]; + } + + ll_sched -> ma_proxy [label="Trigger / Copy"]; + dp_sched -> ma_proxy [label="Thread Exec"]; + + ma_proxy -> mod_ops [label="Invokes"]; + ma_proxy -> ma_mem [label="Manages"]; + ma_ipc -> ma_proxy [label="IPC Events"]; + + mod_ops -> mod_vol; + mod_ops -> mod_eq; + mod_ops -> mod_aec; + mod_ops -> mod_dyn; + + mod_vol -> src_api [style=dashed, label="Read"]; + mod_vol -> snk_api [style=dashed, label="Write"]; + mod_eq -> src_api [style=dashed, label="Read"]; + mod_eq -> snk_api [style=dashed, label="Write"]; + } + +The Core Architectural Concepts +=============================== + +1. **Standardized Module Operations (`module_interface`)**: + A uniform set of function callbacks (`init`, `prepare`, `process`, `reset`, `free`, and `set_configuration`) that every audio algorithm must implement. Because the interface is generic, the algorithm requires no knowledge of whether it is running on a real-time interrupt tick, inside an asynchronous RTOS worker thread, or within an offline simulation testbench. + +2. **Runtime Module Instance (`processing_module`)**: + The runtime state of an instantiated module. It contains instance-specific metadata, negotiated audio format descriptors (sample rate, channel count, sample bit depth), memory pointers, and references to connected audio streams. + +3. **Module Adapter (`module_adapter`)**: + The architectural glue and sandboxing layer. To the pipeline scheduler, the adapter looks like a standard pipeline component. Internally, it manages the module's lifecycle, allocates dedicated memory, handles parameter blobs from host IPC messages, and dispatches audio samples through standardized input and output APIs. + +--- + +2. The Module Adapter & Sandboxing Container +******************************************** + +The **Module Adapter** wraps internal DSP kernels and third-party processing engines, acting as a secure protective sandbox between the untrusted algorithm and the core operating system: + +.. graphviz:: + :caption: Module Adapter Container: Encapsulation, State Control, and IPC Translation + :align: center + + digraph module_adapter_container { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_external { + label = "Pipeline Environment"; + style = "filled,rounded"; + color = "#2c3e50"; + fillcolor = "#ebedef"; + fontname = "Verdana-Bold"; + fontsize = 9; + + pipe_call [label="Pipeline Engine\n- Scheduling triggers\n- Buffer links", fillcolor="#d5dbdb"]; + ipc_cmd [label="Host Driver IPC\n- Set/Get parameter blobs\n- Control sliders", fillcolor="#d5dbdb"]; + } + + subgraph cluster_adapter_box { + label = "Module Adapter Wrapper (Security & Isolation Boundary)"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1e8449"; + + proxy_api [label="Component Interface Proxy\n- Intercepts comp_copy()\n- Intercepts comp_trigger()\n- Validates stream states", fillcolor="#a9dfbf"]; + heap_mgr [label="Isolated Component Heap\n- Dedicated memory pool\n- Object pool tracking\n- Auto-free on teardown", fillcolor="#a9dfbf"]; + ipc_trans [label="IPC Translation Engine\n- Deserializes config blobs\n- Bounds-checks buffer sizes\n- Dispatches to module ops", fillcolor="#a9dfbf"]; + + subgraph cluster_inner_mod { + label = "Audio Processing Kernel"; + style = "filled,rounded"; + color = "#d35400"; + fillcolor = "#fef5e7"; + fontname = "Verdana-Bold"; + fontsize = 9; + fontcolor = "#a04000"; + + inner_state [label="Module Internal State\n- Filter delay lines\n- Biquad coefficients\n- Scratch memory buffers", fillcolor="#fad7a0"]; + inner_kernel [label="Signal Processing Kernel\n(Pure Math / SIMD Transform)", fillcolor="#f39c12", fontcolor="#ffffff", style="filled,bold"]; + + inner_kernel -> inner_state; + } + + proxy_api -> inner_kernel [label="Execute"]; + ipc_trans -> inner_state [label="Apply Config"]; + heap_mgr -> inner_state [label="Allocates"]; + } + + pipe_call -> proxy_api [label="comp_copy()"]; + ipc_cmd -> ipc_trans [label="IPC Config"]; + } + +Adapter Responsibilities +======================== + +* **Scheduler Translation**: Translates pipeline commands (`comp_new`, `comp_prepare`, `comp_copy`, `comp_free`) into clean module callbacks (`init`, `prepare`, `process`, `free`). +* **Memory Isolation**: Restricts module allocations to dedicated component memory heaps so that third-party code cannot corrupt global RTOS heaps. +* **Leak Protection**: Automatically logs and frees any lingering module memory allocations when the component is destroyed. +* **Format Negotiation**: Checks that incoming audio formats meet the module's declared mathematical constraints (e.g., verifying that a 16-bit module does not receive unformatted 32-bit floating-point data). + +--- + +3. Standardized Processing Interface: Source & Sink APIs +******************************************************** + +In traditional audio drivers, processing components often access circular ring buffer memory directly through raw pointers. This tightly couples the algorithm to buffer wrap-around mathematics and DMA alignment quirks. + +The SOF Module Framework decouples algorithms from buffers through the **Source and Sink APIs**. Modules operate in a clean **"Get → Manipulate → Commit/Release"** execution flow: + +.. graphviz:: + :caption: Source and Sink API Execution Pattern + :align: center + + digraph source_sink_flow { + rankdir=TB; + nodesep=0.25; + ranksep=0.35; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + step1 [label="1. Module Triggered\nPipeline scheduler invokes module's process() entry point", fillcolor="#d4e6f1"]; + step2 [label="2. Source Request (source_get_data)\nModule requests N input frames from the Source API\nSource API verifies available samples and returns a contiguous read pointer", fillcolor="#aed6f1"]; + step3 [label="3. Sink Reservation (sink_get_buffer)\nModule requests N output frames from the Sink API\nSink API verifies available space and returns a contiguous write pointer", fillcolor="#aed6f1"]; + step4 [label="4. Execute Audio Algorithm\nModule reads from read pointer, executes mathematical transformations,\nand writes processed samples to write pointer", fillcolor="#2ecc71", fontcolor="#ffffff", style="filled,bold"]; + step5 [label="5. Source Release (source_release_data)\nModule notifies Source API of the exact number of frames consumed,\nadvancing the upstream read pointer", fillcolor="#abebc6"]; + step6 [label="6. Sink Commit (sink_commit_buffer)\nModule notifies Sink API of the exact number of frames written,\nadvancing the downstream write pointer and validating data for consumers", fillcolor="#abebc6"]; + step7 [label="7. Yield to Scheduler\nProcess operation returns status (success or error code) to the adapter", fillcolor="#d4e6f1"]; + + step1 -> step2 -> step3 -> step4 -> step5 -> step6 -> step7; + } + +Source API (Inputs) +=================== + +* Modules request readable frames by invoking ``source_get_data()``. +* The API abstracts circular buffer wrap-around, providing safe contiguous memory blocks. +* Upon completing execution, the module calls ``source_release_data()`` with the exact number of frames consumed. If a module cannot process all available frames during this tick, unconsumed frames remain buffered for the next execution period. + +Sink API (Outputs) +================== + +* Modules reserve writable space by invoking ``sink_get_buffer()``. +* Once processed samples are written into the buffer, the module calls ``sink_commit_buffer()`` with the number of valid produced frames. +* The commit operation makes the newly processed samples immediately visible to downstream components. + +--- + +4. Pin Topologies & Stream Binding +********************************** + +Audio modules connect to other components and buffers through directional pins: + +* **Sink Pins (Inputs)**: Accept audio data streams from upstream components. +* **Source Pins (Outputs)**: Deliver processed audio streams to downstream components. + +.. graphviz:: + :caption: Supported Module Pin Topologies + :align: center + + digraph pin_topologies { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_siso { + label = "Single-Input Single-Output (SISO)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 9; + + siso_in [label="Input Buffer", shape=ellipse, fillcolor="#ffffff"]; + siso_comp [label="In-Line Filter\n(Volume / EQ / DRC / SRC)", fillcolor="#aed6f1"]; + siso_out [label="Output Buffer", shape=ellipse, fillcolor="#ffffff"]; + + siso_in -> siso_comp [label="Sink Pin 0"]; + siso_comp -> siso_out [label="Source Pin 0"]; + } + + subgraph cluster_miso { + label = "Multi-Input Single-Output (MISO)"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 9; + + miso_in1 [label="Stream A Buffer", shape=ellipse, fillcolor="#ffffff"]; + miso_in2 [label="Stream B Buffer", shape=ellipse, fillcolor="#ffffff"]; + miso_comp [label="Audio Mixer\n(Summing Bus)", fillcolor="#a9dfbf"]; + miso_out [label="Mixed Buffer", shape=ellipse, fillcolor="#ffffff"]; + + miso_in1 -> miso_comp [label="Sink Pin 0"]; + miso_in2 -> miso_comp [label="Sink Pin 1"]; + miso_comp -> miso_out [label="Source Pin 0"]; + } + + subgraph cluster_simo { + label = "Single-Input Multi-Output (SIMO)"; + style = "filled,rounded"; + color = "#8e44ad"; + fillcolor = "#f4ecf7"; + fontname = "Verdana-Bold"; + fontsize = 9; + + simo_in [label="Multi-Ch Buffer", shape=ellipse, fillcolor="#ffffff"]; + simo_comp [label="Demux / Splitter\n(Channel Router)", fillcolor="#d7bde2"]; + simo_out1 [label="Ch 0-1 Buffer", shape=ellipse, fillcolor="#ffffff"]; + simo_out2 [label="Ch 2-3 Buffer", shape=ellipse, fillcolor="#ffffff"]; + + simo_in -> simo_comp [label="Sink Pin 0"]; + simo_comp -> simo_out1 [label="Source Pin 0"]; + simo_comp -> simo_out2 [label="Source Pin 1"]; + } + } + +Dynamic Pin Binding +=================== + +Pins are not hard-coded into the firmware executable; they are dynamically bound and unbound at runtime based on topology directives or host IPC commands: + +* **Binding (`comp_bind`)**: Connects an upstream module's source pin to a downstream module's sink pin through an intermediate audio buffer. +* **Unbinding (`comp_unbind`)**: Safely detaches pins when an audio pipeline is torn down or rerouted. + +--- + +5. Module Runtime State Machine +******************************* + +Every processing module is strictly governed by a uniform runtime state machine managed by the `module_adapter`. Modules must adhere to the transitions defined by ``enum module_state``: + +.. graphviz:: + :caption: Module Runtime State Transition Diagram + :align: center + + digraph module_state_machine { + rankdir=TB; + nodesep=0.4; + ranksep=0.4; + node [shape=circle, style="filled", fontname="Verdana-Bold", fontsize=9, width=1.4, height=1.4, fixedsize=true]; + edge [fontname="Verdana", fontsize=8, color="#2c3e50"]; + + node [fillcolor="#eaeded"] MODULE_DISABLED; + node [fillcolor="#d4e6f1"] MODULE_INITIALIZED; + node [fillcolor="#fcf3cf"] MODULE_IDLE; + node [fillcolor="#abebc6"] MODULE_PROCESSING; + + MODULE_DISABLED -> MODULE_INITIALIZED [label="init()\n(Allocates scratch memory,\nparses init config)", color="#2980b9", fontcolor="#2980b9"]; + MODULE_INITIALIZED -> MODULE_DISABLED [label="free()\n(Releases component heap)", color="#c0392b", fontcolor="#c0392b"]; + + MODULE_INITIALIZED -> MODULE_IDLE [label="prepare()\n(Negotiates sample rates,\nclears filter history)", color="#27ae60", fontcolor="#27ae60"]; + MODULE_IDLE -> MODULE_INITIALIZED [label="reset()\n(Flushes stream history)", color="#f39c12", fontcolor="#b7950b"]; + + MODULE_IDLE -> MODULE_PROCESSING [label="trigger(START)\n(Begins audio processing)", color="#27ae60", fontcolor="#27ae60", penwidth=2]; + MODULE_PROCESSING -> MODULE_IDLE [label="trigger(STOP / PAUSE)\n(Suspends processing)", color="#c0392b", fontcolor="#c0392b"]; + } + +State Definitions +================= + +* **`MODULE_DISABLED`**: The module is uninstantiated or has been freed. Zero memory or execution slots are allocated. +* **`MODULE_INITIALIZED`**: The module has successfully executed its `.init()` callback. It has parsed static initialization configuration parameters and allocated necessary internal structures (delay lines, coefficient arrays). +* **`MODULE_IDLE`**: The module has executed `.prepare()`. Stream formats (sample rates, channel maps, sample bit depths) are fully negotiated and agreed upon. The algorithm is ready to stream. +* **`MODULE_PROCESSING`**: The pipeline has issued a `START` trigger. The module's `.process()` function is actively transforming audio buffers on every scheduling tick. + +--- + +6. Parameter & Configuration Management +*************************************** + +Audio processing components require dynamic runtime tuning—such as adjusting equalizer cutoffs, modifying compressor thresholds, or setting speaker protection parameters. + +The Module Framework separates configuration into three primary delivery channels: + +.. graphviz:: + :caption: Configuration Dispatch: Static Blobs, Runtime Blobs, and Scalar Controls + :align: center + + digraph config_dispatch { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_host { + label = "Host User Space (ALSA / UCM2 / sof-ctl)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 9; + + host_init [label="Topology Manifest (.tplg)\n- Static filter defaults", fillcolor="#aed6f1"]; + host_blob [label="Binary Coefficient Blob\n(e.g., 10-Band EQ Matrix)", fillcolor="#aed6f1"]; + host_kctl [label="Mixer Control Switch\n(e.g., Volume Fader / Mute)", fillcolor="#aed6f1"]; + } + + subgraph cluster_ipc { + label = "IPC Messaging Gateway"; + style = "filled,rounded"; + color = "#d35400"; + fillcolor = "#fef5e7"; + fontname = "Verdana-Bold"; + fontsize = 9; + + ipc_init [label="IPC Component New", fillcolor="#fad7a0"]; + ipc_data [label="IPC Set Data (Large Payload)", fillcolor="#fad7a0"]; + ipc_val [label="IPC Set Value (Immediate)", fillcolor="#fad7a0"]; + } + + subgraph cluster_mod { + label = "Processing Module"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 9; + + handler_init [label=".init() Parser\n(Applies default configuration)", fillcolor="#a9dfbf"]; + handler_data [label=".set_configuration() Callback\n(Parses multi-byte filter blobs)", fillcolor="#a9dfbf"]; + handler_kctl [label="Direct Control Binding\n(Updates gain / coefficients in-place)", fillcolor="#a9dfbf"]; + } + + host_init -> ipc_init -> handler_init; + host_blob -> ipc_data -> handler_data; + host_kctl -> ipc_val -> handler_kctl; + } + +1. **Static Initialization Blobs**: Delivered when the module is first instantiated via topology. Specifies initial configurations such as default filter modes or speaker models. +2. **Large Runtime Blobs (Set Data)**: Used for multi-kilobyte binary payloads (e.g., acoustic echo cancellation calibration matrices, custom FIR filter impulse responses). Delivered over shared host-DSP SRAM mailboxes. +3. **Immediate Scalar Values (Set Value)**: High-speed, lightweight commands used for volume faders, mute switches, or channel routing indices without allocation overhead. + +--- + +7. Memory Sandboxing & Leak Protection +************************************** + +To guarantee system stability, SOF isolates module allocations from global RTOS memory pools. This is especially vital when integrating third-party proprietary audio engines or dynamically loaded LLEXT modules: + +.. graphviz:: + :caption: Memory Sandboxing: Global System Heap vs Component Heap with Object Tracking + :align: center + + digraph memory_sandboxing { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_global { + label = "Global RTOS Memory Space"; + style = "filled,rounded"; + color = "#7f8c8d"; + fillcolor = "#f2f4f4"; + fontname = "Verdana-Bold"; + fontsize = 9; + + sys_heap [label="System Global Heap\n(Kernel structs, DMA queues,\ninterrupt vectors)\n\nPROTECTED FROM MODULES", fillcolor="#d5dbdb", style="filled,bold"]; + } + + subgraph cluster_sandbox { + label = "Module Adapter Component Sandbox (dp_heap_user)"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1e8449"; + + comp_heap [label="Dedicated Component Heap\n(Allocated per module instance)", fillcolor="#a9dfbf"]; + + subgraph cluster_objpool { + label = "Object Pool (Tracking Table)"; + style = "filled,rounded"; + color = "#d35400"; + fillcolor = "#fef5e7"; + fontname = "Verdana-Bold"; + fontsize = 8; + + obj1 [label="Tracked Block 1\n(Filter Delay Line)", fillcolor="#fad7a0"]; + obj2 [label="Tracked Block 2\n(Coeff Matrix)", fillcolor="#fad7a0"]; + obj3 [label="Tracked Block 3\n(Scratch Buffer)", fillcolor="#fad7a0"]; + + obj1 -> obj2 -> obj3 [style=invis]; + } + + comp_heap -> obj1; + comp_heap -> obj2; + comp_heap -> obj3; + } + + cleanup [label="Automated Cleanup (mod_free_all)\nOn component destruction, adapter iterates\nthrough Object Pool and frees all tracked\nallocations automatically", fillcolor="#abebc6", shape=note]; + + cluster_objpool -> cleanup [style=dashed, color="#27ae60"]; + } + +Memory Protection Features +========================== + +* **Isolated Allocation Pool (`dp_heap_user`)**: Modules allocate scratch buffers and persistent delay lines from their assigned component heap partition rather than competing with kernel heaps. +* **Tracked Object Pool (`objpool`)**: Every allocation is registered in a tracking pool associated with the `processing_module`. +* **Automatic Garbage Collection on Teardown**: When an audio stream closes, the Module Adapter calls ``mod_free_all()``. Even if a third-party algorithm neglects to free internal scratch buffers during its `.free()` callback, the adapter reclaims every registered memory block automatically, completely preventing memory leaks. + +--- + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level C implementation details, data structures, and function prototypes: + +* **Upstream Module Specification**: Consult the modern module API design document in the SOF repository at `thesofproject/sof: src/module/README.md `_. +* **Module Adapter Design Guide**: Consult the container and sandboxing guide at `thesofproject/sof: src/audio/module_adapter/README.md `_. +* **Core Header Files**: + * `src/include/module/module/interface.h`: Complete declaration of `struct module_interface` and module state definitions. + * `src/include/module/audio/source_api.h`: Source API function prototypes for reading audio frames. + * `src/include/module/audio/sink_api.h`: Sink API function prototypes for committing produced frames. + * `src/audio/module_adapter/module_adapter.c`: Implementation of the proxy container, memory sandboxing, and IPC handlers. + +Related Guides +============== + +* :ref:`pipeline_architecture`: How processing modules are assembled into directed acyclic execution graphs (DAGs). +* :ref:`llext_modules`: Authoring, compiling, and signing dynamic loadable modules using Zephyr LLEXT. +* :ref:`sof_hostless_firmware`: Instantiating static modules in autonomous embedded firmware. +* :ref:`topology2`: Declaring audio widgets and binding modules using ALSA Topology 2.0. diff --git a/developer_guides/firmware/pipeline_architecture.rst b/developer_guides/firmware/pipeline_architecture.rst index 325bf082..1497d939 100644 --- a/developer_guides/firmware/pipeline_architecture.rst +++ b/developer_guides/firmware/pipeline_architecture.rst @@ -109,7 +109,7 @@ Rather than placing all audio processing modules into one monolithic loop, SOF p 2. Audio Modules & Pin Interfaces ********************************* -An **Audio Module** (or component) is the atomic building block of signal processing in SOF. Modules accept incoming audio frames on **Sink Pins** (inputs), process or transform the samples, and produce processed frames on **Source Pins** (outputs). +An **Audio Module** (or component) is the atomic building block of signal processing in SOF. Modules accept incoming audio frames on **Sink Pins** (inputs), process or transform the samples, and produce processed frames on **Source Pins** (outputs). For an in-depth architectural guide on module containers, Source/Sink APIs, and memory sandboxing, see :ref:`module_framework`. .. graphviz:: :caption: Anatomy of an SOF Audio Processing Module @@ -457,6 +457,7 @@ For developers seeking low-level C implementation details, data structures, and Related Guides ============== +* :ref:`module_framework`: The standardized module interface, Source/Sink APIs, and memory sandboxing. * :ref:`topology2`: How pipelines and widgets are declared using ALSA Topology 2.0 configuration classes. * :ref:`sof_hostless_firmware`: How to create static pipelines compiled into ROM for standalone microcontrollers. * :ref:`llext_modules`: Building dynamic loadable modules (LLEXT) that integrate into SOF pipelines. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index e5135e3b..fb5d3295 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -32,10 +32,9 @@ Core Infrastructure & Pipeline ------------------------------ * :ref:`pipeline_architecture` (High-level architecture; also see upstream `pipeline README `_) +* :ref:`module_framework` (High-level architecture; also see upstream `module README `_ & `module adapter README `_) * `Audio Buffer Management `_ * `Scheduler `_ -* `Module Framework `_ -* `Module Adapter & IADK Integration `_ * `IPC Infrastructure (IPC3 & IPC4) `_ * `Firmware Initialization & Boot `_ @@ -82,6 +81,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa :maxdepth: 1 firmware/pipeline_architecture + firmware/module_framework rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 5341143bf68fdda617457ebea6ceb4c1df9c3bc7 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Thu, 17 Sep 2026 22:14:08 +0100 Subject: [PATCH 04/64] docs: developer_guides: add high-level scheduler architecture guide Create developer_guides/firmware/scheduler_architecture.rst providing a comprehensive, high-level overview of SOF's real-time scheduling infrastructure on top of the Zephyr RTOS: - Three-tier scheduling model: Low-Latency (LL), Data Processing (DP), and Thread With Budget (TWB) - Hardware interrupt sources (1ms/10ms timers, DMA buffer interrupts) - Thread-per-task architecture and dynamic Earliest Deadline First (EDF) - Cycle accounting, priority demotion, and LL tick replenishment in TWB - Multi-core scheduling topology, core affinity, and IDC messaging - Real-time task prioritization and deadline tie-breaking - Power management duty cycles, autonomous wait states, and tickless idle - 7 vector Graphviz diagrams illustrating all architectural concepts - Cross-references to upstream thesofproject/sof: src/schedule/README.md Update developer_guides/index.rst, pipeline_architecture.rst, and module_framework.rst with cross-references. Signed-off-by: Liam Girdwood --- .../firmware/module_framework.rst | 1 + .../firmware/pipeline_architecture.rst | 3 +- .../firmware/scheduler_architecture.rst | 570 ++++++++++++++++++ developer_guides/index.rst | 3 +- 4 files changed, 575 insertions(+), 2 deletions(-) create mode 100644 developer_guides/firmware/scheduler_architecture.rst diff --git a/developer_guides/firmware/module_framework.rst b/developer_guides/firmware/module_framework.rst index c1849798..8e0e0b53 100644 --- a/developer_guides/firmware/module_framework.rst +++ b/developer_guides/firmware/module_framework.rst @@ -516,6 +516,7 @@ For developers seeking low-level C implementation details, data structures, and Related Guides ============== +* :ref:`scheduler_architecture`: Multi-tier real-time scheduling (LL, DP, TWB), EDF mechanics, and multi-core execution. * :ref:`pipeline_architecture`: How processing modules are assembled into directed acyclic execution graphs (DAGs). * :ref:`llext_modules`: Authoring, compiling, and signing dynamic loadable modules using Zephyr LLEXT. * :ref:`sof_hostless_firmware`: Instantiating static modules in autonomous embedded firmware. diff --git a/developer_guides/firmware/pipeline_architecture.rst b/developer_guides/firmware/pipeline_architecture.rst index 1497d939..802f70f0 100644 --- a/developer_guides/firmware/pipeline_architecture.rst +++ b/developer_guides/firmware/pipeline_architecture.rst @@ -170,7 +170,7 @@ Modules support different pin topologies depending on their functional role: Audio signal processing has diverse timing requirements. Simple volume adjustment must happen with sub-millisecond determinism to prevent hardware dropouts, whereas complex algorithms like Acoustic Echo Cancellation (AEC) or neural speech enhancement require flexible execution windows. -To resolve these conflicting demands, SOF separates pipeline execution into two primary **Scheduling Domains**: +To resolve these conflicting demands, SOF separates pipeline execution into multiple **Scheduling Domains** (see :ref:`scheduler_architecture` for a comprehensive deep dive): .. list-table:: :widths: 20 40 40 @@ -457,6 +457,7 @@ For developers seeking low-level C implementation details, data structures, and Related Guides ============== +* :ref:`scheduler_architecture`: Multi-tier real-time scheduling (LL, DP, TWB), EDF mechanics, and multi-core execution. * :ref:`module_framework`: The standardized module interface, Source/Sink APIs, and memory sandboxing. * :ref:`topology2`: How pipelines and widgets are declared using ALSA Topology 2.0 configuration classes. * :ref:`sof_hostless_firmware`: How to create static pipelines compiled into ROM for standalone microcontrollers. diff --git a/developer_guides/firmware/scheduler_architecture.rst b/developer_guides/firmware/scheduler_architecture.rst new file mode 100644 index 00000000..ae5d551f --- /dev/null +++ b/developer_guides/firmware/scheduler_architecture.rst @@ -0,0 +1,570 @@ +.. _scheduler_architecture: + +Scheduler Architecture +###################### + +The **Scheduling Infrastructure** in Sound Open Firmware (SOF) is the real-time engine responsible for orchestrating task execution across multi-core Digital Signal Processors (DSPs). Deeply integrated with the underlying **Zephyr RTOS**, SOF utilizes a multi-tiered scheduling model to satisfy contrasting computing demands: deterministic, sub-millisecond low-latency audio streaming alongside heavy, variable-duration algorithmic processing (such as Echo Cancellation, Beamforming, and Machine Learning inference). + +This guide provides a high-level conceptual overview of the three scheduling domains, hardware interrupt triggers, Earliest Deadline First (EDF) mechanics, cycle budgeting, multi-core affinity, and power-saving tickless idle operation without focusing on low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +1. Multi-Tier Scheduling Architecture +************************************* + +Why Audio DSPs Require Multiple Scheduling Domains +================================================== + +Audio signal processing imposes conflicting real-time constraints on embedded DSP systems: + +1. **Deterministic Low-Latency Streaming**: Hardware audio interfaces (I2S, SoundWire, HDA) and Host DMA controllers transfer audio in strict, repetitive time slots (typically 1 ms or sub-millisecond frames). Missing a single deadline causes audible glitches, buffer underruns, or audio dropouts. +2. **Heavy, Variable-Duration Computation**: Algorithms like Acoustic Echo Cancellation (AEC), multi-microphone beamforming, noise suppression, and neural network inference require millions of math operations. Processing times vary dynamically depending on acoustic convergence and input features. + +A single flat scheduling model cannot satisfy both needs. If heavy compute algorithms ran synchronously on the audio interrupt tick, they would delay I/O transfers and cause buffer underruns. Conversely, if all audio transfers ran in standard cooperative OS threads, scheduling jitter would break strict timing guarantees. + +To solve this, SOF implements a **three-tier scheduling architecture** on top of the Zephyr RTOS: + +.. graphviz:: + :caption: Multi-Tier Scheduling Architecture: Hardware Triggers to RTOS Execution + :align: center + + digraph multi_tier_sched { + rankdir=TB; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_hw { + label = "Hardware & Interrupt Sources"; + style = "filled,rounded"; + color = "#7f8c8d"; + fillcolor = "#f2f4f4"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#2c3e50"; + + hw_timer [label="Hardware Timer\n(1ms / 10ms System Tick)", fillcolor="#d5dbdb"]; + hw_dma [label="DMA Controller\n(Buffer Half / Full Interrupts)", fillcolor="#d5dbdb"]; + } + + subgraph cluster_domains { + label = "SOF Scheduling Domains"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1b4f72"; + + subgraph cluster_ll { + label = "Low-Latency (LL) Domain"; + style = "filled,rounded"; + color = "#2471a3"; + fillcolor = "#d4e6f1"; + fontname = "Verdana-Bold"; + fontsize = 9; + + ll_engine [label="LL Scheduler Engine\n(Synchronous Task Walk,\nStrict Priority Order)", fillcolor="#aed6f1"]; + ll_tasks [label="LL Tasks\n(Host DMA, DAI Copy, Volume,\nMixer, Format Convert)", fillcolor="#aed6f1"]; + ll_engine -> ll_tasks [label="Dispatches"]; + } + + subgraph cluster_dp { + label = "Data Processing (DP) Domain"; + style = "filled,rounded"; + color = "#8e44ad"; + fillcolor = "#f4ecf7"; + fontname = "Verdana-Bold"; + fontsize = 9; + + dp_eval [label="DP Readiness Evaluator\n(Buffer Threshold & Space Check)", fillcolor="#d7bde2"]; + dp_tasks [label="DP Tasks (Dedicated Threads)\n(AEC, Beamformer, Noise Suppress,\nML Keyword Spotting)", fillcolor="#d7bde2"]; + dp_eval -> dp_tasks [label="Signals Ready"]; + } + + subgraph cluster_twb { + label = "Thread With Budget (TWB) Domain"; + style = "filled,rounded"; + color = "#d35400"; + fillcolor = "#fef5e7"; + fontname = "Verdana-Bold"; + fontsize = 9; + + twb_budget [label="Cycle Budget Accounting\n(Time-Slice Monitor)", fillcolor="#fad7a0"]; + twb_tasks [label="Budgeted Tasks\n(Background Filters, Diagnostics,\nNon-Critical Compute)", fillcolor="#fad7a0"]; + twb_budget -> twb_tasks [label="Monitors Cycles"]; + } + } + + subgraph cluster_zephyr { + label = "Zephyr RTOS Execution Layer"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1e8449"; + + z_threads [label="Zephyr Kernel Scheduler\n(Preemptive Priority & Earliest Deadline First - EDF)", fillcolor="#a9dfbf"]; + z_idle [label="Tickless Idle / Power Management\n(Autonomous DSP Low-Power Wait States)", fillcolor="#a9dfbf"]; + } + + /* Trigger flows */ + hw_timer -> ll_engine [label="Timer Tick", color="#2471a3", fontcolor="#2471a3", penwidth=1.5]; + hw_dma -> ll_engine [label="DMA Interrupt", color="#2471a3", fontcolor="#2471a3", penwidth=1.5]; + + ll_engine -> dp_eval [label="Post-Run Hook\n(Trigger Eval)", color="#8e44ad", fontcolor="#8e44ad", style=dashed]; + ll_engine -> twb_budget [label="Tick Replenish", color="#d35400", fontcolor="#d35400", style=dashed]; + + ll_tasks -> z_threads [label="Runs in Pinned\nDomain Thread", color="#27ae60"]; + dp_tasks -> z_threads [label="Dedicated Threads\n(EDF Deadlines)", color="#27ae60"]; + twb_tasks -> z_threads [label="Time-Sliced\nThreads", color="#27ae60"]; + z_threads -> z_idle [label="All Work Done", style=dotted]; + } + +The Three Scheduling Domains +============================ + +1. **Low-Latency (LL) Domain**: + + * **Execution Model**: Deterministic, synchronous task execution within a high-priority Zephyr domain thread pinned to each core. + * **Trigger**: Hardware timer ticks (typically 1 ms or 10 ms) or hardware DMA completion interrupts. + * **Characteristics**: Strict priority ordering, sub-millisecond execution deadlines, zero thread-context switching between internal tasks, and hard real-time guarantees. + * **Typical Workloads**: Host DMA transfers, DAI endpoints (I2S, SoundWire), mixers, linear volume controls, and sample format conversions. + +2. **Data Processing (DP) Domain**: + + * **Execution Model**: Asynchronous, multithreaded processing where each DP task runs inside its own dedicated Zephyr RTOS thread. + * **Trigger**: Buffer threshold readiness (when sufficient input frames are present and downstream output space is available), evaluated at the end of each LL tick. + * **Characteristics**: Employs Zephyr's **Earliest Deadline First (EDF)** scheduler. Deadlines are calculated dynamically based on frame sizes and stream sample rates. + * **Typical Workloads**: Acoustic Echo Cancellation (AEC), Time-Domain Fixed Beamforming (TDFB), noise suppression, parametric equalizers with large FIR filter taps, and TensorFlow Lite Micro (TFLM) neural networks. + +3. **Thread With Budget (TWB) Domain**: + + * **Execution Model**: Sandboxed time-sliced execution using Zephyr thread time slicing and hardware cycle accounting. + * **Trigger**: Periodic scheduling with an allocated cycle budget per tick. + * **Characteristics**: Prevents CPU starvation. If a task exceeds its budgeted cycle quota before completing its chunk, the kernel invokes a callback that immediately demotes the thread to a background priority. The budget is replenished on the subsequent LL tick. + * **Typical Workloads**: Background room acoustic calibration, non-critical telemetry, diagnostic probes, and low-priority algorithmic tasks. + +--- + +2. Low-Latency (LL) Scheduler Domain +************************************ + +The Low-Latency scheduler is the real-time backbone of SOF. Designed for minimal latency and jitter, it bypasses generic OS thread context switching for its child tasks by multiplexing all low-latency work within a single, dedicated high-priority Zephyr thread pinned to each DSP core. + +Domain Architecture & Execution Sequence +======================================== + +.. graphviz:: + :caption: Low-Latency Domain Execution Flow: Hardware Interrupt to Task Dispatch + :align: center + + digraph ll_execution { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + step1 [label="1. Hardware Event\n(Timer Tick or\nDMA Interrupt)", fillcolor="#d5dbdb"]; + step2 [label="2. Unblock Domain Thread\n(ll_thread0 / ll_thread1\nhigh-priority semaphore)", fillcolor="#aed6f1"]; + step3 [label="3. Acquire Domain Lock\n(Atomic SMP lock\nsafeguards queue)", fillcolor="#aed6f1"]; + step4 [label="4. Synchronous Task Walk\n(Iterate priority list:\nTask 1 -> Task 2 -> Task 3)", fillcolor="#aed6f1"]; + step5 [label="5. Release Domain Lock\n(Re-arm timer or\nDMA trigger)", fillcolor="#aed6f1"]; + step6 [label="6. Post-Run Hook\n(Trigger DP readiness\n& TWB replenishment)", fillcolor="#d7bde2"]; + step7 [label="7. Thread Sleep\n(Yield to Zephyr kernel;\nawait next tick)", fillcolor="#a9dfbf"]; + + step1 -> step2 -> step3 -> step4 -> step5 -> step6 -> step7; + } + +Key Design Characteristics +========================== + +* **Pinned Core Domains**: Each active DSP core runs an independent LL domain thread (such as ``ll_thread0`` on Core 0 and ``ll_thread1`` on Core 1). This ensures that core-local audio processing never suffers from inter-core cache invalidation or cross-core spinlock contention. +* **Synchronous Task Iteration**: When the domain wakes up, it walks through all queued tasks in strict priority order. Because tasks are invoked via direct C function calls rather than thread yields, context-switching overhead is virtually zero. +* **Deterministic Timing**: Tasks within the LL domain must complete within a fraction of the period window (e.g., within 200 µs of a 1 ms tick), leaving sufficient DSP headroom for data processing threads and low-power sleep states. +* **Post-Run Hook**: At the completion of each LL task walk, the scheduler invokes a post-run hook. This hook triggers readiness evaluations for the Data Processing (DP) and Thread With Budget (TWB) domains. + +LL Task State Machine +===================== + +Tasks registered with the LL scheduler progress through an operational lifecycle: + +.. graphviz:: + :caption: Low-Latency Task Lifecycle and State Transitions + :align: center + + digraph ll_task_states { + rankdir=TB; + nodesep=0.4; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + init [label="INIT\n(Task allocated & initialized)", fillcolor="#eaeded"]; + queued [label="QUEUED\n(Inserted in sorted priority list;\nwaiting for timer/DMA tick)", fillcolor="#d4e6f1"]; + running [label="RUNNING\n(Domain thread executes task callback)", fillcolor="#aed6f1", style="filled,bold"]; + cancel [label="CANCELED\n(Removed from queue via cancel)", fillcolor="#fadbd8"]; + free [label="FREE\n(Deallocated & resources released)", fillcolor="#d5dbdb"]; + + init -> queued [label="schedule_task()"]; + queued -> running [label="Domain Tick"]; + running -> queued [label="Task returns RESCHEDULE\n(Periodic stream)"]; + running -> free [label="Task returns COMPLETED\n(One-shot task)"]; + running -> cancel [label="Stream Stop / Pause"]; + queued -> cancel [label="task_cancel()"]; + cancel -> free [label="task_free()"]; + } + +--- + +3. Data Processing (DP) Scheduler Domain +**************************************** + +While the Low-Latency domain handles time-critical I/O movement, the **Data Processing (DP) domain** manages compute-heavy algorithms that require multi-millisecond or variable execution times. + +Thread-Per-Task Architecture +============================ + +Unlike the LL domain, which serializes all tasks within a single domain thread, **each DP task executes inside its own dedicated Zephyr RTOS thread**. This decoupling ensures that: + +1. A slow or complex algorithm running on one stream cannot block or delay audio streaming on other pipelines. +2. The Zephyr kernel can preempt a running DP thread whenever an LL timer tick or hardware DMA interrupt arrives. +3. Compute tasks can be prioritized dynamically based on their actual consumption deadlines. + +Readiness Evaluation & Earliest Deadline First (EDF) +==================================================== + +The DP scheduling cycle is driven by data availability and buffer space rather than a rigid clock tick: + +.. graphviz:: + :caption: Data Processing (DP) Execution Workflow with Buffer Readiness and EDF + :align: center + + digraph dp_workflow { + rankdir=TB; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_eval { + label = "Readiness Evaluation (End of LL Tick)"; + style = "filled,rounded"; + color = "#8e44ad"; + fillcolor = "#f4ecf7"; + fontname = "Verdana-Bold"; + fontsize = 9; + + chk_src [label="Check Source Ring Buffer\n(Are sufficient input frames present?)", fillcolor="#d7bde2"]; + chk_snk [label="Check Sink Ring Buffer\n(Is sufficient output free space available?)", fillcolor="#d7bde2"]; + chk_gate [label="Readiness Gate\n(Both conditions satisfied?)", fillcolor="#bb8fce", shape=diamond]; + + chk_src -> chk_gate; + chk_snk -> chk_gate; + } + + subgraph cluster_thread { + label = "Dedicated DP Task Thread"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 9; + + calc_dl [label="Compute Dynamic Deadline\n(Absolute timestamp based on frame size & rate)", fillcolor="#aed6f1"]; + set_edf [label="Set Zephyr EDF Deadline\nk_thread_absolute_deadline_set()", fillcolor="#aed6f1"]; + proc_blk [label="Execute Algorithm\n(Process audio chunk in component sandbox)", fillcolor="#aed6f1", style="filled,bold"]; + commit [label="Commit Audio Data\n(Advance buffer read/write pointers)", fillcolor="#aed6f1"]; + + calc_dl -> set_edf -> proc_blk -> commit; + } + + subgraph cluster_wait { + label = "Thread Sleep / Block"; + style = "filled,rounded"; + color = "#7f8c8d"; + fillcolor = "#f2f4f4"; + fontname = "Verdana-Bold"; + fontsize = 9; + + thread_wait [label="Wait on Zephyr Event\n(Sleep until next readiness trigger)", fillcolor="#d5dbdb"]; + } + + chk_gate -> calc_dl [label="Yes (Signal Event)", color="#27ae60", penwidth=1.5]; + chk_gate -> thread_wait [label="No (Remain Sleeping)", color="#c0392b", style=dashed]; + commit -> thread_wait [label="Task Yields"]; + thread_wait -> chk_src [label="Next LL Tick Hook", style=dotted]; + } + +Dynamic Deadline Calculation +============================ + +In real-time audio, a deadline is the point in time when the next buffer consumer (such as the speaker output DMA) will starve if new samples are not delivered. + +The DP scheduler dynamically calculates the task's absolute deadline timestamp based on: + +.. math:: + + \text{Deadline} = \text{Current Time} + \frac{\text{Frames in Buffer}}{\text{Sampling Frequency}} - \text{Safety Margin} + +When the DP thread wakes up, it passes this absolute timestamp to Zephyr via ``k_thread_absolute_deadline_set()``. Zephyr's EDF kernel prioritizes threads whose deadlines are closest to expiring, automatically resolving scheduling conflicts between competing audio streams. + +--- + +4. Thread With Budget (TWB) Scheduler Domain +******************************************** + +The **Thread With Budget (TWB) domain** is designed for non-critical, intensive, or bursty computational tasks where starvation of the primary audio pipelines must be strictly prevented. + +The Challenge of Unbounded Compute +================================== + +Certain audio algorithms—such as acoustic space measurement, complex FIR filter calculation, or machine learning background training—can consume substantial DSP cycles. If a high-priority thread runs without restriction, it can starve lower-priority system tasks or monopolize the processor, preventing other pipelines from meeting their deadlines. + +Time Slicing & Cycle Demotion Model +=================================== + +The TWB domain combines Zephyr RTOS time slicing with hardware cycle accounting: + +.. graphviz:: + :caption: Thread With Budget (TWB) Priority Demotion and Replenishment Cycle + :align: center + + digraph twb_cycle { + rankdir=LR; + nodesep=0.4; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + start [label="Task Scheduled\n(Configured with cycle budget\nderived from OS ticks)", fillcolor="#eaeded"]; + high_pri [label="High Priority State\n(Thread executes audio compute\nwith full CPU share)", fillcolor="#abebc6", style="filled,bold"]; + budget_cb [label="Budget Exhaustion\n(Zephyr callback detects\ncycle limit reached)", fillcolor="#fadbd8"]; + low_pri [label="Demoted State\n(Thread dropped to background priority\nCONFIG_TWB_THREAD_LOW_PRIORITY)", fillcolor="#f5b7b1", style="filled,bold"]; + ll_tick [label="Next LL Tick\n(Hardware tick handler\nreplenishes budget)", fillcolor="#d4e6f1"]; + + start -> high_pri; + high_pri -> budget_cb [label="Cycles Exceeded", color="#c0392b", penwidth=1.5]; + budget_cb -> low_pri [label="Demote Priority", color="#c0392b"]; + low_pri -> ll_tick [label="Awaits LL Tick"]; + ll_tick -> high_pri [label="Restore Priority\n& Reset Cycles", color="#27ae60", penwidth=1.5]; + high_pri -> start [label="Task Completes", style=dotted]; + } + +How TWB Protects System Integrity +================================= + +1. **Cycle Quota Allocation**: When a TWB task is scheduled, its budget is configured in OS ticks via ``k_thread_time_slice_set()``. The runtime converts this into equivalent DSP hardware cycles. +2. **Autonomous Kernel Demotion**: If the task runs continuously and depletes its cycle quota before completing its current unit of work, the Zephyr kernel triggers ``scheduler_twb_task_cb()``. This callback immediately lowers the thread's priority to a background level. +3. **Audio Chain Protection**: In the background state, the demoted task can only run when all LL audio streaming tasks and DP algorithms have completed their work. +4. **Periodic Priority Restoration**: On the subsequent LL timer tick, the scheduler invokes ``scheduler_twb_ll_tick()``. This resets the consumed cycle counter, restores the thread's high priority, and re-enables its time slice. + +--- + +5. Multi-Core Scheduling & Core Affinity +**************************************** + +Modern Intel and partner audio DSPs feature multi-core architectures (Dual-Core, Quad-Core, or Octa-Core). Sound Open Firmware leverages multi-core processing by statically partitioning scheduling domains across physical cores. + +Core Affinity Model +=================== + +.. graphviz:: + :caption: Multi-Core Scheduling Topology: Core 0 (I/O) and Core 1 (Compute Offload) + :align: center + + digraph multicore_sched { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_core0 { + label = "DSP Core 0 (Real-Time I/O Controller)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1b4f72"; + + c0_ll [label="LL Domain (Core 0)\nll_thread0 (High Priority)", fillcolor="#aed6f1"]; + c0_host [label="Host Gateway DMA\n(PCIe / IPC Ingest)", fillcolor="#aed6f1"]; + c0_dai [label="DAI Endpoint\n(I2S / SoundWire Link)", fillcolor="#aed6f1"]; + c0_mixer [label="Real-Time Mixer\n(Fast Audio Summation)", fillcolor="#aed6f1"]; + + c0_ll -> c0_host; + c0_ll -> c0_mixer; + c0_ll -> c0_dai; + } + + subgraph cluster_core1 { + label = "DSP Core 1 (Compute & Algorithm Offload)"; + style = "filled,rounded"; + color = "#8e44ad"; + fillcolor = "#f4ecf7"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#512e5f"; + + c1_ll [label="LL Domain (Core 1)\nll_thread1 (High Priority)", fillcolor="#d7bde2"]; + c1_dp_aec [label="DP Thread: Echo Cancellation\n(AEC / Dynamic Filter)", fillcolor="#d7bde2"]; + c1_dp_bf [label="DP Thread: Beamforming\n(Multi-Mic TDFB)", fillcolor="#d7bde2"]; + c1_twb_ml [label="TWB Thread: Neural Net\n(Keyword Detection / AI)", fillcolor="#fad7a0"]; + + c1_ll -> c1_dp_aec [style=dashed]; + c1_ll -> c1_dp_bf [style=dashed]; + c1_ll -> c1_twb_ml [style=dashed]; + } + + /* Inter-core coordination */ + shared_buf [label="Cross-Core Shared Ring Buffer\n(Cache-coherent shared memory)", fillcolor="#fadbd8", shape=cylinder]; + idc_msg [label="Inter-Domain Communication (IDC)\n(Hardware Doorbell Interrupts)", fillcolor="#fcf3cf", shape=cds]; + + c0_mixer -> shared_buf [color="#e67e22", penwidth=2, label="Audio Samples"]; + shared_buf -> c1_dp_aec [color="#e67e22", penwidth=2, label="Audio Samples"]; + + c0_ll -> idc_msg [color="#d35400", style=dotted, label="Notify Buffer Ready"]; + idc_msg -> c1_ll [color="#d35400", style=dotted, label="Wake Core 1"]; + } + +Principles of Multi-Core Distribution +===================================== + +1. **Dedicated Core 0 for I/O & Host Communications**: + Core 0 typically manages all Host IPC messaging, DMA gateways, and low-latency audio hardware links. Pinning I/O tasks to Core 0 guarantees uninterrupted streaming and immediate host response times. +2. **Compute Offload to Secondary Cores**: + Complex algorithms (AEC, beamforming, ML models) are assigned to secondary cores (Core 1, Core 2, Core 3). This shields real-time audio links on Core 0 from heavy algorithmic processing spikes. +3. **Cross-Core Ring Buffers**: + Data movement between cores occurs through shared circular buffers located in shared DSP memory. Producers and consumers synchronize using cache-coherent read/write pointers. +4. **Inter-Domain Communication (IDC)**: + When Core 0 deposits audio frames into a shared cross-core buffer, it signals the destination core via hardware doorbell interrupts (IDC). This awakens the secondary core's LL domain thread without polling or spinlocks. + +--- + +6. Task Prioritization & Deadline Calculation +********************************************* + +Task Priorities +=============== + +Within each scheduling domain, tasks are assigned explicit priority values: + +.. list-table:: + :widths: 20 20 60 + :header-rows: 1 + + * - Priority Level + - Domain + - Typical Component Assignment + * - **Critical / High** + - LL Domain + - Hardware DAI copy, Host DMA gateway reader/writer, clock synchronization. + * - **Medium** + - LL Domain / DP Domain + - Real-time mixers, standard volume controls, sample rate converters. + * - **Low / Dynamic** + - DP Domain (EDF) + - Asynchronous filter updates, multi-frame acoustic echo cancellation, noise suppression. + * - **Background** + - TWB Domain + - Diagnostic trace DMA, room calibration estimation, power telemetry sampling. + +Queue Processing & Tie-Breaking +=============================== + +When multiple tasks are scheduled within the same domain: + +* In the **LL domain**, tasks are queued in a doubly linked list sorted strictly by priority. The domain thread executes higher-priority tasks first. If multiple tasks share the same priority, they are dispatched in FIFO (First-In, First-Out) arrival order. +* In the **DP domain**, the Zephyr kernel schedules runnable threads using their calculated absolute deadline timestamps. A thread processing a 1 ms frame with an impending 500 µs deadline automatically preempts a thread processing a 10 ms background frame whose deadline is 8 ms away. + +--- + +7. Power Management & Tickless Idle +*********************************** + +Power consumption is critical in modern laptops, smartphones, and embedded audio devices. The SOF scheduling architecture is explicitly designed to maximize the duration the DSP spends in ultra-low-power autonomous wait states. + +The Active vs. Sleep Duty Cycle +=============================== + +.. graphviz:: + :caption: Execution Timeline: Active Processing Window vs. Autonomous Low-Power Sleep + :align: center + + digraph power_timeline { + rankdir=LR; + nodesep=0.2; + ranksep=0.3; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.1,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_1ms { + label = "Standard 1 ms Audio Frame Period (1000 µs)"; + style = "filled,rounded"; + color = "#2c3e50"; + fillcolor = "#f8f9f9"; + fontname = "Verdana-Bold"; + fontsize = 10; + + t_tick [label="Tick Interrupt\n(0 µs)", fillcolor="#d5dbdb"]; + t_ll [label="LL Task Walk\n(0 - 120 µs)\nHost DMA & DAI Copy", fillcolor="#aed6f1", style="filled,bold"]; + t_dp [label="DP Thread Exec\n(120 - 250 µs)\nAEC & Filter Chunk", fillcolor="#d7bde2", style="filled,bold"]; + t_sleep [label="Autonomous Low-Power Sleep Window\n(250 - 1000 µs: 750 µs Duration)\nDSP Core Enters WFI / D0ix Autonomous Clock Gating", fillcolor="#abebc6", style="filled,bold"]; + + t_tick -> t_ll -> t_dp -> t_sleep; + } + + next_tick [label="Next Tick\n(1000 µs)", fillcolor="#d5dbdb"]; + t_sleep -> next_tick; + } + +Tickless Idle Operation +======================= + +When all audio streams are stopped or paused: + +1. **Timer Suppression**: The SOF scheduler works with Zephyr's tickless idle subsystem to suppress periodic hardware timer interrupts. +2. **Autonomous Wait States**: Rather than spinning or polling, the DSP core executes a Wait For Interrupt (``WFI``) instruction, allowing hardware power controllers to lower core voltage, gate DSP clocks, or enter autonomous D0ix states. +3. **Interrupt-Only Wakeup**: The DSP remains quiescent until a hardware event occurs—such as a new host IPC command, a wake-on-voice (WOV) sound detector trigger, or an external jack insertion interrupt. + +--- + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level C implementation details, data structures, and function prototypes: + +* **Upstream Scheduler Specification**: Consult the comprehensive scheduler architecture specification in the SOF repository at `thesofproject/sof: src/schedule/README.md `_. +* **Core Source Files**: + + * ``src/schedule/schedule.c``: Generic scheduler registration, task queuing, and API entry points. + * ``src/schedule/zephyr_ll.c``: Low-Latency scheduler engine and synchronous task dispatch loop. + * ``src/schedule/zephyr_domain.c``: Pinned domain thread initialization and core affinity management. + * ``src/schedule/zephyr_dma_domain.c``: DMA interrupt-driven scheduling domain. + * ``src/schedule/zephyr_dp_schedule.c``: Data Processing scheduler readiness evaluation and event signaling. + * ``src/schedule/zephyr_dp_schedule_thread.c``: Dedicated thread execution and EDF deadline management. + * ``src/schedule/zephyr_twb_schedule.c``: Thread With Budget cycle monitoring and priority demotion callbacks. + +* **Core Header Files**: + + * ``src/include/sof/schedule/schedule.h``: Core scheduler structures and task lifecycle definitions. + * ``src/include/sof/schedule/ll_schedule.h``: Low-Latency domain prototypes. + * ``src/include/sof/schedule/dp_schedule.h``: Data Processing readiness and thread structures. + * ``src/include/sof/schedule/twb_schedule.h``: Thread With Budget constants and time-slice interfaces. + +Related Guides +============== + +* :ref:`pipeline_architecture`: How audio pipelines interact with the scheduling domains to stream data. +* :ref:`module_framework`: The standardized module interface executed by LL and DP scheduler tasks. +* :ref:`sof_hostless_firmware`: Autonomous firmware pipelines and timer configurations on embedded targets. +* :ref:`unit_tests`: Unit testing scheduler components and domain threads using Zephyr Ztest. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index fb5d3295..5a4d6ef6 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -33,8 +33,8 @@ Core Infrastructure & Pipeline * :ref:`pipeline_architecture` (High-level architecture; also see upstream `pipeline README `_) * :ref:`module_framework` (High-level architecture; also see upstream `module README `_ & `module adapter README `_) +* :ref:`scheduler_architecture` (High-level architecture; also see upstream `scheduler README `_) * `Audio Buffer Management `_ -* `Scheduler `_ * `IPC Infrastructure (IPC3 & IPC4) `_ * `Firmware Initialization & Boot `_ @@ -82,6 +82,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/pipeline_architecture firmware/module_framework + firmware/scheduler_architecture rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 497d62a1983750336d47e8f8a966abdd5a9d132a Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Thu, 17 Sep 2026 22:28:16 +0100 Subject: [PATCH 05/64] docs: developer_guides: add high-level audio buffer management guide Create developer_guides/firmware/audio_buffer_management.rst providing a comprehensive, high-level overview of SOF's audio buffer subsystem: - Lockless Single-Producer Single-Consumer (SPSC) ring buffer mechanics - Modulo offset wrap-around arithmetic resolving empty vs full states - Buffer sizing criterion (2 * max(IBS, OBS)) and asynchronous decoupling - Multi-tier DSP memory hierarchy (L1 TCM, L2 HP-SRAM, LP-SRAM, Host DRAM) - Local mode vs cross-core shared mode with cache write-back/invalidation - Sample container formats (S16_LE, S24_4LE, S32_LE, FLOAT) and SIMD/DMA alignment - In-place zero-copy processing vs intermediate buffering - Buffer overrun/underrun (XRUN) detection and automated self-healing recovery - 7 vector Graphviz diagrams illustrating all architectural concepts - Cross-references to upstream thesofproject/sof: src/audio/buffers/README.md Update developer_guides/index.rst, pipeline_architecture.rst, module_framework.rst, and scheduler_architecture.rst with cross-references. Signed-off-by: Liam Girdwood --- .../firmware/audio_buffer_management.rst | 624 ++++++++++++++++++ .../firmware/module_framework.rst | 1 + .../firmware/pipeline_architecture.rst | 3 +- .../firmware/scheduler_architecture.rst | 1 + developer_guides/index.rst | 3 +- 5 files changed, 630 insertions(+), 2 deletions(-) create mode 100644 developer_guides/firmware/audio_buffer_management.rst diff --git a/developer_guides/firmware/audio_buffer_management.rst b/developer_guides/firmware/audio_buffer_management.rst new file mode 100644 index 00000000..c503df9f --- /dev/null +++ b/developer_guides/firmware/audio_buffer_management.rst @@ -0,0 +1,624 @@ +.. _audio_buffer_management: + +Audio Buffer Management +####################### + +The **Audio Buffer Management** subsystem in Sound Open Firmware (SOF) provides the foundational memory and data-transport infrastructure that connects audio processing components into streaming pipelines. By abstracting raw memory allocation, circular pointer math, multi-core cache coherency, and format alignment, the buffer subsystem enables real-time audio streams to flow deterministically across heterogeneous DSP memory architectures. + +This guide provides a high-level conceptual overview of circular ring buffers, lockless single-producer single-consumer (SPSC) mechanics, memory tiers, cache synchronization, sample interleaving, and automated self-healing recovery without focusing on low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +1. Audio Buffer Architecture Overview +************************************* + +Why Real-Time Audio Requires Specialized Buffer Management +=========================================================== + +Unlike general-purpose computing where data buffers can be resized or queued dynamically, embedded audio processing operates under uncompromising real-time constraints: + +1. **Jitter Absorption**: Audio hardware Direct Memory Access (DMA) controllers demand a constant, uninterrupted stream of samples. Buffers absorb transient execution jitter caused by high-priority interrupts, host operating system scheduling delays, or variable algorithmic execution times. +2. **Clock Domain & Period Decoupling**: Components in an audio pipeline often execute at different chunk sizes or period rates (for example, a 1 ms low-latency I/O component feeding a 10 ms acoustic echo canceler). Buffers decouple these mismatched consumption and production rhythms. +3. **Multi-Core Isolation**: In multi-core DSPs, audio buffers act as the shared memory conduits connecting tasks running on different physical cores without requiring coarse-grained cross-core spinlocks. +4. **Hardware DMA Alignment**: Audio interfaces (I2S, SoundWire, HDA) transfer samples in burst transactions that mandate strict memory alignment (e.g., 64-byte or 128-byte boundaries) to achieve maximum memory bus throughput. + +High-Level Architecture +======================= + +.. graphviz:: + :caption: High-Level Audio Buffer Architecture: Decoupling Producers and Consumers + :align: center + + digraph audio_buffer_arch { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_prod { + label = "Upstream Component (Producer)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1b4f72"; + + prod_comp [label="Producer Module\n(Host Copier / Volume / EQ)", fillcolor="#aed6f1"]; + sink_api [label="Sink API\n(sink_get_buffer / commit)", fillcolor="#aed6f1", style="filled,bold"]; + prod_comp -> sink_api [label="Renders\nSamples"]; + } + + subgraph cluster_buffer { + label = "Circular Ring Buffer Container"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1e8449"; + + buf_mem [label="Audio Sample Storage\n(Allocated in SRAM / DRAM)", fillcolor="#a9dfbf", shape=cylinder]; + buf_meta [label="Atomic State Variables\n_write_offset (Producer)\n_read_offset (Consumer)", fillcolor="#a9dfbf"]; + buf_mem -> buf_meta [style=invis]; + } + + subgraph cluster_cons { + label = "Downstream Component (Consumer)"; + style = "filled,rounded"; + color = "#8e44ad"; + fillcolor = "#f4ecf7"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#512e5f"; + + src_api [label="Source API\n(source_get_data / release)", fillcolor="#d7bde2", style="filled,bold"]; + cons_comp [label="Consumer Module\n(Mixer / AEC / DAI Copier)", fillcolor="#d7bde2"]; + src_api -> cons_comp [label="Consumes\nSamples"]; + } + + sink_api -> buf_mem [label="Writes Audio Data\n& Advances _write_offset", color="#2980b9", penwidth=1.5]; + buf_mem -> src_api [label="Reads Audio Data\n& Advances _read_offset", color="#8e44ad", penwidth=1.5]; + } + +The Buffer Abstraction Evolution +================================ + +Sound Open Firmware has evolved its buffer implementation across architectural generations: + +* **Legacy Component Buffers (``comp_buffer``)**: Used in Pipeline 1.0, where buffers were tightly coupled to component devices via linked lists (``source_list`` and ``sink_list``) and relied on direct pointer arithmetic and shared structures. +* **Modern Ring Buffers (``ring_buffer``)**: Introduced in Pipeline 2.0, providing completely asynchronous, lockless Single-Producer Single-Consumer (SPSC) circular queues with independent read and write offsets, explicit cache coherency management, and pluggable Source/Sink APIs. + +--- + +2. Circular Ring Buffers & Lockless SPSC Mechanics +************************************************** + +The foundation of SOF audio streaming is the **Lockless Circular (Ring) Buffer**. In high-performance audio DSPs, acquiring mutexes or spinlocks during audio frame processing introduces unacceptable jitter and risks inter-core priority inversions. SOF solves this by using a Single-Producer Single-Consumer (SPSC) lockless design. + +The Lockless Architecture +========================= + +A ring buffer connects exactly one data producer to exactly one data consumer. Thread-safety and multi-core safety are achieved through two simple architectural principles: + +1. **Only Two Shared State Variables**: + * ``_write_offset``: Represents the cumulative position where the producer writes new samples. It is modified **exclusively** by the producer. + * ``_read_offset``: Represents the cumulative position where the consumer reads samples. It is modified **exclusively** by the consumer. +2. **Atomic 32-Bit Operations**: On modern DSP architectures (Tensilica Xtensa, ARM Cortex-M, RISC-V), 32-bit aligned memory writes and reads are atomic instructions. Because neither component writes to the other component's offset variable, no locks or critical sections are required. + +Resolving the "Buffer Full vs. Buffer Empty" Ambiguity +====================================================== + +In classical circular buffers with an index spanning from ``0`` to ``buffer_size - 1``, when ``write_offset == read_offset``, the system cannot distinguish between a **completely empty** buffer and a **completely full** buffer without maintaining a secondary counter. + +SOF employs an elegant mathematical solution: + +.. graphviz:: + :caption: Circular Ring Buffer Traversal: Resolving Full vs Empty using Double-Size Virtual Offsets + :align: center + + digraph ring_buffer_math { + rankdir=TB; + nodesep=0.4; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_virtual { + label = "Virtual Offset Range (0 to 2 * buffer_size)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1b4f72"; + + v_empty [label="Empty Condition\n_write_offset == _read_offset\n(Available Data = 0)", fillcolor="#d4e6f1"]; + v_data [label="Partially Filled\nAvailable Data = (_write_offset - _read_offset) % (2 * buffer_size)\nFree Space = buffer_size - Available Data", fillcolor="#aed6f1"]; + v_full [label="Full Condition\n_write_offset == _read_offset + buffer_size\n(Free Space = 0)", fillcolor="#d4e6f1"]; + v_empty -> v_data -> v_full [style=invis]; + } + + subgraph cluster_physical { + label = "Physical DSP Memory Buffer (0 to buffer_size - 1)"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1e8449"; + + phys_map [label="Physical Memory Address Calculation\nPhysical Offset = Offset % buffer_size\nMemory Pointer = data_buffer_start + Physical Offset", fillcolor="#a9dfbf", style="filled,bold"]; + } + + subgraph cluster_circular { + label = "Circular Ring Traversal"; + style = "filled,rounded"; + color = "#d35400"; + fillcolor = "#fef5e7"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#a04000"; + + cell0 [label="Cell 0\n[Start]", fillcolor="#fad7a0"]; + cell1 [label="Cell 1", fillcolor="#fad7a0"]; + cell2 [label="Cell 2", fillcolor="#fad7a0"]; + cell3 [label="Cell 3\n[End]", fillcolor="#fad7a0"]; + + cell0 -> cell1 -> cell2 -> cell3; + cell3 -> cell0 [label="Wrap Around", color="#d35400", style=dashed]; + } + + v_data -> phys_map [label="Modulo Mapping"]; + phys_map -> cell0 [label="Accesses Array"]; + } + +1. **Double-Size Virtual Range**: Both ``_write_offset`` and ``_read_offset`` are allowed to increment continuously from ``0`` up to ``2 * buffer_size``. +2. **Deterministic State Detection**: + + * When ``_write_offset == _read_offset``, the buffer is **strictly empty**. + * When ``_write_offset == _read_offset + buffer_size``, the buffer is **strictly full**. + +3. **Physical Addressing**: When reading or writing sample bytes in physical memory, the address is calculated using the modulo operator: + +.. math:: + + \text{Physical Offset} = \text{Offset} \pmod{\text{buffer\_size}} + +This mathematical formulation completely eliminates ambiguous states, avoids secondary count variables, and guarantees glitch-free concurrency across cores. + +--- + +3. Buffer Sizing, Chunk Ratios & Asynchronous Decoupling +******************************************************** + +The Minimum Sizing Criterion +============================ + +Audio streams connect processing blocks that consume and produce data in different chunk sizes. To guarantee that neither component blocks or starves, SOF enforces a mathematical sizing guideline: + +.. math:: + + \text{Buffer Size} \ge 2 \times \max(\text{IBS}, \text{OBS}) + +* **IBS (Input Buffer Size)**: The maximum audio chunk size (in bytes or frames) consumed by the downstream component during each execution step. +* **OBS (Output Buffer Size)**: The maximum audio chunk size (in bytes or frames) produced by the upstream component during each execution step. + +Why Twice the Maximum Chunk Size? +================================= + +Consider an asynchronous scenario where the producer writes 3 frames and the consumer reads 5 frames: + +.. graphviz:: + :caption: Asynchronous Buffer Occupancy Over Time (Unequal IBS and OBS Ratios) + :align: center + + digraph buffer_occupancy { + rankdir=LR; + nodesep=0.2; + ranksep=0.3; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=8, margin="0.1,0.05"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + c0 [label="Cycle 0\nBuffer: 0 frames\nProducer starts", fillcolor="#eaeded"]; + c3 [label="Cycle 3\nProduce 3 frames\nBuffer: 3 frames", fillcolor="#d4e6f1"]; + c6 [label="Cycle 6\nProduce 3 frames\nBuffer: 6 frames\n(Consumer ready)", fillcolor="#aed6f1"]; + c7 [label="Cycle 7\nConsume 5 frames\nBuffer: 1 frame", fillcolor="#d7bde2"]; + c9 [label="Cycle 9\nProduce 3 frames\nBuffer: 4 frames", fillcolor="#aed6f1"]; + c12 [label="Cycle 12\nProduce 3 + Consume 5\nBuffer Peak: 7 frames", fillcolor="#f5b7b1", style="filled,bold"]; + c15 [label="Cycle 15\nProduce 3 + Consume 5\nBuffer: 0 frames", fillcolor="#abebc6"]; + + c0 -> c3 -> c6 -> c7 -> c9 -> c12 -> c15; + } + +Even when average input and output throughput are identical, scheduling latency and thread preemption mean that producer and consumer execution intervals will drift. Allocating at least ``2 * max(IBS, OBS)`` ensures that the producer always has sufficient free space to write its chunk, and the consumer always has sufficient buffered samples to satisfy its read request. + +Topology 2.0 Buffer Declaration +=============================== + +In ALSA Topology 2.0 configuration files (such as ``tools/topology/topology2/include/components/buffer.conf``), buffers are instantiated with explicit period multiples and capability flags: + +.. list-table:: + :widths: 25 25 50 + :header-rows: 1 + + * - Parameter + - Typical Values + - Architectural Purpose + * - **periods** + - ``2``, ``4``, ``8`` + - Number of audio periods buffered (e.g., 2 periods for low-latency, 4–8 for host DMA). + * - **caps** + - ``host``, ``dai``, ``comp``, ``pass`` + - Declares memory placement constraints (e.g., L2 HP-SRAM vs. DMA-accessible memory). + * - **size** + - Automatically computed + - Computed dynamically as ``period_bytes * periods``. + +--- + +4. DSP Memory Tiers & Cache Coherency +************************************* + +Modern audio DSPs (such as Intel cAVS and ACE architectures) feature heterogeneous memory hierarchies with differing access latencies, power profiles, and caching behaviors. + +The DSP Memory Hierarchy +======================== + +.. graphviz:: + :caption: DSP Memory Tiers: Access Latency vs Storage Capacity + :align: center + + digraph memory_tiers { + rankdir=TB; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_l1 { + label = "Tier 1: Core-Local Scratchpad (L1 TCM)"; + style = "filled,rounded"; + color = "#c0392b"; + fillcolor = "#f9ebea"; + fontname = "Verdana-Bold"; + fontsize = 9; + + t_l1 [label="L1 Tightly-Coupled Memory (TCM)\nSingle-cycle latency, private to individual DSP core.\nUsed for module stack, scratch registers, and FIR coefficient delay lines.", fillcolor="#f5b7b1"]; + } + + subgraph cluster_l2 { + label = "Tier 2: High-Performance System SRAM (L2 HP-SRAM)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 9; + + t_l2 [label="L2 High-Performance SRAM (HP-SRAM)\nMulti-banked shared SRAM accessible by all DSP cores and DMA controllers.\nPrimary storage for active ring buffers, module state, and IPC mailboxes.", fillcolor="#aed6f1"]; + } + + subgraph cluster_lp { + label = "Tier 3: Low-Power System SRAM (LP-SRAM)"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 9; + + t_lp [label="Low-Power SRAM (LP-SRAM)\nRetains memory during DSP low-power wait states (D0ix).\nHosts wake-on-voice (WoV) buffers and low-power streaming queues.", fillcolor="#a9dfbf"]; + } + + subgraph cluster_host { + label = "Tier 4: Host Memory (Host DRAM)"; + style = "filled,rounded"; + color = "#7f8c8d"; + fillcolor = "#f2f4f4"; + fontname = "Verdana-Bold"; + fontsize = 9; + + t_dram [label="Host System DRAM (PCIe / Shared DMA Windows)\nGigabyte-scale capacity with high access latency.\nHosts circular ALSA ring buffers managed via Host DMA gateways.", fillcolor="#d5dbdb"]; + } + + t_l1 -> t_l2 [label="Cache Miss / Spilling", style=dashed]; + t_l2 -> t_lp [label="Power Tier Migration", style=dashed]; + t_l2 -> t_dram [label="Host DMA Transfers", color="#2980b9", penwidth=1.5]; + } + +Local Mode vs. Shared Mode +========================== + +The SOF buffer management subsystem automatically configures buffers into one of two operational modes: + +1. **Local Mode (Intra-Core)**: + + * Used when both the producer and consumer components execute on the **same DSP core**. + * The ring buffer structure and audio sample payload reside in local cached SRAM. + * **Zero Cache Overhead**: The CPU core reads and writes directly from L1 cache without issuing cache invalidations or flushes. + +2. **Shared Mode (Cross-Core)**: + + * Used when the producer executes on Core 0 and the consumer executes on Core 1 (or between DSP cores and hardware DMA controllers). + * Because each DSP core maintains its own local L1 data cache, hardware memory lines can quickly become desynchronized. + * SOF enforces cache coherency through explicit kernel primitives: + +.. graphviz:: + :caption: Cross-Core Shared Buffer Synchronization and Cache Coherency + :align: center + + digraph cache_coherency { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_c0 { + label = "DSP Core 0 (Producer Core)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1b4f72"; + + c0_write [label="1. Render Samples\n(Write audio to L1 Cache)", fillcolor="#aed6f1"]; + c0_wb [label="2. Write-Back Cache\ndcache_writeback_region()\n(Flushes dirty lines to SRAM)", fillcolor="#aed6f1", style="filled,bold"]; + c0_write -> c0_wb; + } + + subgraph cluster_sram { + label = "Shared L2 HP-SRAM"; + style = "filled,rounded"; + color = "#d35400"; + fillcolor = "#fef5e7"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#a04000"; + + shared_mem [label="Physical Shared Ring Buffer\n(Audio Samples + Modulo Offsets)", fillcolor="#fad7a0", shape=cylinder]; + } + + subgraph cluster_c1 { + label = "DSP Core 1 (Consumer Core)"; + style = "filled,rounded"; + color = "#8e44ad"; + fillcolor = "#f4ecf7"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#512e5f"; + + c1_inv [label="3. Invalidate Cache\ndcache_invalidate_region()\n(Discards stale L1 lines)", fillcolor="#d7bde2", style="filled,bold"]; + c1_read [label="4. Consume Samples\n(Fetches fresh data from SRAM)", fillcolor="#d7bde2"]; + c1_inv -> c1_read; + } + + c0_wb -> shared_mem [label="Flush Dirty Lines", color="#2980b9", penwidth=1.5]; + shared_mem -> c1_inv [label="Read Updated Memory", color="#8e44ad", penwidth=1.5]; + } + +--- + +5. Audio Formats, Interleaving & SIMD Memory Alignment +****************************************************** + +Audio samples inside a buffer must adhere to specific bit-depth containerization and channel arrangements to maximize processing efficiency. + +Interleaved vs. Planar (Non-Interleaved) Formats +================================================ + +.. graphviz:: + :caption: Interleaved vs Planar Multi-Channel Audio Packing in Memory + :align: center + + digraph audio_packing { + rankdir=TB; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_interleaved { + label = "Interleaved Stereo Stream (L / R Frame Sequence)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 9; + + i_mem [label="Byte 0: Left[0] | Byte 4: Right[0] | Byte 8: Left[1] | Byte 12: Right[1] | Byte 16: Left[2] | Byte 20: Right[2]", fillcolor="#aed6f1", shape=record]; + i_desc [label="Standard for I2S, SoundWire, HDA DMA, and simple Volume/Mute processing", fillcolor="#d4e6f1"]; + i_mem -> i_desc [style=invis]; + } + + subgraph cluster_planar { + label = "Planar (Non-Interleaved) Multi-Channel Stream"; + style = "filled,rounded"; + color = "#8e44ad"; + fillcolor = "#f4ecf7"; + fontname = "Verdana-Bold"; + fontsize = 9; + + p_left [label="Plane 0 (Left): | Left[0] | Left[1] | Left[2] | Left[3] | Left[4] |", fillcolor="#d7bde2", shape=record]; + p_right [label="Plane 1 (Right): | Right[0] | Right[1] | Right[2] | Right[3] | Right[4] |", fillcolor="#d7bde2", shape=record]; + p_desc [label="Ideal for Frequency-Domain FFTs, Multi-Mic Beamforming, and SIMD Vector Math", fillcolor="#e8daef"]; + p_left -> p_right -> p_desc [style=invis]; + } + } + +Sample Container Formats +======================== + +Audio samples are packaged into standardized container sizes: + +* **16-bit in 16-bit Container (``S16_LE``)**: Compact storage (2 bytes per sample); ideal for low-power voice capture and standard Bluetooth links. +* **24-bit in 32-bit Container (``S24_4LE``)**: High-resolution audio where 24 active bits are placed in the most significant bits (MSB) of a 32-bit word, with the lowest 8 bits zero-padded. This enables direct 32-bit math without pre-shifting. +* **32-bit Fixed-Point (``S32_LE``)**: Full 32-bit dynamic range audio used for professional studio pipelines and high-dynamic-range mixers. +* **32-bit IEEE Floating-Point (``FLOAT``)**: Single-precision floating point used in complex acoustic algorithms (e.g. Valve Steam Audio 3D spatializer, AEC, and neural networks). + +SIMD & DMA Alignment Rules +========================== + +To achieve maximum performance on DSP SIMD engines (Tensilica HiFi 3/4/5, ARM Helium, RISC-V Vector): + +1. **Cacheline Boundary Alignment**: Buffer base addresses and period chunk sizes are aligned to the DSP architecture's cacheline boundary (typically 64 or 128 bytes). This prevents partial cacheline invalidation penalties. +2. **SIMD Vector Alignment**: Digital Signal Processors fetch multiple samples simultaneously using SIMD load instructions (such as 128-bit or 256-bit wide registers). Misaligned buffer offsets force the processor to issue multiple unaligned memory accesses, degrading processing throughput. + +--- + +6. Dynamic Lifecycle, Zero-Copy & Inter-Pipeline Routing +******************************************************** + +The Buffer Lifecycle +==================== + +Buffers progress through an operational lifecycle synchronized with the parent pipeline state machine: + +1. **Instantiation & Allocation**: The buffer structure is created from the topology configuration and assigned an initial capacity in the target memory pool (L2 HP-SRAM or LP-SRAM). +2. **Binding & Connection**: The buffer connects upstream components via their Sink APIs and downstream components via their Source APIs. +3. **Parameter Preparation (``prepare``)**: During the stream prepare phase, the pipeline engine negotiates channel counts, sample rates, and sample containers, configuring the buffer's effective frame size and byte alignment. +4. **Streaming (``ACTIVE``)**: During active playback or capture, the buffer transfers samples, advancing its internal read and write offsets continuously. +5. **Reset & Teardown**: When the stream stops, the buffer resets its offsets to zero and reclaims or re-initializes memory. + +Zero-Copy Optimization +====================== + +In simple pipelines where consecutive components share identical audio formats, SOF employs **In-Place (Zero-Copy) Processing**: + +.. graphviz:: + :caption: In-Place Processing vs Intermediate Double Buffering + :align: center + + digraph zero_copy { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_inplace { + label = "In-Place Zero-Copy Optimization"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 9; + + zp_buf [label="Single Shared Buffer\n(Allocated once in SRAM)", fillcolor="#a9dfbf", shape=cylinder]; + zp_vol [label="Volume Module\n(Modifies samples in-place)", fillcolor="#a9dfbf"]; + zp_mute [label="Mute Module\n(Inspects/modifies same buffer)", fillcolor="#a9dfbf"]; + + zp_buf -> zp_vol [label="Direct Ptr"]; + zp_vol -> zp_mute [label="Passes Same Ptr"]; + } + + subgraph cluster_standard { + label = "Standard Intermediate Buffering"; + style = "filled,rounded"; + color = "#7f8c8d"; + fillcolor = "#f2f4f4"; + fontname = "Verdana-Bold"; + fontsize = 9; + + sb_buf1 [label="Buffer 1", fillcolor="#d5dbdb", shape=cylinder]; + sb_src [label="Sample Rate Converter\n(Produces new rate/size)", fillcolor="#d5dbdb"]; + sb_buf2 [label="Buffer 2", fillcolor="#d5dbdb", shape=cylinder]; + + sb_buf1 -> sb_src [label="Reads"]; + sb_src -> sb_buf2 [label="Writes"]; + } + } + +When components do not alter the sample rate or channel count (e.g. Volume followed by Mute), the downstream module modifies samples directly inside the upstream buffer's memory without allocating an intermediate buffer. Intermediate buffers are only introduced when format transformations occur (such as sample rate conversion, channel mixing, or cross-core routing). + +--- + +7. Buffer Overruns, Underruns (XRUNs) & Self-Healing +***************************************************** + +An **XRUN** is an abnormal streaming state where real-time synchronization breaks down. In audio processing, an XRUN immediately results in audible pops, clicks, or silence. + +The Anatomy of an XRUN +====================== + +* **Buffer Underrun (Starvation)**: + + * Occurs when the consumer (such as the speaker output DMA) arrives to read audio frames, but the producer has not yet delivered them (``Available Data == 0``). + * The hardware DMA engine is forced to replay old samples or emit zeroes, causing an audible drop or glitch. + +* **Buffer Overrun (Overflow)**: + + * Occurs when the producer (such as the microphone input DMA) produces new audio frames, but the consumer has not emptied the buffer (``Free Space < Chunk Size``). + * The new audio frames overwrite unread samples, causing corrupted waveforms or packet loss. + +Automated Self-Healing Recovery +=============================== + +Rather than letting an XRUN destabilize the DSP firmware or hang audio streams, Sound Open Firmware implements an automated **Self-Healing Recovery** mechanism: + +.. graphviz:: + :caption: Automated Buffer XRUN Detection and Self-Healing Recovery Sequence + :align: center + + digraph xrun_recovery { + rankdir=TB; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + s1 [label="1. Normal Streaming State\n(Periodic read and write operations maintain safe latency margin)", fillcolor="#abebc6"]; + s2 [label="2. XRUN Event Triggered\n(Hardware DMA starvation or queue space exhaustion detected)", fillcolor="#fadbd8", style="filled,bold"]; + s3 [label="3. Pipeline Enters XRUN_PAUSED\n(Processing temporarily halted to prevent reading corrupted memory)", fillcolor="#f5b7b1"]; + s4 [label="4. Buffer Flush & Pointer Resynchronization\n(Stale samples cleared; _read_offset and _write_offset reset to initial offset)", fillcolor="#f5b7b1", style="filled,bold"]; + s5 [label="5. Component Re-Preparation\n(Filter delay lines and stream parameters refreshed)", fillcolor="#d4e6f1"]; + s6 [label="6. Automatic Stream Resumption\n(Pipeline triggers START; streaming seamlessly recovers)", fillcolor="#aed6f1"]; + + s1 -> s2 [label="Latency Spike", color="#c0392b", penwidth=1.5]; + s2 -> s3; + s3 -> s4; + s4 -> s5; + s5 -> s6; + s6 -> s1 [label="Stable Audio", color="#27ae60", penwidth=1.5]; + } + +1. **Immediate Detection**: The buffer monitoring logic flags the condition and notifies the parent pipeline engine. +2. **State Freeze (``XRUN_PAUSED``)**: The pipeline transitions into an isolated pause state to protect downstream audio filters from feeding on junk memory. +3. **Pointer Resynchronization**: Read and write offsets are reinitialized to establish a safe initial phase margin (typically one full period offset). +4. **Stale Sample Cleansing**: Corrupted or incomplete frame fragments are zeroed out to eliminate residual pops or speaker thumps. +5. **Seamless Resumption**: The pipeline issues an internal start event, restoring clean audio streaming without requiring application or driver restarts. + +--- + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level C implementation details, data structures, and function prototypes: + +* **Upstream Buffer Specification**: Consult the core buffer architecture documentation in the SOF repository at `thesofproject/sof: src/audio/buffers/README.md `_. +* **Core Source Files**: + + * ``src/audio/buffers/ring_buffer.c``: Implementation of the lockless asynchronous circular ring buffer and double-size modulo offset math. + * ``src/audio/buffers/audio_buffer.c``: Base audio buffer class initialization and format configuration. + * ``src/audio/buffers/comp_buffer.c``: Legacy component buffer connectors and list operations. + * ``src/audio/pipeline/pipeline-xrun.c``: XRUN detection and self-healing recovery handlers. + +* **Core Header Files**: + + * ``src/include/sof/audio/ring_buffer.h``: Ring buffer data structures, SPSC offsets, and modulo wrap-around constants. + * ``src/include/sof/audio/audio_buffer.h``: Base buffer structure and format callback declarations. + * ``src/include/sof/audio/buffer.h``: Comprehensive buffer macros, trace handlers, and legacy ``comp_buffer`` declarations. + * ``src/include/sof/audio/audio_stream.h``: Audio stream configuration descriptors and channel parameters. + +Related Guides +============== + +* :ref:`pipeline_architecture`: How audio buffers interconnect components into directed acyclic graphs (DAGs). +* :ref:`module_framework`: The standardized module interface that consumes and produces audio samples through Source and Sink APIs. +* :ref:`scheduler_architecture`: Real-time scheduling domains (LL, DP, TWB) that drive buffer read and write intervals. +* :ref:`topology2`: Declaring buffer sizes, capabilities, and period counts in ALSA Topology 2.0 configuration files. diff --git a/developer_guides/firmware/module_framework.rst b/developer_guides/firmware/module_framework.rst index 8e0e0b53..59e70849 100644 --- a/developer_guides/firmware/module_framework.rst +++ b/developer_guides/firmware/module_framework.rst @@ -516,6 +516,7 @@ For developers seeking low-level C implementation details, data structures, and Related Guides ============== +* :ref:`audio_buffer_management`: Lockless circular ring buffers, multi-tier DSP memory (SRAM/DRAM), and cache coherency. * :ref:`scheduler_architecture`: Multi-tier real-time scheduling (LL, DP, TWB), EDF mechanics, and multi-core execution. * :ref:`pipeline_architecture`: How processing modules are assembled into directed acyclic execution graphs (DAGs). * :ref:`llext_modules`: Authoring, compiling, and signing dynamic loadable modules using Zephyr LLEXT. diff --git a/developer_guides/firmware/pipeline_architecture.rst b/developer_guides/firmware/pipeline_architecture.rst index 802f70f0..eb0f412e 100644 --- a/developer_guides/firmware/pipeline_architecture.rst +++ b/developer_guides/firmware/pipeline_architecture.rst @@ -250,7 +250,7 @@ Execution Comparison 4. Data Movement & Buffer Queues ******************************** -Audio samples move through the pipeline via continuous **Circular Ring Buffers**. Rather than allocating dynamic memory packets on every audio tick, SOF pre-allocates cache-aligned circular memory pools during pipeline initialization. +Audio samples move through the pipeline via continuous **Circular Ring Buffers** (see :ref:`audio_buffer_management` for a comprehensive deep dive into lockless SPSC mechanics, sizing criteria, and DSP memory tiers). Rather than allocating dynamic memory packets on every audio tick, SOF pre-allocates cache-aligned circular memory pools during pipeline initialization. The Producer-Consumer Model =========================== @@ -458,6 +458,7 @@ Related Guides ============== * :ref:`scheduler_architecture`: Multi-tier real-time scheduling (LL, DP, TWB), EDF mechanics, and multi-core execution. +* :ref:`audio_buffer_management`: Lockless circular ring buffers, multi-tier DSP memory (SRAM/DRAM), and cache coherency. * :ref:`module_framework`: The standardized module interface, Source/Sink APIs, and memory sandboxing. * :ref:`topology2`: How pipelines and widgets are declared using ALSA Topology 2.0 configuration classes. * :ref:`sof_hostless_firmware`: How to create static pipelines compiled into ROM for standalone microcontrollers. diff --git a/developer_guides/firmware/scheduler_architecture.rst b/developer_guides/firmware/scheduler_architecture.rst index ae5d551f..b09dc672 100644 --- a/developer_guides/firmware/scheduler_architecture.rst +++ b/developer_guides/firmware/scheduler_architecture.rst @@ -564,6 +564,7 @@ For developers seeking low-level C implementation details, data structures, and Related Guides ============== +* :ref:`audio_buffer_management`: Lockless circular ring buffers, multi-tier DSP memory (SRAM/DRAM), and cache coherency. * :ref:`pipeline_architecture`: How audio pipelines interact with the scheduling domains to stream data. * :ref:`module_framework`: The standardized module interface executed by LL and DP scheduler tasks. * :ref:`sof_hostless_firmware`: Autonomous firmware pipelines and timer configurations on embedded targets. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 5a4d6ef6..383df20f 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -34,7 +34,7 @@ Core Infrastructure & Pipeline * :ref:`pipeline_architecture` (High-level architecture; also see upstream `pipeline README `_) * :ref:`module_framework` (High-level architecture; also see upstream `module README `_ & `module adapter README `_) * :ref:`scheduler_architecture` (High-level architecture; also see upstream `scheduler README `_) -* `Audio Buffer Management `_ +* :ref:`audio_buffer_management` (High-level architecture; also see upstream `buffer README `_) * `IPC Infrastructure (IPC3 & IPC4) `_ * `Firmware Initialization & Boot `_ @@ -83,6 +83,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/pipeline_architecture firmware/module_framework firmware/scheduler_architecture + firmware/audio_buffer_management rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 5447c4c38910c9238410c1b6415b15c8ed47bbc5 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Fri, 18 Sep 2026 10:54:52 +0100 Subject: [PATCH 06/64] docs: developer_guides: add high-level IPC infrastructure guide Add a comprehensive, high-level developer guide covering the Sound Open Firmware (SOF) Inter-Processor Communication (IPC) infrastructure across IPC3 and IPC4 generations. The guide covers: - The dual planes of inter-processor communication (control vs telemetry). - Shared memory mailbox architecture (Windows 0 to 3) and doorbell handshake protocols. - Core framework and deferred Zephyr k_work queue thread handoff. - Protocol generational comparison (IPC3 flat scalar model vs IPC4 dynamic compound object model). - Dynamic pipeline lifecycle and pin binding in IPC4. - Firmware-initiated asynchronous notifications (stream positions, XRUNs, panic dumps). - Multi-core IPC routing and Inter-Domain Communication (IDC) with Core 0 as the host gateway. - Upstream code references and related architecture links. Includes 7 custom vector Graphviz SVG diagrams illustrating system architecture, mailbox handshakes, Zephyr queue handoffs, IPC3 vs IPC4 structure, pipeline binding sequences, async telemetry, and multi-core IDC topology. Signed-off-by: Liam Girdwood --- .../firmware/audio_buffer_management.rst | 1 + .../firmware/ipc_infrastructure.rst | 589 ++++++++++++++++++ .../firmware/module_framework.rst | 1 + .../firmware/pipeline_architecture.rst | 1 + .../firmware/scheduler_architecture.rst | 1 + developer_guides/index.rst | 3 +- 6 files changed, 595 insertions(+), 1 deletion(-) create mode 100644 developer_guides/firmware/ipc_infrastructure.rst diff --git a/developer_guides/firmware/audio_buffer_management.rst b/developer_guides/firmware/audio_buffer_management.rst index c503df9f..57f9913c 100644 --- a/developer_guides/firmware/audio_buffer_management.rst +++ b/developer_guides/firmware/audio_buffer_management.rst @@ -618,6 +618,7 @@ For developers seeking low-level C implementation details, data structures, and Related Guides ============== +* :ref:`ipc_infrastructure`: Host-to-DSP messaging, hardware mailbox windows, and dynamic IPC4 compound commands. * :ref:`pipeline_architecture`: How audio buffers interconnect components into directed acyclic graphs (DAGs). * :ref:`module_framework`: The standardized module interface that consumes and produces audio samples through Source and Sink APIs. * :ref:`scheduler_architecture`: Real-time scheduling domains (LL, DP, TWB) that drive buffer read and write intervals. diff --git a/developer_guides/firmware/ipc_infrastructure.rst b/developer_guides/firmware/ipc_infrastructure.rst new file mode 100644 index 00000000..0cfe2c8d --- /dev/null +++ b/developer_guides/firmware/ipc_infrastructure.rst @@ -0,0 +1,589 @@ +.. _ipc_infrastructure: + +IPC Infrastructure (IPC3 & IPC4) +################################ + +The **Inter-Processor Communication (IPC)** infrastructure in Sound Open Firmware (SOF) is the primary messaging conduit and control plane bridging the host operating system (mainline Linux ASoC drivers, Windows audio subsystems) and the Digital Signal Processor (DSP) firmware. It coordinates audio pipeline topologies, runtime module parameter updates, hardware interface configurations, stream power states, and real-time diagnostic telemetry. + +This guide provides a high-level conceptual overview of the IPC messaging framework, hardware mailbox windows, doorbell interrupt handshakes, deferred Zephyr work queues, protocol evolution from IPC3 to IPC4, dynamic module binding, asynchronous telemetry, and multi-core Inter-Domain Communication (IDC) without focusing on low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +1. IPC Infrastructure & Communication Model +******************************************* + +The Dual Planes of Inter-Processor Communication +================================================ + +In modern audio systems, the DSP operates as an autonomous processor requiring tightly coordinated, bidirectional communication with the host kernel: + +1. **The Control Plane (Host to DSP)**: + * **Pipeline Topology Instantiation**: Dynamically assembling audio pipelines, allocating memory buffers, and binding processing components. + * **Parameter Configuration**: Applying volume curves, equalizer filter coefficients, dynamic range compressor profiles, and microphone calibration blobs. + * **Stream State Machine**: Transitioning audio streams through operational states (``PREPARE``, ``START``, ``PAUSE``, ``STOP``, ``RESET``). + * **Power Management**: Coordinating clock scaling, core sleep states, and host D0ix runtime power transitions. + +2. **The Telemetry & Event Plane (DSP to Host)**: + * **Stream Position Tracking**: High-frequency DMA buffer pointer updates allowing the host ALSA subsystem to maintain accurate audio-video synchronization without host polling. + * **XRUN Alerts**: Instantaneous notifications when an audio buffer underrun (starvation) or overrun (overflow) occurs. + * **Diagnostic Traces & Crash Telemetry**: Streaming real-time debug log packets and exception backtraces directly into host trace buffers. + +System-Level Architecture +========================= + +.. graphviz:: + :caption: System-Level IPC Architecture: Host Driver to DSP Firmware Dispatch + :align: center + + digraph ipc_system_arch { + rankdir=TB; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_host { + label = "Host Operating System (Linux Kernel / Windows)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1b4f72"; + + host_alsa [label="ALSA / ASoC Core\n(snd-soc-core / PCM Stream Ops)", fillcolor="#aed6f1"]; + host_drv [label="SOF Host Driver (snd-sof)\n(IPC Protocol Encoder / Decoder)", fillcolor="#aed6f1", style="filled,bold"]; + host_pci [label="PCIe / Shim Transport Layer\n(Bar Mapping & Interrupt Dispatch)", fillcolor="#aed6f1"]; + + host_alsa -> host_drv -> host_pci; + } + + subgraph cluster_hw { + label = "Hardware Mailbox & Doorbell Interconnect (PCIe BARs / SRAM)"; + style = "filled,rounded"; + color = "#7f8c8d"; + fillcolor = "#f2f4f4"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#2c3e50"; + + mbox_in [label="Mailbox Window 1: Inbox (Host -> DSP)\n(Command Payloads & Parameter Blobs)", fillcolor="#d5dbdb", shape=cylinder]; + mbox_out [label="Mailbox Window 0: Outbox (DSP -> Host)\n(Replies, Notifications & Boot Info)", fillcolor="#d5dbdb", shape=cylinder]; + doorbells [label="Hardware Doorbells\nHost Doorbell (IPC IRQ to DSP)\nDSP Doorbell (Done/Reply IRQ to Host)", fillcolor="#bdc3c7"]; + } + + subgraph cluster_dsp { + label = "DSP Firmware Architecture (SOF on Zephyr RTOS)"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1e8449"; + + dsp_isr [label="Mailbox ISR\n(Catches Doorbell IRQ & Validates)", fillcolor="#a9dfbf"]; + dsp_work [label="Zephyr Work Queue (k_work)\n(Deferred Thread Processing)", fillcolor="#a9dfbf", style="filled,bold"]; + dsp_core [label="Core IPC Framework\n(ipc-common.c: Dispatcher & State Machine)", fillcolor="#a9dfbf"]; + + subgraph cluster_protocols { + label = "Protocol-Specific Handlers"; + style = "filled,rounded"; + color = "#d35400"; + fillcolor = "#fef5e7"; + fontname = "Verdana-Bold"; + fontsize = 9; + + ipc3_hdl [label="IPC3 Handler\n(Scalar Commands: Stream, DAI, PM)", fillcolor="#fad7a0"]; + ipc4_hdl [label="IPC4 Handler\n(Dynamic Objects: Pipeline, Module, Bind)", fillcolor="#fad7a0", style="filled,bold"]; + } + + dsp_isr -> dsp_work [label="Enqueues"]; + dsp_work -> dsp_core [label="Executes"]; + dsp_core -> ipc3_hdl [label="IPC3 Msg"]; + dsp_core -> ipc4_hdl [label="IPC4 Msg"]; + } + + host_pci -> mbox_in [label="Writes Payload", color="#2980b9", penwidth=1.5]; + host_pci -> doorbells [label="Rings Host Doorbell", color="#2980b9", penwidth=1.5]; + doorbells -> dsp_isr [label="Hardware IRQ", color="#c0392b", penwidth=1.5]; + + ipc3_hdl -> mbox_out [label="Writes Reply", style=dashed, color="#27ae60"]; + ipc4_hdl -> mbox_out [label="Writes Reply", style=dashed, color="#27ae60"]; + dsp_core -> doorbells [label="Rings DSP Doorbell", color="#27ae60", penwidth=1.5]; + doorbells -> host_pci [label="Reply IRQ", color="#27ae60", penwidth=1.5]; + mbox_out -> host_pci [label="Reads Status", color="#2980b9", style=dashed]; + } + +--- + +2. Hardware Mailbox Architecture & Memory Windows +************************************************* + +Inter-processor messaging relies on dedicated **Shared SRAM Windows** mapped directly across PCIe Base Address Registers (BARs) on the host and accessible over the DSP system interconnect. + +Shared Memory Mailbox Windows +============================= + +Modern SOF platforms partition shared SRAM into distinct functional memory windows: + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Window + - Direction + - Architectural Purpose + * - **Window 0 (Outbox & Status)** + - DSP to Host + - Stores firmware reply payloads, asynchronous notifications, boot status words, and firmware version descriptors. + * - **Window 1 (Inbox)** + - Host to DSP + - Receives incoming host command headers, large parameter configuration blobs, and pipeline state commands. + * - **Window 2 (Debug & Traces)** + - DSP to Host + - Real-time debug log buffer accessed by host logging daemons (such as ``sof-logger`` or trace DMA). + * - **Window 3 (Stream Payloads)** + - Bidirectional + - Hosts large coefficient matrices (e.g. 10-band equalizer filter tables) and page-table descriptors for host DMA gateways. + +The Doorbell Interrupt Handshake Protocol +========================================= + +To coordinate memory access without race conditions, the host and DSP follow a strict **Doorbell Handshake Protocol**: + +.. graphviz:: + :caption: Bidirectional Hardware Mailbox and Doorbell Handshake Sequence + :align: center + + digraph doorbell_handshake { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_h2d { + label = "Host-to-DSP Command Transaction"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 9; + + h1 [label="1. Host writes command payload\ninto Mailbox Window 1 (Inbox)", fillcolor="#aed6f1"]; + h2 [label="2. Host asserts Host Doorbell IRQ\n(Sets Busy bit in PCIe register)", fillcolor="#aed6f1"]; + h3 [label="3. DSP ISR catches interrupt,\nclears IRQ & schedules work", fillcolor="#a9dfbf"]; + h4 [label="4. DSP processes command,\nwrites reply to Window 0 (Outbox)", fillcolor="#a9dfbf"]; + h5 [label="5. DSP asserts Done / Reply IRQ\n(Clears Busy bit; rings Host IRQ)", fillcolor="#a9dfbf", style="filled,bold"]; + h6 [label="6. Host catches reply IRQ,\nreads Window 0 & releases lock", fillcolor="#aed6f1"]; + + h1 -> h2 -> h3 -> h4 -> h5 -> h6; + } + } + +1. **Atomic Ownership**: While the Busy bit is asserted, the host is barred from overwriting the inbox. Ownership belongs exclusively to the DSP. +2. **Deterministic Acknowledgment**: The DSP signals completion by asserting the Done interrupt and writing status codes directly into Window 0, ensuring that the host driver never experiences mailbox data corruption. + +--- + +3. Core Framework & Zephyr Thread Handoff +***************************************** + +Why IPC Processing is Decoupled from Interrupts +=============================================== + +When the host triggers a mailbox doorbell interrupt, the DSP responds inside a hardware **Interrupt Service Routine (ISR)**. However, executing the entire IPC message within the ISR is strictly forbidden in real-time audio systems: + +* **Real-Time Latency Spikes**: Parsing complex pipeline topologies, allocating dynamic heaps, or configuring DAI clocks requires thousands of cycles. If executed inside an ISR, audio DMA interrupts would be delayed, causing immediate audio glitches and buffer underruns. +* **Blocking & DMA Waits**: Certain commands require waiting for DMA page table synchronization or inter-core responses. Interrupt service routines cannot sleep or block. + +Deferred Work Queue Architecture +================================ + +Sound Open Firmware solves this by delegating all command handling to the **Zephyr Work Queue subsystem** (``k_work``): + +.. graphviz:: + :caption: Mailbox ISR to Zephyr Work Queue Handoff and Message State Machine + :align: center + + digraph isr_handoff { + rankdir=TB; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_isr { + label = "Hardware Interrupt Context (Immediate, Zero Delay)"; + style = "filled,rounded"; + color = "#c0392b"; + fillcolor = "#f9ebea"; + fontname = "Verdana-Bold"; + fontsize = 9; + + irq_step1 [label="1. Hardware Mailbox IRQ Fires", fillcolor="#f5b7b1"]; + irq_step2 [label="2. Read Primary Header Word\n(Validates message boundaries)", fillcolor="#f5b7b1"]; + irq_step3 [label="3. Acknowledge Hardware Level\n(Clears interrupt latch)", fillcolor="#f5b7b1"]; + irq_step4 [label="4. Enqueue Work Item into Zephyr\nk_work_submit(&ipc->ipc_work)", fillcolor="#f5b7b1", style="filled,bold"]; + + irq_step1 -> irq_step2 -> irq_step3 -> irq_step4; + } + + subgraph cluster_thread { + label = "Thread Context (Zephyr Kernel Work Queue: ipc_work_handler)"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 9; + + th_step1 [label="5. Worker Thread Awakens\n(Runs at high cooperative priority)", fillcolor="#a9dfbf"]; + th_step2 [label="6. Decode Command & Dispatch\n(Routes to IPC3 or IPC4 handler)", fillcolor="#a9dfbf"]; + th_step3 [label="7. Execute Graph / Module Operation\n(Pipeline build, bind, or parameter update)", fillcolor="#a9dfbf", style="filled,bold"]; + th_step4 [label="8. Complete Transaction\n(Writes reply & rings Host Doorbell)", fillcolor="#a9dfbf"]; + + th_step1 -> th_step2 -> th_step3 -> th_step4; + } + + irq_step4 -> th_step1 [label="Context Switch", color="#27ae60", penwidth=1.5]; + } + +Message Lifecycle & Backpressure Handling +========================================= + +Firmware-initiated messages (such as notifications or stream position updates) are governed by an internal state machine: + +1. **State Progression**: Messages transition through ``UNREGISTERED`` $\rightarrow$ ``QUEUED`` $\rightarrow$ ``PROCESSING`` $\rightarrow$ ``ACK_PENDING`` $\rightarrow$ ``COMPLETED``. +2. **Outbox Message Queueing**: If the DSP needs to send an asynchronous notification while the hardware mailbox is already occupied by a previous pending message, the core IPC framework places the new message onto an internal transmission list (``ipc_msg_send``), preventing message loss under heavy host bus traffic. + +--- + +4. Protocol Generations: IPC3 vs. IPC4 +************************************** + +Sound Open Firmware supports two major generations of the Inter-Processor Communication protocol. While older hardware architectures use IPC3, all modern Intel platforms (Tiger Lake, Meteor Lake, Arrow Lake, Panther Lake) and contemporary designs utilize IPC4. + +Architectural Comparison +======================== + +.. graphviz:: + :caption: Structural Comparison: IPC3 Flat Scalar Model vs IPC4 Dynamic Compound Object Model + :align: center + + digraph ipc_comparison { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_ipc3 { + label = "IPC3: Static Scalar Model (Legacy)"; + style = "filled,rounded"; + color = "#7f8c8d"; + fillcolor = "#f2f4f4"; + fontname = "Verdana-Bold"; + fontsize = 9; + + ipc3_hdr [label="sof_ipc_cmd_hdr\n(Global Command Type + Size)", fillcolor="#d5dbdb"]; + ipc3_pcm [label="SOF_IPC_GLB_STREAM_MSG\n(pcm_params, trigger, position)", fillcolor="#d5dbdb"]; + ipc3_dai [label="SOF_IPC_GLB_DAI_MSG\n(dai_config, ssp/hda config)", fillcolor="#d5dbdb"]; + ipc3_topo [label="Static Graph Deployment\n(Topology loaded monolithically at probe)", fillcolor="#bdc3c7", style="filled,bold"]; + + ipc3_hdr -> ipc3_pcm; + ipc3_hdr -> ipc3_dai; + ipc3_pcm -> ipc3_topo; + } + + subgraph cluster_ipc4 { + label = "IPC4: Dynamic Compound Object Model (Modern)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 9; + + ipc4_hdr [label="64-Bit Primary Compact Header\n(Type, Rsp, Target, Status, Ext)", fillcolor="#aed6f1", style="filled,bold"]; + ipc4_ppl [label="Pipeline Management\n(new_pipeline, set_state, delete)", fillcolor="#aed6f1"]; + ipc4_mod [label="Dynamic Modules\n(init_instance, set/get_params)", fillcolor="#aed6f1"]; + ipc4_bind [label="Dynamic Pin Binding\n(ipc4_bind / ipc4_unbind)", fillcolor="#aed6f1", style="filled,bold"]; + + ipc4_hdr -> ipc4_ppl; + ipc4_hdr -> ipc4_mod; + ipc4_hdr -> ipc4_bind; + } + } + +Key Differences +=============== + +.. list-table:: + :widths: 20 40 40 + :header-rows: 1 + + * - Dimension + - IPC3 (Scalar Architecture) + - IPC4 (Compound Object Architecture) + * - **Topology Model** + - **Static**: Entire pipeline graph is compiled into a monolithic topology binary and parsed at driver probe. + - **Dynamic**: Pipelines and modules are constructed, bound, and torn down dynamically at runtime via individual IPC commands. + * - **Component Addressing** + - Global 32-bit component IDs assigned statically by the topology compiler. + - Modular 32-bit Tuple: ``module_id`` (algorithm type UUID) combined with an ``instance_id`` (unique runtime instance). + * - **Command Density** + - Scalar: Each operation requires a separate round-trip command/response handshake. + - Compound: Multiple operations (create pipeline, instantiate modules, bind pins) can be batched in a single transaction. + * - **Memory Footprint** + - Graph nodes and buffers are pre-allocated statically during system boot. + - Memory heaps are allocated and reclaimed on-demand as audio streams open and close. + +--- + +5. Pipeline Lifecycle & Dynamic Graph Control +********************************************* + +In IPC4, the host operating system dynamically constructs, connects, and controls the audio processing graph: + +Dynamic Graph Instantiation Flow +================================ + +.. graphviz:: + :caption: IPC4 Dynamic Pipeline Construction and Streaming Sequence + :align: center + + digraph ipc4_lifecycle { + rankdir=TB; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + s1 [label="1. Create Pipeline (ipc4_new_pipeline)\nHost defines pipeline ID, execution priority, and core affinity", fillcolor="#d4e6f1"]; + s2 [label="2. Instantiate Modules (ipc4_init_module_instance)\nDSP allocates component memory sandbox and initializes algorithm state", fillcolor="#d4e6f1"]; + s3 [label="3. Bind Component Pins (ipc4_bind)\nHost links Source Pin of Module A to Sink Pin of Module B via intermediate ring buffer", fillcolor="#aed6f1", style="filled,bold"]; + s4 [label="4. Configure Parameters (ipc4_set_module_params)\nHost delivers coefficient matrices, volume curves, and audio format descriptors", fillcolor="#d4e6f1"]; + s5 [label="5. Set Pipeline State (ipc4_set_pipeline_state)\nTransitions pipeline: INIT -> PAUSED -> RUNNING", fillcolor="#abebc6", style="filled,bold"]; + s6 [label="6. Audio Streaming\nScheduler domains (LL / DP) process audio frames across circular buffers", fillcolor="#abebc6"]; + s7 [label="7. Teardown (ipc4_unbind & ipc4_delete_pipeline)\nPipeline halted, memory sandbox reclaimed, and buffers deallocated", fillcolor="#fadbd8"]; + + s1 -> s2 -> s3 -> s4 -> s5 -> s6 -> s7; + } + +Core State Machine Integration +============================== + +The host controls pipeline progression by sending ``ipc4_set_pipeline_state()`` commands. The IPC framework maps these high-level host requests directly into SOF core state machine triggers: + +* **``IPC4_PIPELINE_STATE_RESET``** $\rightarrow$ Re-initializes buffers and resets filter delay lines. +* **``IPC4_PIPELINE_STATE_PAUSED``** $\rightarrow$ Halts active processing while preserving audio parameters and buffer memory. +* **``IPC4_PIPELINE_STATE_RUNNING``** $\rightarrow$ Dispatches ``COMP_TRIGGER_START``, enabling real-time timer or DMA interrupts. +* **``IPC4_PIPELINE_STATE_EOS``** $\rightarrow$ Signals End-Of-Stream, allowing remaining samples in ring buffers to drain cleanly without truncation. + +--- + +6. Firmware-Initiated Notifications & Telemetry +*********************************************** + +While commands flow from Host to DSP, the IPC infrastructure also provides a high-efficiency path for **Firmware-Initiated Asynchronous Notifications** (DSP to Host). + +Asynchronous Telemetry Flow +=========================== + +.. graphviz:: + :caption: Firmware-Initiated Asynchronous Notification Architecture + :align: center + + digraph notification_flow { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_events { + label = "DSP Event Generators"; + style = "filled,rounded"; + color = "#8e44ad"; + fillcolor = "#f4ecf7"; + fontname = "Verdana-Bold"; + fontsize = 9; + + ev_pos [label="Position Reporter\n(Stream DMA sample offset)", fillcolor="#d7bde2"]; + ev_xrun [label="XRUN Monitor\n(Buffer underrun / overrun)", fillcolor="#d7bde2"]; + ev_panic [label="Exception Handler\n(Crash dump & register state)", fillcolor="#f5b7b1"]; + } + + subgraph cluster_queue { + label = "Notification Management (notification_pool.c)"; + style = "filled,rounded"; + color = "#d35400"; + fillcolor = "#fef5e7"; + fontname = "Verdana-Bold"; + fontsize = 9; + + pool_mgr [label="Notification Pool Allocator\n(Pre-allocated descriptors)", fillcolor="#fad7a0"]; + tx_queue [label="Outbox Transmission Queue\n(Buffers notifications if mailbox busy)", fillcolor="#fad7a0", style="filled,bold"]; + pool_mgr -> tx_queue; + } + + subgraph cluster_outbox { + label = "Mailbox Outbox & Host IRQ"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 9; + + mb_out [label="Window 0 (Outbox SRAM)\nWrites notification payload", fillcolor="#a9dfbf", shape=cylinder]; + mb_irq [label="Assert DSP Doorbell IRQ\nSignals Host PCIe interrupt", fillcolor="#a9dfbf", style="filled,bold"]; + mb_out -> mb_irq; + } + + ev_pos -> tx_queue [label="Periodic"]; + ev_xrun -> tx_queue [label="Immediate"]; + ev_panic -> tx_queue [label="Fatal"]; + + tx_queue -> mb_out [label="Dispatches to SRAM"]; + } + +Notification Types & Purpose +============================ + +1. **Stream Position Updates**: + * Sent periodically as hardware DMA transfers audio frames to/from host memory. + * Updates host ALSA ring buffer pointers, allowing user-space applications to track playback timing with microsecond accuracy. +2. **XRUN Notifications**: + * Instantly alerts the host kernel if an audio underrun or overrun occurs, enabling the host driver to log diagnostics and initiate recovery. +3. **Firmware Panic & Error Reports**: + * In the rare event of a CPU exception, watchdog timeout, or kernel assert, the exception handler formats a panic descriptor containing CPU register states, execution backtraces, and memory faults into Window 0 before resetting the DSP. + +--- + +7. Multi-Core IPC & Inter-Domain Communication (IDC) +**************************************************** + +Modern Intel and partner DSPs feature multi-core architectures (Dual-Core, Quad-Core, or Octa-Core). However, the physical PCIe mailbox hardware and doorbell interrupt registers are physically routed **only to Core 0**. + +Core 0 as the Central Host Gateway +================================== + +Core 0 acts as the central gateway for all external host communication: + +* All incoming host doorbell interrupts are caught exclusively by Core 0's mailbox ISR. +* All outgoing notifications and replies must be written to Window 0 by Core 0. + +Inter-Domain Communication (IDC) Architecture +============================================= + +When the host issues an IPC command targeting a pipeline, audio module, or power state located on a secondary core (such as Core 1, Core 2, or Core 3), SOF utilizes **Inter-Domain Communication (IDC)**: + +.. graphviz:: + :caption: Multi-Core IPC Routing Topology: Core 0 (Host Gateway) and Core 1 (Secondary Core) via IDC + :align: center + + digraph idc_topology { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_c0 { + label = "DSP Core 0 (Host Gateway & Primary Dispatcher)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1b4f72"; + + c0_isr [label="Mailbox ISR\n(Catches Host Doorbell)", fillcolor="#aed6f1"]; + c0_dec [label="Core Target Decoder\n(Detects command targets Core 1)", fillcolor="#aed6f1"]; + c0_idc [label="IDC Sender\n(Writes IDC shared mailbox\n& rings Inter-Core Doorbell)", fillcolor="#aed6f1", style="filled,bold"]; + c0_reply [label="Host Reply Aggregator\n(Writes Window 0 & rings Host IRQ)", fillcolor="#aed6f1"]; + + c0_isr -> c0_dec -> c0_idc; + c0_reply -> c0_isr [style=invis]; + } + + subgraph cluster_shared { + label = "Inter-Core Shared Memory (HP-SRAM)"; + style = "filled,rounded"; + color = "#d35400"; + fillcolor = "#fef5e7"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#a04000"; + + idc_msg [label="IDC Message Structure\n(Shared Memory Buffer)", fillcolor="#fad7a0", shape=cylinder]; + idc_irq [label="Hardware Inter-Core Doorbell\n(DSP Architectural IRQ)", fillcolor="#fad7a0"]; + } + + subgraph cluster_c1 { + label = "DSP Core 1 (Secondary Compute Core)"; + style = "filled,rounded"; + color = "#8e44ad"; + fillcolor = "#f4ecf7"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#512e5f"; + + c1_isr [label="IDC ISR\n(Catches Core 0 Doorbell)", fillcolor="#d7bde2"]; + c1_work [label="IDC Worker Thread\n(Executes target module operation)", fillcolor="#d7bde2", style="filled,bold"]; + c1_ack [label="IDC Reply\n(Signals completion back to Core 0)", fillcolor="#d7bde2"]; + + c1_isr -> c1_work -> c1_ack; + } + + c0_idc -> idc_msg [label="Write Payload", color="#2980b9", penwidth=1.5]; + c0_idc -> idc_irq [label="Assert IRQ", color="#2980b9", penwidth=1.5]; + idc_irq -> c1_isr [label="Hardware Interrupt", color="#c0392b", penwidth=1.5]; + idc_msg -> c1_work [label="Read Payload", color="#8e44ad", style=dashed]; + + c1_ack -> c0_reply [label="IDC Return Status", color="#27ae60", penwidth=1.5]; + } + +1. **Transparent Routing**: The host driver remains completely agnostic to core partitioning. The host targets a module by ID; Core 0's IPC framework transparently resolves which core owns the module. +2. **IDC Doorbell Interrupts**: Core 0 copies the message payload into shared inter-core SRAM and triggers a hardware inter-core interrupt to awaken Core 1. +3. **Status Aggregation**: When Core 1 finishes processing the command, it returns an acknowledgment via IDC. Core 0 aggregates the response and completes the transaction to the host. + +--- + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level C implementation details, data structures, and function prototypes: + +* **Upstream IPC Specifications**: + * Core IPC framework architecture: `thesofproject/sof: src/ipc/README.md `_. + * IPC3 scalar architecture: `thesofproject/sof: src/ipc/ipc3/README.md `_. + * IPC4 dynamic object architecture: `thesofproject/sof: src/ipc/ipc4/README.md `_. + +* **Core Source Files**: + + * ``src/ipc/ipc-common.c``: Core message state machine, dispatcher, and outbox queue management. + * ``src/ipc/ipc-zephyr.c``: Zephyr work queue thread handoff (``ipc_work_handler``). + * ``src/ipc/ipc3/handler.c``: IPC3 global command dispatcher (stream, DAI, PM). + * ``src/ipc/ipc4/handler-kernel.c``: IPC4 primary header parser, global pipeline state engine, and module dispatcher. + * ``src/ipc/ipc4/ams_helpers.c``: IPC4 dynamic module instantiation and pin binding helpers. + * ``src/ipc/notification_pool.c``: Pre-allocated asynchronous notification pool allocator. + +* **Core Header Files**: + + * ``src/include/ipc/header.h``: Common IPC message header definitions and command enums. + * ``src/include/ipc/stream.h``: Stream parameter, trigger, and position payload definitions. + * ``src/include/ipc/topology.h``: Topology IPC structures and component creation payloads. + * ``src/include/sof/ipc/schedule.h``: Scheduling domain integration with IPC work queues. + +Related Guides +============== + +* :ref:`pipeline_architecture`: How IPC commands dynamically create, prepare, and trigger audio pipelines. +* :ref:`module_framework`: How IPC parameter blobs configure processing modules and runtime algorithms. +* :ref:`scheduler_architecture`: Real-time scheduling domains (LL, DP, TWB) that coordinate with IPC work queues. +* :ref:`audio_buffer_management`: Allocating and binding circular ring buffers during IPC pipeline construction. +* :ref:`topology2`: How ALSA Topology 2.0 configuration files generate IPC topology commands. diff --git a/developer_guides/firmware/module_framework.rst b/developer_guides/firmware/module_framework.rst index 59e70849..a0aba5df 100644 --- a/developer_guides/firmware/module_framework.rst +++ b/developer_guides/firmware/module_framework.rst @@ -516,6 +516,7 @@ For developers seeking low-level C implementation details, data structures, and Related Guides ============== +* :ref:`ipc_infrastructure`: Host-to-DSP messaging, hardware mailbox windows, and dynamic IPC4 compound commands. * :ref:`audio_buffer_management`: Lockless circular ring buffers, multi-tier DSP memory (SRAM/DRAM), and cache coherency. * :ref:`scheduler_architecture`: Multi-tier real-time scheduling (LL, DP, TWB), EDF mechanics, and multi-core execution. * :ref:`pipeline_architecture`: How processing modules are assembled into directed acyclic execution graphs (DAGs). diff --git a/developer_guides/firmware/pipeline_architecture.rst b/developer_guides/firmware/pipeline_architecture.rst index eb0f412e..4232450b 100644 --- a/developer_guides/firmware/pipeline_architecture.rst +++ b/developer_guides/firmware/pipeline_architecture.rst @@ -457,6 +457,7 @@ For developers seeking low-level C implementation details, data structures, and Related Guides ============== +* :ref:`ipc_infrastructure`: Host-to-DSP messaging, hardware mailbox windows, and dynamic IPC4 compound commands. * :ref:`scheduler_architecture`: Multi-tier real-time scheduling (LL, DP, TWB), EDF mechanics, and multi-core execution. * :ref:`audio_buffer_management`: Lockless circular ring buffers, multi-tier DSP memory (SRAM/DRAM), and cache coherency. * :ref:`module_framework`: The standardized module interface, Source/Sink APIs, and memory sandboxing. diff --git a/developer_guides/firmware/scheduler_architecture.rst b/developer_guides/firmware/scheduler_architecture.rst index b09dc672..201b5325 100644 --- a/developer_guides/firmware/scheduler_architecture.rst +++ b/developer_guides/firmware/scheduler_architecture.rst @@ -564,6 +564,7 @@ For developers seeking low-level C implementation details, data structures, and Related Guides ============== +* :ref:`ipc_infrastructure`: Host-to-DSP messaging, hardware mailbox windows, and dynamic IPC4 compound commands. * :ref:`audio_buffer_management`: Lockless circular ring buffers, multi-tier DSP memory (SRAM/DRAM), and cache coherency. * :ref:`pipeline_architecture`: How audio pipelines interact with the scheduling domains to stream data. * :ref:`module_framework`: The standardized module interface executed by LL and DP scheduler tasks. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 383df20f..5e43efbe 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -35,7 +35,7 @@ Core Infrastructure & Pipeline * :ref:`module_framework` (High-level architecture; also see upstream `module README `_ & `module adapter README `_) * :ref:`scheduler_architecture` (High-level architecture; also see upstream `scheduler README `_) * :ref:`audio_buffer_management` (High-level architecture; also see upstream `buffer README `_) -* `IPC Infrastructure (IPC3 & IPC4) `_ +* :ref:`ipc_infrastructure` (High-level architecture; also see upstream `IPC README `_) * `Firmware Initialization & Boot `_ Audio Processing Modules & Algorithms @@ -84,6 +84,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/module_framework firmware/scheduler_architecture firmware/audio_buffer_management + firmware/ipc_infrastructure rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 7d88e965d15f28471c8024f824b4c0be74cb2402 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Fri, 18 Sep 2026 14:46:02 +0100 Subject: [PATCH 07/64] docs: developer_guides: add high-level firmware init and boot guide Add a comprehensive, high-level developer architecture guide covering the Sound Open Firmware (SOF) initialization and boot subsystem. The guide covers: - End-to-end boot lifecycle across host staging, hardware boot ROM, Zephyr kernel initialization, SOF primary core bringup, and host ready synchronization. - Extended firmware manifest (.fw_metadata) structure and pre-boot host discovery. - Zephyr RTOS multi-stage kernel initialization levels (EARLY, PRE_KERNEL_1, PRE_KERNEL_2, POST_KERNEL, APPLICATION) and the rationale for SYS_INIT(sof_init, POST_KERNEL, 99). - Primary core platform bringup (primary_core_init) covering context allocation, DMA trace buffering, system notifiers, runtime PM, schedulers, DMACs, IPC, and AltBootManifest LP-SRAM unpacking. - Host-firmware boot synchronization and FW Ready handshake protocols (IPC3 vs IPC4) in Mailbox Window 0, along with boot timeout detection. - Multi-core secondary core boot flow, dynamic power state assessment (check_restore for cold boot vs D0ix retention wake), and IDC. - Power state lifecycles and wake transitions (D3 cold boot, D0 active, D0ix low-power retention, and S0ix/S3 suspend), including LLEXT dynamic library preservation. - Upstream code references and related architecture links. Includes 7 custom vector Graphviz SVG diagrams illustrating all boot phases, binary manifest structures, Zephyr hook levels, platform bringup, mailbox handshakes, secondary core activation, and power states. Signed-off-by: Liam Girdwood --- .../firmware/audio_buffer_management.rst | 1 + developer_guides/firmware/fw_init_boot.rst | 504 ++++++++++++++++++ .../firmware/ipc_infrastructure.rst | 1 + .../firmware/pipeline_architecture.rst | 1 + .../firmware/scheduler_architecture.rst | 1 + developer_guides/index.rst | 3 +- 6 files changed, 510 insertions(+), 1 deletion(-) create mode 100644 developer_guides/firmware/fw_init_boot.rst diff --git a/developer_guides/firmware/audio_buffer_management.rst b/developer_guides/firmware/audio_buffer_management.rst index 57f9913c..0daa12d3 100644 --- a/developer_guides/firmware/audio_buffer_management.rst +++ b/developer_guides/firmware/audio_buffer_management.rst @@ -622,4 +622,5 @@ Related Guides * :ref:`pipeline_architecture`: How audio buffers interconnect components into directed acyclic graphs (DAGs). * :ref:`module_framework`: The standardized module interface that consumes and produces audio samples through Source and Sink APIs. * :ref:`scheduler_architecture`: Real-time scheduling domains (LL, DP, TWB) that drive buffer read and write intervals. +* :ref:`fw_init_boot`: Boot flow, hardware mailbox FW Ready handshake, and Zephyr initialization. * :ref:`topology2`: Declaring buffer sizes, capabilities, and period counts in ALSA Topology 2.0 configuration files. diff --git a/developer_guides/firmware/fw_init_boot.rst b/developer_guides/firmware/fw_init_boot.rst new file mode 100644 index 00000000..23f6b837 --- /dev/null +++ b/developer_guides/firmware/fw_init_boot.rst @@ -0,0 +1,504 @@ +.. _fw_init_boot: + +Firmware Initialization & Boot Architecture +########################################### + +The **Firmware Initialization & Boot** subsystem in Sound Open Firmware (SOF) governs the complete sequence through which the audio Digital Signal Processor (DSP) transitions from an unpowered or quiescent hardware state into a fully initialized, real-time audio computing engine. + +Operating as an embedded real-time system across diverse silicon architectures (Intel CAVS/ACE, NXP i.MX, AMD ACP, and embedded microcontrollers like ESP32 and Teensy), SOF couples low-level hardware bootstrap sequences with the **Zephyr RTOS** kernel lifecycle, multi-tier platform hardware bringup, host driver synchronization handshakes, and multi-core power restoration. + +This guide provides a comprehensive, high-level architectural walkthrough of the firmware initialization and boot framework without delving into low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +.. _fw_boot_lifecycle: + +1. End-to-End Boot & Initialization Lifecycle +********************************************* + +Bringing an audio DSP from host power-on to active audio stream processing spans multiple distinct execution domains: host operating system orchestration, DSP hardware boot ROM, Zephyr RTOS kernel initialization, SOF primary core initialization, application thread startup, and host-firmware synchronization. + +The Five Architectural Phases of Boot +===================================== + +1. **Host Driver Pre-Boot Staging**: The host operating system (e.g., Linux mainline ALSA/ASoC driver) parses the signed firmware ELF binary, inspects embedded metadata headers, allocates host DMA buffers (or Isolated Memory Regions / IMR), programs DSP base address registers (BARs), and deasserts the hardware DSP core reset latch. +2. **DSP Hardware Boot ROM Execution**: The DSP's embedded on-chip ROM begins executing on Core 0. The ROM powers up internal SRAM banks, configures early clock trees, validates cryptographic signatures and hash manifests, configures DSP memory management page tables, copies the firmware image from host memory into DSP SRAM, and vectors execution to the operating system entry point (``_start``). +3. **Zephyr RTOS Kernel Bringup**: The Zephyr operating system initializes processor registers, zeroes BSS, unpacks initialized data sections, initializes architectural exception vectors, and progresses through deterministic kernel initialization stages (``EARLY``, ``PRE_KERNEL_1``, ``PRE_KERNEL_2``, and ``POST_KERNEL``). +4. **SOF Core & Platform Subsystem Initialization**: Registered at Zephyr's ``POST_KERNEL`` stage, SOF's entry function (``sof_init()``) executes on Core 0. It sets up logging and DMA trace buffers, initializes system-wide notifiers, configures runtime power management, invokes platform-specific peripheral drivers (clocks, DMACs, IPC mailboxes, audio schedulers), and unpacks secondary core storage manifests. +5. **Application Main Handoff & Host Ready Handshake**: Zephyr transitions execution to the application main thread (``sof_app_main()``). SOF verifies library integrity (such as dynamically restored LLEXT components), writes firmware status and ABI details to the hardware mailbox, asserts the host interrupt, and transitions to the active running state, awaiting host IPC audio pipeline commands. + +.. graphviz:: + :caption: End-to-End SOF Boot Flow & System Lifecycle from Host Driver Staging to Audio Readiness + + digraph fw_boot_flow { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_host { + label="Host Operating System (Linux Kernel ALSA/ASoC SOF Driver)"; + style="filled,rounded"; + fillcolor="#F7FAFC"; + color="#CBD5E0"; + + h1 [label="Parse Firmware ELF Binary\nInspect Extended Manifest (.fw_metadata)", fillcolor="#EDF2F7", color="#CBD5E0"]; + h2 [label="Stage Firmware into Host DMA / IMR\nProgram DSP BARs & Power Registers", fillcolor="#EDF2F7", color="#CBD5E0"]; + h3 [label="Deassert DSP Hardware Reset Latch\nStart DSP Boot Timeout Monitor", fillcolor="#EDF2F7", color="#CBD5E0"]; + h4 [label="Receive Mailbox FW Ready Interrupt\nVerify ABI & Register Sound Card", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + h1 -> h2 -> h3; + } + + subgraph cluster_rom { + label="DSP Hardware Boot ROM (Core 0)"; + style="filled,rounded"; + fillcolor="#FFF5F5"; + color="#FEB2B2"; + + r1 [label="Hardware Reset Vector\nInit Early Clocks, Cache & Internal SRAM", fillcolor="#FED7D7", color="#E53E3E"]; + r2 [label="Validate Cryptographic Signature\nVerify Hash Manifest & Manifest Headers", fillcolor="#FED7D7", color="#E53E3E"]; + r3 [label="Program MMU/MPU Page Tables\nDMA Load SOF Image into DSP SRAM/TCM", fillcolor="#FED7D7", color="#E53E3E"]; + r4 [label="Branch to Operating System Entry Point\nJump to Zephyr _start Vector", fillcolor="#FEB2B2", color="#C53030"]; + + r1 -> r2 -> r3 -> r4; + } + + subgraph cluster_zephyr { + label="Zephyr RTOS Initialization Stages"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + z1 [label="Architecture Setup (crt0.S)\nClear BSS, Copy .data, Init Vectors", fillcolor="#BEE3F8", color="#3182CE"]; + z2 [label="Zephyr PRE_KERNEL Stages\nInit CPU, Interrupt Controllers & Timers", fillcolor="#BEE3F8", color="#3182CE"]; + z3 [label="Zephyr POST_KERNEL Stage\nTrigger Registered Drivers & SYS_INIT Hooks", fillcolor="#90CDF4", color="#2B6CB0"]; + + z1 -> z2 -> z3; + } + + subgraph cluster_sof { + label="Sound Open Firmware Subsystems (Core 0)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + s1 [label="SOF Framework Hook: sof_init()\nprimary_core_init(sof)", fillcolor="#C6F6D5", color="#38A169"]; + s2 [label="Subsystem Bringup: trace_init(),\ninit_system_notify(), pm_runtime_init()", fillcolor="#E6FFFA", color="#319795"]; + s3 [label="Platform Bringup: platform_init()\nClocks, Schedulers (EDF, LL, DP), IPC, DMAC", fillcolor="#E6FFFA", color="#319795"]; + s4 [label="Component Registry & Unpack:\nsys_comp_init(), lp_sram_unpack()", fillcolor="#E6FFFA", color="#319795"]; + s5 [label="Application Entry: sof_app_main()\nstart_complete() -> boot_complete()", fillcolor="#9AE6B4", color="#2F855A", fontcolor="#1C4532"]; + s6 [label="Write Mailbox FW Ready & Status\nRaise Host Doorbell Interrupt", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + s1 -> s2 -> s3 -> s4 -> s5 -> s6; + } + + h3 -> r1 [label="Reset Deassert", color="#E53E3E", style="dashed"]; + r4 -> z1 [label="Vector Jump", color="#3182CE"]; + z3 -> s1 [label="SYS_INIT(POST_KERNEL, 99)", color="#2B6CB0"]; + s6 -> h4 [label="Mailbox Doorbell Interrupt (FW Ready)", color="#276749", style="bold"]; + } + +--- + +.. _ext_manifest_architecture: + +2. Extended Firmware Manifest & Host Pre-Boot Discovery +******************************************************* + +Before the DSP hardware is taken out of reset, the host operating system must discover firmware capabilities, ABI compatibility constraints, memory window geometries, and debugging parameters. + +SOF accomplishes this via the **Extended Firmware Manifest**, an embedded data structure placed directly into the dedicated ``.fw_metadata`` section of the compiled firmware ELF binary (implemented in ``src/init/ext_manifest.c``). + +Manifest Structure & Header Elements +==================================== + +The extended manifest consists of a contiguous sequence of self-describing structured elements. Each element begins with a standard header (``ext_man_elem_header``) containing an element type identifier and a total element payload byte length: + +* **Firmware Version (``ext_man_fw_version``)**: Exposes the major, minor, micro, build tag, and cryptographic Git commit hash of the compiled firmware binary. The host uses this to verify driver compatibility before downloading. +* **Compiler & Toolchain Version (``ext_man_cc_version``)**: Contains the compiler name, toolchain version, and build timestamp (e.g., LLVM/Clang or Cadence XCC) used to build the image. +* **Extraction Probe Support (``ext_man_probe_support``)**: Informs the host driver whether live trace probe DMA extraction points are enabled and provides buffer sizing limits for real-time telemetry streaming. +* **Debug ABI Specification (``ext_man_dbg_abi``)**: Declares the user-space debugger and probe ABI version (such as dictionary-based log extraction schemas used by ``smex`` and ``sof-logger``). +* **Configuration Dictionary (``ext_man_config_data``)**: A key-value array of hardware and build configuration constants, including maximum IPC message sizes (``SOF_IPC_MSG_MAX_SIZE``), memory window offsets, and platform capabilities. + +.. graphviz:: + :caption: Extended Manifest (`.fw_metadata`) Binary Layout and Pre-Boot Host Parsing Flow + + digraph ext_manifest_layout { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_elf { + label="Compiled SOF Firmware ELF Binary"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + elf_hdr [label="Standard ELF Header\n& Program Headers", fillcolor="#FFFFFF", color="#CBD5E0"]; + text_sec [label="Executable Code\n.text, .literal", fillcolor="#FFFFFF", color="#CBD5E0"]; + data_sec [label="Initialized Data\n.data, .rodata", fillcolor="#FFFFFF", color="#CBD5E0"]; + + subgraph cluster_meta { + label="Section: .fw_metadata"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + em_hdr [label="ext_man_header\nMagic: 0x3e456d78\nTotal Manifest Size", fillcolor="#FAF089", color="#B7791F"]; + em_ver [label="ext_man_fw_version\nMajor, Minor, Micro\nGit Commit Hash", fillcolor="#FAF089", color="#B7791F"]; + em_cc [label="ext_man_cc_version\nToolchain: Clang / XCC\nBuild Description", fillcolor="#FAF089", color="#B7791F"]; + em_prb [label="ext_man_probe_support\nProbe Extraction Limits\nTrace DMA Capabilities", fillcolor="#FAF089", color="#B7791F"]; + em_dbg [label="ext_man_dbg_abi\nDebugger ABI Version\nLog Schema Hashes", fillcolor="#FAF089", color="#B7791F"]; + em_cfg [label="ext_man_config_data\nKey-Value Configuration\nMax IPC Size, Windows", fillcolor="#FAF089", color="#B7791F"]; + + em_hdr -> em_ver -> em_cc -> em_prb -> em_dbg -> em_cfg; + } + } + + subgraph cluster_host_driver { + label="Host Linux ASoC Driver (snd-sof)"; + style="filled,rounded"; + fillcolor="#E6FFFA"; + color="#319795"; + + h_parse [label="Manifest Parser (sof_ext_man_parse)\nExtracts Metadata Before DSP Power-On", fillcolor="#B2F5EA", color="#319795"]; + h_compat [label="Version & ABI Verification\nMatch Kernel Driver Capabilities", fillcolor="#B2F5EA", color="#319795"]; + h_prep [label="Allocate Mailbox & Trace Buffers\nConfigure Stream DMA Windows", fillcolor="#B2F5EA", color="#319795"]; + + h_parse -> h_compat -> h_prep; + } + + em_hdr -> h_parse [label="Host Pre-Boot Inspection", color="#319795", style="dashed"]; + } + +Because the host driver inspects this manifest directly from the binary file prior to downloading code into the DSP, mismatched firmware builds or incompatible ABI revisions are intercepted immediately, preventing kernel panics or DSP hangs. + +--- + +.. _zephyr_boot_stages: + +3. Zephyr RTOS Multi-Stage Initialization +***************************************** + +Sound Open Firmware is natively constructed upon the **Zephyr RTOS**. Zephyr utilizes a deterministic, multi-level initialization table where drivers, core kernel primitives, and application subsystems are systematically registered and executed using the ``SYS_INIT()`` macro. + +Deterministic Initialization Levels +=================================== + +Zephyr defines five sequential initialization levels: + +1. **EARLY**: Low-level platform hardware initialization executed before any OS abstractions exist. No kernel structures or memory allocators are available. +2. **PRE_KERNEL_1**: Core CPU architecture features, basic interrupt controllers, and essential hardware console devices are brought online. No thread scheduling or kernel synchronization primitives exist. +3. **PRE_KERNEL_2**: High-resolution hardware system timers, memory management units (MMU/MPU), and hardware clock trees are initialized. +4. **POST_KERNEL**: The Zephyr kernel is fully operational. Dynamic memory allocators, thread creation, semaphores, and inter-thread messaging primitives are ready. Device drivers and middleware services initialize during this level. +5. **APPLICATION**: Executed after all kernel and device driver subsystems are ready, immediately prior to invoking the main application thread. + +.. graphviz:: + :caption: Zephyr RTOS Multi-Stage Initialization Pipeline and SOF SYS_INIT Integration + + digraph zephyr_stages { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + z_early [label="Level 1: EARLY\nLow-level SoC pinmux, early silicon clocks\n(No OS features available)", fillcolor="#EDF2F7", color="#CBD5E0"]; + z_pk1 [label="Level 2: PRE_KERNEL_1\nCPU registers, vector tables, interrupt controller\nHardware console / early UART", fillcolor="#EDF2F7", color="#CBD5E0"]; + z_pk2 [label="Level 3: PRE_KERNEL_2\nSystem tick timer (HPET/DSP timer), MMU/MPU tables\nClock domain managers", fillcolor="#EDF2F7", color="#CBD5E0"]; + z_post [label="Level 4: POST_KERNEL\nKernel Core Active: Heaps, Threads, Mutexes, Workqueues\nDevice Drivers, Audio Hardware Peripherals", fillcolor="#BEE3F8", color="#3182CE"]; + + subgraph cluster_sof_hook { + label="SOF Entry Hook: SYS_INIT(sof_init, POST_KERNEL, 99)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + sof_entry [label="sof_init() (src/init/init.c)\nExecutes at POST_KERNEL Priority 99\nGuarantees Full OS Infrastructure Ready", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + } + + z_app [label="Level 5: APPLICATION\nApplication-level services, background monitors", fillcolor="#EDF2F7", color="#CBD5E0"]; + z_main [label="Application Thread: main() -> sof_app_main()\nStart Real-Time Audio Tasks & IPC Mailbox Handoff", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + z_early -> z_pk1 -> z_pk2 -> z_post; + z_post -> sof_entry [label="POST_KERNEL Execution Order"]; + sof_entry -> z_app; + z_app -> z_main; + } + +The Rationale for `POST_KERNEL, 99` +=================================== + +SOF explicitly binds its primary initialization entry point via: + +.. code-block:: c + + /* Registered in src/init/init.c */ + SYS_INIT(sof_init, POST_KERNEL, 99); + +Selecting ``POST_KERNEL`` at priority level ``99`` (the lowest priority within that stage) guarantees that: + +* All hardware buses, DMA controllers, and interrupt routing controllers registered by Zephyr drivers have finished their initialization. +* The Zephyr kernel heap allocator is fully operational, allowing SOF to dynamically allocate its global context structures and buffer descriptors. +* Zephyr thread creation and synchronization APIs (such as ``k_work_queue`` and ``k_thread``) are ready for SOF's deferred IPC handler and real-time audio schedulers. +* The SOF initialization code runs synchronously to completion on Core 0 *before* Zephyr switches execution to user application threads. + +--- + +.. _primary_core_platform_init: + +4. Primary Core Platform Initialization (`primary_core_init`) +************************************************************* + +When Zephyr invokes ``sof_init()``, control transitions immediately to ``primary_core_init()`` in ``src/init/init.c``. This function orchestrates the deterministic bringup of SOF's internal audio subsystem and invokes hardware-specific platform initializers. + +Primary Core Initialization Stages +================================== + +.. graphviz:: + :caption: Primary Core (`primary_core_init`) Execution Flow & Platform Subsystem Bringup Sequence + + digraph primary_core_flow { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + pc1 [label="1. Context Allocation\nAllocate global 'struct sof' context\nBind command arguments and runtime pointers", fillcolor="#EDF2F7", color="#CBD5E0"]; + pc2 [label="2. Logging & DMA Tracing (trace_init)\nConfigure Zephyr log timestamps (k_cycle_get_32)\nAllocate DMA trace buffer; print firmware version banner", fillcolor="#EBF8FF", color="#3182CE"]; + pc3 [label="3. System Notification & Power (pm_runtime_init)\nInitialize system-wide notification dispatch queue\nConfigure runtime power management & idle states", fillcolor="#EBF8FF", color="#3182CE"]; + pc4 [label="4. Platform Bringup (platform_init)\nPlatform clock init & dynamic KCPS budgeting\nInitialize Schedulers: EDF, LL Timer Domain, DP, TWB\nConfigure System Agent, DMACs, IPC Mailbox & Watchdog", fillcolor="#FEFCBF", color="#D69E2E"]; + pc5 [label="5. AltBootManifest Unpack (lp_sram_unpack)\nUnpack LP-SRAM text/data sections for secondary cores\nFlush data cache to memory (dcache_writeback_region)", fillcolor="#E2E8F0", color="#A0AEC0"]; + pc6 [label="6. Audio Registry & Component Setup\nRegister built-in audio components (sys_comp_init)\nInitialize pipeline position offsets (pipeline_posn_init)", fillcolor="#F0FFF4", color="#38A169"]; + pc7 [label="7. Task Loop Handoff (task_main_start)\nComplete primary core setup; enter ready state", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + pc1 -> pc2 -> pc3 -> pc4 -> pc5 -> pc6 -> pc7; + } + +1. **Global Context Setup**: Allocates and binds the singleton ``struct sof`` firmware context, which anchors pointers to memory pools, platform configurations, and audio schedulers. +2. **Logging, Timestamps, and Trace Buffering**: Configures Zephyr's logging timestamp source to the high-resolution hardware cycle counter (``k_cycle_get_32()`` or 64-bit system ticks). Initializes the circular DMA trace buffer (``trace_init()``) and prints the official firmware ABI, build hash, and version banner. +3. **System Notifiers & Runtime Power Management**: Initializes the asynchronous system notification bus (``init_system_notify()``) used for inter-component messaging (such as clock changes and audio underrun broadcasts). Brings up runtime power management (``pm_runtime_init()``) to prepare low-power idle policies. +4. **Platform Hardware Bringup (``platform_init()``)**: Calls the platform-specific hardware initialization routine (e.g., ``src/platform/intel/ace/platform.c`` or ``cavs/platform.c``): + - **Clocks & KCPS**: Configures DSP clock frequencies and initializes the kilo-cycles-per-second (KCPS) dynamic frequency scaling budget. + - **Audio Schedulers**: Instantiates the Earliest Deadline First (EDF) scheduler, the Low-Latency (LL) timer domain, the Data Processing (DP) preemptive thread scheduler, and the Thread With Budget (TWB) scheduler. + - **System Agent**: Configures periodic background health monitors (``sa_init()``) and hardware watchdog timers. + - **Audio DMACs**: Initializes host and peripheral DMA controllers (HD-Audio DMA, GPDMA). + - **Host IPC & IDC**: Allocates shared SRAM mailbox windows (Windows 0 to 3) and configures Inter-Domain Communication (IDC) for multi-core DSPs. +5. **AltBootManifest Unpacking (``lp_sram_unpack()``)**: On platforms where secondary cores lack hardware boot ROMs, the primary core parses the linker-generated ``AltBootManifest`` to copy secondary core executable code and read-only data into Low-Power SRAM (LP-SRAM), followed by data cache write-back flushing. +6. **Component Registry & Pipeline Setup**: Registers built-in processing modules (Volume, Mixer, SRC, EQ) into the component factory table (``sys_comp_init()``) and initializes stream position tracking structures. + +--- + +.. _host_fw_handshake: + +5. Host-Firmware Boot Synchronization & FW Ready Handshake +********************************************************** + +Once the primary core completes internal hardware bringup, it must formally notify the host operating system that the DSP is operational and ready to accept audio stream commands. The host and firmware synchronize through the hardware mailbox and doorbell interrupt registers. + +Protocol Generational Differences: IPC3 vs IPC4 +================================================ + +The handshake mechanism differs fundamentally between protocol generations: + +.. graphviz:: + :caption: Host-Firmware Boot Synchronization & FW Ready Handshake (IPC3 vs IPC4) + + digraph fw_ready_handshake { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_ipc3 { + label="IPC3 Boot Handshake (Static Topology)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + i3_dsp [label="DSP Core 0 completes boot\nConstructs struct sof_ipc_fw_ready\n(Version, Flags, Window Offsets)", fillcolor="#FFFFFF", color="#CBD5E0"]; + i3_win [label="Writes payload to Mailbox Window 0\nRaises Host Doorbell Interrupt", fillcolor="#BEE3F8", color="#3182CE"]; + i3_hst [label="Host receives FW_READY interrupt\nReads Window 0 memory structure\nValidates ABI; Loads Topology", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + i3_dsp -> i3_win -> i3_hst [color="#3182CE"]; + } + + subgraph cluster_ipc4 { + label="IPC4 Boot Handshake (Dynamic Object Model)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + i4_dsp [label="DSP Core 0 completes boot\nWrites ABI version to fw_reg.abi_ver\nUpdates FW State: FW_STATUS_READY", fillcolor="#FFFFFF", color="#D69E2E"]; + i4_win [label="Sets Mailbox Window 0 Status Register\nFires Host Notification Interrupt", fillcolor="#FAF089", color="#B7791F"]; + i4_hst [label="Host detects FW_STATUS_READY\nReads Window 0 base registers\nSends IPC4 Base FW Capabilities Query", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + i4_dsp -> i4_win -> i4_hst [color="#B7791F"]; + } + } + +* **IPC3 Handshake Protocol**: + 1. The DSP constructs a structured ``sof_ipc_fw_ready`` message containing ABI major/minor versions, build tags, and an array of memory window descriptors (defining the base offsets and lengths of Windows 0, 1, 2, and 3). + 2. The DSP writes this message directly into Mailbox Window 0 (the Outbox) and rings the host doorbell interrupt. + 3. The host driver's ISR reads Window 0, verifies ABI compatibility, records mailbox memory geometries, clears its boot watchdog timer, and proceeds to parse and download the monolithic topology binary. +* **IPC4 Handshake Protocol**: + 1. The DSP writes the ABI version of the firmware register layout into the ``abi_ver`` field of the firmware status structure within Mailbox Window 0. + 2. The DSP updates the firmware status register to ``SOF_IPC4_FW_STATUS_READY``. + 3. The host driver detects this state transition (via either an interrupt or status register polling), cancels the boot timeout, and issues an initial IPC4 ``GLB_GET_FW_VERSION`` or capabilities query to dynamically discover audio pipeline and module parameters. + +Boot Timeout Protection +======================= + +During boot, the host driver starts a hardware boot timeout monitor (typically 2 to 5 seconds). If the DSP boot ROM, cryptographic validation, or firmware initialization encounters a fatal crash: + +1. The DSP writes panic code dumps, exception vectors, and stack frames into Mailbox Window 0 before halting. +2. If the DSP hangs completely without writing to the mailbox, the host boot timer expires. +3. The host driver logs a boot failure error, captures the DSP register dump, triggers a hardware power-cycle or reset sequence, and prevents sound card registration from hanging the host operating system. + +--- + +.. _multicore_secondary_init: + +6. Multi-Core Initialization & Secondary Core Boot +************************************************** + +Modern audio DSPs (such as Intel cAVS 2.5, ACE 1.5, ACE 2.0, and ACE 3.0) feature multi-core symmetric multiprocessing (SMP) clusters (Dual-Core, Quad-Core, or Octa-Core). To conserve power, secondary cores are kept in low-power power-gated states during early boot and are powered up on demand. + +The Secondary Core Boot Flow +============================ + +When an audio pipeline requires processing capacity beyond Core 0, the host or primary core powers up secondary cores (Core 1, Core 2, Core 3): + +1. **Power Domain Activation**: Core 0 writes to the platform power management control registers to ungated clocks and energize the secondary core's power well. +2. **Zephyr SMP Core Bringup**: The secondary core vectors out of reset into Zephyr's secondary CPU startup stub. +3. **State Assessment (``check_restore()``)**: The secondary core executes ``secondary_core_init()`` in ``src/init/init.c``. It immediately evaluates whether this boot is a **Cold Boot** or a **Power Restore** (e.g., resuming from low-power D0ix retention where memory remained energized): + - If persistent structures (schedulers, notifiers, IDC contexts) are already present in shared memory, ``check_restore()`` returns true, invoking ``secondary_core_restore()``. This bypasses re-allocation, preventing memory leaks and preserving pipeline state. + - If memory was unpowered, the core proceeds with a full cold boot initialization. + +.. graphviz:: + :caption: Secondary Core Boot, Power State Assessment (`check_restore`), and Dynamic Activation Flow + + digraph secondary_core_flow { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + sc1 [label="Core 0 Power Request\nEnergize Secondary Core Power Well & Release Reset", fillcolor="#EDF2F7", color="#CBD5E0"]; + sc2 [label="Secondary Core Starts: secondary_core_init()\nExecute Early CPU Register Initialization", fillcolor="#EBF8FF", color="#3182CE"]; + sc_check [label="check_restore() Evaluation\nAre Schedulers & IDC Contexts already allocated?", shape=diamond, fillcolor="#FEFCBF", color="#D69E2E"]; + + subgraph cluster_restore { + label="Low-Power Retention Wake"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + sc_rest [label="secondary_core_restore()\nSkip Structure Re-Allocation\nRe-enable Core Interrupts & IDC", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + subgraph cluster_cold { + label="Full Cold Boot Initialization"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + sc_not [label="Initialize Core Notifiers\ninit_system_notify(sof)", fillcolor="#FFFFFF", color="#CBD5E0"]; + sc_ll [label="Initialize Core Schedulers\nLL Timer Domain & LL DMA Domain", fillcolor="#FFFFFF", color="#CBD5E0"]; + sc_dp [label="Initialize DP Scheduler\nscheduler_dp_init()", fillcolor="#FFFFFF", color="#CBD5E0"]; + sc_idc [label="Initialize IDC Communications\nidc_init() & AMS Messaging Service", fillcolor="#FFFFFF", color="#CBD5E0"]; + sc_clk [label="Adjust Core Clock Budget\ncore_kcps_adjust(cpu_id, SECONDARY_BASE)", fillcolor="#FFFFFF", color="#CBD5E0"]; + + sc_not -> sc_ll -> sc_dp -> sc_idc -> sc_clk; + } + + sc_ready [label="Secondary Core Enters Idle Loop\nReady to Accept IDC Pipeline Processing Tasks", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + sc1 -> sc2 -> sc_check; + sc_check -> sc_rest [label="True (D0ix Retention Wake)"]; + sc_check -> sc_not [label="False (Cold Boot)"]; + sc_rest -> sc_ready; + sc_clk -> sc_ready; + } + +Cold Boot Subsystem Configuration +================================= + +During a cold boot, the secondary core configures its own local resources: + +* **Local Core Notifiers**: Registers local core notification queues for intra-core event handling. +* **Independent Low-Latency (LL) Domain**: Sets up dedicated per-core timer domains and DMA domains, allowing the secondary core to drive real-time audio tasks without lock contention with Core 0. +* **Local Data Processing (DP) Scheduler**: Initializes preemptive thread pools for compute-heavy audio algorithms. +* **Inter-Domain Communication (IDC)**: Binds hardware doorbell interrupts between Core 0 and the secondary core, allowing Core 0 to forward host IPC commands and synchronize audio scheduling across cores. +* **Dynamic KCPS Budget**: Adjusts core clock frequencies to match its active processing workload. + +--- + +.. _power_states_boot_lifecycles: + +7. Power State Lifecycles & Wake Transitions +******************************************** + +Firmware initialization occurs not only during system power-on, but also across runtime power state transitions. SOF coordinates with the host operating system to optimize energy efficiency through dynamic power management. + +Power States & Transition Topologies +==================================== + +The DSP transitions across three principal operational states: + +1. **D3 (Cold / Powered Off)**: The entire DSP power well is severed. All internal SRAM contents, registers, and cache lines are completely lost. Waking from D3 requires a complete cold boot: host binary download, DSP ROM cryptographic validation, Zephyr initialization, and full SOF platform bringup. +2. **D0 (Active / Operational)**: The DSP is fully powered. Core 0 and optional secondary cores actively execute audio pipelines, process DMA interrupts, and handle host IPC transactions. +3. **D0ix (Low-Power Idle / Retention)**: When no audio streams are active (or when streams enter extended pause), the DSP transitions into low-power idle. High-Performance SRAM (HP-SRAM) banks are dynamically powered down, and essential context is preserved in Low-Power SRAM (LP-SRAM) or Host DRAM. Secondary cores are powered off. Waking from D0ix bypasses full image download, executing a fast-restore path that re-enables clocks and restores execution in microseconds. + +.. graphviz:: + :caption: Power State Lifecycle Transitions, Wake Sequences, and Context Preservation + + digraph power_states { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.4, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + d3 [label="D3: Fully Powered Off\nPower wells severed; SRAM lost\nZero power draw", fillcolor="#FED7D7", color="#E53E3E", fontcolor="#742A2A"]; + d0 [label="D0: Fully Active Streaming\nCore 0 Active; Secondary Cores Enabled\nFull Audio Processing & DMA Streaming", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + d0ix [label="D0ix: Low-Power Retention Idle\nSecondary cores powered down; HP-SRAM gated\nContext retained in LP-SRAM / Host DRAM", fillcolor="#FEFCBF", color="#D69E2E", fontcolor="#744210"]; + + d3 -> d0 [label="Cold Boot Sequence (Full Init)\nHost DMA download -> ROM verify -> Zephyr -> SOF\nLatency: ~50-150 ms", color="#3182CE", style="bold"]; + d0 -> d3 [label="Host Driver Unbind / System Shutdown\nFlush DMA, save persistent stats, sever power", color="#E53E3E"]; + + d0 -> d0ix [label="Stream Pause / Inactivity Timeout\nSave context to LP-SRAM/DRAM; gate HP-SRAM\nLatency: ~1 ms", color="#D69E2E"]; + d0ix -> d0 [label="Fast Restore Wake (check_restore == True)\nPower up HP-SRAM; skip memory re-allocation\nLatency: ~5-15 µs", color="#38A169", style="bold"]; + } + +LLEXT Dynamic Library Restoration +================================= + +When waking from low-power states where HP-SRAM banks were powered down, dynamically loaded Linkable Loadable Extension (LLEXT) modules must be preserved without requiring the host to re-download shared libraries over PCIe. + +SOF's LLEXT manager (``llext_manager_restore_from_dram()``) caches module text and data sections in host-backed DRAM or non-volatile LP-SRAM. During the wake sequence, the manager automatically verifies image checksums and restores the module code directly into DSP execution memory before the host ready handshake is signaled, ensuring seamless audio playback resumption. + +--- + +.. _upstream_init_references: + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level implementation details, data structure definitions, and linker scripts: + +* **Upstream DSP Initialization Specifications**: + - `thesofproject/sof: src/init/README.md `_ + - `thesofproject/sof: src/platform/intel/ace/platform.c `_ +* **Core Firmware Implementation Files**: + - ``src/init/init.c``: Primary and secondary core initialization logic, ``sof_init()`` hook, and version banners. + - ``src/init/ext_manifest.c``: Extended firmware manifest structure definitions, header parsers, and metadata tables. + - ``zephyr/wrapper.c``: Zephyr application handoff stubs, ``sof_app_main()``, and ``boot_complete()`` signaling. + - ``src/include/sof/init.h``: Global firmware context definitions and initialization function prototypes. + - ``src/include/sof/trace/trace-boot.h``: Boot-time trace point macros and debug markers. + +Related Subsystem Architecture Guides +===================================== + +* :ref:`ipc_infrastructure`: How the host and DSP exchange control messages and synchronize boot state via hardware mailboxes. +* :ref:`scheduler_architecture`: Real-time scheduling domains (LL, DP, TWB) initialized during platform bringup. +* :ref:`audio_buffer_management`: Ring buffer sizing, memory hierarchies (TCM, HP-SRAM, LP-SRAM), and cache operations. +* :ref:`pipeline_architecture`: Dynamic audio processing graph construction following boot completion. +* :ref:`module_framework`: Audio component lifecycle, module adapters, and parameter configuration. diff --git a/developer_guides/firmware/ipc_infrastructure.rst b/developer_guides/firmware/ipc_infrastructure.rst index 0cfe2c8d..d9199ce6 100644 --- a/developer_guides/firmware/ipc_infrastructure.rst +++ b/developer_guides/firmware/ipc_infrastructure.rst @@ -586,4 +586,5 @@ Related Guides * :ref:`module_framework`: How IPC parameter blobs configure processing modules and runtime algorithms. * :ref:`scheduler_architecture`: Real-time scheduling domains (LL, DP, TWB) that coordinate with IPC work queues. * :ref:`audio_buffer_management`: Allocating and binding circular ring buffers during IPC pipeline construction. +* :ref:`fw_init_boot`: Boot flow, hardware mailbox FW Ready handshake, and Zephyr initialization. * :ref:`topology2`: How ALSA Topology 2.0 configuration files generate IPC topology commands. diff --git a/developer_guides/firmware/pipeline_architecture.rst b/developer_guides/firmware/pipeline_architecture.rst index 4232450b..0b1cfaf5 100644 --- a/developer_guides/firmware/pipeline_architecture.rst +++ b/developer_guides/firmware/pipeline_architecture.rst @@ -461,6 +461,7 @@ Related Guides * :ref:`scheduler_architecture`: Multi-tier real-time scheduling (LL, DP, TWB), EDF mechanics, and multi-core execution. * :ref:`audio_buffer_management`: Lockless circular ring buffers, multi-tier DSP memory (SRAM/DRAM), and cache coherency. * :ref:`module_framework`: The standardized module interface, Source/Sink APIs, and memory sandboxing. +* :ref:`fw_init_boot`: Boot flow, hardware mailbox FW Ready handshake, and Zephyr initialization. * :ref:`topology2`: How pipelines and widgets are declared using ALSA Topology 2.0 configuration classes. * :ref:`sof_hostless_firmware`: How to create static pipelines compiled into ROM for standalone microcontrollers. * :ref:`llext_modules`: Building dynamic loadable modules (LLEXT) that integrate into SOF pipelines. diff --git a/developer_guides/firmware/scheduler_architecture.rst b/developer_guides/firmware/scheduler_architecture.rst index 201b5325..cee61df6 100644 --- a/developer_guides/firmware/scheduler_architecture.rst +++ b/developer_guides/firmware/scheduler_architecture.rst @@ -568,5 +568,6 @@ Related Guides * :ref:`audio_buffer_management`: Lockless circular ring buffers, multi-tier DSP memory (SRAM/DRAM), and cache coherency. * :ref:`pipeline_architecture`: How audio pipelines interact with the scheduling domains to stream data. * :ref:`module_framework`: The standardized module interface executed by LL and DP scheduler tasks. +* :ref:`fw_init_boot`: Boot flow, hardware mailbox FW Ready handshake, and Zephyr initialization. * :ref:`sof_hostless_firmware`: Autonomous firmware pipelines and timer configurations on embedded targets. * :ref:`unit_tests`: Unit testing scheduler components and domain threads using Zephyr Ztest. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 5e43efbe..27e2a0cb 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -36,7 +36,7 @@ Core Infrastructure & Pipeline * :ref:`scheduler_architecture` (High-level architecture; also see upstream `scheduler README `_) * :ref:`audio_buffer_management` (High-level architecture; also see upstream `buffer README `_) * :ref:`ipc_infrastructure` (High-level architecture; also see upstream `IPC README `_) -* `Firmware Initialization & Boot `_ +* :ref:`fw_init_boot` (High-level architecture; also see upstream `init README `_) Audio Processing Modules & Algorithms ------------------------------------- @@ -85,6 +85,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/scheduler_architecture firmware/audio_buffer_management firmware/ipc_infrastructure + firmware/fw_init_boot rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 1228606ca050340a7c82b46c72ccf84c3bf1b3ca Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Fri, 18 Sep 2026 15:27:57 +0100 Subject: [PATCH 08/64] docs: developer_guides: add high-level volume module guide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a comprehensive, high-level developer architecture guide covering the Sound Open Firmware (SOF) Volume Control module (PGA widget). The guide covers: - System-level architecture & signal flow across asynchronous control plane (ALSA mixer, IPC3/IPC4) and hard real-time data plane. - Fixed-point gain scaling and saturation arithmetic across protocol generations (IPC3 Q8.16 vs IPC4 Q1.31) and audio formats. - Smooth volume ramping and zipper noise elimination comparing linear ramping with smooth Windows S-curve / Hann fades and adaptive update rate selection (125 µs to 1000 µs). - Zero-crossing muting and pop suppression mechanics using lookahead waveform analysis, plus stateful unmuting. - Zero-overhead unity gain passthrough mode decision flow for 0 dB unmodified playback. - Real-time in-line peak meter telemetry (COMP_PEAK_VOL) and shared Mailbox Window 0 synchronization for zero-IPC host VU meters. - SIMD vector processing parallelism comparing portable generic C with Tensilica Xtensa HiFi 3, HiFi 4 (4-way vector), and HiFi 5 (8-way vector). - Upstream code references and related architecture links. Includes 7 custom vector Graphviz SVG diagrams illustrating architecture, fixed-point math, ramping curves, zero-crossing muting, passthrough logic, peak telemetry, and SIMD vector bandwidth. Signed-off-by: Liam Girdwood --- .../firmware/module_framework.rst | 1 + .../firmware/pipeline_architecture.rst | 1 + developer_guides/firmware/volume_module.rst | 553 ++++++++++++++++++ developer_guides/index.rst | 3 +- 4 files changed, 557 insertions(+), 1 deletion(-) create mode 100644 developer_guides/firmware/volume_module.rst diff --git a/developer_guides/firmware/module_framework.rst b/developer_guides/firmware/module_framework.rst index a0aba5df..44a4ccc0 100644 --- a/developer_guides/firmware/module_framework.rst +++ b/developer_guides/firmware/module_framework.rst @@ -520,6 +520,7 @@ Related Guides * :ref:`audio_buffer_management`: Lockless circular ring buffers, multi-tier DSP memory (SRAM/DRAM), and cache coherency. * :ref:`scheduler_architecture`: Multi-tier real-time scheduling (LL, DP, TWB), EDF mechanics, and multi-core execution. * :ref:`pipeline_architecture`: How processing modules are assembled into directed acyclic execution graphs (DAGs). +* :ref:`volume_module`: Comprehensive architecture of the canonical volume control module, ramping, and SIMD optimization. * :ref:`llext_modules`: Authoring, compiling, and signing dynamic loadable modules using Zephyr LLEXT. * :ref:`sof_hostless_firmware`: Instantiating static modules in autonomous embedded firmware. * :ref:`topology2`: Declaring audio widgets and binding modules using ALSA Topology 2.0. diff --git a/developer_guides/firmware/pipeline_architecture.rst b/developer_guides/firmware/pipeline_architecture.rst index 0b1cfaf5..7d6c414b 100644 --- a/developer_guides/firmware/pipeline_architecture.rst +++ b/developer_guides/firmware/pipeline_architecture.rst @@ -461,6 +461,7 @@ Related Guides * :ref:`scheduler_architecture`: Multi-tier real-time scheduling (LL, DP, TWB), EDF mechanics, and multi-core execution. * :ref:`audio_buffer_management`: Lockless circular ring buffers, multi-tier DSP memory (SRAM/DRAM), and cache coherency. * :ref:`module_framework`: The standardized module interface, Source/Sink APIs, and memory sandboxing. +* :ref:`volume_module`: Per-channel gain scaling, smooth ramping, zero-crossing muting, and SIMD acceleration. * :ref:`fw_init_boot`: Boot flow, hardware mailbox FW Ready handshake, and Zephyr initialization. * :ref:`topology2`: How pipelines and widgets are declared using ALSA Topology 2.0 configuration classes. * :ref:`sof_hostless_firmware`: How to create static pipelines compiled into ROM for standalone microcontrollers. diff --git a/developer_guides/firmware/volume_module.rst b/developer_guides/firmware/volume_module.rst new file mode 100644 index 00000000..8d1f29b7 --- /dev/null +++ b/developer_guides/firmware/volume_module.rst @@ -0,0 +1,553 @@ +.. _volume_module: + +Volume Control Module Architecture +################################## + +The **Volume Control Module** (implemented in ``src/audio/volume/``) is the core audio processing component in Sound Open Firmware responsible for per-channel amplitude scaling, smooth volume ramping, pop-free zero-crossing muting, real-time peak metering telemetry, and zero-overhead passthrough optimization. + +Represented as a Programmable Gain Amplifier (PGA) widget in ALSA Topology, the volume module operates across both playback pipelines (post-mix main faders, stream attenuation, multi-channel speaker balancing) and capture pipelines (microphone preamplification, digital gain boost). + +This guide provides a comprehensive, high-level architectural walkthrough of the volume module, its fixed-point mathematics, pop-suppression algorithms, SIMD hardware acceleration, and host telemetry pipelines without delving into low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +.. _volume_signal_flow: + +1. System-Level Architecture & Signal Flow +****************************************** + +The volume module functions as a Single-Input Single-Output (SISO) audio processing component conforming to the standardized SOF **Module Adapter** framework. It bridges host control interfaces (ALSA mixer faders, PulseAudio, PipeWire, Windows audio controls) with the real-time DSP audio streaming pipeline. + +Dual-Plane Architectural Separation +==================================== + +The module operates across two strictly decoupled execution planes: + +* **Control Plane (Asynchronous)**: Receives volume adjustments, mute toggles, and ramping parameters from the host driver via IPC3 (``SOF_IPC_COMP_SET_VALUE``) or IPC4 (``VOLUME`` and ``GAIN`` compound parameter blocks). The control plane converts host dB values into internal fixed-point multipliers, calculates ramping coefficients, and updates internal target states without stalling real-time audio threads. +* **Data Plane (Hard Real-Time)**: Invoked periodically by the Low-Latency (LL) or Data Processing (DP) scheduler on each audio processing tick. It pulls PCM frames from the input circular ring buffer, applies fixed-point vector multiplication or passthrough routing, tracks peak signal envelopes, and writes scaled samples into the output circular buffer. + +.. graphviz:: + :caption: System-Level Volume Module Architecture & Signal Processing Chain + + digraph volume_architecture { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_control_plane { + label="Control Plane: Host Mixer & IPC Interface"; + style="filled,rounded"; + fillcolor="#F7FAFC"; + color="#CBD5E0"; + + host_cmd [label="Host Audio Server / ALSA Mixer\nVolume Fader Change / Mute Toggle", fillcolor="#EDF2F7", color="#CBD5E0"]; + ipc_rx [label="IPC Handler (IPC3/IPC4)\nUnpack Target Gains & Ramp Durations", fillcolor="#EDF2F7", color="#CBD5E0"]; + cfg_calc [label="Module Adapter Config (volume_set_config)\nConvert dB to Fixed-Point (Q8.16 / Q1.31)\nInit Ramp Curve & Target Arrays", fillcolor="#BEE3F8", color="#3182CE"]; + + host_cmd -> ipc_rx -> cfg_calc; + } + + subgraph cluster_data_plane { + label="Data Plane: Real-Time Audio Processing Core"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + in_buf [label="Input Circular Ring Buffer\n(cir_buf_source)", fillcolor="#E2E8F0", color="#A0AEC0"]; + mode_sw [label="Processing Mode Selector\nCheck Passthrough vs Active Gain", shape=diamond, fillcolor="#FEFCBF", color="#D69E2E"]; + + subgraph cluster_engine { + label="Gain & Pop Suppression Engines"; + style="filled,rounded"; + fillcolor="#FFFFFF"; + color="#E2E8F0"; + + ramp_eng [label="Ramping Engine\nLinear or Windows S-Curve Fade\nAdaptive 125 µs - 1000 µs Tick", fillcolor="#FAF089", color="#B7791F"]; + zc_eng [label="Zero-Crossing Detector\nFind y(t) ~ 0 on Mute / Unmute", fillcolor="#FAF089", color="#B7791F"]; + simd_mul [label="SIMD Vector Multiplier\nHiFi 3 / HiFi 4 / HiFi 5 / Generic\nPer-Channel Scaling with Saturation", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + peak_trk [label="In-Line Peak Metering\nTrack |x_n| Maximum per Channel", fillcolor="#E6FFFA", color="#319795"]; + + ramp_eng -> simd_mul; + zc_eng -> simd_mul; + simd_mul -> peak_trk; + } + + pass_path [label="Zero-Overhead Passthrough\nDirect Sample Transfer (0 dB Unity Gain)", fillcolor="#EBF8FF", color="#3182CE"]; + out_buf [label="Output Circular Ring Buffer\n(cir_buf_sink)", fillcolor="#E2E8F0", color="#A0AEC0"]; + + in_buf -> mode_sw; + mode_sw -> ramp_eng [label="Gain != 0 dB\nor Ramping"]; + mode_sw -> pass_path [label="Gain == 0 dB\n(Idle)"]; + peak_trk -> out_buf; + pass_path -> out_buf; + } + + subgraph cluster_telemetry { + label="Telemetry Plane: Mailbox Status"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + mbox_win [label="Shared Mailbox Window 0\n(ipc4_peak_volume_regs)\nZero-IPC Host Level Polling", fillcolor="#FEFCBF", color="#D69E2E"]; + } + + cfg_calc -> ramp_eng [label="New Target Gain", color="#3182CE", style="dashed"]; + peak_trk -> mbox_win [label="Periodic SW Reg Write", color="#319795", style="dashed"]; + } + +--- + +.. _fixed_point_scaling: + +2. Fixed-Point Gain Scaling & Saturation Mathematics +**************************************************** + +Digital signal processors execute audio processing predominantly in integer or fixed-point arithmetic to achieve maximum power efficiency and deterministic cycle latency. Sound Open Firmware employs standardized fixed-point fractional formats tailored to protocol generations and silicon capabilities. + +Fixed-Point Gain Representations +================================ + +The numeric format used to represent volume multipliers depends on the IPC protocol generation: + +* **IPC3 Generation (Q8.16 Format)**: + + - 8-bit signed integer component and 16-bit fractional component. + - Represents linear gain factors from :math:`0.0` (digital silence) up to :math:`128.0` (+42.14 dB gain). + - Unity gain (:math:`0\text{ dB}`) is represented exactly by :math:`2^{16} = 65536` (``0x00010000``). + - Dynamic range spans from :math:`-138.47\text{ dB}` to :math:`+42.14\text{ dB}`. + +* **IPC4 Generation (Q1.31 Format)**: + + - 1-bit sign and 31-bit fractional precision. + - Represents attenuation factors from :math:`0.0` (silence) up to :math:`1.0` (:math:`0\text{ dB}` unity gain). + - Unity gain (:math:`0\text{ dB}`) is represented by ``INT32_MAX`` (``0x7FFFFFFF``). + - Firmware converts or scales Q1.31 multipliers to internal Q1.23 or native 32-bit registers depending on target SIMD architecture requirements. + +Multiplication & Saturation Protection +====================================== + +When scaling an audio sample :math:`x_n` by gain factor :math:`G`, fixed-point multiplication requires bit-shifting and saturation clamping to prevent integer wraparound: + +.. math:: + + y_n = \text{clamp}\left( \frac{x_n \times G}{2^{Q_y}}, \text{MIN\_VAL}, \text{MAX\_VAL} \right) + +If a volume fader applies positive gain (:math:`G > 1.0`), the resulting amplitude can exceed the maximum container range (e.g., :math:`+32767` for 16-bit audio or :math:`+2^{31}-1` for 32-bit audio). Rather than permitting numerical overflow—which would invert positive wave crests into negative troughs and cause catastrophic acoustic distortion—SOF applies hardware-accelerated **saturation arithmetic** to clamp peaks to full-scale maximum. + +.. graphviz:: + :caption: Fixed-Point Gain Scaling & Saturation Arithmetic + + digraph fixed_point_math { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_sample [label="Audio Sample x_n\nS16_LE / S24_4LE / S32_LE", fillcolor="#EDF2F7", color="#CBD5E0"]; + gain_val [label="Gain Multiplier G\nQ8.16 (IPC3) / Q1.31 (IPC4)", fillcolor="#EDF2F7", color="#CBD5E0"]; + wide_mul [label="64-Bit Wide Multiply\nP = x_n * G\n(Preserves High Precision)", fillcolor="#BEE3F8", color="#3182CE"]; + norm_shf [label="Fixed-Point Normalization\nRight-shift by Q_y bits\n(Align to Container)", fillcolor="#BEE3F8", color="#3182CE"]; + sat_chk [label="Saturation Clamp\nCheck Overflow Bounds\n[MIN_VAL, MAX_VAL]", fillcolor="#FEFCBF", color="#D69E2E"]; + out_sample [label="Output Sample y_n\nScaled & Clamped Sample", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + in_sample -> wide_mul; + gain_val -> wide_mul; + wide_mul -> norm_shf -> sat_chk -> out_sample; + } + +--- + +.. _smooth_ramping: + +3. Smooth Volume Ramping & Zipper Noise Elimination +*************************************************** + +When a user adjusts a volume slider or an application changes audio levels, applying the new gain immediately within a single audio frame produces an instantaneous step discontinuity in the waveform. + +The Physics of Zipper Noise +=========================== + +An abrupt amplitude jump introduces high-frequency harmonic distortion known as **zipper noise** or audible clicking: + +* The ear perceives rapid discrete volume steps as high-frequency clicks. +* The faster the transition, the more pronounced the acoustic artifact. + +To eliminate zipper noise, Sound Open Firmware interpolates volume transitions across dozens or hundreds of frames using smooth **volume ramping**. + +Ramping Curves: Linear vs Windows S-Curve Fade +============================================== + +SOF provides two configurable ramping algorithms: + +1. **Linear Ramping (``COMP_VOLUME_LINEAR_RAMP``)**: + + - Steps gain by a constant increment :math:`\Delta G` per frame. + - Low computational complexity, ideal for resource-constrained microcontrollers. + - While vastly superior to instantaneous steps, linear ramping has non-zero second derivatives (:math:`d^2A/dt^2 \neq 0`) at the inflection points where ramping starts and stops, which can produce subtle clicks on high-fidelity audio equipment. + +2. **Windows S-Curve / Hann Fade (``COMP_VOLUME_WINDOWS_FADE``)**: + + - Employs a trigonometric S-curve (raised cosine / Hann window envelope). + - Provides smooth, continuous first and second derivatives at both the launch and landing points of the transition. + - Completely eliminates click artifacts by easing into the ramp and easing out as the target volume is reached. + +.. graphviz:: + :caption: Pop-Free Volume Ramping Curves and Audio Waveform Smoothing + + digraph ramping_curves { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_step { + label="Unbuffered Step Transition (Audible Zipper Pop)"; + style="filled,rounded"; + fillcolor="#FFF5F5"; + color="#FEB2B2"; + + step_wave [label="Instantaneous Gain Change\nWaveform exhibits vertical edge discontinuity\nHigh-frequency acoustic click / pop artifact", fillcolor="#FED7D7", color="#E53E3E", fontcolor="#742A2A"]; + } + + subgraph cluster_linear { + label="Linear Ramp (COMP_VOLUME_LINEAR_RAMP)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + lin_wave [label="Constant Slope Interpolation (dG/dt = const)\nGradual gain change across transition window\nMinor inflection corners at start / end", fillcolor="#E2E8F0", color="#A0AEC0"]; + } + + subgraph cluster_scurve { + label="Smooth Windows S-Curve Fade (COMP_VOLUME_WINDOWS_FADE)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + scurve_wave [label="Raised Cosine / Hann Window Envelope\nContinuous first & second derivatives (Smooth Easing)\nCompletely pop-free studio-grade transition", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + step_wave -> lin_wave [label="Introduce Interpolation", color="#3182CE"]; + lin_wave -> scurve_wave [label="Apply Windowed Easing", color="#38A169", style="bold"]; + } + +Adaptive Ramping Update Intervals +================================= + +Calculating a new gain factor on every individual audio sample (e.g. 48,000 times per second per channel) imposes unnecessary CPU overhead. Conversely, updating the gain too infrequently (e.g. once every 10 ms) re-introduces zipper artifacts. + +SOF resolves this trade-off using an **adaptive update rate engine**: + +* **Fast Ramps (< 32 ms)**: Gain values update every **125 µs** (``VOL_RAMP_UPDATE_FASTEST_US``) to preserve smoothness during rapid fader movements. +* **Medium Ramps (32 ms to 64 ms)**: Gain updates every **250 µs** (``VOL_RAMP_UPDATE_FAST_US``). +* **Slow Ramps (64 ms to 128 ms)**: Gain updates every **500 µs** (``VOL_RAMP_UPDATE_SLOW_US``). +* **Extended Fades (> 128 ms)**: Gain updates every **1000 µs** (``VOL_RAMP_UPDATE_SLOWEST_US``), minimizing DSP cycle consumption. + +--- + +.. _zero_crossing_mute: + +4. Zero-Crossing Muting & Pop Suppression +***************************************** + +When an audio stream is muted, stopping playback immediately or ramping to silence across 50 ms presents conflicting trade-offs: + +* **Immediate Cutoff**: If playback is severed mid-wave while the waveform is at peak amplitude, the sudden drop to zero produces a loud pop. +* **Gradual Ramp**: In emergency mute scenarios or low-latency telephony, a 50 ms ramp introduces unacceptable latency. + +SOF solves this dilemma via **Zero-Crossing Detection** (``vol_zc_get_s16`` and ``vol_zc_get_s24``). + +Zero-Crossing Detection Mechanics +================================= + +Before applying an immediate mute, the volume module analyzes upcoming frames within the circular buffer to detect the precise sample where the audio waveform crosses the zero-amplitude baseline (:math:`y(t) \approx 0`): + +1. **Buffer Lookahead**: The detector inspects the current frame buffer across all active channels. +2. **Sign Change Detection**: It computes the channel sample sum and monitors for a sign bit inversion (``sum ^ prev_sum < 0``). +3. **Mute Synchronization**: The module continues processing samples at the current volume until the zero-crossing frame is reached. +4. **Clean Cutoff**: Gain drops to zero exactly at the zero crossing. Because the signal amplitude is already zero, no DC step discontinuity occurs, producing an immediate, pop-free mute. + +.. graphviz:: + :caption: Zero-Crossing Mute vs Immediate Cutoff Waveform Comparison + + digraph zero_crossing { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_bad { + label="Immediate Mid-Wave Cutoff (Pop Occurs)"; + style="filled,rounded"; + fillcolor="#FFF5F5"; + color="#FEB2B2"; + + b1 [label="Signal at Peak Amplitude (+Vpeak)\nHost issues MUTE command", fillcolor="#FED7D7", color="#E53E3E"]; + b2 [label="Immediate Cut to 0\nStep discontinuity from +Vpeak to 0\nAcoustic Pop / Click generated", fillcolor="#FEB2B2", color="#C53030", fontcolor="#742A2A"]; + + b1 -> b2; + } + + subgraph cluster_good { + label="SOF Zero-Crossing Mute (Pop-Free)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + g1 [label="Signal at Peak Amplitude (+Vpeak)\nHost issues MUTE command", fillcolor="#E6FFFA", color="#319795"]; + g2 [label="vol_zc_get() Lookahead\nScan buffer for sign change (sum ^ prev_sum < 0)", fillcolor="#FAF089", color="#B7791F"]; + g3 [label="Mute Applied at Zero Crossing (y(t) == 0)\nZero step delta = Zero acoustic pop", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + g1 -> g2 -> g3; + } + } + +Stateful Unmuting +================= + +When unmuting, the module does not instantaneously jump to the previous volume. Instead, it retrieves the cached target volume (``tvolume``) saved prior to the mute event and launches a smooth S-curve ramp from silence up to the target level, preventing startling auditory spikes. + +--- + +.. _passthrough_mode: + +5. Zero-Overhead Unity Gain Passthrough Mode +******************************************** + +In many operating scenarios—such as standard desktop playback where application faders are set to 100% (:math:`0\text{ dB}`)—the volume module is not actively altering signal amplitudes. + +Executing 48,000 vector multiplications per second on unmodified audio samples wastes processor cycles and drains battery power. Sound Open Firmware incorporates an automated **Zero-Overhead Passthrough** subsystem. + +The Passthrough Decision Matrix +=============================== + +During pipeline preparation and after every volume transition, the module evaluates its operational state: + +.. graphviz:: + :caption: Zero-Overhead Unity Gain Passthrough Decision Flow + + digraph passthrough_eval { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + eval_start [label="Volume Module State Evaluation\n(After parameter update or ramp completion)", fillcolor="#EDF2F7", color="#CBD5E0"]; + cond_gain [label="Are all channels at Unity Gain (0 dB)?\nvolume[ch] == VOL_ZERO_DB", shape=diamond, fillcolor="#FEFCBF", color="#D69E2E"]; + cond_mute [label="Are all channels Unmuted?\nmuted[ch] == false", shape=diamond, fillcolor="#FEFCBF", color="#D69E2E"]; + cond_ramp [label="Is Ramping Engine Idle?\nramp_finished == true", shape=diamond, fillcolor="#FEFCBF", color="#D69E2E"]; + cond_peak [label="Is Peak Metering Disabled?\nCONFIG_COMP_PEAK_VOL == 0", shape=diamond, fillcolor="#FEFCBF", color="#D69E2E"]; + + subgraph cluster_active { + label="Active Processing Mode"; + style="filled,rounded"; + fillcolor="#FFF5F5"; + color="#FEB2B2"; + + act_ptr [label="Bind scale_vol to SIMD Multiplier\n(volume_hifi4 / volume_generic)\nExecute vector gain scaling", fillcolor="#FED7D7", color="#E53E3E"]; + } + + subgraph cluster_pass { + label="Passthrough Mode"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + pass_ptr [label="Bind scale_vol to passthrough_func\nDirect sample copy or zero-copy buffer pointer handoff\nZero arithmetic operations = Zero CPU overhead", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + eval_start -> cond_gain; + cond_gain -> cond_mute [label="Yes"]; + cond_gain -> act_ptr [label="No"]; + + cond_mute -> cond_ramp [label="Yes"]; + cond_mute -> act_ptr [label="No"]; + + cond_ramp -> cond_peak [label="Yes"]; + cond_ramp -> act_ptr [label="No"]; + + cond_peak -> pass_ptr [label="Yes\n(All Conditions Met)"]; + cond_peak -> act_ptr [label="No\n(Peak Meter Active)"]; + } + +When passthrough mode is active, the function pointer ``scale_vol`` is bound directly to ``passthrough_func``. In shared-buffer pipeline topologies, this can even be optimized into a zero-copy buffer handoff, completely bypassing memory copy operations. + +--- + +.. _peak_metering_telemetry: + +6. Real-Time Peak Meter Telemetry (`COMP_PEAK_VOL`) +*************************************************** + +Operating systems and user applications frequently display live audio visualizers, volume unit (VU) meters, and clipping warning indicators. In conventional audio stacks, measuring peak amplitude requires either a dedicated DSP visualizer module or streaming raw audio back to the host CPU, consuming substantial bus bandwidth. + +The volume module integrates an efficient **In-Line Peak Metering** subsystem (``peak_volume.h``) that computes peak amplitudes during volume scaling at zero additional memory traversal cost. + +In-Line Peak Tracking Pipeline +============================== + +.. graphviz:: + :caption: Real-Time Peak Meter Telemetry Pipeline and Shared Memory Synchronization + + digraph peak_meter_flow { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_dsp { + label="DSP Firmware Processing Loop (volume_hifi4_with_peakvol)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + s_mul [label="SIMD Gain Scaling (4 samples / cycle)\ny_n = clamp((x_n * G) >> Q)", fillcolor="#C6F6D5", color="#38A169"]; + s_peak [label="Absolute Peak Comparison\npeak[ch] = max(peak[ch], |y_n|)\n(Tracked inside SIMD vector registers)", fillcolor="#9AE6B4", color="#2F855A", fontcolor="#1C4532"]; + s_acc [label="Accumulation Window Counter\nAccumulate peaks across N audio periods", fillcolor="#E6FFFA", color="#319795"]; + s_wr [label="Periodic Mailbox Sync: peak_vol_update()\nWrite peak_regs to Mailbox Window 0\nmailbox_sw_regs_write(mailbox_offset, ...)", fillcolor="#FEFCBF", color="#D69E2E"]; + + s_mul -> s_peak -> s_acc -> s_wr; + } + + subgraph cluster_hw_mailbox { + label="DSP Hardware Shared Memory (SRAM)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + win0_regs [label="Mailbox Window 0 (Software Registers)\nstruct ipc4_peak_volume_regs\n[Ch0 Peak | Ch1 Peak | Ch2 Peak | ...]", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + } + + subgraph cluster_host_ui { + label="Host Operating System (Linux ALSA / PipeWire / Windows)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + h_poll [label="Host Telemetry Reader / VU Meter UI\nDirect Memory-Mapped BAR Read\n(Zero PCIe Doorbell Interrupts, Zero DSP Wakeups)", fillcolor="#BEE3F8", color="#3182CE"]; + h_disp [label="GUI Visualizer / ALSA Mixer Level Display\nSmooth 60 fps VU meter animation", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + h_poll -> h_disp; + } + + s_wr -> win0_regs [label="DMA / Memory Store", color="#D69E2E"]; + win0_regs -> h_poll [label="Host BAR Memory Read", color="#3182CE", style="dashed"]; + } + +1. **Simultaneous Peak Tracking**: As samples pass through the SIMD gain multiplier, the absolute value :math:`|y_n|` is compared against the running channel peak register in parallel vector execution units. +2. **Decoupled Reporting Rate**: Peak values accumulate over a configurable number of periods (e.g. 10 ms to 50 ms) to match display refresh rates. +3. **Zero-IPC Mailbox Synchronization**: At each reporting interval, the DSP writes the peak register structure directly into **Mailbox Window 0** (shared SRAM). +4. **Non-Intrusive Host Polling**: The host audio server (or user-space VU meter) reads the peak values directly from host memory-mapped I/O (MMIO). No IPC interrupts are fired, and sleeping DSP cores are never awakened to service telemetry queries. + +--- + +.. _simd_acceleration: + +7. SIMD Vector Acceleration Across Architectures +************************************************ + +Audio streams contain millions of samples per second across multi-channel topologies (Stereo, 5.1, 7.1, or Ambisonics). To minimize cycle counts and thermal dissipation, SOF provides highly specialized Single Instruction Multiple Data (SIMD) implementations. + +Architectural SIMD Implementations +================================== + +.. graphviz:: + :caption: SIMD Vector Processing Parallelism across Processor Architectures + + digraph simd_comparison { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_generic { + label="Generic C Implementation (volume_generic.c)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + gen_desc [label="Portable Scalar C Code\n1 sample processed per loop iteration\nTarget: ARM Cortex-M, RISC-V, Host Simulator", fillcolor="#FFFFFF", color="#CBD5E0"]; + } + + subgraph cluster_hifi3 { + label="Cadence Xtensa HiFi 3 (volume_hifi3.c)"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + h3_desc [label="Dual 32-bit SIMD Registers (ae_p24x2s / ae_s32x2)\n2 samples processed per cycle\nDual 64-bit load/store memory operations", fillcolor="#BEE3F8", color="#3182CE"]; + } + + subgraph cluster_hifi4 { + label="Cadence Xtensa HiFi 4 (volume_hifi4.c)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + h4_desc [label="Quad 32-bit Vector Units (ae_int32x4)\n4 samples processed per instruction cycle\n128-bit aligned vector load/store operations\nHardware saturation & vector peak comparison", fillcolor="#FAF089", color="#B7791F"]; + } + + subgraph cluster_hifi5 { + label="Cadence Xtensa HiFi 5 (volume_hifi5.c)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + h5_desc [label="Octa 32-bit Vector Engine\n8 samples processed concurrently\nDual 128-bit memory buses (256 bits/cycle)\nMaximum throughput for multi-channel TDM", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + gen_desc -> h3_desc [label="2x Vector Throughput", color="#3182CE"]; + h3_desc -> h4_desc [label="2x Vector Throughput (4x Total)", color="#B7791F"]; + h4_desc -> h5_desc [label="2x Vector Throughput (8x Total)", color="#38A169", style="bold"]; + } + +Key SIMD Architectural Features +=============================== + +* **HiFi 4 Implementation (``volume_hifi4.c``)**: + + - Employs 128-bit vector registers (``ae_int32x4``) holding four 32-bit audio samples. + - Loads four audio samples and four volume multipliers simultaneously. + - Executes four 32x32 multiply-accumulate operations in a single clock cycle with automated hardware saturation. + - Computes four-way absolute peak tracking without branching or pipeline stalls. + +* **HiFi 5 Implementation (``volume_hifi5.c``)**: + + - Doubles vector execution bandwidth, processing eight 32-bit audio samples per instruction cycle. + - Utilizes dual 128-bit load/store units to feed vector arithmetic units without memory wait states. + +* **Generic Portable Fallback (``volume_generic.c``)**: + + - Provides a clean, highly portable C reference implementation utilizing standard integer division and 64-bit multiplication. + - Guarantees complete cross-architecture compatibility for platforms without Tensilica DSP extensions (e.g. PJRC Teensy 4.1 ARM Cortex-M7, Espressif ESP32-P4 RISC-V). + +--- + +.. _upstream_volume_references: + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level implementation details, vector assembly intrinsics, and configuration structures: + +* **Upstream Volume Specification**: + - `thesofproject/sof: src/audio/volume/README.md `_ +* **Core Firmware Source Files**: + - ``src/audio/volume/volume.c``: Core volume component logic, lifecycle callbacks, and zero-crossing detection. + - ``src/audio/volume/volume.h``: Component state structures (``struct vol_data``), gain constants, and update thresholds. + - ``src/audio/volume/peak_volume.h``: In-line peak metering structures and telemetry definitions. + - ``src/audio/volume/volume_generic.c``: Portable scalar reference implementation. + - ``src/audio/volume/volume_hifi3.c``: Tensilica Xtensa HiFi 3 SIMD vector implementation. + - ``src/audio/volume/volume_hifi4.c``: Tensilica Xtensa HiFi 4 SIMD vector implementation. + - ``src/audio/volume/volume_hifi5.c``: Tensilica Xtensa HiFi 5 SIMD vector implementation. + - ``src/audio/volume/volume_ipc3.c``: IPC3 configuration unpacker and control handler. + - ``src/audio/volume/volume_ipc4.c``: IPC4 volume and gain parameter handlers. +* **Topology Definitions**: + - ``tools/topology/topology2/include/components/volume.conf``: ALSA Topology 2 configuration class for PGA volume widgets. + +Related Subsystem Architecture Guides +===================================== + +* :ref:`module_framework`: The standardized module interface, Source/Sink APIs, and memory sandboxing that wraps the volume component. +* :ref:`pipeline_architecture`: How volume modules are chained with copiers, mixers, and equalizers in audio processing DAGs. +* :ref:`audio_buffer_management`: Lockless circular ring buffers supplying samples to the volume processing functions. +* :ref:`ipc_infrastructure`: Control plane protocols and mailbox window communication for volume parameter updates. +* :ref:`scheduler_architecture`: Real-time scheduling domains (LL and DP) driving periodic volume processing calls. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 27e2a0cb..a9a32640 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -41,7 +41,7 @@ Core Infrastructure & Pipeline Audio Processing Modules & Algorithms ------------------------------------- -* `Volume Control `_ +* :ref:`volume_module` (High-level architecture; also see upstream `volume README `_) * `Mixer & Mixin / Mixout `_ * `Sample Rate Converter (SRC) `_ & `ASRC `_ * `Parametric EQ (FIR) `_ & `EQ (IIR) `_ @@ -86,6 +86,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/audio_buffer_management firmware/ipc_infrastructure firmware/fw_init_boot + firmware/volume_module rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 1938a189545557e43a6990b393fede6ab06e3b12 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Fri, 18 Sep 2026 15:48:55 +0100 Subject: [PATCH 09/64] docs: developer_guides: add high-level mixin/mixout architecture guide Add a comprehensive, high-level developer architecture guide covering the Sound Open Firmware (SOF) Mixin / Mixout audio processing subsystem. The guide covers: - Decoupled mixing paradigm comparing legacy monolithic mixers (rigid scheduling, single-core bottlenecks) with the asynchronous Mixin and Mixout decoupled architecture. - Multi-stream fan-out (up to 3 output sinks per Mixin for speakers, headphones, and AEC reference loopbacks) and fan-in (up to 8 input sources per Mixout). - Direct-to-sink zero-intermediate-buffer in-place accumulation mechanics, eliminating intermediate FIFO allocations and halving memory bus bandwidth. - Asynchronous scheduling coordination and pending frames tracking across independent pipeline rates, with autonomous silence generation on source stream starvation. - Per-sink gain scaling (10-bit fractional precision) and flexible channel remapping nibble masks. - SIMD vector accumulation and hardware saturation clamping comparing generic C with Tensilica Xtensa HiFi 3 and HiFi 5 (8-way vector). - Mixin telemetry, rate-limited underrun reporting to prevent IPC message flooding, and End-of-Stream (EOS) DAI latency flushing. - Upstream code references and related architecture links. Includes 7 custom vector Graphviz SVG diagrams illustrating architecture, routing topologies, in-place accumulation, pending frames state machine, per-sink engine, SIMD vector saturation, and telemetry workflows. Signed-off-by: Liam Girdwood --- developer_guides/firmware/mixin_mixout.rst | 524 ++++++++++++++++++ .../firmware/module_framework.rst | 1 + developer_guides/firmware/volume_module.rst | 1 + developer_guides/index.rst | 3 +- 4 files changed, 528 insertions(+), 1 deletion(-) create mode 100644 developer_guides/firmware/mixin_mixout.rst diff --git a/developer_guides/firmware/mixin_mixout.rst b/developer_guides/firmware/mixin_mixout.rst new file mode 100644 index 00000000..4c674dad --- /dev/null +++ b/developer_guides/firmware/mixin_mixout.rst @@ -0,0 +1,524 @@ +.. _mixin_mixout: + +Mixin / Mixout Audio Processing Architecture +############################################ + +The **Mixin / Mixout** subsystem (implemented in ``src/audio/mixin_mixout/``) provides the decoupled, multi-stream audio mixing and distribution architecture in Sound Open Firmware. + +In modern audio systems, mixing multiple concurrent streams (such as media playback, navigation alerts, voice calls, and notification chimes) while routing them to disparate output endpoints (main speakers, headphones, and Acoustic Echo Cancellation loopback references) demands a flexible, non-blocking architecture. + +Rather than relying on a legacy monolithic mixer that forces rigid scheduling across pipelines, SOF decomposes audio mixing into paired, asynchronously coordinated components: **Mixin** and **Mixout**. + +This guide provides a comprehensive, high-level architectural walkthrough of the Mixin / Mixout subsystem, its direct-to-sink zero-intermediate-buffer accumulation mechanics, asynchronous pending frame tracking, per-sink gain attenuation, SIMD vector saturation, and underrun telemetry without delving into low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +.. _decoupled_mixing_paradigm: + +1. Decoupled Mixing Paradigm: Monolithic Mixer vs Mixin / Mixout +**************************************************************** + +Traditional audio DSP architectures implement audio mixing using a single, monolithic Multi-Input Single-Output (MISO) mixer component. While conceptually straightforward, the monolithic mixer introduces severe architectural bottlenecks in modern multi-rate, multi-core audio DSPs. + +Limitations of the Legacy Monolithic Mixer +========================================== + +In a monolithic mixer architecture (e.g., ``src/audio/mixer/``): + +* **Tightly Coupled Scheduling**: All upstream source pipelines must be synchronously locked to the exact same scheduling clock tick and period (e.g., 1 ms). If one client pipeline executes at a different cadence (e.g., 4 ms or 10 ms), the mixer stalls or suffers from buffer starvation. +* **Core Affinity Bottlenecks**: A monolithic mixer component resides on a single DSP core. Mixing streams originated from different cores requires complex cross-core synchronization and frequent inter-processor interrupts (IDC), creating memory bus contention. +* **Rigid Buffer Locking**: All input streams compete for buffer access within a single component processing pass. If one audio application experiences jitter or pauses, the entire mixer can block, causing audible glitches across all other running streams. + +The Decoupled Mixin / Mixout Solution +===================================== + +SOF resolves these challenges by separating the mixing function into two complementary modules: + +1. **Mixin Component**: Serves as the terminal output endpoint of individual client and application pipelines. It executes independently within its own pipeline scheduling domain, consuming source audio and mixing it directly into target mixout buffers. +2. **Mixout Component**: Serves as the origin and primary synchronization point of downstream output pipelines (e.g., post-processing, equalization, and hardware digital audio interfaces). It coordinates buffer availability, ensures all connected mixins have contributed their data, and commits mixed audio frames downstream. + +.. graphviz:: + :caption: Architectural Comparison: Monolithic Mixer vs Decoupled Mixin / Mixout Paradigm + + digraph mixer_comparison { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_mono { + label="Legacy Monolithic Mixer (Tightly Coupled & Synchronous)"; + style="filled,rounded"; + fillcolor="#FFF5F5"; + color="#FEB2B2"; + + m_s1 [label="Pipeline 1: Media\n(Locked 1 ms Tick)", fillcolor="#FED7D7", color="#E53E3E"]; + m_s2 [label="Pipeline 2: Alerts\n(Locked 1 ms Tick)", fillcolor="#FED7D7", color="#E53E3E"]; + m_s3 [label="Pipeline 3: Voice\n(Locked 1 ms Tick)", fillcolor="#FED7D7", color="#E53E3E"]; + + mono_mix [label="Monolithic Mixer Component\nSingle Core Execution / Synchronous Lock\n(Stalls if any single input starves)", fillcolor="#FEB2B2", color="#C53030", fontcolor="#742A2A"]; + mono_out [label="Downstream Sink / Speaker", fillcolor="#FED7D7", color="#E53E3E"]; + + m_s1 -> mono_mix; + m_s2 -> mono_mix; + m_s3 -> mono_mix; + mono_mix -> mono_out; + } + + subgraph cluster_decoupled { + label="Modern Decoupled Mixin / Mixout Paradigm (Asynchronous & Non-Blocking)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + d_s1 [label="Pipeline 1: Media\n(Independent Period)\n[Mixin A]", fillcolor="#E6FFFA", color="#319795"]; + d_s2 [label="Pipeline 2: Alerts\n(Independent Period)\n[Mixin B]", fillcolor="#E6FFFA", color="#319795"]; + d_s3 [label="Pipeline 3: Voice\n(Independent Period)\n[Mixin C]", fillcolor="#E6FFFA", color="#319795"]; + + d_mixout [label="Downstream Mixout Component\nAutonomous Coordinator & Buffer Committer\n(Pads silence on starvation; non-blocking)", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + d_out [label="Downstream Sink / Speaker Pipeline", fillcolor="#9AE6B4", color="#2F855A", fontcolor="#1C4532"]; + + d_s1 -> d_mixout [label="Asynchronous In-Place Mix", color="#319795"]; + d_s2 -> d_mixout [label="Asynchronous In-Place Mix", color="#319795"]; + d_s3 -> d_mixout [label="Asynchronous In-Place Mix", color="#319795"]; + d_mixout -> d_out; + } + } + +--- + +.. _routing_topologies: + +2. Fan-Out & Fan-In Routing Topologies +************************************** + +Real-world audio systems require complex many-to-many audio routing: a single media stream may need to play simultaneously on internal speakers, external headphones, and an echo cancellation reference monitor, while the main speakers simultaneously combine media, navigation voice guidance, and notification sounds. + +Multi-Sink Fan-Out (Mixin Capabilities) +======================================= + +Each Mixin module supports up to **3 independent output queues (sinks)** (``IPC4_MIXIN_MODULE_MAX_OUTPUT_QUEUES``): + +* **Sink 0**: Primary output path routed to the Main Speakers Mixout. +* **Sink 1**: Secondary output path routed to the Headphone Mixout. +* **Sink 2**: Reference loopback path routed to an Acoustic Echo Cancellation (AEC) Mixout for real-time acoustic echo suppression. + +Multi-Source Fan-In (Mixout Capabilities) +========================================= + +Each Mixout module accepts up to **8 concurrent input queues (sources)** (``IPC4_MIXOUT_MODULE_MAX_INPUT_QUEUES``): + +* Collects and mixes up to 8 active Mixin streams simultaneously. +* Each connected Mixin stream can operate with distinct channel counts, independent gain attenuation factors, and custom channel remapping matrices. + +.. graphviz:: + :caption: Multi-Stream Fan-In & Fan-Out Routing Matrix (3 Sinks per Mixin, 8 Sources per Mixout) + + digraph routing_matrix { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_mixins { + label="Client Source Pipelines (Mixins: Up to 3 Output Sinks Each)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + m_media [label="Media Player\n[Mixin 1]", fillcolor="#FFFFFF", color="#CBD5E0"]; + m_nav [label="Navigation Prompts\n[Mixin 2]", fillcolor="#FFFFFF", color="#CBD5E0"]; + m_phone [label="Cellular Voice Call\n[Mixin 3]", fillcolor="#FFFFFF", color="#CBD5E0"]; + } + + subgraph cluster_mixouts { + label="Destination Downstream Pipelines (Mixouts: Up to 8 Input Sources Each)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + mo_spk [label="Main Speakers Mixout\n(Media + Nav + Phone)", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + mo_hp [label="Headphone Mixout\n(Media + Phone)", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + mo_aec [label="AEC Echo Ref Mixout\n(Media Ref Loopback)", fillcolor="#FEFCBF", color="#D69E2E", fontcolor="#744210"]; + } + + m_media -> mo_spk [label="Sink 0 (100% Vol)", color="#3182CE"]; + m_media -> mo_hp [label="Sink 1 (100% Vol)", color="#3182CE"]; + m_media -> mo_aec [label="Sink 2 (Ref Loop)", color="#D69E2E", style="dashed"]; + + m_nav -> mo_spk [label="Sink 0 (Ducked)", color="#805AD5"]; + + m_phone -> mo_spk [label="Sink 0 (Voice)", color="#38A169"]; + m_phone -> mo_hp [label="Sink 1 (Voice)", color="#38A169"]; + } + +--- + +.. _direct_to_sink_mixing: + +3. Direct-to-Sink Zero-Intermediate-Buffer Mixing Mechanics +*********************************************************** + +In a conventional multi-component audio pipeline, interconnecting multiple producers to a single consumer typically requires dedicated First-In First-Out (FIFO) intermediate ring buffers between every connection. + +The Cost of Intermediate Buffering +================================== + +If 8 Mixins were connected to 3 Mixouts using intermediate buffers: + +* The system would require :math:`8 \times 3 = 24` distinct intermediate circular buffers. +* In SRAM-constrained DSPs, allocating 24 separate audio buffers (each several kilobytes) severely fragments and depletes high-performance memory. +* Every audio sample would be copied twice: first from Mixin into the intermediate FIFO, and then from the FIFO into the downstream pipeline, doubling memory bus bandwidth and cache thrashing. + +The SOF Direct-to-Sink Accumulation Solution +============================================ + +Sound Open Firmware completely eliminates intermediate buffers between Mixins and Mixouts. Mixing is performed directly inside the **Mixout sink buffer**: + +1. **First Mixin Execution**: When the first active Mixin executes (``mixin_process()``), it detects that the Mixout sink buffer has no data present (``mixed_frames == 0``). It acquires the Mixout sink buffer (``sink_get_buffer()``) and directly copies its source audio into the buffer. +2. **Subsequent Mixin Executions**: When subsequent connected Mixins execute within the same period, they detect that audio frames already exist in the Mixout sink buffer. Rather than overwriting, they read the existing samples, add their scaled source samples to the buffer (accumulating), and write the sum back into the Mixout sink buffer. +3. **Mixout Commit**: When all connected Mixins have completed their processing, ``mixout_process()`` simply commits the accumulated audio buffer downstream (``sink_commit_buffer()``). + +.. graphviz:: + :caption: Direct-to-Sink In-Place Accumulation Sequence without Intermediate Buffers + + digraph direct_mixing { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_step1 { + label="Step 1: First Mixin (Mixin A: Media) Executes"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + s1_in [label="Mixin A Source Buffer\n[Sample A0, A1, A2, ...]", fillcolor="#FFFFFF", color="#CBD5E0"]; + s1_buf [label="Mixout Sink Buffer (Empty: mixed_frames = 0)\nDirect Copy: Sink[n] = A[n]", fillcolor="#BEE3F8", color="#3182CE"]; + + s1_in -> s1_buf [label="Copy Source to Sink", color="#3182CE"]; + } + + subgraph cluster_step2 { + label="Step 2: Second Mixin (Mixin B: Voice) Executes"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + s2_in [label="Mixin B Source Buffer\n[Sample B0, B1, B2, ...]", fillcolor="#FFFFFF", color="#D69E2E"]; + s2_buf [label="Mixout Sink Buffer (Contains Data A)\nIn-Place Add: Sink[n] = clamp(Sink[n] + B[n])", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + + s2_in -> s2_buf [label="Accumulate & Write Back", color="#B7791F"]; + } + + subgraph cluster_step3 { + label="Step 3: Mixout Process Executes"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + s3_buf [label="Mixout Sink Buffer (Contains A + B)\nsink_commit_buffer(mixed_frames)\nHandoff to Post-Processing / Speaker DAI", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + s1_buf -> s2_buf [style="dashed", color="#A0AEC0"]; + s2_buf -> s3_buf [style="dashed", color="#38A169"]; + } + +By performing mixing directly in the destination buffer, SOF eliminates intermediate FIFO allocations entirely and cuts memory bus read/write traffic by **50%**. + +--- + +.. _asynchronous_scheduling: + +4. Asynchronous Scheduling & Pending Frames Tracking +**************************************************** + +In a multi-pipeline audio DSP graph, different pipelines may execute under different scheduling conditions: + +* Mixin pipelines may run under the Low-Latency (LL) 1 ms timer domain, while the Mixout pipeline runs under a 4 ms or 10 ms DMA domain. +* Mixin and Mixout may reside on different DSP cores, communicating across core boundaries. + +The Pending Frames Mechanism (``struct pending_frames``) +======================================================== + +Because Mixins consume source audio during ``mixin_process()``, but sink audio cannot be committed until ``mixout_process()`` runs, there is an inherent temporal phase offset between consumption and production: + +* For each connected Mixin $\leftrightarrow$ Mixout pair, the Mixout maintains a ``struct pending_frames`` entry. +* When a Mixin processes and writes frames into the Mixout buffer, it increments its ``pending_frames->frames`` counter. +* When ``mixout_process()`` runs, it evaluates the pending frames across all connected active Mixins: + + .. math:: + + \text{Frames to Produce} = \min_{k \in \text{Active Mixins}} \left( \text{pending\_frames}_k \right) + +* After committing the data downstream, ``mixout_process()`` decrements the pending frame counters by the committed amount. + +.. graphviz:: + :caption: Asynchronous Pipeline Execution and Pending Frames State Machine + + digraph pending_frames_flow { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + m_tick [label="Mixin Pipeline Tick (1 ms Period)\nmixin_process() runs\nConsumes source data; writes to Mixout buffer", fillcolor="#EDF2F7", color="#CBD5E0"]; + p_inc [label="Update Pending Counter:\npending_frames[mixin] += frames_copied\nmixed_frames updated", fillcolor="#BEE3F8", color="#3182CE"]; + + mo_tick [label="Mixout Pipeline Tick (e.g. DMA Callback)\nmixout_process() runs", fillcolor="#FEFCBF", color="#D69E2E"]; + mo_calc [label="Calculate Production Limit:\nframes_to_produce = min(pending_frames of all active mixins)", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + mo_com [label="Commit Buffer Downstream:\nsink_commit_buffer(frames_to_produce)\nDecrement pending_frames for all mixins", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + m_tick -> p_inc; + p_inc -> mo_tick [style="dashed", color="#A0AEC0"]; + mo_tick -> mo_calc -> mo_com; + } + +Autonomous Silence Generation on Starvation +=========================================== + +A critical challenge in audio mixing occurs when one input stream pauses, ends, or encounters an underrun while other streams remain active: + +* If the system waited for the starved stream to produce data, the entire Mixout pipeline would stall, causing audio dropouts across all active streams. +* SOF resolves this via **autonomous silence injection**: If an active Mixin has zero source frames available (``source_avail_frames == 0``), it invokes ``silence()``. +* The Mixin automatically fills its unmixed portion of the Mixout buffer with zeros, allowing the other connected Mixins to mix normally and ensuring continuous, glitch-free audio playback. + +--- + +.. _per_sink_gain_remapping: + +5. Per-Sink Gain Attenuation & Channel Remapping Engine +******************************************************* + +Each connection between a Mixin and a destination Mixout can have unique acoustic requirements. For example, navigation speech routed to the driver's speaker may require full volume, while simultaneously being ducked by -12 dB when routed to passenger speakers. + +IPC4 Mixer Mode Configuration (``struct ipc4_mixer_mode_sink_config``) +====================================================================== + +SOF allows independent gain and channel matrix configuration on every Mixin output queue: + +* **10-Bit Fractional Gain Attenuation (``gain``)**: + + - Gain is expressed as a 16-bit integer ranging from ``0x0`` (silence) to ``0x400`` (1024, representing :math:`1.0` or :math:`0\text{ dB}` unity gain). + - Samples are scaled by multiplying by ``gain`` and right-shifting by 10 bits: + + .. math:: + + y_n = \frac{x_n \times \text{gain}}{1024} + +* **Flexible Channel Remapping (``output_channel_map``)**: + + - A 32-bit bitfield where each 4-bit nibble corresponds to an output destination channel, storing the index of the source channel to copy. + - A nibble value of ``0xF`` designates that the output channel should be left unmodified. + - Enables dynamic downmixing (e.g. Stereo L/R downmixed to Mono center), channel duplication, or surround channel routing per destination Mixout. + +.. graphviz:: + :caption: Independent Per-Sink Gain Attenuation & Channel Remapping Engine + + digraph sink_engine { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_stereo [label="Mixin Source Stream\nStereo (Ch0: Left, Ch1: Right)", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_q0 { + label="Output Queue 0 (Speakers Mixout)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + q0_map [label="Channel Map: Normal Stereo\nOut0 <- In0, Out1 <- In1", fillcolor="#FFFFFF", color="#CBD5E0"]; + q0_gain [label="Gain: 0x400 (Unity / 0 dB)\nFull amplitude", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + q0_map -> q0_gain; + } + + subgraph cluster_q1 { + label="Output Queue 1 (Mono AEC Loopback)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + q1_map [label="Channel Map: Mono Downmix\nOut0 <- (In0 + In1) / 2", fillcolor="#FFFFFF", color="#D69E2E"]; + q1_gain [label="Gain: 0x200 (-6 dB Attenuation)\nPrevent AEC distortion", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + + q1_map -> q1_gain; + } + + in_stereo -> q0_map; + in_stereo -> q1_map; + } + +--- + +.. _simd_vector_saturation: + +6. SIMD Vector Acceleration & Saturation Arithmetic +*************************************************** + +Mixing audio streams requires summing digital samples across multiple channels and sources: + +.. math:: + + S_{\text{mixed}}[n] = \sum_{k=1}^{N} \left( x_k[n] \times G_k \right) + +If multiple high-amplitude signals are summed simultaneously, the resulting values can easily exceed the container limits (:math:`+32767` for 16-bit, :math:`+2^{31}-1` for 32-bit). Numerical overflow causes catastrophic acoustic clipping. + +Hardware Saturation Protection +============================== + +To prevent integer wraparound, SOF's mixing kernels apply **saturation arithmetic**: + +.. math:: + + S_{\text{clamped}}[n] = \text{clamp}\left( S_{\text{mixed}}[n], \text{MIN\_VAL}, \text{MAX\_VAL} \right) + +SIMD Vector Implementations +=========================== + +To maximize throughput and minimize battery consumption, SOF provides architecture-specific SIMD implementations: + +* **Generic Portable C (``mixin_mixout_generic.c``)**: Portable scalar C reference with 64-bit integer accumulators and clamping, running on ARM Cortex-M and RISC-V. +* **Cadence Xtensa HiFi 3 (``mixin_mixout_hifi3.c``)**: Vectorized 24-bit and 32-bit SIMD instructions with automated hardware saturation. +* **Cadence Xtensa HiFi 5 (``mixin_mixout_hifi5.c``)**: 8-way 32-bit vector engine processing eight audio samples per clock cycle, utilizing dual 128-bit memory load/store operations. + +.. graphviz:: + :caption: SIMD Vector Accumulation and Saturation Clamping Architecture + + digraph simd_mix { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_inputs { + label="Concurrent Audio Streams (32-bit Samples)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + in_a [label="Stream A (Media): [A0, A1, A2, A3, A4, A5, A6, A7]", fillcolor="#FFFFFF", color="#CBD5E0"]; + in_b [label="Stream B (Voice): [B0, B1, B2, B3, B4, B5, B6, B7]", fillcolor="#FFFFFF", color="#CBD5E0"]; + } + + subgraph cluster_hifi5 { + label="Xtensa HiFi 5 SIMD Vector Execution Engine"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + v_mul [label="8-Way Parallel Vector Gain Scaling\nV_A = (A * G_A) >> 10\nV_B = (B * G_B) >> 10", fillcolor="#BEE3F8", color="#3182CE"]; + v_add [label="8-Way Vector Addition with Saturation\nSum = sat_add(V_A, V_B)", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + v_sat [label="Hardware Saturation Clamp\nValues clamped to [INT32_MIN, INT32_MAX]", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + v_mul -> v_add -> v_sat; + } + + out_mix [label="Mixed 8-Sample Vector Output\nDirectly committed to Mixout Sink Buffer", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + in_a -> v_mul; + in_b -> v_mul; + v_sat -> out_mix; + } + +--- + +.. _telemetry_underrun_eos: + +7. Telemetry, Underrun Rate-Limiting & End-of-Stream (EOS) +********************************************************** + +Because the Mixin is the terminal component of client pipelines, it is the primary observer of stream starvation and playback completion. + +Underrun Detection & Rate-Limiting +================================== + +When a client application fails to supply audio frames on time, the Mixin detects buffer exhaustion (``source_avail_frames == 0``): + +* To alert the host operating system, the Mixin generates an asynchronous underrun notification (``send_mixer_underrun_notif_msg()``). +* **Notification Flooding Prevention**: If an application remains starved for multiple seconds, sending notifications on every 1 ms frame tick would overwhelm the host IPC queue with thousands of messages. +* SOF applies **underrun rate-limiting** (``underrun_notification_period``, defaulting to 10 periods): Underrun notifications are throttled, delivering timely diagnostics without flooding the host driver. + +End-of-Stream (EOS) Delay Compensation +====================================== + +When an audio track finishes playback, the pipeline enters the End-of-Stream (EOS) state: + +* Signaling EOS immediately when the last sample reaches the Mixin would cause the host driver to stop the sound card while audio samples are still traversing downstream buffers and the hardware Digital-to-Analog Converter (DAC). +* SOF queries the physical latency of downstream components (``pipeline_get_dai_comp_latency()``). +* The Mixin delays emitting the final EOS notification until the remaining samples have cleared all downstream FIFOs and physically exited the speakers, guaranteeing that audio tracks are never prematurely truncated. + +.. graphviz:: + :caption: Mixin Telemetry, Rate-Limited Underrun Reporting, and EOS Flushing Sequence + + digraph mixin_telemetry { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + cond_state [label="Mixin Event Evaluation\nCheck Buffer State & Available Frames", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_underrun { + label="Starvation / Underrun Branch"; + style="filled,rounded"; + fillcolor="#FFF5F5"; + color="#FEB2B2"; + + u_check [label="source_avail_frames == 0?\nIncrement last_reported_underrun", fillcolor="#FED7D7", color="#E53E3E"]; + u_rate [label="Has period threshold expired?\nlast_reported_underrun >= notification_period", shape=diamond, fillcolor="#FEFCBF", color="#D69E2E"]; + u_send [label="Send IPC Underrun Notification\nReset notification counter", fillcolor="#FEB2B2", color="#C53030", fontcolor="#742A2A"]; + + u_check -> u_rate; + u_rate -> u_send [label="Yes (Throttled Alert)"]; + } + + subgraph cluster_eos { + label="End-of-Stream (EOS) Branch"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + e_check [label="AUDIOBUF_STATE_END_OF_STREAM\nQuery DAI latency (pipeline_get_dai_comp_latency)", fillcolor="#E6FFFA", color="#319795"]; + e_delay [label="Countdown delay periods as samples flush\neos_delay_periods == 0", shape=diamond, fillcolor="#FEFCBF", color="#D69E2E"]; + e_send [label="Send IPC EOS Completed Notification\nHost safely powers down stream", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + e_check -> e_delay; + e_delay -> e_send [label="Flushed through DAC"]; + } + + cond_state -> u_check [label="Starvation Detected"]; + cond_state -> e_check [label="Track Finished"]; + } + +--- + +.. _upstream_mixin_references: + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level implementation details, vector assembly intrinsics, and configuration structures: + +* **Upstream Mixin / Mixout Specification**: + - `thesofproject/sof: src/audio/mixin_mixout/README.md `_ + - `thesofproject/sof: src/audio/mixer/README.md `_ +* **Core Firmware Source Files**: + - ``src/audio/mixin_mixout/mixin_mixout.c``: Core Mixin and Mixout lifecycle, direct-to-sink accumulation, and pending frame tracking. + - ``src/audio/mixin_mixout/mixin_mixout.h``: IPC4 mixer mode configuration structures, gain shift definitions, and function prototypes. + - ``src/audio/mixin_mixout/mixin_mixout_generic.c``: Portable scalar C mixing implementation. + - ``src/audio/mixin_mixout/mixin_mixout_hifi3.c``: Cadence Tensilica Xtensa HiFi 3 SIMD vector mixing kernel. + - ``src/audio/mixin_mixout/mixin_mixout_hifi5.c``: Cadence Tensilica Xtensa HiFi 5 8-way SIMD vector mixing kernel. +* **Topology Definitions**: + - ``tools/topology/topology2/include/components/mixin.conf``: ALSA Topology 2 configuration class for Mixin widgets. + - ``tools/topology/topology2/include/components/mixout.conf``: ALSA Topology 2 configuration class for Mixout widgets. + +Related Subsystem Architecture Guides +===================================== + +* :ref:`volume_module`: Per-channel gain scaling, smooth ramping, zero-crossing muting, and SIMD acceleration. +* :ref:`module_framework`: The standardized module interface, Source/Sink APIs, and memory sandboxing that wraps Mixin and Mixout components. +* :ref:`pipeline_architecture`: How audio pipelines are connected across Mixin and Mixout endpoints into complex DAGs. +* :ref:`audio_buffer_management`: Ring buffer sizing, lockless single-producer single-consumer mechanics, and cross-core memory operations. +* :ref:`scheduler_architecture`: Real-time scheduling domains (LL and DP) driving independent Mixin and Mixout pipeline executions. +* :ref:`ipc_infrastructure`: Control plane protocols and dynamic pin binding for Mixin and Mixout components. diff --git a/developer_guides/firmware/module_framework.rst b/developer_guides/firmware/module_framework.rst index 44a4ccc0..16fa6a2e 100644 --- a/developer_guides/firmware/module_framework.rst +++ b/developer_guides/firmware/module_framework.rst @@ -521,6 +521,7 @@ Related Guides * :ref:`scheduler_architecture`: Multi-tier real-time scheduling (LL, DP, TWB), EDF mechanics, and multi-core execution. * :ref:`pipeline_architecture`: How processing modules are assembled into directed acyclic execution graphs (DAGs). * :ref:`volume_module`: Comprehensive architecture of the canonical volume control module, ramping, and SIMD optimization. +* :ref:`mixin_mixout`: Multi-stream audio distribution, fan-out/fan-in routing, and direct-to-sink accumulation. * :ref:`llext_modules`: Authoring, compiling, and signing dynamic loadable modules using Zephyr LLEXT. * :ref:`sof_hostless_firmware`: Instantiating static modules in autonomous embedded firmware. * :ref:`topology2`: Declaring audio widgets and binding modules using ALSA Topology 2.0. diff --git a/developer_guides/firmware/volume_module.rst b/developer_guides/firmware/volume_module.rst index 8d1f29b7..195b8c08 100644 --- a/developer_guides/firmware/volume_module.rst +++ b/developer_guides/firmware/volume_module.rst @@ -548,6 +548,7 @@ Related Subsystem Architecture Guides * :ref:`module_framework`: The standardized module interface, Source/Sink APIs, and memory sandboxing that wraps the volume component. * :ref:`pipeline_architecture`: How volume modules are chained with copiers, mixers, and equalizers in audio processing DAGs. +* :ref:`mixin_mixout`: Multi-stream audio distribution, fan-out/fan-in routing, and direct-to-sink accumulation. * :ref:`audio_buffer_management`: Lockless circular ring buffers supplying samples to the volume processing functions. * :ref:`ipc_infrastructure`: Control plane protocols and mailbox window communication for volume parameter updates. * :ref:`scheduler_architecture`: Real-time scheduling domains (LL and DP) driving periodic volume processing calls. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index a9a32640..f8868a13 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -42,7 +42,7 @@ Audio Processing Modules & Algorithms ------------------------------------- * :ref:`volume_module` (High-level architecture; also see upstream `volume README `_) -* `Mixer & Mixin / Mixout `_ +* :ref:`mixin_mixout` (High-level architecture; also see upstream `mixin_mixout README `_ & `mixer README `_) * `Sample Rate Converter (SRC) `_ & `ASRC `_ * `Parametric EQ (FIR) `_ & `EQ (IIR) `_ * `Dynamic Range Compressor (DRC) `_ & `Multiband DRC `_ @@ -87,6 +87,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/ipc_infrastructure firmware/fw_init_boot firmware/volume_module + firmware/mixin_mixout rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 6a3234c50a0f6a5f0dcb32b3fd36d1d3fa38af2c Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Fri, 18 Sep 2026 16:05:44 +0100 Subject: [PATCH 10/64] docs: developer_guides: add high-level src and asrc architecture guide Add a comprehensive, high-level developer architecture guide for Sample Rate Conversion (SRC & ASRC) in Sound Open Firmware. Covers: - Architectural taxonomy: Synchronous SRC (locked rational ratios M/N) vs Asynchronous ASRC (independent crystals and continuous drift). - Synchronous polyphase FIR filter bank decomposition, subfilter phase commutator, and zero-elimination optimization. - Multi-stage conversion and latency optimization (factoring 44.1 kHz to 48 kHz into two stages with halfband filters to reduce latency by >75%). - Asynchronous Farrow filter structure with polynomial approximation and continuous fractional sample delay parameter mu. - Closed-loop drift estimation and buffer watermark tracking controller. - Push-mode (playback/transmit) vs pull-mode (capture/receive) execution topologies across audio endpoints. - SIMD vector acceleration across Cadence Xtensa HiFi 3, HiFi 4, HiFi 5, and generic scalar C implementations. - 7 native vector Graphviz SVG diagrams. - Cross-references in developer_guides/index.rst, pipeline_architecture.rst, and algorithms/src/sample_rate_conversion.rst. Signed-off-by: Liam Girdwood --- .../algorithms/src/sample_rate_conversion.rst | 7 + .../firmware/pipeline_architecture.rst | 2 + developer_guides/firmware/src_asrc.rst | 560 ++++++++++++++++++ developer_guides/index.rst | 3 +- 4 files changed, 571 insertions(+), 1 deletion(-) create mode 100644 developer_guides/firmware/src_asrc.rst diff --git a/developer_guides/algorithms/src/sample_rate_conversion.rst b/developer_guides/algorithms/src/sample_rate_conversion.rst index 22c3655b..ebd6f539 100644 --- a/developer_guides/algorithms/src/sample_rate_conversion.rst +++ b/developer_guides/algorithms/src/sample_rate_conversion.rst @@ -3,6 +3,13 @@ Sample Rate Conversion ###################### +.. seealso:: + + For a high-level firmware architectural overview of both Synchronous (SRC) and + Asynchronous (ASRC) converters—including multi-stage factorization, continuous + Farrow drift compensation, push vs pull topologies, and SIMD acceleration—see + :ref:`src_asrc`. + Introduction ************ diff --git a/developer_guides/firmware/pipeline_architecture.rst b/developer_guides/firmware/pipeline_architecture.rst index 7d6c414b..3cfd8a6f 100644 --- a/developer_guides/firmware/pipeline_architecture.rst +++ b/developer_guides/firmware/pipeline_architecture.rst @@ -462,6 +462,8 @@ Related Guides * :ref:`audio_buffer_management`: Lockless circular ring buffers, multi-tier DSP memory (SRAM/DRAM), and cache coherency. * :ref:`module_framework`: The standardized module interface, Source/Sink APIs, and memory sandboxing. * :ref:`volume_module`: Per-channel gain scaling, smooth ramping, zero-crossing muting, and SIMD acceleration. +* :ref:`mixin_mixout`: Multi-pipeline audio mixing, stream splitting, dynamic clock domain decoupling, and matrix routing. +* :ref:`src_asrc`: Synchronous polyphase conversion, asynchronous Farrow drift tracking, push/pull topologies, and SIMD acceleration. * :ref:`fw_init_boot`: Boot flow, hardware mailbox FW Ready handshake, and Zephyr initialization. * :ref:`topology2`: How pipelines and widgets are declared using ALSA Topology 2.0 configuration classes. * :ref:`sof_hostless_firmware`: How to create static pipelines compiled into ROM for standalone microcontrollers. diff --git a/developer_guides/firmware/src_asrc.rst b/developer_guides/firmware/src_asrc.rst new file mode 100644 index 00000000..098cd7ac --- /dev/null +++ b/developer_guides/firmware/src_asrc.rst @@ -0,0 +1,560 @@ +.. _src_asrc: + +Sample Rate Conversion Architecture (SRC & ASRC) +################################################ + +The **Sample Rate Conversion** subsystem in Sound Open Firmware provides real-time sampling frequency transformations across heterogeneous audio streams, mixing buses, and hardware peripherals. + +Modern audio platforms must seamlessly interconnect disparate sample rates: 44.1 kHz (compact discs, MP3, AAC), 48 kHz (standard pro-audio, video, Bluetooth SBC/mSBC), 16 kHz (speech recognition, voice wake-word engines), and 96/192 kHz (high-resolution DACs). Furthermore, when interfacing with independent external hardware clocks (such as USB Audio Class, Bluetooth LE Audio, HDMI/DisplayPort PLLs, or external codecs), clock frequencies drift over time, requiring continuous fractional compensation. + +SOF addresses these challenges through two specialized architectural components: + +1. **Synchronous Sample Rate Converter (SRC)** (``src/audio/src/``): Converts sample rates by exact, mathematically locked rational ratios (:math:`M / N`) using a multi-stage polyphase FIR filter bank. +2. **Asynchronous Sample Rate Converter (ASRC)** (``src/audio/asrc/``): Converts sample rates across independent, unsynchronized clock domains using a polynomial Farrow filter structure coupled with a closed-loop drift tracking controller. + +This guide provides a comprehensive, high-level architectural walkthrough of the SRC and ASRC subsystems, polyphase filter banks, Farrow polynomial structures, push/pull modes, and SIMD hardware acceleration without delving into low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +.. _src_asrc_taxonomy: + +1. Sample Rate Conversion in Audio Systems +****************************************** + +Audio systems process signals sampled at discrete time intervals. When two connected components operate at different sampling frequencies, digital sample rate conversion is required to change the effective time interval between samples without introducing audible acoustic distortion, aliasing, or spectral imaging. + +Synchronous vs Asynchronous Taxonomy +==================================== + +The fundamental distinction between SRC and ASRC lies in whether the input and output sampling clocks share a locked timebase: + +* **Synchronous Sample Rate Conversion (SRC)**: + + - Input rate :math:`F_{\text{in}}` and output rate :math:`F_{\text{out}}` are derived from the same physical clock tree or share an exact, immutable rational ratio :math:`M / N`. + - For every :math:`N` input frames consumed, exactly :math:`M` output frames are produced. + - Ideal for intra-DSP conversions (e.g. upsampling a 44.1 kHz MP3 stream to the 48 kHz pipeline mixing bus, or downsampling 48 kHz microphone audio to 16 kHz for a voice keyword recognizer). + +* **Asynchronous Sample Rate Conversion (ASRC)**: + + - Input and output sampling clocks originate from independent physical crystal oscillators (e.g. a host PC USB clock vs an embedded DSP oscillator, or an external S/PDIF transceiver vs an internal audio PLL). + - Due to physical manufacturing tolerances and thermal fluctuations, crystals exhibit drift (typically 20 to 100 parts per million). Over seconds or minutes, clock drift causes cumulative sample count discrepancies that lead to buffer underrun or overrun. + - ASRC continuously tracks clock drift and applies time-varying fractional interpolation, maintaining a constant buffer watermark without audible pitch distortion or clicks. + +.. graphviz:: + :caption: Architectural Taxonomy: Synchronous SRC (Fixed M/N Ratio) vs Asynchronous ASRC (Drifting Clocks) + + digraph src_taxonomy { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_src { + label="Synchronous SRC (Locked Rational Ratio M / N)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + src_in [label="Input Stream (F_in)\ne.g., 44.1 kHz (Shared System PLL)", fillcolor="#FFFFFF", color="#CBD5E0"]; + src_core [label="Synchronous Polyphase FIR Engine\nFixed M/N ratio (e.g., 160/147)\nConsumes N samples -> Produces M samples", fillcolor="#BEE3F8", color="#3182CE"]; + src_out [label="Output Stream (F_out)\ne.g., 48.0 kHz (Locked Timebase)", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + src_in -> src_core -> src_out [color="#3182CE"]; + } + + subgraph cluster_asrc { + label="Asynchronous ASRC (Drifting Clocks & Closed-Loop Tracking)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + asrc_in [label="Input Stream (Clock Domain A)\ne.g., USB / Bluetooth LC3 Clock", fillcolor="#FFFFFF", color="#D69E2E"]; + asrc_core [label="Farrow Polynomial Engine\nContinuous Fractional Delay µ in [0, 1)\nVariable conversion ratio tracked dynamically", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + asrc_out [label="Output Stream (Clock Domain B)\ne.g., Local DSP Audio Interface Clock", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + asrc_pll [label="Closed-Loop Drift Estimator\nMonitors Buffer Fill Watermark\nDynamically adjusts time step µ", fillcolor="#FED7D7", color="#E53E3E", fontcolor="#742A2A"]; + + asrc_in -> asrc_core -> asrc_out [color="#B7791F"]; + asrc_core -> asrc_pll [label="Watermark Feedback", style="dashed", color="#E53E3E"]; + asrc_pll -> asrc_core [label="Adjust Fractional Step µ", style="dashed", color="#E53E3E"]; + } + } + +--- + +.. _polyphase_src_architecture: + +2. Synchronous SRC: Polyphase Filter Bank Architecture +****************************************************** + +In classic digital signal processing textbooks, sample rate conversion by a rational fraction :math:`M / N` is described as a three-step sequence: upsampling by :math:`M` (inserting :math:`M-1` zero-valued samples), low-pass filtering to eliminate spectral imaging, and downsampling by :math:`N` (retaining every :math:`N`-th sample). + +The Inefficiency of Textbook Interpolation +========================================== + +Implementing textbook zero-stuffing directly in DSP firmware would be disastrously inefficient: + +* For a conversion from 44.1 kHz to 48 kHz (:math:`M/N = 160/147`), upsampling by 160 requires inserting 159 zeros between every sample, inflating the intermediate sample rate to :math:`44.1 \times 160 = 7.056\text{ MHz}`. +* Over 99% of multiply-accumulate operations in the FIR filter would multiply filter coefficients by zero, wasting immense processor power. + +The Polyphase Filter Bank Optimization +====================================== + +Sound Open Firmware implements the **Polyphase Filter Bank** decomposition. In a polyphase architecture: + +1. **Subfilter Decomposition**: The large prototype low-pass FIR filter is mathematically partitioned into :math:`M` smaller subfilters (phases), where subfilter :math:`p` contains coefficients :math:`h[kM + p]`. +2. **Zero Elimination**: Because the locations of non-zero input samples are known deterministically, zero-valued samples are never inserted or computed. +3. **Lowest-Rate Filtering**: Filtering operations execute directly at the input sample rate. For each output sample required, the engine selects the appropriate polyphase subfilter branch and computes a short dot-product against historical input samples stored in a circular delay line. + +.. graphviz:: + :caption: Polyphase Filter Bank Interpolation and Decimation Mechanics in Synchronous SRC + + digraph polyphase_flow { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_data [label="Input Audio Stream\n(F_in = 48 kHz)\nSample x[n]", fillcolor="#EDF2F7", color="#CBD5E0"]; + d_line [label="Circular Delay Line\n(Historical Samples)\n[x_n, x_{n-1}, x_{n-2}, ...]", fillcolor="#BEE3F8", color="#3182CE"]; + + subgraph cluster_bank { + label="Polyphase Subfilter Bank (M Phases)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + f0 [label="Phase 0: h[0], h[M], h[2M]...", fillcolor="#FAF089", color="#B7791F"]; + f1 [label="Phase 1: h[1], h[M+1], h[2M+1]...", fillcolor="#FAF089", color="#B7791F"]; + f2 [label="Phase 2: h[2], h[M+2], h[2M+2]...", fillcolor="#FAF089", color="#B7791F"]; + fm [label="Phase M-1: h[M-1], h[2M-1]...", fillcolor="#FAF089", color="#B7791F"]; + + f0 -> f1 -> f2 -> fm [style="invis"]; + } + + phase_mux [label="Phase Selector / Commutator\nAdvance phase index by N modulo M", fillcolor="#FEFCBF", color="#D69E2E"]; + out_data [label="Output Audio Stream\n(F_out = 44.1 kHz)\nSample y[m]", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + in_data -> d_line; + d_line -> f0; + d_line -> f1; + d_line -> f2; + d_line -> fm; + f0 -> phase_mux; + f1 -> phase_mux; + f2 -> phase_mux; + fm -> phase_mux; + phase_mux -> out_data; + } + +Quality and Memory Footprint Profiles +===================================== + +SOF allows system integrators to configure filter complexity via compile-time Kconfig settings: + +* **Standard Profile (``COMP_SRC_STD``)**: Studio-grade conversion quality. Exceeds 120 dB signal-to-noise ratio (SNR) with stopband rejection greater than 100 dB and passband ripple below 0.001 dB, ideal for high-fidelity 24-bit/32-bit music playback. +* **Small Profile (``COMP_SRC_SMALL``)**: Balanced profile offering ~100 dB SNR while halving filter coefficient memory tables. +* **Tiny / Lite Profiles (``COMP_SRC_TINY`` / ``COMP_SRC_LITE``)**: Ultra-compact filter sets tailored for memory-constrained microcontrollers and speech streams (e.g. 16 kHz to 48 kHz). + +--- + +.. _multistage_conversion: + +3. Multi-Stage Conversion & Latency Optimization +************************************************ + +Converting between sample rates in the same family (e.g. 48 kHz to 96 kHz, ratio :math:`2/1`, or 32 kHz to 48 kHz, ratio :math:`3/2`) requires low-order filters and introduces minimal latency. + +However, cross-family conversions (such as 44.1 kHz to 48 kHz) present severe mathematical challenges. The exact ratio is: + +.. math:: + + \frac{F_{\text{out}}}{F_{\text{in}}} = \frac{48000}{44100} = \frac{160}{147} + +The Latency Penalty of Single-Stage Conversion +============================================== + +In a single-stage converter: + +* The numerator :math:`M = 160` and denominator :math:`N = 147` require processing blocks of 147 input samples producing 160 output samples. +* At 44.1 kHz, 147 samples corresponds to **3.33 ms of algorithmic latency**, and steep anti-aliasing filter requirements inflate the total delay line buffer requirement to over **13.3 ms**. +* For real-time communications, gaming, and interactive audio, 13 ms of added latency is unacceptable. + +The Two-Stage Factored Pipeline Solution +======================================== + +SOF resolves this by factoring difficult conversion fractions into **two cascaded stages**: + +.. math:: + + \frac{160}{147} = \frac{32}{21} \times \frac{5}{7} \quad \text{or} \quad \text{Stage 1 (Polyphase)} \times \text{Stage 2 (Halfband Filter)} + +.. graphviz:: + :caption: Multi-Stage Conversion Pipeline (e.g. 44.1 kHz to 48 kHz Factored into 2 Stages with Halfband Filters) + + digraph multistage_src { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_stream [label="Input Audio Stream\n44.1 kHz", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_stage1 { + label="Stage 1: Fractional Polyphase Filter"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + s1_core [label="Polyphase Converter (32/21)\n21 input frames granularity\n20 output frames granularity", fillcolor="#BEE3F8", color="#3182CE"]; + } + + inter_buf [label="Intermediate Buffer\n(Small FIFO: ~64 samples)", fillcolor="#E2E8F0", color="#A0AEC0"]; + + subgraph cluster_stage2 { + label="Stage 2: Halfband Resampling Filter"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + s2_core [label="Halfband Filter (5/7 or 2x)\n50% of coefficients are Zero\nZero MAC overhead on even taps", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + out_stream [label="Output Audio Stream\n48.0 kHz", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + in_stream -> s1_core; + s1_core -> inter_buf [label="Intermediate Rate"]; + inter_buf -> s2_core; + s2_core -> out_stream; + } + +Benefits of Two-Stage Conversion +================================ + +* **Shorter Buffer Latency**: Granularity drops from 147 frames to 21 frames, reducing buffer latency by more than **75%**. +* **Computational Efficiency**: Halfband filters possess symmetric impulse responses where nearly 50% of the coefficients are exactly zero, eliminating half of the required multiplication operations. +* **Reduced Memory Footprint**: Total filter coefficient storage and delay line allocations are dramatically lower than a monolithic 160/147 single-stage filter. + +--- + +.. _farrow_asrc_architecture: + +4. Asynchronous ASRC: Farrow Filter Structure +********************************************* + +When audio traverses independent clock domains—such as streaming over Bluetooth (where the remote earbud clock is slightly slower than the phone's clock) or recording from a USB microphone—the ratio between input and output sample rates is not a fixed rational number. Instead, the ratio drifts continuously over time: + +.. math:: + + R(t) = \frac{F_{\text{out}}(t)}{F_{\text{in}}(t)} = R_0 + \Delta R(t) + +The Failure of Conventional Polyphase Filters for Drift +======================================================= + +A polyphase filter bank requires precomputed coefficient tables for a specific, fixed rational fraction :math:`M / N`. If the clock drifts by even 15 parts per million, the true conversion fraction changes into an irrational or unmanageably large ratio, making static polyphase tables useless. + +The Farrow Polynomial Structure +=============================== + +The Asynchronous Sample Rate Converter (implemented in ``src/audio/asrc/asrc_farrow.c``) uses the **Farrow structure**. Invented by Cecil W. Farrow, this architecture evaluates continuous fractional delay filtering: + +1. **Polynomial Approximation**: The impulse response :math:`h(t)` of the continuous interpolation filter is approximated by a set of :math:`P`-th order polynomials over each sample interval: + + .. math:: + + h(t) \approx \sum_{k=0}^{P} c_k(n) \cdot \mu^k + + where :math:`\mu \in [0, 1)` represents the **fractional sample delay** (the exact sub-sample time offset where the output sample lies between two input samples). + +2. **Parallel Fixed Subfilters**: The filter coefficients :math:`c_k(n)` are fixed and precomputed at compile time. The input audio signal is passed through :math:`P+1` parallel fixed FIR filter branches. +3. **Continuous Polynomial Interpolation**: The outputs of the parallel branches are multiplied by successive powers of :math:`\mu` using Horner's rule: + + .. math:: + + y(t) = C_0 + \mu \cdot \left( C_1 + \mu \cdot \left( C_2 + \mu \cdot C_3 \right) \right) + +.. graphviz:: + :caption: Asynchronous Farrow Filter Structure with Continuous Fractional Delay Parameter µ + + digraph farrow_structure { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_pcm [label="Input Stream x[n]\n(Rate F_in)", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_branches { + label="Parallel Fixed FIR Filter Branches"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + b0 [label="FIR Branch C_0\n(Fixed Coefficients)", fillcolor="#FAF089", color="#B7791F"]; + b1 [label="FIR Branch C_1\n(Fixed Coefficients)", fillcolor="#FAF089", color="#B7791F"]; + b2 [label="FIR Branch C_2\n(Fixed Coefficients)", fillcolor="#FAF089", color="#B7791F"]; + b3 [label="FIR Branch C_3\n(Fixed Coefficients)", fillcolor="#FAF089", color="#B7791F"]; + + b0 -> b1 -> b2 -> b3 [style="invis"]; + } + + subgraph cluster_horner { + label="Horner Polynomial Evaluation Engine"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + h_mul [label="Polynomial Summation:\ny = C_0 + µ*(C_1 + µ*(C_2 + µ*C_3))\nArbitrary Fractional Delay", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + mu_param [label="Fractional Delay µ in [0, 1)\n(From Drift Estimator)", fillcolor="#FED7D7", color="#E53E3E", fontcolor="#742A2A"]; + out_pcm [label="Output Stream y(t)\n(Rate F_out)", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + in_pcm -> b0; + in_pcm -> b1; + in_pcm -> b2; + in_pcm -> b3; + b0 -> h_mul; + b1 -> h_mul; + b2 -> h_mul; + b3 -> h_mul; + mu_param -> h_mul [label="Continuous µ", color="#E53E3E", style="bold"]; + h_mul -> out_pcm; + } + +Because :math:`\mu` is a continuous floating-point or high-precision fixed-point value, the Farrow structure can synthesize output samples at **any arbitrary sub-sample time point** on the fly, requiring zero table updates or DSP filter recalculations. + +--- + +.. _drift_estimation_control: + +5. Closed-Loop Drift Estimation & Buffer Watermark Control +********************************************************** + +While the Farrow structure provides the mathematical capability to interpolate at any arbitrary time offset :math:`\mu`, the ASRC requires an intelligent control system to determine what :math:`\mu` should be at every moment in time. + +The Closed-Loop Feedback Architecture +===================================== + +The ASRC subsystem implements a closed-loop **Drift Estimator** and watermark tracking controller: + +1. **Watermark Monitoring**: The controller continuously tracks the fill level (number of available frames) in the secondary buffer. +2. **Phase Error Calculation**: If the output clock is running faster than nominal, the secondary buffer level gradually falls below the target watermark. If the output clock is running slower, the buffer level rises. +3. **Fractional Step Adjustment**: The drift estimator calculates the exact clock phase error :math:`\Delta \mu` and updates the step increment: + + .. math:: + + \mu_{n+1} = (\mu_n + \Delta t) \pmod 1.0 + +4. **Zero-Crossing Compensation**: By continuously nudging :math:`\Delta t`, the controller maintains a stable, steady-state buffer watermark, preventing buffer starvation and buffer overflow indefinitely. + +.. graphviz:: + :caption: Closed-Loop Drift Estimation & Buffer Watermark Control in ASRC + + digraph asrc_feedback { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + s_in [label="Input Ring Buffer (Source)", fillcolor="#EDF2F7", color="#CBD5E0"]; + s_eng [label="ASRC Farrow Resampling Engine\n(asrc_farrow.c)", fillcolor="#BEE3F8", color="#3182CE"]; + s_out [label="Output Ring Buffer (Sink)", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_ctrl { + label="Closed-Loop ASRC Drift Controller"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + c_wm [label="Watermark Monitor\nRead sink_get_free_size(sink)\nTarget: 50% buffer capacity", fillcolor="#FAF089", color="#B7791F"]; + c_err [label="Error Discriminator & Filter\ne(t) = Current_Level - Target_Level\nLow-pass filter jitter & short spikes", fillcolor="#FAF089", color="#B7791F"]; + c_pll [label="Ratio Adjustment (update_drift)\nModulate fractional time step µ\nSmooth sub-ppm adjustment", fillcolor="#FAF089", color="#B7791F"]; + + c_wm -> c_err -> c_pll; + } + + s_in -> s_eng -> s_out; + s_out -> c_wm [label="Buffer Fill Level", style="dashed", color="#D69E2E"]; + c_pll -> s_eng [label="Continuous Step µ", style="bold", color="#E53E3E"]; + } + +--- + +.. _push_pull_modes: + +6. Push-Mode vs Pull-Mode Operational Topologies +************************************************ + +Because rate conversion alters the relationship between consumed and produced frames, an ASRC component cannot simultaneously satisfy fixed-size buffer constraints on both its input and output pins. + +To accommodate different audio streaming directions, the ASRC provides two operational modes: + +Push-Mode Operation (Playback / Transmit) +========================================= + +* **Operation**: The caller feeds a **fixed number of input frames** into the ASRC on each period (e.g. 48 frames). +* **Production**: Depending on the instantaneous clock drift and fractional ratio, the ASRC produces a **variable number of output frames** (e.g. 47, 48, or 49 frames). +* **Topology**: Used in playback pipelines where the DSP pushes audio toward an external digital audio interface (such as a Bluetooth controller or external DAC) that dictates the downstream clock. The output is coupled to a circular ring buffer to absorb production variance. + +Pull-Mode Operation (Capture / Receive) +======================================= + +* **Operation**: The downstream consumer requests a **fixed number of output frames** on each period. +* **Consumption**: The ASRC pulls a **variable number of input frames** from its input ring buffer to synthesize the requested output block. +* **Topology**: Used in capture pipelines where an external peripheral (e.g. a USB microphone or S/PDIF receiver) produces samples at its own hardware clock, and the DSP pulls audio into a synchronous processing graph. + +.. graphviz:: + :caption: Push-Mode vs Pull-Mode Execution Topologies across Audio Interfaces + + digraph push_pull { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_push { + label="Push-Mode Topology (Playback / Transmit Use Case)"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + p_src [label="Host / Decoder Pipeline\n(Supplies Fixed N Input Frames)", fillcolor="#FFFFFF", color="#CBD5E0"]; + p_asrc [label="ASRC Push Engine (process_push32)\nConsumes exactly N frames", fillcolor="#BEE3F8", color="#3182CE"]; + p_ring [label="Output Circular Ring Buffer\n(Absorbs Variable M Output Frames)", fillcolor="#FEFCBF", color="#D69E2E"]; + p_sink [label="External Peripheral / Bluetooth LC3\n(Independent Clock Domain)", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + p_src -> p_asrc -> p_ring -> p_sink; + } + + subgraph cluster_pull { + label="Pull-Mode Topology (Capture / Receive Use Case)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + l_src [label="External USB Mic / S/PDIF Receiver\n(Independent Clock Domain)", fillcolor="#FFFFFF", color="#CBD5E0"]; + l_ring [label="Input Circular Ring Buffer\n(Holds Variable N Incoming Frames)", fillcolor="#FEFCBF", color="#D69E2E"]; + l_asrc [label="ASRC Pull Engine (process_pull32)\nSynthesizes exactly M requested frames", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + l_sink [label="Downstream Processing / Host DMA\n(Requests Fixed M Output Frames)", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + l_src -> l_ring -> l_asrc -> l_sink; + } + } + +--- + +.. _simd_src_asrc: + +7. SIMD Vector Acceleration Across DSP Architectures +**************************************************** + +Both polyphase FIR filtering (in SRC) and polynomial Farrow evaluation (in ASRC) are computationally intensive, requiring dozens of multiply-accumulate operations per sample across multi-channel streams. + +SOF provides highly optimized vector assembly implementations tailored for Tensilica Xtensa DSP architectures: + +* **Cadence Tensilica Xtensa HiFi 3 (``src_hifi3.c``, ``asrc_farrow_hifi3.c``)**: + + - Utilizes 64-bit dual multiply-accumulate instructions (``AE_MULAA32RA``, ``AE_S32X2``). + - Processes two 32-bit channels or samples in parallel with hardware saturation. + +* **Cadence Tensilica Xtensa HiFi 4 (``src_hifi4.c``)**: + + - Employs 128-bit SIMD registers (``ae_int32x4``) executing four 32x32 multiplications per clock cycle. + - Leverages circular buffer address pointers (``AE_L32X4_XC``) to advance delay line pointers without scalar address math. + +* **Cadence Tensilica Xtensa HiFi 5 (``src_hifi5.c``, ``asrc_farrow_hifi5.c``)**: + + - 8-way vector processing engine executing eight 32-bit multiply-accumulate operations concurrently. + - Dual 128-bit memory load buses ensure that filter coefficients and audio delay lines are fetched with zero cache wait states. + +* **Generic Portable C Reference (``src_generic.c``, ``asrc_farrow_generic.c``)**: + + - Clean, portable scalar C implementations designed for non-Xtensa platforms (e.g. ARM Cortex-M7 on Teensy 4.1, RISC-V on ESP32-P4). + +.. graphviz:: + :caption: SIMD Vector Processing & Circular Delay Line Buffering across Hardware Architectures + + digraph simd_src { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_gen { + label="Generic Scalar C (src_generic.c / asrc_farrow_generic.c)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + g_core [label="Portable Scalar Execution\n1 sample per loop iteration\nTarget: ARM Cortex-M, RISC-V, Simulator", fillcolor="#FFFFFF", color="#CBD5E0"]; + } + + subgraph cluster_hf3 { + label="Xtensa HiFi 3 (src_hifi3.c / asrc_farrow_hifi3.c)"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + h3_core [label="Dual 32-bit Vector Engine\n2 samples processed per cycle\nDual 64-bit load/store instructions", fillcolor="#BEE3F8", color="#3182CE"]; + } + + subgraph cluster_hf4 { + label="Xtensa HiFi 4 (src_hifi4.c)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + h4_core [label="Quad 32-bit Vector Engine (128-bit)\n4 samples processed per instruction cycle\nVector circular delay addressing", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + } + + subgraph cluster_hf5 { + label="Xtensa HiFi 5 (src_hifi5.c / asrc_farrow_hifi5.c)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + h5_core [label="Octa 32-bit Vector Engine (256-bit bus)\n8 samples processed per cycle\nMaximum throughput for multi-channel audio", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + g_core -> h3_core [label="2x Speedup", color="#3182CE"]; + h3_core -> h4_core [label="2x Speedup (4x Total)", color="#B7791F"]; + h4_core -> h5_core [label="2x Speedup (8x Total)", color="#38A169", style="bold"]; + } + +--- + +.. _upstream_src_references: + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level implementation details, filter coefficient tables, and MATLAB tuning scripts: + +* **Upstream Component Specifications**: + - `thesofproject/sof: src/audio/src/README.md `_ + - `thesofproject/sof: src/audio/asrc/README.md `_ +* **Synchronous SRC Firmware Files**: + - ``src/audio/src/src_common.c``: Core polyphase staging and state machine. + - ``src/audio/src/src_common.h``: Stage descriptors (``struct src_stage``), parameter structs, and circular buffer pointers. + - ``src/audio/src/src_generic.c``: Portable scalar C polyphase filter. + - ``src/audio/src/src_hifi3.c``: Cadence Tensilica Xtensa HiFi 3 SIMD kernel. + - ``src/audio/src/src_hifi4.c``: Cadence Tensilica Xtensa HiFi 4 SIMD kernel. + - ``src/audio/src/src_hifi5.c``: Cadence Tensilica Xtensa HiFi 5 SIMD kernel. + - ``src/audio/src/src_ipc4.c``: IPC4 parameter handlers and configuration blobs. +* **Asynchronous ASRC Firmware Files**: + - ``src/audio/asrc/asrc.c``: ASRC module interface and lifecycle management. + - ``src/audio/asrc/asrc_farrow.c``: Farrow polynomial interpolation and push/pull processing. + - ``src/audio/asrc/asrc_farrow.h``: Farrow filter structures and buffer mode definitions. + - ``src/audio/asrc/asrc_farrow_hifi3.c``: HiFi 3 SIMD vector implementation. + - ``src/audio/asrc/asrc_farrow_hifi5.c``: HiFi 5 SIMD vector implementation. +* **Topology Definitions**: + - ``tools/topology/topology2/include/components/src.conf``: ALSA Topology 2 configuration class for SRC widgets. + - ``tools/topology/topology2/include/components/asrc.conf``: ALSA Topology 2 configuration class for ASRC widgets. +* **Filter Design & Coefficient Tuning**: + - :ref:`sample_rate_conversion`: Detailed MATLAB and GNU Octave script runbook for generating custom polyphase FIR filter tables. + +Related Subsystem Architecture Guides +===================================== + +* :ref:`volume_module`: Per-channel gain scaling, smooth ramping, and zero-crossing muting. +* :ref:`mixin_mixout`: Decoupled multi-stream mixing, fan-out/fan-in routing, and direct-to-sink accumulation. +* :ref:`module_framework`: The standardized module interface, Source/Sink APIs, and memory sandboxing that wraps SRC and ASRC components. +* :ref:`audio_buffer_management`: Ring buffer sizing, lockless single-producer single-consumer mechanics, and delay line memory allocation. +* :ref:`pipeline_architecture`: How sample rate converters are integrated into directed acyclic audio graphs (DAGs). diff --git a/developer_guides/index.rst b/developer_guides/index.rst index f8868a13..b6b787cc 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -43,7 +43,7 @@ Audio Processing Modules & Algorithms * :ref:`volume_module` (High-level architecture; also see upstream `volume README `_) * :ref:`mixin_mixout` (High-level architecture; also see upstream `mixin_mixout README `_ & `mixer README `_) -* `Sample Rate Converter (SRC) `_ & `ASRC `_ +* :ref:`src_asrc` (High-level architecture; also see upstream `SRC README `_ & `ASRC README `_) * `Parametric EQ (FIR) `_ & `EQ (IIR) `_ * `Dynamic Range Compressor (DRC) `_ & `Multiband DRC `_ * `Crossover `_ @@ -88,6 +88,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/fw_init_boot firmware/volume_module firmware/mixin_mixout + firmware/src_asrc rimage/index.rst firmware/llext_modules firmware/hostless_firmware From a0bbabafe1f6922878e3e35d8d25b32032f83a6f Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Fri, 18 Sep 2026 16:15:02 +0100 Subject: [PATCH 11/64] docs: developer_guides: add high-level eq fir and iir architecture guide Add a comprehensive, high-level developer architecture guide for Equalizers (EQ FIR & EQ IIR) in Sound Open Firmware. Covers: - Filter taxonomy and architectural trade-offs: feedforward transversal FIR (linear phase, constant group delay, unconditional stability) vs recursive cascaded biquad IIR (ultra-low latency, minimum phase, analog filter emulation). - Finite Impulse Response (FIR) architecture: discrete-time convolution, circular delay line buffer management, and linear phase symmetric tap folding optimization (reducing multiplications by 50%). - Infinite Impulse Response (IIR) architecture: sensitivity of high-order monolithic polynomials, cascaded Second-Order Sections (SOS / biquads), and Direct Form I (DF1) with 64-bit accumulators preventing limit cycles and internal overflow on low-frequency poles. - Parametric equalizer topologies based on the Audio EQ Cookbook: peaking bells, shelving filters, high-pass/low-pass roll-offs, surgical notch filters, and flat passthrough sections. - Real-time dynamic parameter updates via IPC configuration blobs, fragment reassembly with comp_data_blob_handler, safety validation, and atomic pointer swapping for glitchless filter updates. - Multi-channel processing with per-channel independent response mapping (assign_response[]) and ALSA Topology 2 component integration. - SIMD vector acceleration across Cadence Tensilica Xtensa HiFi 3, HiFi 4, HiFi 5, and generic portable scalar C fallbacks. - 7 native vector Graphviz SVG diagrams. - Cross-references in developer_guides/index.rst, pipeline_architecture.rst, and algorithms/eq/equalizers_tuning.rst. Signed-off-by: Liam Girdwood --- .../algorithms/eq/equalizers_tuning.rst | 7 + developer_guides/firmware/eq_fir_iir.rst | 573 ++++++++++++++++++ .../firmware/pipeline_architecture.rst | 1 + developer_guides/index.rst | 3 +- 4 files changed, 583 insertions(+), 1 deletion(-) create mode 100644 developer_guides/firmware/eq_fir_iir.rst diff --git a/developer_guides/algorithms/eq/equalizers_tuning.rst b/developer_guides/algorithms/eq/equalizers_tuning.rst index 45335c60..9b237ffd 100644 --- a/developer_guides/algorithms/eq/equalizers_tuning.rst +++ b/developer_guides/algorithms/eq/equalizers_tuning.rst @@ -3,6 +3,13 @@ Equalizers, IIR and FIR ####################### +.. seealso:: + + For a high-level firmware architectural overview of both Finite Impulse Response (FIR) + and Infinite Impulse Response (IIR) equalizers—including transversal filter structures, + Direct Form I biquad cascades, parametric filter topologies, dynamic IPC blob swapping, + and SIMD acceleration—see :ref:`eq_fir_iir`. + .. contents:: :depth: 3 diff --git a/developer_guides/firmware/eq_fir_iir.rst b/developer_guides/firmware/eq_fir_iir.rst new file mode 100644 index 00000000..a2369ca0 --- /dev/null +++ b/developer_guides/firmware/eq_fir_iir.rst @@ -0,0 +1,573 @@ +.. _eq_fir_iir: + +Equalizer Architecture (EQ FIR & EQ IIR) +######################################## + +The **Equalization** subsystem in Sound Open Firmware provides real-time frequency response shaping, acoustic correction, and dynamic tone control across heterogeneous audio pipelines and physical transducers. + +In modern audio systems, physical transducers—such as laptop micro-speakers, smartphone earpieces, and digital microphone arrays—inevitably suffer from non-ideal acoustical characteristics: mechanical cavity resonances, enclosure-induced high-frequency roll-off, and limited low-frequency bass extension. Furthermore, room acoustics, listener preferences, and voice intelligibility algorithms require precise, low-latency spectral filtering. + +SOF addresses these challenges through two specialized, complementary equalizer components: + +1. **Finite Impulse Response (FIR) Equalizer** (``eq_fir``, ``src/audio/eq_fir/``): Feedforward transversal filter engine providing exact linear-phase response, constant group delay, and arbitrary magnitude shaping without phase distortion. +2. **Infinite Impulse Response (IIR) Equalizer** (``eq_iir``, ``src/audio/eq_iir/``): Recursive feedback filter engine implementing cascaded second-order sections (biquads) for ultra-low algorithmic latency, minimal memory footprint, and classic parametric tone shaping (peaking bells, shelves, and passbands). + +This guide provides a comprehensive, high-level architectural walkthrough of the EQ FIR and EQ IIR subsystems, filter topologies, biquad cascades, dynamic parameter updates, and SIMD hardware acceleration without delving into low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +.. _eq_taxonomy: + +1. Equalization in Audio Systems & Filter Taxonomy +************************************************** + +Audio equalization modifies the balance of frequency components within an audio signal. In SOF, equalization is applied across several core audio use cases: + +* **Speaker Frequency Response Correction**: Flattening peaky mechanical resonances and boosting attenuated frequency bands to produce natural, transparent sound reproduction within mass-market industrial designs. +* **Microphone Acoustic Flattening**: Correcting frequency deviations across MEMS digital and analog microphone capsules prior to Acoustic Echo Cancellation (AEC) and directional beamforming. +* **Parametric User Tone Controls**: Implementing interactive user-facing equalizers (e.g. 10-band graphic equalizers, bass boost, speech enhancement, and treble tone controls). +* **Driver Protection & Rumble Filtering**: Rolling off sub-audible frequencies below speaker excursion limits to prevent mechanical damage and voice coil burnout. + +FIR vs IIR Architectural Taxonomy +================================= + +The choice between FIR and IIR equalizers involves architectural trade-offs between phase linearity, algorithmic latency, computational complexity, and memory utilization: + +* **Finite Impulse Response (FIR) Equalizer**: + + - **Structure**: Feedforward transversal delay line with no feedback paths. The impulse response settles to exactly zero after :math:`L` samples. + - **Phase Response**: Exact linear phase with constant group delay :math:`\tau = (L - 1) / 2` samples across all frequencies. Preserves transient waveforms without phase dispersion. + - **Stability**: Unconditionally stable. All transfer function poles reside at the origin (:math:`z = 0`). + - **Resource Cost**: Higher computational load (requires :math:`L` multiply-accumulate operations per sample) and larger delay line memory. Algorithmic latency is proportional to filter length. + +* **Infinite Impulse Response (IIR) Equalizer**: + + - **Structure**: Recursive feedback network where the current output depends on both past inputs and past outputs. The impulse response decays asymptotically over time. + - **Phase Response**: Minimum-phase response with frequency-dependent group delay, mimicking analog RC/RLC active filter circuits. + - **Latency**: Ultra-low algorithmic latency (typically a fraction of a sample), making it ideal for interactive communications and gaming. + - **Resource Cost**: Exceptionally efficient (only 5 coefficients and 4 state variables per second-order biquad). However, poles must be carefully bounded within the unit circle to guarantee stability. + +.. graphviz:: + :caption: Architectural Taxonomy: Feedforward Transversal FIR vs Recursive Cascaded Biquad IIR + + digraph eq_taxonomy { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_fir { + label="FIR Equalizer (Feedforward Transversal Architecture)"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + fir_in [label="Input Sample x[n]", fillcolor="#FFFFFF", color="#CBD5E0"]; + fir_delay [label="Tapped Delay Line\nx[n-1], x[n-2], ..., x[n-L+1]\nUnconditionally Stable (Poles at Origin)", fillcolor="#BEE3F8", color="#3182CE"]; + fir_mac [label="Tap Multipliers (h[0] .. h[L-1])\nSymmetric Tap Pre-Addition\nConstant Group Delay (Linear Phase)", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + fir_out [label="Output Sample y[n]\nPreserved Transient Waveforms", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + fir_in -> fir_delay -> fir_mac -> fir_out [color="#3182CE"]; + } + + subgraph cluster_iir { + label="IIR Equalizer (Recursive Biquad Cascade Architecture)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + iir_in [label="Input Sample x[n]", fillcolor="#FFFFFF", color="#D69E2E"]; + iir_biquad [label="Cascaded Biquad Stages (SOS)\nFeedforward Zeros (b0, b1, b2)\nFeedback Poles (a1, a2)\nPoles bounded inside Unit Circle (|z| < 1)", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + iir_out [label="Output Sample y[n]\nUltra-Low Algorithmic Latency\nAnalog Emulation (Minimum Phase)", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + iir_in -> iir_biquad -> iir_out [color="#B7791F"]; + } + } + +--- + +.. _fir_architecture: + +2. Finite Impulse Response (FIR) Equalizer Architecture +******************************************************* + +The FIR Equalizer component (``src/audio/eq_fir/eq_fir.c``) applies digital filtering by computing discrete-time convolution between the incoming audio stream and a pre-designed impulse response vector :math:`h[k]`: + +.. math:: + + y[n] = \sum_{k=0}^{L-1} h[k] \cdot x[n-k] + +where :math:`L` represents the filter length (number of taps). + +Circular Delay Line Management +============================== + +To compute convolution across successive audio frames without copying memory blocks, SOF maintains a circular delay line for each audio channel: + +1. **Circular Addressing**: Historical input samples are stored in a contiguous RAM buffer. When new samples arrive, they overwrite the oldest samples using circular pointer indexing. +2. **Multi-Channel Separation**: Each audio channel maintains its own independent delay line buffer, sized according to the longest configured filter across the system. +3. **Zero Overhead**: Circular buffer pointer arithmetic avoids memory shift operations (``memmove``), keeping memory bus activity strictly proportional to audio frame sizes. + +Linear Phase Symmetry Optimization +================================== + +Most acoustic equalization curves require linear phase to prevent phase smearing across stereo and surround sound fields. A filter has linear phase if and only if its impulse response exhibits even symmetry (:math:`h[k] = h[L-1-k]`) or odd anti-symmetry (:math:`h[k] = -h[L-1-k]`). + +SOF exploits this mathematical property through **symmetric tap folding**: + +.. math:: + + y[n] = h\left[\frac{L-1}{2}\right] \cdot x\left[n - \frac{L-1}{2}\right] + \sum_{k=0}^{\frac{L-3}{2}} h[k] \cdot \Big( x[n-k] + x[n - L + 1 + k] \Big) + +By pre-adding the symmetric past and current input samples before multiplying by the shared coefficient :math:`h[k]`, the total number of multiplication operations is reduced by **50%** (from :math:`L` down to :math:`L/2` multiplications per sample). + +.. graphviz:: + :caption: FIR Transversal Tap Delay Line with Linear Phase Symmetric Tap Folding Optimization + + digraph fir_structure { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_x [label="Input Audio x[n]\n(Current Sample)", fillcolor="#EDF2F7", color="#CBD5E0"]; + d_line [label="Circular Delay Line Buffer\nx[n], x[n-1], x[n-2], ... x[n-L+1]", fillcolor="#BEE3F8", color="#3182CE"]; + + subgraph cluster_fold { + label="Symmetric Tap Folding Engine (50% Multiplication Reduction)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + pre_add [label="Pairwise Pre-Adders:\n(x[n-k] + x[n-L+1+k])", fillcolor="#FAF089", color="#B7791F"]; + mult [label="Coefficient Multipliers:\nh[k] * (Sum)\nQ1.15 Fixed-Point Coeffs", fillcolor="#FAF089", color="#B7791F"]; + acc [label="64-Bit Accumulator\nSum across all folded taps", fillcolor="#FAF089", color="#B7791F"]; + + pre_add -> mult -> acc; + } + + post_sh [label="Output Scaler & Shift\nApply out_shift & Saturation", fillcolor="#E2E8F0", color="#A0AEC0"]; + out_y [label="Equalized Audio y[n]\nLinear Phase (Zero Distortion)", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + in_x -> d_line; + d_line -> pre_add; + acc -> post_sh -> out_y; + } + +--- + +.. _iir_architecture: + +3. Infinite Impulse Response (IIR) Equalizer Architecture & Biquad Cascades +*************************************************************************** + +The IIR Equalizer component (``src/audio/eq_iir/eq_iir.c``) implements frequency shaping using recursive difference equations. The general transfer function of an :math:`N`-th order IIR filter is a ratio of polynomials: + +.. math:: + + H(z) = \frac{\sum_{k=0}^{N} b_k z^{-k}}{1 + \sum_{k=1}^{N} a_k z^{-k}} + +The Sensitivity Hazard of High-Order Monolithic Filters +======================================================= + +Directly implementing a high-order polynomial filter (e.g. 10th or 20th order) in digital signal processing hardware is notoriously dangerous: + +* The roots of high-order polynomials are hypersensitive to small perturbations in filter coefficients caused by fixed-point quantization. +* Tiny round-off errors can push poles outside the complex unit circle (:math:`|z| \ge 1.0`), causing catastrophic instability, oscillation, and rail-to-rail digital clipping. + +Cascaded Second-Order Sections (SOS / Biquads) +============================================== + +To ensure absolute numerical stability, SOF factors all high-order IIR filters into a cascade of independent **Second-Order Sections (SOS)**, commonly known as **Biquads**: + +.. math:: + + H(z) = \prod_{k=1}^{K} H_k(z) = \prod_{k=1}^{K} \frac{b_{0,k} + b_{1,k} z^{-1} + b_{2,k} z^{-2}}{1 + a_{1,k} z^{-1} + a_{2,k} z^{-2}} + +Each biquad section isolates a single conjugate pair of poles and zeros: + +* **Poles within Unit Circle**: Stability is verified algebraically for each biquad individually by checking that :math:`|a_{2,k}| < 1` and :math:`|a_{1,k}| < 1 + a_{2,k}`. +* **Octave Band Coverage**: SOF supports cascading up to 11 biquads in series (a 22nd-order filter), sufficient to cover all 11 octave bands across the 20 Hz – 20 kHz audio spectrum. + +Direct Form I (DF1) Implementation Mechanics +============================================ + +SOF implements biquads using **Direct Form I (DF1)** with 64-bit accumulators: + +1. **Independent State Variables**: Direct Form I maintains separate delay histories for input samples (:math:`x[n-1], x[n-2]`) and output samples (:math:`y[n-1], y[n-2]`). +2. **64-Bit Internal Accumulation**: All five product terms (:math:`b_0 x[n] + b_1 x[n-1] + b_2 x[n-2] - a_1 y[n-1] - a_2 y[n-2]`) accumulate into a high-precision 64-bit accumulator with guard bits before rounding and shifting. +3. **Limit Cycle Immunity**: In low-frequency narrow-band equalization (such as deep bass boosts at 40 Hz), poles lie extremely close to :math:`z = 1.0`. Direct Form II structures can suffer from internal node overflow and limit cycle oscillations. Direct Form I with 64-bit accumulation completely avoids internal node overflow. + +.. graphviz:: + :caption: Cascaded Direct Form I (DF1) Second-Order Section (Biquad) Processing Chain with 64-bit Accumulator + + digraph iir_biquad_cascade { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_pcm [label="Input Audio x[n]\n(From Pipeline Buffer)", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_bq0 { + label="Biquad Stage 0 (e.g. Bass Shelf / Low Cut)"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + df1_x0 [label="Input State\nx0[n-1], x0[n-2]", fillcolor="#FFFFFF", color="#BEE3F8"]; + df1_c0 [label="Feedforward (b0, b1, b2)\nFeedback (-a1, -a2)\n64-Bit Accumulator", fillcolor="#BEE3F8", color="#3182CE"]; + df1_y0 [label="Output State\ny0[n-1], y0[n-2]", fillcolor="#FFFFFF", color="#BEE3F8"]; + + df1_x0 -> df1_c0 -> df1_y0; + } + + subgraph cluster_bq1 { + label="Biquad Stage 1 (e.g. Parametric Peaking Bell)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + df1_c1 [label="Biquad 1 DF1 Engine\nIndependent Poles/Zeros\n64-Bit Accumulator", fillcolor="#FAF089", color="#B7791F"]; + } + + subgraph cluster_bqk { + label="Biquad Stage K-1 (e.g. Treble Shelf)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + df1_ck [label="Biquad K-1 DF1 Engine\nFinal Shaping Section\nHeadroom Scaler", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + out_pcm [label="Equalized Audio y[n]\n(Ultra-Low Latency)", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + in_pcm -> df1_x0; + df1_y0 -> df1_c1 [label="Intermediate SOS"]; + df1_c1 -> df1_ck [label="Cascaded SOS", style="dashed"]; + df1_ck -> out_pcm; + } + +--- + +.. _parametric_eq_topologies: + +4. Parametric Equalizer Topologies & Biquad Filter Types +******************************************************** + +By configuring the five coefficients (:math:`b_0, b_1, b_2, a_1, a_2`) of each biquad section, the SOF IIR equalizer implements all classic parametric filter types defined in the Audio EQ Cookbook: + +* **Peaking / Bell Filter**: + + - Provides selective boost or attenuation centered around a target frequency :math:`f_0`. + - Configured via Center Frequency (:math:`f_0`), Quality Factor (:math:`Q` or bandwidth in octaves), and Gain (:math:`G` in dB). + - Primary tool for eliminating sharp speaker resonance peaks and acoustic cavity dips. + +* **Low-Shelf & High-Shelf Filters**: + + - Boosts or attenuates all frequencies below (low-shelf) or above (high-shelf) a transition corner frequency with a smooth plateau response. + - Used for classic bass and treble tone controls. + +* **High-Pass Filter (HPF) & Low-Pass Filter (LPF)**: + + - 12 dB/octave attenuation slope per biquad (cascaded to form 24 dB/oct or 48 dB/oct Butterworth, Linkwitz-Riley, or Chebyshev filters). + - HPF blocks sub-audible DC offsets and speaker rumble; LPF blocks ultrasonic noise above the audible band. + +* **Band-Pass (BPF) & Notch (Band-Stop) Filters**: + + - BPF isolates a specific frequency band for feature detection or wake-word preprocessing. + - Notch filters provide deep, surgical attenuation (e.g. -40 dB) at a specific frequency to eliminate electrical mains hum (50 Hz / 60 Hz) or microphone feedback howling. + +* **Flat / Neutral Biquad**: + + - Configured with :math:`b_0 = 1.0, \text{gain} = 1.0` and all other coefficients zero. + - Acts as a zero-overhead passthrough section for unused biquad slots in a generic configuration. + +.. graphviz:: + :caption: Parametric EQ Biquad Filter Types and Characteristic Frequency Response Curves + + digraph parametric_types { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_types { + label="Parametric Biquad Library (Audio EQ Cookbook Topologies)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + t_bell [label="Peaking / Bell Filter\nBoost/Cut around Center Frequency f0\nAdjustable Q (Bandwidth) & Gain (dB)", fillcolor="#BEE3F8", color="#3182CE"]; + t_shelf [label="Low / High Shelving Filters\nSmooth plateau boost/attenuation\nBass & Treble User Tone Controls", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + t_pass [label="High-Pass (HPF) & Low-Pass (LPF)\n12 dB / 24 dB / 48 dB per octave slopes\nRumble filtering & Tweeter protection", fillcolor="#FED7D7", color="#E53E3E", fontcolor="#742A2A"]; + t_notch [label="Band-Stop / Notch Filter\nSurgical high-Q narrow attenuation\n50/60 Hz Mains Hum & Howl Suppression", fillcolor="#E9D8FD", color="#805AD5", fontcolor="#44337A"]; + t_flat [label="Flat / Passthrough Section\nb0 = 1.0, Gain = 1.0 (Neutral)\nUnused cascade slots bypass", fillcolor="#FFFFFF", color="#CBD5E0"]; + + t_pass -> t_shelf -> t_bell -> t_notch -> t_flat [style="invis"]; + } + } + +--- + +.. _dynamic_updates: + +5. Dynamic Parameter Updates & Configuration Blobs +************************************************** + +Equalizers must adapt dynamically to user actions (e.g. moving a graphic equalizer slider in an audio control panel) and environmental context (e.g. switching between built-in laptop speakers and an external dock). + +The Component Blob Handler Framework +==================================== + +SOF delivers equalizer parameters from the Linux host driver using **Component Configuration Blobs** managed by the ``comp_data_blob_handler`` infrastructure: + +1. **IPC Delivery**: The host sends serialized configuration blobs via IPC3 (``SOF_IPC_COMP_SET_DATA``) or IPC4 (``SET_LARGE_CONFIG`` with dedicated component UUID). +2. **Fragmented Assembly**: If a filter configuration exceeds the maximum single IPC mailbox window, the blob handler transparently reassembles incoming multi-part packet fragments. +3. **Pre-Validation Hook**: Before applying any changes to the running audio stream, the blob handler invokes the component's validator callback (``eq_fir_init_coef()`` with ``fir == NULL`` or ``eq_iir_validate_config()``). + +Atomic Swapping & Glitchless Transitions +======================================== + +Applying an unvalidated or corrupt filter configuration can crash the DSP or generate destructive acoustic pops. SOF enforces strict atomic updating: + +* **Validation Bounds**: The validator checks payload byte length, verifies channel count matches active stream configuration, and ensures filter taps or biquad counts do not exceed hardware limits. +* **Delay Line Reallocation**: If the new configuration requires more taps or biquads than currently allocated, new RAM buffers are allocated before releasing the previous ones. +* **Atomic State Pointer Swap**: The running audio thread continues executing using the existing filter configuration until the new configuration is fully prepared in memory. Once ready, active state pointers are swapped atomically between audio periods. +* **Glitchless Crossfading**: Filter states prevent DC discontinuities and audible pops during runtime adjustment. + +.. graphviz:: + :caption: Dynamic IPC Configuration Blob Handling, Safe Validation, and Active Coefficient Swapping + + digraph dynamic_config_flow { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + host_ipc [label="Host ALSA / PipeWire User Interface\nSends EQ Profile via IPC Blob", fillcolor="#EDF2F7", color="#CBD5E0"]; + blob_mgr [label="Component Blob Handler (comp_data_blob_handler)\nFragment Reassembly & Staging", fillcolor="#BEE3F8", color="#3182CE"]; + + subgraph cluster_val { + label="Pre-Validation & Safety Checks"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + v_check [label="Validation Callback (eq_validate)\nCheck Payload Sizing & Header Magic\nVerify Channel Bounds & Stability Limits", fillcolor="#FAF089", color="#B7791F"]; + v_alloc [label="Shadow Allocation\nAllocate new delay lines in DSP RAM\nPre-compute Q2.30 / Q1.15 coefficient tables", fillcolor="#FAF089", color="#B7791F"]; + + v_check -> v_alloc [label="Valid"]; + } + + subgraph cluster_exec { + label="Active Audio Processing Loop"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + swap_ptr [label="Atomic State Swap\nSwap active coefficient & delay pointers\nZero pipeline interruption", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + run_eng [label="Active Filtering Engine\nExecutes with updated EQ curve\nGlitchless acoustic transition", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + swap_ptr -> run_eng; + } + + host_ipc -> blob_mgr -> v_check; + v_alloc -> swap_ptr [label="Atomic Swap Trigger", color="#38A169", style="bold"]; + } + +--- + +.. _multichannel_topology: + +6. Multi-Channel Processing & ALSA Topology Integration +******************************************************* + +Real-world consumer hardware rarely features acoustically identical speaker channels. In thin laptops, the left speaker is often constrained by the internal battery while the right speaker sits next to a thermal exhaust vent, causing significant differences in frequency response. + +Independent Channel Response Assignment +======================================= + +SOF equalizers solve this via **Channel Response Mapping**: + +* **Response Definition Pool**: A single configuration blob can define multiple distinct filter responses (up to 8 independent FIR or IIR responses). +* **Channel Assignment Vector (``assign_response[]``)**: A mapping array assigns which response curve applies to each audio channel: + + .. code-block:: text + + assign_response = [0, 1] # Left channel -> Curve 0, Right channel -> Curve 1 + +* **Selective Passthrough**: Channels assigned an index of ``-1`` bypass the filter engine entirely, passing unmodified audio through high-speed memory copies (``audio_stream_copy()``). + +ALSA Topology 2 Integration +=========================== + +Equalizer modules are declared in ALSA Topology 2 files using the ``eqfir.conf`` and ``eqiir.conf`` component classes: + +* **Effect Widget**: Instantiated with widget type ``effect`` and dedicated component UUIDs: + + - **FIR Equalizer UUID**: ``e7:0c:a9:43:a5:f3:df:41:ac:06:ba:98:65:1a:e6:a3`` + - **IIR Equalizer UUID**: ``e6:c0:50:51:f9:27:c8:4e:83:51:c7:05:b6:42:d1:2f`` + +* **Static ROM Initialization**: Default speaker and microphone tuning blobs can be embedded directly into compiled topology binaries (``.bin``), ensuring optimal audio quality immediately upon system boot before userspace drivers initialize. + +.. graphviz:: + :caption: Multi-Channel Response Assignment and ALSA Topology 2 Widget Integration + + digraph multichannel_topology { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_stream [label="Multi-Channel Audio\n(e.g. Stereo Stream)", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_map { + label="Channel Response Assignment (assign_response[])"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + ch0 [label="Channel 0 (Left)\nAssign: Response 0\n(Left Speaker Profile)", fillcolor="#FAF089", color="#B7791F"]; + ch1 [label="Channel 1 (Right)\nAssign: Response 1\n(Right Speaker Profile)", fillcolor="#FAF089", color="#B7791F"]; + } + + subgraph cluster_filters { + label="Filter Engine Instances"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + f0 [label="FIR / IIR Response 0\nTuned for Left Cavity", fillcolor="#BEE3F8", color="#3182CE"]; + f1 [label="FIR / IIR Response 1\nTuned for Right Cavity", fillcolor="#BEE3F8", color="#3182CE"]; + } + + out_stream [label="Equalized Stereo Audio\nBalanced Acoustic Output", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + in_stream -> ch0; + in_stream -> ch1; + ch0 -> f0; + ch1 -> f1; + f0 -> out_stream; + f1 -> out_stream; + } + +--- + +.. _simd_eq_acceleration: + +7. SIMD Vector Acceleration Across DSP Architectures +**************************************************** + +Multi-channel equalization with dense FIR tap lines (e.g. 128 taps across 4 channels = 512 multiply-accumulates per frame) or cascaded IIR biquads (11 biquads = 55 MACs per frame per channel) demands substantial processor throughput. + +SOF provides optimized vector assembly kernels across target DSP architectures: + +* **Cadence Tensilica Xtensa HiFi 3 (``fir_hifi3.c``, ``iir_df1_hifi3.c``)**: + + - Utilizes 64-bit dual multiply-accumulate instructions (``AE_MULAA32RA``, ``AE_S32X2``). + - Processes two 32-bit audio samples concurrently with hardware saturation. + +* **Cadence Tensilica Xtensa HiFi 4 (``iir_df1_hifi4.c``)**: + + - Employs 128-bit SIMD registers executing four 32x32 multiplications per cycle. + - Leverages vector circular pointer instructions (``AE_L32X4_XC``) to advance delay line indices with zero scalar addressing overhead. + +* **Cadence Tensilica Xtensa HiFi 5 (``fir_hifi5.c``, ``iir_df1_hifi5.c``)**: + + - Octa 32-bit vector processing engine (256-bit data bus) executing eight 32-bit multiply-accumulate operations in parallel. + - Dual memory load buses allow simultaneously fetching filter coefficients and audio delay buffers in a single clock cycle. + +* **Generic Portable C Reference (``fir_generic.c``, ``iir_df1_generic.c``)**: + + - Clean, portable scalar C implementations designed for non-Xtensa platforms (e.g. ARM Cortex-M7 on Teensy 4.1, RISC-V on ESP32-P4). + +.. graphviz:: + :caption: SIMD Vector Processing and Circular Delay Line Buffering across DSP Architectures + + digraph simd_eq { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_gen { + label="Generic Scalar C (fir_generic.c / iir_df1_generic.c)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + g_core [label="Portable Scalar C Loop\n1 sample per iteration\nTarget: ARM Cortex-M, RISC-V, Simulator", fillcolor="#FFFFFF", color="#CBD5E0"]; + } + + subgraph cluster_hf3 { + label="Xtensa HiFi 3 (fir_hifi3.c / iir_df1_hifi3.c)"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + h3_core [label="Dual 32-bit Vector Engine\n2 samples processed per cycle\n64-bit dual MAC instructions", fillcolor="#BEE3F8", color="#3182CE"]; + } + + subgraph cluster_hf4 { + label="Xtensa HiFi 4 (iir_df1_hifi4.c)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + h4_core [label="Quad 32-bit Vector Engine (128-bit)\n4 samples processed per instruction cycle\nCircular delay line auto-wrapping", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + } + + subgraph cluster_hf5 { + label="Xtensa HiFi 5 (fir_hifi5.c / iir_df1_hifi5.c)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + h5_core [label="Octa 32-bit Vector Engine (256-bit bus)\n8 samples processed per cycle\nDual 128-bit memory buses for coefficients & delay line", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + g_core -> h3_core [label="2x Speedup", color="#3182CE"]; + h3_core -> h4_core [label="2x Speedup (4x Total)", color="#B7791F"]; + h4_core -> h5_core [label="2x Speedup (8x Total)", color="#38A169", style="bold"]; + } + +--- + +.. _upstream_eq_references: + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level implementation details, filter coefficient structures, and acoustic tuning scripts: + +* **Upstream Component Specifications**: + - `thesofproject/sof: src/audio/eq_fir/README.md `_ + - `thesofproject/sof: src/audio/eq_iir/README.md `_ +* **FIR Equalizer Source Files**: + - ``src/audio/eq_fir/eq_fir.c``: Component initialization, channel assignment, and buffer copying. + - ``src/audio/eq_fir/eq_fir.h``: FIR private data structures (``struct comp_data``) and format function pointers. + - ``src/include/user/fir.h``: FIR user configuration structures (``struct sof_fir_coef_data``). + - ``src/math/fir_generic.c``: Portable scalar C FIR convolution kernel. + - ``src/math/fir_hifi3.c``: Tensilica Xtensa HiFi 3 SIMD vector kernel. + - ``src/math/fir_hifi5.c``: Tensilica Xtensa HiFi 5 octa-vector kernel. +* **IIR Equalizer Source Files**: + - ``src/audio/eq_iir/eq_iir.c``: Component lifecycle, blob validation, and processing dispatch. + - ``src/audio/eq_iir/eq_iir.h``: IIR private structures and biquad state headers. + - ``src/include/user/eq.h``: IIR biquad structures (``struct sof_eq_iir_biquad``) and configuration headers (``struct sof_eq_iir_config``). + - ``src/math/iir_df1_generic.c``: Portable scalar C Direct Form I biquad cascade. + - ``src/math/iir_df1_hifi3.c``: Tensilica Xtensa HiFi 3 SIMD biquad kernel. + - ``src/math/iir_df1_hifi4.c``: Tensilica Xtensa HiFi 4 SIMD biquad kernel. + - ``src/math/iir_df1_hifi5.c``: Tensilica Xtensa HiFi 5 SIMD biquad kernel. +* **Topology Definitions**: + - ``tools/topology/topology2/include/components/eqfir.conf``: ALSA Topology 2 configuration class for FIR widgets. + - ``tools/topology/topology2/include/components/eqiir.conf``: ALSA Topology 2 configuration class for IIR widgets. +* **Acoustic Measurement & Filter Tuning**: + - :ref:`equalizers_tuning`: Comprehensive acoustic measurement runbook for tuning speaker equalizers using calibrated reference microphones, sine sweeps, and Octave/MATLAB scripts. + +Related Subsystem Architecture Guides +===================================== + +* :ref:`volume_module`: Per-channel gain scaling, smooth ramping, zero-crossing muting, and volume controls preceding/following equalizers. +* :ref:`src_asrc`: Sample rate conversion architecture handling fixed and drifting clocks across heterogeneous pipelines. +* :ref:`mixin_mixout`: Multi-stream audio mixing and distribution across post-equalizer loudspeaker and headphone buses. +* :ref:`module_framework`: The standardized module interface, Source/Sink APIs, and memory sandboxing wrapping FIR and IIR equalizers. +* :ref:`audio_buffer_management`: Lockless circular ring buffers, multi-tier DSP memory, and cache coherency. +* :ref:`pipeline_architecture`: How equalizer modules are integrated into directed acyclic audio graphs (DAGs). diff --git a/developer_guides/firmware/pipeline_architecture.rst b/developer_guides/firmware/pipeline_architecture.rst index 3cfd8a6f..340dfad1 100644 --- a/developer_guides/firmware/pipeline_architecture.rst +++ b/developer_guides/firmware/pipeline_architecture.rst @@ -464,6 +464,7 @@ Related Guides * :ref:`volume_module`: Per-channel gain scaling, smooth ramping, zero-crossing muting, and SIMD acceleration. * :ref:`mixin_mixout`: Multi-pipeline audio mixing, stream splitting, dynamic clock domain decoupling, and matrix routing. * :ref:`src_asrc`: Synchronous polyphase conversion, asynchronous Farrow drift tracking, push/pull topologies, and SIMD acceleration. +* :ref:`eq_fir_iir`: Finite and Infinite Impulse Response equalizers, linear-phase FIR tap folding, Direct Form I biquad cascades, and dynamic IPC blob updates. * :ref:`fw_init_boot`: Boot flow, hardware mailbox FW Ready handshake, and Zephyr initialization. * :ref:`topology2`: How pipelines and widgets are declared using ALSA Topology 2.0 configuration classes. * :ref:`sof_hostless_firmware`: How to create static pipelines compiled into ROM for standalone microcontrollers. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index b6b787cc..2d869908 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -44,7 +44,7 @@ Audio Processing Modules & Algorithms * :ref:`volume_module` (High-level architecture; also see upstream `volume README `_) * :ref:`mixin_mixout` (High-level architecture; also see upstream `mixin_mixout README `_ & `mixer README `_) * :ref:`src_asrc` (High-level architecture; also see upstream `SRC README `_ & `ASRC README `_) -* `Parametric EQ (FIR) `_ & `EQ (IIR) `_ +* :ref:`eq_fir_iir` (High-level architecture; also see upstream `FIR README `_ & `IIR README `_) * `Dynamic Range Compressor (DRC) `_ & `Multiband DRC `_ * `Crossover `_ * `DC Blocker `_ @@ -89,6 +89,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/volume_module firmware/mixin_mixout firmware/src_asrc + firmware/eq_fir_iir rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 82ff0021ab75b1655f3749db3b07b4c0c9cec4e4 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Fri, 18 Sep 2026 16:38:26 +0100 Subject: [PATCH 12/64] docs: developer_guides: add high-level drc and multiband drc architecture guide Add a comprehensive, high-level developer architecture guide for Dynamic Range Compression (DRC & Multi-Band DRC) in Sound Open Firmware. Covers: - Dynamic range compression principles and use cases: speaker excursion protection, dialogue intelligibility, and microphone capture dynamics. - Static transfer characteristic: threshold, quadratic soft knee, compression ratio, and makeup gain. - Single-band DRC processing architecture: signal path vs sidechain detector path, lookahead pre-delay circular buffers (up to 512 frames), and division-based sub-block processing (32 frames). - Envelope ballistics and adaptive multi-segment release curve: fast attack transient capture vs adaptive non-linear recovery (kA through kE) preventing harmonic distortion and audible pumping/breathing. - The spectral pumping hazard and the multi-band DRC paradigm: frequency partitioning to eliminate wideband ducking caused by heavy bass energy. - Multi-band compound 4-stage processing pipeline: Emphasis EQ (2-biquad IIR), Linkwitz-Riley 4th-order (LR4) crossover bank (flat 0 dB sum, zero phase error), parallel independent DRC band engines, and De-emphasis summation filter. - Dynamic parameter updates via IPC configuration blobs, multi-packet staging, and ALSA Topology 2 component widgets (drc.conf, multiband_drc.conf). - SIMD vector acceleration across Tensilica Xtensa HiFi 3, HiFi 4, HiFi 5, and generic portable scalar C fallbacks. - 7 native vector Graphviz SVG diagrams. - Cross-references in developer_guides/index.rst and pipeline_architecture.rst. Signed-off-by: Liam Girdwood --- .../firmware/drc_multiband_drc.rst | 575 ++++++++++++++++++ .../firmware/pipeline_architecture.rst | 1 + developer_guides/index.rst | 3 +- 3 files changed, 578 insertions(+), 1 deletion(-) create mode 100644 developer_guides/firmware/drc_multiband_drc.rst diff --git a/developer_guides/firmware/drc_multiband_drc.rst b/developer_guides/firmware/drc_multiband_drc.rst new file mode 100644 index 00000000..241e5fc4 --- /dev/null +++ b/developer_guides/firmware/drc_multiband_drc.rst @@ -0,0 +1,575 @@ +.. _drc_multiband_drc: + +Dynamic Range Compression Architecture (DRC & Multi-Band DRC) +############################################################# + +The **Dynamic Range Compression** subsystem in Sound Open Firmware provides real-time acoustic loudness management, speaker excursion protection, dialogue intelligibility enhancement, and audio leveling across heterogeneous playback and capture streams. + +Audio signals in real-world environments present extreme dynamic variations: whisper-quiet dialogue alternating with deafening explosions in movie soundtracks, wide acoustic swings in digital microphone voice capture, and high-energy bass peaks that overdrive compact micro-speaker diaphragms. Without dynamic management, high-amplitude transients cause severe acoustic distortion, amplifier clipping, and voice coil thermal damage, while low-amplitude nuances remain inaudible. + +SOF addresses these dynamics through two specialized, complementary components: + +1. **Dynamic Range Compressor (DRC)** (``src/audio/drc/``): A full-featured single-band compressor featuring lookahead pre-delay buffering, quadratic soft-knee smoothing, adaptive multi-segment release ballistics, and division-based sub-block envelope processing. +2. **Multi-Band Dynamic Range Compressor (Multi-Band DRC)** (``src/audio/multiband_drc/``): A compound multi-stage processing component that splits the audio spectrum into 2, 3, or 4 discrete frequency bands using Linkwitz-Riley 4th-order (LR4) crossover filters, compresses each band independently to eliminate spectral pumping, and recombines the bands through emphasis and de-emphasis equalization. + +This guide provides a comprehensive, high-level architectural walkthrough of single-band DRC, lookahead mechanics, envelope ballistics, multi-band Linkwitz-Riley splitting, dynamic IPC configuration, and SIMD hardware acceleration without delving into low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +.. _drc_principles: + +1. Dynamic Range Compression in Audio Systems +********************************************* + +Dynamic range compression narrows the span between the quietest and loudest portions of an audio signal. Unlike static gain or volume scaling, compression is an active, level-dependent non-linear operation: low-level signals pass through unmodified (or amplified), while signals exceeding a predetermined threshold are attenuated according to a mathematical transfer function. + +Core Audio Use Cases in SOF +=========================== + +* **Micro-Speaker Protection & Excursion Limiting**: Compact transducers in laptops, smartphones, and monitors have strict physical excursion limits. High-energy low-frequency bursts can force the voice coil beyond its linear magnetic gap, causing harsh bottoming-out distortion or permanent mechanical failure. DRC applies peak limiting and compression to tame dangerous transients. +* **Speech Intelligibility & Dialogue Leveling**: In movies, podcasts, and teleconferencing, listeners frequently struggle to hear soft voices without cranking the volume—only to be overwhelmed when sound effects or loud participants speak. DRC compresses peak levels and applies makeup gain to lift quiet speech into an audible, comfortable zone. +* **Microphone Voice Capture Dynamics**: Digital and analog microphones capture signals ranging from soft ambient whispers to loud vocal shouts. DRC prevents analog-to-digital converter (ADC) saturation and clipping while maintaining consistent speech levels for automatic speech recognition (ASR) engines. + +Static Transfer Characteristic & Parameters +=========================================== + +The static compression curve defines the relationship between input level (:math:`X_{\text{dB}}`) and output level (:math:`Y_{\text{dB}}`): + +1. **Threshold (:math:`T_{\text{dB}}`)**: The input level above which compression begins. Below the threshold, the transfer function has a 1:1 slope (linear unity gain). +2. **Soft Knee (:math:`W_{\text{dB}}`)**: A smooth transition region surrounding the threshold. Rather than transitioning abruptly from unity gain to compression (a "hard knee"), SOF employs a quadratic polynomial curve over a knee width of :math:`W_{\text{dB}}`. This eliminates sharp slope discontinuities that produce audible harmonic distortion. +3. **Compression Ratio (:math:`R:1`)**: The degree of attenuation applied to signals above the knee. A ratio of :math:`4:1` means that for every 4 dB increase in input level above the threshold, the output level only increases by 1 dB (slope :math:`1/R = 0.25`). Very high ratios (e.g. :math:`20:1` to :math:`\infty:1`) configure the compressor as a brickwall limiter. +4. **Main Makeup Gain**: Post-compression linear amplification applied to restore overall average loudness lost during peak reduction. + +.. graphviz:: + :caption: Static Dynamic Range Compression Transfer Function: Threshold, Soft Knee, Ratio, and Makeup Gain + + digraph drc_curve { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_regions { + label="Compression Characteristic Curve (Input dB vs Output dB)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + reg_lin [label="Linear Region (Below Threshold)\nInput < Threshold\nSlope = 1:1 (Unity Gain, No Compression)", fillcolor="#FFFFFF", color="#CBD5E0"]; + reg_knee [label="Soft Knee Region (Threshold ± Knee/2)\nQuadratic Spline Interpolation\nSmooth parabolic transition, zero slope discontinuity", fillcolor="#FEFCBF", color="#D69E2E", fontcolor="#744210"]; + reg_comp [label="Compressed Region (Above Knee)\nInput > Threshold + Knee/2\nSlope = 1 / Ratio (e.g. 4:1 or 20:1 Limiting)", fillcolor="#BEE3F8", color="#3182CE"]; + reg_gain [label="Main Makeup Gain\nPost-compression linear amplification\nRestores perceived audio loudness", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + reg_lin -> reg_knee -> reg_comp -> reg_gain; + } + } + +--- + +.. _single_band_drc_architecture: + +2. Single-Band DRC Processing Architecture +****************************************** + +The single-band DRC component (``src/audio/drc/drc.c``) decouples audio streaming from level detection by utilizing a dedicated sidechain detector path and a lookahead pre-delay buffer. + +Signal Path vs Sidechain Detector Path +====================================== + +The compressor splits incoming audio into two parallel branches: + +1. **The Signal Path**: Carries the audio samples that will eventually be delivered to the output. These samples pass through a circular lookahead pre-delay buffer before being scaled by the calculated compressor gain. +2. **The Sidechain Detector Path**: Analyzes the instantaneous amplitude of the audio signal, evaluates peak and RMS signal envelopes, maps levels through the static compression curve, and calculates the target attenuation. + +Lookahead Pre-Delay Buffering +============================= + +A fundamental challenge in dynamic range compression is that loud acoustic transients (such as the initial crack of a snare drum or gun shot) rise in a fraction of a millisecond. If the compressor only reacts after detecting the transient, the leading edge of the burst leaks through unattenuated, causing amplifier clipping: + +* **Pre-Delay Circular Buffer (``pre_delay_buffers``)**: SOF introduces a small, configurable delay into the signal path (up to 512 frames, typically 5 to 10 ms at 48 kHz). +* **Transient Anticipation**: Because the sidechain detector inspects incoming samples before they exit the pre-delay buffer, the envelope generator begins ramping down compressor gain *before* the transient peak reaches the output gain multiplier. +* **Overshoot Prevention**: Transient peaks are smoothly captured and compressed without requiring harsh, zero-attack brickwall clipping. + +Division-Based Sub-Block Processing +=================================== + +Calculating logarithmic decibel conversions, exponential envelope decay curves, and quadratic knee formulas for every single audio sample would impose prohibitive MIPS overhead on embedded DSP cores. + +SOF optimizes this via **Division-Based Processing**: + +* **Division Frames (``DRC_DIVISION_FRAMES = 32``)**: Heavy envelope calculations (such as target gain and exponential attack/release rates) execute once every 32 audio frames (~0.67 ms at 48 kHz). +* **Sample-by-Sample Linear Interpolation**: Across the 32 frames of each division, the compressor applies smooth linear interpolation between the current gain and the target gain, delivering artifact-free volume modulation with minimal processing overhead. + +.. graphviz:: + :caption: Single-Band DRC Processing Architecture: Lookahead Pre-Delay, Sidechain Detector, and Envelope Gain Application + + digraph drc_architecture { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_pcm [label="Input Audio x[n]\n(From Pipeline Buffer)", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_sidechain { + label="Sidechain Detector & Gain Computer Path"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + det_peak [label="Peak / Envelope Detector\ndrc_update_detector_average()\nEvaluates signal energy in dB", fillcolor="#FAF089", color="#B7791F"]; + det_calc [label="Compression Curve & Knee\nEvaluates Threshold, Knee, Ratio\nDetermines Target Gain (Q2.30)", fillcolor="#FAF089", color="#B7791F"]; + det_ball [label="Ballistics Generator (Division)\ndrc_update_envelope() (Every 32 Frames)\nComputes Attack / Adaptive Release Rate", fillcolor="#FAF089", color="#B7791F"]; + + det_peak -> det_calc -> det_ball; + } + + subgraph cluster_signal { + label="Delayed Audio Signal Path"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + p_delay [label="Lookahead Pre-Delay Buffer\npre_delay_buffers[ch] (up to 512 frames)\nAnticipates incoming transients", fillcolor="#BEE3F8", color="#3182CE"]; + } + + vca_gain [label="Gain Multiplier (VCA)\ndrc_compress_output()\nSmooth interpolated sample scaling", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + out_pcm [label="Compressed Audio y[n]\n(Zero Transient Overshoot)", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + in_pcm -> p_delay; + in_pcm -> det_peak; + det_ball -> vca_gain [label="Interpolated Gain", color="#D69E2E", style="bold"]; + p_delay -> vca_gain [label="Delayed Audio"]; + vca_gain -> out_pcm; + } + +--- + +.. _envelope_ballistics: + +3. Envelope Ballistics & Adaptive Release Mechanics +*************************************************** + +The dynamic response of a compressor over time is governed by its **ballistics**: how quickly it attenuates the signal when a loud sound occurs (**Attack**), and how smoothly it restores gain once the loud sound ceases (**Release**). + +Attack Ballistics (Transient Capture) +===================================== + +* **Attack Time**: The duration required for the compressor to apply gain reduction after the input crosses above the threshold. +* **Fast Response**: Attack times are typically fast (1 ms to 10 ms) to prevent high-amplitude peaks from damaging speaker hardware or clipping downstream DACs. +* **Logarithmic Envelope Tracking**: Gain reduction follows an exponential decay towards the target attenuation, ensuring rapid initial clamping. + +The Pitfalls of Conventional Static Release +=========================================== + +Selecting a static release time constant involves a severe compromise: + +* **If Release is Too Fast**: Following a bass note or vocal peak, the gain recovers so rapidly that it amplifies the low-frequency waveform cycles themselves, introducing severe harmonic distortion and audible "breathing" or noise-pumping artifacts. +* **If Release is Too Slow**: A single brief snare drum crack causes the entire audio track to drop in volume and remain suppressed for hundreds of milliseconds, creating a sluggish, muffled presentation. + +SOF Adaptive Multi-Segment Release Curve +======================================== + +SOF addresses this challenge by implementing an **adaptive non-linear release curve** governed by parameterized polynomial coefficients (:math:`kA, kB, kC, kD, kE`): + +1. **Short-Duration Transients**: If a loud peak lasts only a few milliseconds, the release curve executes a rapid recovery, instantly restoring natural volume without sluggishness. +2. **Sustained Loud Passages**: If the audio signal remains consistently loud over an extended period, the compressor smoothly transitions into a slower, gentler release mode. This prevents rapid gain fluctuations across low-frequency cycles, eliminating distortion while maintaining transparent acoustic leveling. + +.. graphviz:: + :caption: Envelope Ballistics: Fast Attack Transient Protection vs Adaptive Non-Linear Release Recovery + + digraph ballistics { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_attack { + label="Attack Phase (Transient Onset)"; + style="filled,rounded"; + fillcolor="#FED7D7"; + color="#E53E3E"; + + atk_det [label="Signal Crosses Above Threshold\nImmediate transient detection via lookahead", fillcolor="#FFFFFF", color="#CBD5E0"]; + atk_drp [label="Rapid Gain Attenuation (1 - 10 ms)\nSuppresses peak energy before speaker overload", fillcolor="#FED7D7", color="#E53E3E", fontcolor="#742A2A"]; + + atk_det -> atk_drp; + } + + subgraph cluster_release { + label="Adaptive Release Phase (Post-Transient Recovery)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + rel_eval [label="Adaptive Release Evaluator (kA, kB, kC, kD, kE)\nMeasures duration and depth of gain compression", fillcolor="#FFFFFF", color="#CBD5E0"]; + rel_fast [label="Fast Release Branch\nShort transient burst -> Rapid gain recovery\nPrevents muffled audio and restores clarity", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + rel_slow [label="Slow Release Branch\nSustained loud passage -> Gentle smooth recovery\nEliminates harmonic distortion & breathing artifacts", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + + rel_eval -> rel_fast [label="Short Peak"]; + rel_eval -> rel_slow [label="Sustained Passage"]; + } + + atk_drp -> rel_eval [label="Signal Drops Below Threshold", color="#4A5568", style="dashed"]; + } + +--- + +.. _multiband_drc_paradigm: + +4. The Multi-Band DRC Paradigm & Spectral Pumping Elimination +************************************************************* + +While single-band DRC provides effective dynamics control for speech and narrow-band sources, wideband complex audio (such as contemporary music, gaming, and cinematic soundtracks) reveals its inherent limitation: **Spectral Pumping**. + +The Spectral Pumping Hazard +=========================== + +In a single-band compressor, gain reduction is governed by the total wideband signal energy: + +* In almost all acoustic genres, low-frequency sounds (bass guitars, kick drums, synthetic sub-bass) carry vastly more physical energy than mid-frequency vocals or high-frequency cymbals. +* When a heavy kick drum hits, the single-band detector detects a massive energy surge and aggressively attenuates the compressor gain across the entire audio spectrum. +* Consequently, the mid-range vocals and high-frequency hi-hats are audibly "ducked" and dragged down in volume with every bass drum hit. This unmusical breathing effect is known as **spectral pumping**. + +The Multi-Band Solution +======================= + +Multi-Band Dynamic Range Compression (``src/audio/multiband_drc/``) eliminates spectral pumping by partitioning the continuous audio spectrum into distinct, isolated frequency bands: + +1. **Acoustic Isolation**: The low-frequency bass energy is separated from mid-frequency vocals and high-frequency cymbals. +2. **Independent Compressor Engines**: Each band processes audio through its own dedicated DRC instance with specialized parameter tuning: + - **Low Band (Bass)**: Configured with a low threshold, high ratio, and fast attack to clamp speaker-damaging diaphragm excursions. + - **Mid Band (Vocals & Instruments)**: Configured with a gentle ratio and transparent release to lift dialogue without altering musical warmth. + - **High Band (Treble & Cymbals)**: Tuned as a fast limiter/de-esser to eliminate harsh sibilance without affecting midrange presence. +3. **Transparent Recombination**: The independently compressed bands are mixed together, preserving full dynamic punch and vocal clarity simultaneously. + +.. graphviz:: + :caption: The Spectral Pumping Hazard: Wideband Compression Ducking vs Multi-Band Frequency Isolation + + digraph spectral_pumping { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_single { + label="Single-Band Compressor (Spectral Pumping Hazard)"; + style="filled,rounded"; + fillcolor="#FED7D7"; + color="#E53E3E"; + + s_in [label="Input: Heavy Bass Drum + Quiet Vocal + High Cymbals", fillcolor="#FFFFFF", color="#CBD5E0"]; + s_det [label="Wideband Energy Detector\nDominated by massive low-frequency bass energy", fillcolor="#FAF089", color="#B7791F"]; + s_gain [label="Single Wideband Gain Attenuation\nPulls down ENTIRE audio spectrum", fillcolor="#FED7D7", color="#E53E3E", fontcolor="#742A2A"]; + s_out [label="Output: Vocal and cymbals audibly duck and pump with each bass kick", fillcolor="#FFFFFF", color="#E53E3E"]; + + s_in -> s_det -> s_gain -> s_out; + } + + subgraph cluster_multi { + label="Multi-Band Compressor (Isolated Dynamic Control)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + m_in [label="Input: Heavy Bass Drum + Quiet Vocal + High Cymbals", fillcolor="#FFFFFF", color="#CBD5E0"]; + m_split [label="Linkwitz-Riley (LR4) Crossover Splitter\nSeparates Bass, Mids, and Highs into isolated paths", fillcolor="#BEE3F8", color="#3182CE"]; + m_b0 [label="Low Band DRC\nTames heavy bass excursion", fillcolor="#C6F6D5", color="#38A169"]; + m_b1 [label="Mid Band DRC\nPreserves crystal clear vocals (No Ducking)", fillcolor="#C6F6D5", color="#38A169"]; + m_b2 [label="High Band DRC\nTames harsh cymbal sibilance", fillcolor="#C6F6D5", color="#38A169"]; + m_sum [label="Output Summation\nNatural, punchy, uncompromised audio reproduction", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + m_in -> m_split; + m_split -> m_b0 -> m_sum; + m_split -> m_b1 -> m_sum; + m_split -> m_b2 -> m_sum; + } + } + +--- + +.. _multiband_drc_pipeline: + +5. Multi-Band DRC Compound Pipeline Architecture +************************************************ + +The Multi-Band DRC component (``src/audio/multiband_drc/multiband_drc.c``) is structured as a **compound 4-stage processing pipeline**: + +Stage 1: Emphasis Equalizer (Pre-Filter) +======================================== + +Before splitting the signal into frequency bands, audio passes through an **Emphasis Equalizer** consisting of two cascaded IIR biquad filters: + +* Shapes the spectral distribution to compensate for frequency-dependent acoustic anomalies in the physical enclosure. +* Pre-conditions the signal to optimize crossover splitting efficiency. +* Can be bypassed (set to neutral passthrough) if external upstream equalization is present. + +Stage 2: Linkwitz-Riley 4th-Order (LR4) Crossover Bank +====================================================== + +The audio spectrum is split into 2, 3, or 4 discrete bands using a **Linkwitz-Riley 4th-order (LR4)** crossover filter bank: + +* **Acoustic Summation Perfection**: An LR4 crossover is formed by cascading two 2nd-order Butterworth filters. At the crossover frequency :math:`f_c`, both the low-pass and high-pass branches are attenuated by exactly :math:`-6\text{ dB}`, resulting in a perfectly flat combined magnitude response (:math:`0\text{ dB}`) upon summation. +* **Zero Phase Difference**: The low-pass and high-pass outputs are strictly in phase (:math:`0^\circ` or :math:`360^\circ` phase difference) across the transition band, completely eliminating destructive comb filtering, phase cancellation notches, or acoustic lobing. + +Stage 3: Parallel Independent DRC Engines +========================================= + +Each frequency band feeds an independent instance of the single-band DRC engine: + +* Each band maintains its own lookahead pre-delay buffer, threshold, knee, ratio, attack time, and adaptive release curves. +* Bands operate in parallel, independently modulating their respective frequency slices. + +Stage 4: Summation & De-Emphasis Equalizer +========================================== + +The outputs of the parallel DRC engines are summed sample-by-sample and routed through a **De-Emphasis Equalizer**: + +* A 2-biquad IIR filter network that mirrors the pre-emphasis curve, restoring the overall tonal balance. +* Delivers a single, cohesive, high-dynamic output stream to downstream audio endpoints. + +.. graphviz:: + :caption: Multi-Band DRC Compound Pipeline Architecture: Emphasis, LR4 Crossover, Parallel DRC Engines, Summation, and De-Emphasis + + digraph multiband_pipeline { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_audio [label="Input Stream x[n]\n(Single Source)", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_emp { + label="Stage 1: Emphasis EQ"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + eq_emp [label="Emphasis Equalizer\n2-Biquad IIR Cascade\nSpectral Pre-Conditioning", fillcolor="#BEE3F8", color="#3182CE"]; + } + + subgraph cluster_xover { + label="Stage 2: LR4 Crossover Splitter"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + xo_bank [label="Linkwitz-Riley (LR4) Bank\nCascaded Butterworth pairs\nFlat 0 dB magnitude sum\nZero inter-band phase error", fillcolor="#FAF089", color="#B7791F"]; + } + + subgraph cluster_drcs { + label="Stage 3: Parallel DRC Band Engines"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + drc_b0 [label="Band 0 DRC (Lows / Bass)\nLookahead + High Ratio\nExcursion Protection", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + drc_b1 [label="Band 1 DRC (Midrange)\nGentle Ratio + Soft Knee\nVocal Transparency", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + drc_b2 [label="Band 2 DRC (Highs / Treble)\nFast Limiting & De-Esser\nHigh-Frequency Smoothing", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + drc_b0 -> drc_b1 -> drc_b2 [style="invis"]; + } + + sum_node [label="Sample-by-Sample\nBand Summation (+)", fillcolor="#EDF2F7", color="#A0AEC0"]; + + subgraph cluster_deemp { + label="Stage 4: De-Emphasis EQ"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + eq_deemp [label="De-Emphasis Equalizer\n2-Biquad IIR Cascade\nTonal Balance Restoration", fillcolor="#BEE3F8", color="#3182CE"]; + } + + out_audio [label="Output Stream y[n]\n(Single Sink)", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + in_audio -> eq_emp -> xo_bank; + xo_bank -> drc_b0 [label="Low Band"]; + xo_bank -> drc_b1 [label="Mid Band"]; + xo_bank -> drc_b2 [label="High Band"]; + drc_b0 -> sum_node; + drc_b1 -> sum_node; + drc_b2 -> sum_node; + sum_node -> eq_deemp -> out_audio; + } + +--- + +.. _drc_dynamic_updates_topology: + +6. Dynamic Parameter Updates & ALSA Topology 2 Integration +********************************************************** + +Dynamic range compressors must accommodate runtime adjustments from host applications, such as switching between "Movie", "Night Mode", and "Voice" audio presets in userspace sound managers. + +Component Configuration Blobs +============================= + +Compressor parameters are packaged into serialized binary blobs managed by the ``comp_data_blob_handler`` framework: + +* **Single-Band DRC Config (``struct sof_drc_config``)**: Contains the single-band threshold, knee width, compression ratio, lookahead pre-delay time, division frames, and adaptive release coefficients (:math:`kA` through :math:`kE`). +* **Multi-Band DRC Config (``struct sof_multiband_drc_config``)**: A compound configuration structure encompassing the number of active bands (up to 4), emphasis/de-emphasis biquad coefficients, Linkwitz-Riley crossover biquad coefficients, and an array of independent DRC parameter blocks (one for each active frequency band). +* **Multi-Packet Staging**: Large multi-band configuration blobs exceeding a single IPC mailbox window are transparently reassembled in memory before being validated and applied atomically between audio periods. + +ALSA Topology 2 Integration +=========================== + +Both DRC components are declared as native audio effect widgets in ALSA Topology 2: + +* **Single-Band DRC Widget (``drc.conf``)**: + + - Widget Type: ``effect`` + - UUID: ``da:e4:6e:b3:6f:00:f9:47:a0:6d:fe:cb:e2:d8:b6:ce`` + - Control Binding: Features an ALSA mixer switch control (control index 0) allowing userspace to dynamically enable or bypass the compressor (``drc_default_pass()``). + +* **Multi-Band DRC Widget (``multiband_drc.conf``)**: + + - Widget Type: ``effect`` + - UUID: ``56:22:9f:0d:4f:8e:b3:47:84:48:23:9a:33:4f:11:91`` + - Control Binding: Exposes switch controls for global bypass (``multiband_drc_default_pass()``) and crossover configuration. + +.. graphviz:: + :caption: Dynamic IPC Configuration Blob Handling, Parameter Staging, and ALSA Topology 2 Integration + + digraph drc_topology_flow { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + host_drv [label="Host ALSA Driver / Userspace Audio Server\nSends DRC / Multi-Band Preset via IPC", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_blob { + label="Blob Handler & Parameter Staging"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + b_rx [label="comp_data_blob_handler\nFragment Reassembly & Offset Tracking", fillcolor="#FAF089", color="#B7791F"]; + b_val [label="Parameter Validation Hook\nCheck Band Counts (<= 4), Buffer Limits & Taps\nVerify Stability of Crossover & EQ Biquads", fillcolor="#FAF089", color="#B7791F"]; + b_swap [label="Atomic State Transition\nRe-allocate lookahead pre-delay buffers if needed\nAtomic pointer swap at period boundary", fillcolor="#FAF089", color="#B7791F"]; + + b_rx -> b_val -> b_swap; + } + + subgraph cluster_topo { + label="ALSA Topology 2 Effect Widgets"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + w_drc [label="drc.conf Widget\nUUID: da:e4:6e:b3:...\nBypass Switch Control", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + w_mdrc [label="multiband_drc.conf Widget\nUUID: 56:22:9f:0d:...\nMulti-Band Compound Effect", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + w_drc -> w_mdrc [style="invis"]; + } + + host_drv -> b_rx; + b_swap -> w_drc [color="#38A169", style="bold"]; + b_swap -> w_mdrc [color="#38A169", style="bold"]; + } + +--- + +.. _simd_drc_acceleration: + +7. SIMD Vector Acceleration Across DSP Architectures +**************************************************** + +Processing multi-channel audio through dynamic compressors involves significant mathematical throughput: circular lookahead buffer indexing, logarithmic decibel energy extraction, exponential envelope smoothing, crossover filtering, and multi-band gain scaling. + +SOF provides architecture-specific SIMD vector acceleration: + +* **Tensilica Xtensa HiFi 3 & HiFi 4 (``drc_hifi4.c``, ``drc_math_hifi3.c``)**: + + - Vectorized lookahead buffer read and write operations advancing circular indices without scalar address math. + - SIMD vector gain application multiplying four 32-bit audio samples concurrently with hardware saturation. + - Fast fixed-point mathematical approximations for logarithmic decibel calculation and exponential decay curves using CORDIC and polynomial lookup tables (LUTs). + +* **Tensilica Xtensa HiFi 5**: + + - 8-way 32-bit vector processing engine (256-bit bus) accelerating multi-band crossover splitting and parallel DRC compression stages. + - Dual memory load buses allow simultaneously loading audio delay lines and compressor gain coefficients in a single clock cycle. + +* **Generic Portable Scalar C (``drc_generic.c``, ``multiband_drc_generic.c``)**: + + - Clean, portable scalar C implementations designed for non-Xtensa platforms (e.g. ARM Cortex-M7 on Teensy 4.1, RISC-V on ESP32-P4). + +.. graphviz:: + :caption: SIMD Vector Acceleration and Fixed-Point Math Approximations across DSP Architectures + + digraph simd_drc { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_gen { + label="Generic Scalar C (drc_generic.c / multiband_drc_generic.c)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + g_core [label="Portable Scalar C Loop\n1 sample per iteration\nTarget: ARM Cortex-M, RISC-V, Simulator", fillcolor="#FFFFFF", color="#CBD5E0"]; + } + + subgraph cluster_hf3 { + label="Xtensa HiFi 3 / HiFi 4 (drc_hifi4.c / drc_math_hifi3.c)"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + h3_core [label="Quad 32-bit Vector Engine\nVector circular pre-delay buffering\nCORDIC / LUT log-exp approximations", fillcolor="#BEE3F8", color="#3182CE"]; + } + + subgraph cluster_hf5 { + label="Xtensa HiFi 5 (Octa Vector Acceleration)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + h5_core [label="Octa 32-bit Vector Engine (256-bit bus)\n8 samples processed concurrently\nAccelerates parallel multi-band DRC filtering", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + g_core -> h3_core [label="2x - 4x Speedup", color="#3182CE"]; + h3_core -> h5_core [label="2x Speedup (8x Total)", color="#38A169", style="bold"]; + } + +--- + +.. _upstream_drc_references: + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level implementation details, mathematical structures, and tuning scripts: + +* **Upstream Component Specifications**: + - `thesofproject/sof: src/audio/drc/README.md `_ + - `thesofproject/sof: src/audio/multiband_drc/README.md `_ +* **Single-Band DRC Source Files**: + - ``src/audio/drc/drc.c``: Component initialization, lifecycle, and buffer dispatch. + - ``src/audio/drc/drc.h``: DRC state definitions (``struct drc_state``), pre-delay buffer management, and division masks. + - ``src/audio/drc/drc_algorithm.h``: Core algorithm prototypes (detector averaging, envelope updating, and compression scaling). + - ``src/audio/drc/drc_user.h``: Parameter definitions (``struct sof_drc_params``, ``struct sof_drc_config``). + - ``src/audio/drc/drc_generic.c``: Portable scalar C compression kernel. + - ``src/audio/drc/drc_hifi4.c``: Tensilica Xtensa HiFi 4 SIMD vector implementation. + - ``src/audio/drc/drc_math_hifi3.c``: Xtensa HiFi 3 fixed-point math acceleration. +* **Multi-Band DRC Source Files**: + - ``src/audio/multiband_drc/multiband_drc.c``: Compound component lifecycle, state reset, and memory allocation. + - ``src/audio/multiband_drc/multiband_drc.h``: Multi-band state (``struct multiband_drc_state``) encompassing emphasis, crossover, DRCs, and deemphasis. + - ``src/audio/multiband_drc/user/multiband_drc.h``: Multi-band configuration structures (``struct sof_multiband_drc_config``). + - ``src/audio/multiband_drc/multiband_drc_generic.c``: Compound pipeline execution loop. +* **Topology Definitions**: + - ``tools/topology/topology2/include/components/drc.conf``: ALSA Topology 2 configuration class for single-band DRC widgets. + - ``tools/topology/topology2/include/components/multiband_drc.conf``: ALSA Topology 2 configuration class for multi-band DRC widgets. +* **MATLAB / Octave Tuning Scripts**: + - ``src/audio/drc/tune/sof_example_drc.m``: Interactive script for tuning threshold, knee, ratio, and attack/release curves. + - ``src/audio/multiband_drc/tune/sof_example_multiband_drc.m``: Tuning script for configuring multi-band crossover frequencies and per-band compressor profiles. + +Related Subsystem Architecture Guides +===================================== + +* :ref:`volume_module`: Per-channel gain scaling, smooth volume ramping, and zero-crossing muting. +* :ref:`eq_fir_iir`: Finite and Infinite Impulse Response equalizers, linear-phase filtering, and biquad cascades. +* :ref:`src_asrc`: Sample rate conversion architecture handling fixed and drifting clocks across heterogeneous audio interfaces. +* :ref:`mixin_mixout`: Multi-stream mixing and audio distribution preceding or following dynamics processing. +* :ref:`module_framework`: The standardized module interface, Source/Sink APIs, and memory sandboxing wrapping DRC components. +* :ref:`pipeline_architecture`: How DRC widgets are integrated into directed acyclic audio graphs (DAGs). diff --git a/developer_guides/firmware/pipeline_architecture.rst b/developer_guides/firmware/pipeline_architecture.rst index 340dfad1..d4a6bded 100644 --- a/developer_guides/firmware/pipeline_architecture.rst +++ b/developer_guides/firmware/pipeline_architecture.rst @@ -465,6 +465,7 @@ Related Guides * :ref:`mixin_mixout`: Multi-pipeline audio mixing, stream splitting, dynamic clock domain decoupling, and matrix routing. * :ref:`src_asrc`: Synchronous polyphase conversion, asynchronous Farrow drift tracking, push/pull topologies, and SIMD acceleration. * :ref:`eq_fir_iir`: Finite and Infinite Impulse Response equalizers, linear-phase FIR tap folding, Direct Form I biquad cascades, and dynamic IPC blob updates. +* :ref:`drc_multiband_drc`: Single-band and multi-band dynamic range compression, lookahead pre-delay buffers, adaptive release ballistics, and Linkwitz-Riley crossover splitting. * :ref:`fw_init_boot`: Boot flow, hardware mailbox FW Ready handshake, and Zephyr initialization. * :ref:`topology2`: How pipelines and widgets are declared using ALSA Topology 2.0 configuration classes. * :ref:`sof_hostless_firmware`: How to create static pipelines compiled into ROM for standalone microcontrollers. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 2d869908..39be4e36 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -45,7 +45,7 @@ Audio Processing Modules & Algorithms * :ref:`mixin_mixout` (High-level architecture; also see upstream `mixin_mixout README `_ & `mixer README `_) * :ref:`src_asrc` (High-level architecture; also see upstream `SRC README `_ & `ASRC README `_) * :ref:`eq_fir_iir` (High-level architecture; also see upstream `FIR README `_ & `IIR README `_) -* `Dynamic Range Compressor (DRC) `_ & `Multiband DRC `_ +* :ref:`drc_multiband_drc` (High-level architecture; also see upstream `DRC README `_ & `Multiband DRC README `_) * `Crossover `_ * `DC Blocker `_ * `Time-Domain Fixed Beamformer (TDFB) `_ @@ -90,6 +90,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/mixin_mixout firmware/src_asrc firmware/eq_fir_iir + firmware/drc_multiband_drc rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 8dbf62c49af392703877ab62800ef6e4ed347345 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Fri, 18 Sep 2026 16:56:15 +0100 Subject: [PATCH 13/64] docs: developer_guides: add high-level crossover filter architecture guide Add a comprehensive, high-level developer architecture guide for the Crossover Filter in Sound Open Firmware. Covers: - Electro-acoustic motivations: physical transducer frequency limits (subwoofers, woofers, midrange, tweeters) and active digital DSP crossovers vs passive analog crossovers. - Linkwitz-Riley 4th-order (LR4) filter theory: flaws of conventional Butterworth crossovers (+3 dB acoustic bump and 90 degree phase quadrature), cascaded Butterworth pairs (-6 dB at fc), flat 0 dB magnitude summation, and 360/0 degree in-phase acoustic alignment. - Crossover split topologies: 2-way splitting, 3-way splitting with the LR4 all-pass phase equalization trick (crossover_generic_lr4_merge), and 4-way symmetrical tree decomposition. - Direct Form I (DF1) cascaded biquad implementation mechanics with 64-bit accumulators and Q2.30 coefficient formatting. - 1-to-N multi-sink buffer distribution (bsinks[], assign_sinks[]), ALSA Topology 2 component integration (crossover.conf), and IPC4 dynamic pin indexing (init_config = 1). - System-level deployment: standalone multi-amplifier bi-amping/tri-amping vs embedded spectral splitting within Multi-Band DRC. - SIMD vector acceleration across Cadence Tensilica Xtensa HiFi 3, HiFi 4, HiFi 5, and generic portable scalar C fallbacks. - 7 native vector Graphviz SVG diagrams. - Cross-references in developer_guides/index.rst and pipeline_architecture.rst. Signed-off-by: Liam Girdwood --- developer_guides/firmware/crossover.rst | 588 ++++++++++++++++++ .../firmware/pipeline_architecture.rst | 1 + developer_guides/index.rst | 3 +- 3 files changed, 591 insertions(+), 1 deletion(-) create mode 100644 developer_guides/firmware/crossover.rst diff --git a/developer_guides/firmware/crossover.rst b/developer_guides/firmware/crossover.rst new file mode 100644 index 00000000..9de4dd37 --- /dev/null +++ b/developer_guides/firmware/crossover.rst @@ -0,0 +1,588 @@ +.. _crossover: + +Crossover Filter Architecture +############################# + +The **Crossover Filter** subsystem in Sound Open Firmware provides spectral band splitting, multi-driver transducer routing, and frequency-domain decomposition across active loudspeaker systems and multi-band audio processing pipelines. + +In acoustic engineering, physical speaker transducers are bounded by rigid physical and mechanical constraints: large-diameter woofers excel at moving large volumes of air to reproduce low-frequency bass but possess too much cone inertia to oscillate rapidly at high frequencies without severe breakup distortion. Conversely, miniature, lightweight tweeters reproduce delicate high-frequency transients effortlessly, but undergo destructive excursion and voice coil burnout if subjected to high-energy bass frequencies. + +To overcome these constraints, high-fidelity audio systems employ **Multi-Way Loudspeakers** (such as 2-way woofer/tweeter systems, 3-way sub/mid/tweeter setups, or 4-way full-range towers). The SOF Crossover component acts as the digital frequency division engine, cleanly partitioning a wideband input audio stream into multiple dedicated frequency bands tailored to individual acoustic drivers or downstream processing components. + +SOF implements active digital crossovers using **Linkwitz-Riley 4th-Order (LR4)** filter networks configured in 2-way, 3-way, and 4-way topologies, providing steep 24 dB/octave attenuation slopes, flat magnitude summation, and in-phase acoustic alignment without comb filtering. + +This guide provides a comprehensive, high-level architectural walkthrough of the Crossover filter subsystem, Linkwitz-Riley filter theory, multi-way splitting topologies with all-pass phase alignment, 1-to-N multi-sink buffer distribution, and SIMD hardware acceleration without delving into low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +.. _crossover_principles: + +1. Electro-Acoustic Motivations & Crossover Principles +****************************************************** + +An audio crossover is an electrical or digital filter network that splits an incoming wideband audio signal into multiple frequency bands tailored to specific transducers or processors: + +* **Subwoofer Band (< 80 Hz)**: Extremely high excursion, omnidirectional deep bass reproduction. +* **Woofer / Bass Band (80 Hz – 1 kHz)**: Low-to-midrange bass punch, drum transients, and lower vocal registers. +* **Midrange Band (1 kHz – 4 kHz)**: Critical human vocal fundamentals, speech clarity, and instrumental harmonics. +* **Tweeter / High Band (> 4 kHz)**: High-frequency sibilance, cymbal brilliance, and spatial airiness. + +Passive Analog Crossovers vs Active Digital Crossovers +====================================================== + +Traditionally, multi-driver speaker cabinets rely on **passive analog crossovers** placed inside the loudspeaker cabinet between a single power amplifier and the physical drivers: + +* **Limitations of Passive Analog Crossovers**: + + - **Power Dissipation & Thermal Drift**: Passive crossovers utilize large inductors with high DC resistance and electrolytic capacitors that dissipate amplifier power as heat. Component heating causes filter values to drift significantly during loud listening sessions. + - **Damping Factor Loss**: Inductors placed in series with woofers degrade the amplifier's electrical damping factor, resulting in loose, uncontrolled bass ringing. + - **Component Tolerances & Phase Smearing**: Real-world passive component tolerances (often 5% to 10%) cause unpredictable phase shifts, irregular impedance curves, and destructive acoustic notches at the crossover frequency. + +* **Advantages of Active DSP Crossovers in SOF**: + + - **Pristine Digital Domain Splitting**: Frequency division occurs inside the DSP firmware before digital-to-analog conversion and power amplification (bi-amping, tri-amping, or quad-amping). + - **Zero Power Loss & Perfect Damping**: Power amplifiers connect directly to driver voice coils with zero intervening passive circuitry, maximizing electrical damping and acoustic efficiency. + - **Mathematical Precision & Stability**: Digital filter coefficients operate with mathematical exactness, unaffected by temperature, component aging, or electrical tolerances. + - **Steep 24 dB/Octave Roll-Offs**: Active DSP filters easily achieve steep 4th-order Linkwitz-Riley slopes that would require prohibitively bulky, expensive, and lossy passive components. + +.. graphviz:: + :caption: Active DSP Crossover vs Passive Analog Crossover Architectures in Multi-Driver Loudspeakers + + digraph crossover_taxonomy { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_passive { + label="Legacy Passive Crossover (Post-Amplifier Analog Domain)"; + style="filled,rounded"; + fillcolor="#FED7D7"; + color="#E53E3E"; + + p_src [label="Host Audio Stream", fillcolor="#FFFFFF", color="#CBD5E0"]; + p_dac [label="Single DAC & Pre-Amp", fillcolor="#FFFFFF", color="#CBD5E0"]; + p_amp [label="Single Power Amplifier\n(Must amplify entire wideband spectrum)", fillcolor="#FED7D7", color="#E53E3E"]; + p_xov [label="Passive LC Filter Network\nBulky inductors & capacitors\nPower loss & thermal drift", fillcolor="#FED7D7", color="#E53E3E", fontcolor="#742A2A"]; + p_spk1 [label="Woofer Driver", fillcolor="#FFFFFF", color="#CBD5E0"]; + p_spk2 [label="Tweeter Driver", fillcolor="#FFFFFF", color="#CBD5E0"]; + + p_src -> p_dac -> p_amp -> p_xov; + p_xov -> p_spk1 [label="Lows (Damping Lost)"]; + p_xov -> p_spk2 [label="Highs"]; + } + + subgraph cluster_active { + label="SOF Active DSP Crossover (Pre-Amplifier Digital Domain)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + a_src [label="Host Audio Stream", fillcolor="#FFFFFF", color="#CBD5E0"]; + a_xov [label="SOF Crossover Component (crossover.c)\nLinkwitz-Riley 4th-Order (LR4) Digital Engine\nSteep 24 dB/oct slope, 0 dB flat sum, in-phase", fillcolor="#BEE3F8", color="#3182CE"]; + a_amp1 [label="Dedicated Woofer DAC & Amp\nDirect voice coil connection\nMaximum electrical damping", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + a_amp2 [label="Dedicated Tweeter DAC & Amp\nLow-noise linear amplification\nZero bass excursion hazard", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + a_spk1 [label="Woofer Driver\n(Tight, punchy bass)", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + a_spk2 [label="Tweeter Driver\n(Crisp, distortion-free highs)", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + a_src -> a_xov; + a_xov -> a_amp1 [label="Low Band"]; + a_xov -> a_amp2 [label="High Band"]; + a_amp1 -> a_spk1; + a_amp2 -> a_spk2; + } + } + +--- + +.. _lr4_filter_theory: + +2. Linkwitz-Riley 4th-Order (LR4) Filter Theory & Phase Alignment +***************************************************************** + +Selecting the mathematical filter topology for an active acoustic crossover is critical. In audio textbooks, Butterworth filters are renowned for their maximally flat passband response. However, when applied to multi-driver acoustic crossovers, traditional Butterworth filters exhibit severe acoustic flaws. + +The Flaws of Conventional Butterworth Crossovers +================================================ + +* **The +3 dB Acoustic Bump**: Standard Butterworth low-pass and high-pass filters intersect at their :math:`-3\text{ dB}` half-power points. While uncorrelated signals (such as independent white noise sources) sum flat, coherent audio signals (such as musical notes spanning the crossover frequency) sum in voltage: :math:`1/\sqrt{2} + 1/\sqrt{2} = \sqrt{2} \approx +3\text{ dB}`. This produces an unnatural, audible acoustic peak at the crossover frequency :math:`f_c`. +* **Phase Quadrature & Acoustic Lobing**: Butterworth filters produce a :math:`90^\circ` phase difference between their low-pass and high-pass outputs at :math:`f_c`. When sound radiates into a room from physically separated speaker drivers, this :math:`90^\circ` phase disparity causes the primary acoustic radiation lobe to tilt off-axis, creating destructive comb filtering and acoustic notches whenever the listener moves vertically. + +The Linkwitz-Riley (LR4) Innovation +=================================== + +To solve these acoustic dilemmas, acoustic pioneers Siegfried Linkwitz and Russ Riley designed the **Linkwitz-Riley** filter topology. In Sound Open Firmware, all active crossovers are implemented as **4th-Order Linkwitz-Riley (LR4)** networks: + +1. **Cascaded Butterworth Pairs**: An LR4 filter is constructed by cascading two identical 2nd-order Butterworth filters in series: + + .. math:: + + H_{\text{LR4}}(z) = \Big( H_{\text{Butterworth 2nd}}(z) \Big)^2 + +2. **Flat 0 dB Magnitude Summation**: Because each 2nd-order stage contributes :math:`-3\text{ dB}` of attenuation at :math:`f_c`, the cascaded LR4 low-pass and high-pass filters are both down by exactly :math:`-6\text{ dB}` at the crossover frequency: + + .. math:: + + |H_{\text{LP}}(j\omega_c)| = 0.5 \quad (-6\text{ dB}), \qquad |H_{\text{HP}}(j\omega_c)| = 0.5 \quad (-6\text{ dB}) + + When the low-pass and high-pass acoustic outputs sum in the air, their coherent combination is mathematically flat: + + .. math:: + + |H_{\text{LP}}(j\omega) + H_{\text{HP}}(j\omega)| = 1.0 \quad (0\text{ dB}) \quad \forall \omega + +3. **Strict In-Phase Acoustic Alignment**: The phase difference between the low-pass and high-pass outputs of an LR4 filter is exactly :math:`360^\circ` (or :math:`0^\circ` modulo :math:`360^\circ`) across all frequencies. Because the drivers operate perfectly in phase across the transition band, the acoustic radiation pattern remains centered along the horizontal listening axis with zero vertical lobing tilt. +4. **Steep 24 dB/Octave Roll-Off**: The 4th-order slope provides rapid attenuation outside the passband, shielding fragile tweeters from low-frequency excursion damage and eliminating high-frequency woofer cone breakup resonances. + +.. graphviz:: + :caption: Linkwitz-Riley 4th-Order (LR4) Magnitude Summation (-6 dB at fc) and In-Phase Acoustic Alignment + + digraph lr4_theory { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_mag { + label="LR4 Magnitude & Phase Alignment Characteristics"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + p_lp [label="Low-Pass LR4 Branch (Woofer)\n-6 dB Cutoff at fc\n24 dB / Octave Attenuation Slope", fillcolor="#BEE3F8", color="#3182CE"]; + p_hp [label="High-Pass LR4 Branch (Tweeter)\n-6 dB Cutoff at fc\n24 dB / Octave Attenuation Slope", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + p_sum [label="Acoustic Magnitude Summation\n0.5 + 0.5 = 1.0 -> Perfectly Flat 0 dB Response\nZero passband ripple, zero crossover bump", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + p_pha [label="Phase Alignment & Spatial Polar Symmetry\nPhase Difference = 360° (Strictly In-Phase)\nZero off-axis lobing tilt, zero comb filtering notches", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + p_lp -> p_sum [label="-6 dB at fc"]; + p_hp -> p_sum [label="-6 dB at fc"]; + p_sum -> p_pha [label="Coherent Radiation", style="bold", color="#276749"]; + } + } + +--- + +.. _crossover_topologies: + +3. Crossover Topologies: 2-Way, 3-Way, and 4-Way Splitting +********************************************************** + +Sound Open Firmware supports three fundamental crossover topologies configured via parameter blobs and topology tokens: + +2-Way Crossover Topology (Woofer + Tweeter) +=========================================== + +* **Structure**: Splits wideband audio at a single cutoff frequency :math:`f_c` using one low-pass LR4 filter and one high-pass LR4 filter. +* **Filter Count**: 2 LR4 filters (each composed of 2 biquads in series, totaling 4 biquads per channel). +* **Outputs**: Output 0 (Low / Woofer) and Output 1 (High / Tweeter). +* **Use Cases**: Standard stereo bookshelf speakers, two-way studio monitors, and dual-driver laptop audio systems. + +3-Way Crossover Topology & The All-Pass Phase Equalization Trick +================================================================ + +In a 3-way crossover, audio is partitioned into three bands: Low (Sub/Woofer), Mid (Midrange driver), and High (Tweeter) across two cutoff frequencies (:math:`f_{c1}, f_{c2}`): + +* **The Asymmetric Phase Dilemma**: + + - The incoming signal is first split into a Low branch and a High branch at :math:`f_{c1}` using LR4 pair 0 (``LP0`` and ``HP0``). + - The high branch is subsequently split at :math:`f_{c2}` using LR4 pair 2 (``LP2`` and ``HP2``), yielding the Midrange and Tweeter outputs. + - Notice that the Midrange and Tweeter signals pass through **two sequential LR4 filters**, while the Low signal only passes through **one LR4 filter** (``LP0``). + - Because each LR4 filter introduces a phase shift, passing through two filters rotates the phase of Mid and High by :math:`360^\circ` relative to Low, causing a catastrophic :math:`180^\circ` phase inversion between Low and Mid! + +* **The All-Pass Merger Solution**: + + - To restore phase coherence, SOF routes the Low branch through an auxiliary LR4 filter pair (``LP1`` and ``HP1``) and immediately sums their outputs back together (``crossover_generic_lr4_merge()``). + - Because an LR4 low-pass and high-pass sum to a flat magnitude of 1.0, this operation acts as a pure **all-pass filter**: it leaves the magnitude of the Low band completely unaltered while introducing the exact phase shift and group delay of an additional LR4 stage! + - Consequently, all three output bands pass through exactly two LR4 stages, guaranteeing strict phase alignment across all crossover regions. + +4-Way Crossover Topology (Sub + Woofer + Mid + Tweeter) +======================================================= + +* **Structure**: A fully symmetrical 2-stage tree decomposition across three cutoff frequencies (:math:`f_{c1}, f_{c2}, f_{c3}`): + - Stage 1: Splits the wideband signal into Low-Mid and Mid-High branches using LR4 pair 1 (``LP1``, ``HP1``). + - Stage 2: Low-Mid is split into Sub and Woofer using LR4 pair 0 (``LP0``, ``HP0``); Mid-High is split into Midrange and Tweeter using LR4 pair 2 (``LP2``, ``HP2``). +* **Filter Count**: 6 LR4 filters (12 biquads per channel). +* **Inherent Phase Alignment**: Because every signal path traverses exactly two sequential LR4 stages, phase delays are inherently identical across all four bands without requiring auxiliary phase-correction networks. + +.. graphviz:: + :caption: Crossover Split Topologies: 2-Way, 3-Way (with All-Pass Phase Merger), and 4-Way Tree Decomposition + + digraph topologies { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.45]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_2way { + label="2-Way Crossover (1 Cutoff fc)"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + x2_in [label="Input x[n]", fillcolor="#FFFFFF", color="#CBD5E0"]; + x2_lp0 [label="LR4 LP0 (fc)", fillcolor="#BEE3F8", color="#3182CE"]; + x2_hp0 [label="LR4 HP0 (fc)", fillcolor="#BEE3F8", color="#3182CE"]; + x2_out0 [label="LOW (Woofer)", fillcolor="#C6F6D5", color="#38A169"]; + x2_out1 [label="HIGH (Tweeter)", fillcolor="#C6F6D5", color="#38A169"]; + + x2_in -> x2_lp0 -> x2_out0; + x2_in -> x2_hp0 -> x2_out1; + } + + subgraph cluster_3way { + label="3-Way Crossover (2 Cutoffs: fc1, fc2) with All-Pass Phase Merger"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + x3_in [label="Input x[n]", fillcolor="#FFFFFF", color="#CBD5E0"]; + x3_lp0 [label="LR4 LP0 (fc1)", fillcolor="#FAF089", color="#B7791F"]; + x3_hp0 [label="LR4 HP0 (fc1)", fillcolor="#FAF089", color="#B7791F"]; + + x3_mrg [label="All-Pass Phase Merger\n(LP1 + HP1 Summation)\nEqualizes group delay", fillcolor="#FED7D7", color="#E53E3E", fontcolor="#742A2A"]; + x3_lp2 [label="LR4 LP2 (fc2)", fillcolor="#FAF089", color="#B7791F"]; + x3_hp2 [label="LR4 HP2 (fc2)", fillcolor="#FAF089", color="#B7791F"]; + + x3_out0 [label="LOW (Sub/Woofer)", fillcolor="#C6F6D5", color="#38A169"]; + x3_out1 [label="MID (Midrange)", fillcolor="#C6F6D5", color="#38A169"]; + x3_out2 [label="HIGH (Tweeter)", fillcolor="#C6F6D5", color="#38A169"]; + + x3_in -> x3_lp0 -> x3_mrg -> x3_out0; + x3_in -> x3_hp0; + x3_hp0 -> x3_lp2 -> x3_out1; + x3_hp0 -> x3_hp2 -> x3_out2; + } + + subgraph cluster_4way { + label="4-Way Crossover (3 Cutoffs: fc1, fc2, fc3) Symmetrical Tree"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + x4_in [label="Input x[n]", fillcolor="#FFFFFF", color="#CBD5E0"]; + x4_lp1 [label="LR4 LP1 (fc2)", fillcolor="#C6F6D5", color="#38A169"]; + x4_hp1 [label="LR4 HP1 (fc2)", fillcolor="#C6F6D5", color="#38A169"]; + + x4_lp0 [label="LR4 LP0 (fc1)", fillcolor="#C6F6D5", color="#38A169"]; + x4_hp0 [label="LR4 HP0 (fc1)", fillcolor="#C6F6D5", color="#38A169"]; + x4_lp2 [label="LR4 LP2 (fc3)", fillcolor="#C6F6D5", color="#38A169"]; + x4_hp2 [label="LR4 HP2 (fc3)", fillcolor="#C6F6D5", color="#38A169"]; + + x4_out0 [label="SUB", fillcolor="#68D391", color="#276749"]; + x4_out1 [label="WOOFER", fillcolor="#68D391", color="#276749"]; + x4_out2 [label="MID", fillcolor="#68D391", color="#276749"]; + x4_out3 [label="TWEETER", fillcolor="#68D391", color="#276749"]; + + x4_in -> x4_lp1; + x4_in -> x4_hp1; + x4_lp1 -> x4_lp0 -> x4_out0; + x4_lp1 -> x4_hp0 -> x4_out1; + x4_hp1 -> x4_lp2 -> x4_out2; + x4_hp1 -> x4_hp2 -> x4_out3; + } + } + +--- + +.. _df1_mechanics: + +4. Direct Form I Biquad Cascade Implementation Mechanics +******************************************************** + +Each 4th-order Linkwitz-Riley filter is implemented in DSP firmware by cascading two identical 2nd-order Direct Form I (DF1) biquad stages in series. + +Direct Form I Difference Equations +================================== + +For each biquad section, the output is computed via the standard difference equation: + +.. math:: + + y[n] = b_0 x[n] + b_1 x[n-1] + b_2 x[n-2] - a_1 y[n-1] - a_2 y[n-2] + +State Variables & Accumulator Precision +======================================= + +* **Independent Delay States**: Direct Form I stores two input state variables (:math:`x[n-1], x[n-2]`) and two output state variables (:math:`y[n-1], y[n-2]`). For an LR4 filter (two biquads), exactly 4 delay slots are allocated per filter (``CROSSOVER_NUM_DELAYS_LR4 = 4``). +* **64-Bit Internal Accumulation**: All product terms accumulate into a 64-bit register with guard bits before rounding and shifting. This prevents internal overflow and avoids limit cycle oscillations near low-frequency cutoff points. +* **Fixed-Point Formatting**: + - Filter coefficients (:math:`a_1, a_2, b_0, b_1, b_2`) are represented in high-precision :math:`Q2.30` fixed-point format. + - Headroom and gain normalization are controlled via per-section ``output_shift`` and ``output_gain`` (:math:`Q2.14`). + +.. graphviz:: + :caption: Cascaded Biquad Implementation of an LR4 Filter with 64-Bit Accumulation + + digraph biquad_cascade { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_x [label="Audio Input x[n]", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_bq1 { + label="Biquad Stage 1 (2nd-Order Butterworth)"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + bq1_core [label="Direct Form I Engine\nFeedforward (b0, b1, b2)\nFeedback (-a1, -a2)\n64-Bit Accumulator", fillcolor="#BEE3F8", color="#3182CE"]; + } + + subgraph cluster_bq2 { + label="Biquad Stage 2 (2nd-Order Butterworth)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + bq2_core [label="Direct Form I Engine\nIdentical Coefficients\nHeadroom Scaler (out_shift)\n64-Bit Accumulator", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + out_y [label="LR4 Output y[n]\n(24 dB / Octave Slope)", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + in_x -> bq1_core; + bq1_core -> bq2_core [label="Intermediate z[n]"]; + bq2_core -> out_y; + } + +--- + +.. _multisink_topology_ipc4: + +5. Multi-Sink Routing, ALSA Topology 2 & IPC4 Pin Indexing +********************************************************** + +Unlike standard 1-in-1-out audio effect widgets (such as Volume or Equalizer), the Crossover module is an inherently **1-to-N multi-sink stream splitter**: it consumes a single wideband input stream and simultaneously drives multiple independent sink buffers. + +Multi-Sink Buffer Management +============================ + +* **Sink Array (``bsinks[]``)**: The crossover processing function receives an array of output stream buffers corresponding to the number of configured bands (2, 3, or 4). +* **Sink Assignment Vector (``assign_sinks[]``)**: A parameter array maps logical crossover frequency outputs to destination sink pipeline IDs: + + .. code-block:: text + + assign_sinks[0] = 0 # Logical Low band -> Sink Buffer 0 (Woofer Pipeline) + assign_sinks[1] = 1 # Logical High band -> Sink Buffer 1 (Tweeter Pipeline) + +* **Passthrough Fallback Mode**: When ``num_sinks == 1`` or when the component is disabled via ALSA mixer controls, the module operates in passthrough mode (``crossover_default_pass()``), replicating input frames across output buffers with zero filtering overhead. + +IPC4 Dynamic Pin Indexing +========================= + +In SOF IPC4, modules are dynamically bound by connecting source pins to sink pins across independent processing modules. Because the Crossover component produces multiple output pins dynamically, the IPC4 firmware requires upfront knowledge of output pin indices before pipeline instantiation: + +* **Early Initialization Config (``init_config = 1``)**: In ``crossover.toml``, the Crossover module sets ``init_config = 1``, instructing the build system to append the extended base configuration (``base_cfg_ext``) to the module initialization IPC payload. +* **Pin Binding**: This upfront payload informs the IPC4 runtime how many output pins are active, enabling the host driver to bind downstream pipeline widgets directly to individual crossover frequency bands. + +ALSA Topology 2 Integration +=========================== + +The Crossover widget is declared in ALSA Topology 2 configuration files using ``tools/topology/topology2/include/components/crossover.conf``: + +* **Widget Type**: ``effect`` +* **Component UUID**: ``d1:9a:8c:94:6a:80:31:41:ad:6c:b2:bd:a9:e3:5a:9f`` +* **Static ROM Initialization**: Default crossover cutoff frequencies, biquad coefficients, and sink routing maps can be compiled directly into the topology binary (``.bin``), establishing active speaker frequency division immediately upon hardware boot. + +.. graphviz:: + :caption: 1-to-N Multi-Sink Buffer Distribution and ALSA Topology 2 / IPC4 Output Pin Binding + + digraph multisink_binding { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_buf [label="Single Wideband Input Stream\n(Pipeline Buffer)", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_comp { + label="Crossover Splitter Widget (crossover.conf)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + x_eng [label="Crossover Engine (crossover.c)\nLR4 Filter Bank Splitter\nassign_sinks[] Routing Table", fillcolor="#FAF089", color="#B7791F"]; + p_out0 [label="Output Pin 0 (Low Band)", fillcolor="#FFFFFF", color="#B7791F"]; + p_out1 [label="Output Pin 1 (High Band)", fillcolor="#FFFFFF", color="#B7791F"]; + + x_eng -> p_out0; + x_eng -> p_out1; + } + + subgraph cluster_sinks { + label="Downstream Sink Pipelines / DAI Endpoints"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + pipe_w [label="Woofer Pipeline / DAI\n(Smart Amp I2S Channel 0)", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + pipe_t [label="Tweeter Pipeline / DAI\n(Smart Amp I2S Channel 1)", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + in_buf -> x_eng; + p_out0 -> pipe_w [label="IPC4 Pin Binding 0", color="#38A169", style="bold"]; + p_out1 -> pipe_t [label="IPC4 Pin Binding 1", color="#38A169", style="bold"]; + } + +--- + +.. _system_integration_multiband: + +6. System-Level Deployment: Multi-Amp Systems vs Multi-Band DRC +*************************************************************** + +The SOF Crossover engine serves two primary architectural deployment models across audio products: + +Model A: Standalone Multi-Amplifier Loudspeaker Systems +======================================================== + +In high-end laptops, automotive audio, and smart speakers, the Crossover operates as an autonomous 1-to-N stream splitter: + +* The input audio stream is split into discrete bands that exit the DSP through separate digital audio interfaces (e.g. multi-channel SoundWire or TDM I2S). +* Each band is routed to a dedicated physical amplifier chip optimized for that specific driver (e.g. high-current Class-D amplifier for woofers, ultra-low-noise amplifier for tweeters). +* Features per-channel independent volume ramping, limiter protection, and speaker EQ. + +Model B: Embedded Spectral Splitting within Multi-Band DRC +========================================================== + +In compact single-amplifier systems, the Crossover operates as an internal component embedded inside the **Multi-Band Dynamic Range Compressor** (``src/audio/multiband_drc/``): + +* The LR4 crossover filter bank partitions the signal into sub-bands internally without exposing multiple external sink pins. +* Each band is compressed independently by parallel DRC instances to eliminate spectral pumping. +* The bands are recombined into a single wideband output stream delivered to a single shared speaker amplifier. + +.. graphviz:: + :caption: System-Level Acoustic Deployment: Standalone Multi-Amping vs Multi-Band DRC Subsystem + + digraph system_deployment { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_dep_a { + label="Deployment Model A: Standalone Multi-Amping (Multi-Sink Architecture)"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + a_in [label="Media Playback Stream", fillcolor="#FFFFFF", color="#CBD5E0"]; + a_xov [label="Crossover Widget (1-to-N Splitter)\nMultiple Output Pins", fillcolor="#BEE3F8", color="#3182CE"]; + a_amp0 [label="Hardware Amp 0: Woofer", fillcolor="#C6F6D5", color="#38A169"]; + a_amp1 [label="Hardware Amp 1: Tweeter", fillcolor="#C6F6D5", color="#38A169"]; + + a_in -> a_xov; + a_xov -> a_amp0 [label="Low Pin"]; + a_xov -> a_amp1 [label="High Pin"]; + } + + subgraph cluster_dep_b { + label="Deployment Model B: Embedded Crossover in Multi-Band DRC (Single-Sink)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + b_in [label="Media Playback Stream", fillcolor="#FFFFFF", color="#CBD5E0"]; + b_mdrc [label="Multi-Band DRC Widget (multiband_drc.c)\nInternal LR4 Crossover -> Parallel DRCs -> Summation", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + b_amp [label="Single Shared Hardware Amplifier & Speaker", fillcolor="#C6F6D5", color="#38A169"]; + + b_in -> b_mdrc -> b_amp [label="Single Wideband Output"]; + } + } + +--- + +.. _simd_crossover_acceleration: + +7. SIMD Vector Acceleration Across DSP Architectures +**************************************************** + +Processing multi-channel audio through up to 6 LR4 filters (12 cascaded biquads per channel) imposes significant computational demands on embedded DSP cores. + +SOF optimizes the crossover filtering pipeline through dedicated vector implementations: + +* **Cadence Tensilica Xtensa HiFi 3 & HiFi 4**: + + - Vectorized biquad filtering utilizing 64-bit dual multiply-accumulate instructions (``AE_MULAA32RA``). + - Processes multiple audio channels or biquad sections in parallel with hardware saturation. + - Automatic circular delay indexing without scalar pointer branching. + +* **Cadence Tensilica Xtensa HiFi 5**: + + - 8-way 32-bit vector processing engine (256-bit bus) accelerating parallel multi-channel crossover splits. + - Dual 128-bit memory load buses allow simultaneously fetching filter coefficients and audio delay buffers in a single clock cycle. + +* **Generic Portable Scalar C (``crossover_generic.c``)**: + + - Clean, portable scalar C implementations designed for non-Xtensa platforms (e.g. ARM Cortex-M7 on Teensy 4.1, RISC-V on ESP32-P4). + +.. graphviz:: + :caption: SIMD Vector Processing across Hardware Architectures + + digraph simd_crossover { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_gen { + label="Generic Scalar C (crossover_generic.c)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + g_core [label="Portable Scalar C Loop\n1 sample per iteration\nTarget: ARM Cortex-M, RISC-V, Simulator", fillcolor="#FFFFFF", color="#CBD5E0"]; + } + + subgraph cluster_hf3 { + label="Xtensa HiFi 3 / HiFi 4"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + h3_core [label="Dual / Quad 32-bit Vector Engine\nParallel Direct Form I biquads\n64-bit dual MAC instructions", fillcolor="#BEE3F8", color="#3182CE"]; + } + + subgraph cluster_hf5 { + label="Xtensa HiFi 5 (Octa Vector Engine)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + h5_core [label="Octa 32-bit Vector Engine (256-bit bus)\n8 samples processed per cycle\nDual 128-bit memory buses for coefficients & delays", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + g_core -> h3_core [label="2x - 4x Speedup", color="#3182CE"]; + h3_core -> h5_core [label="2x Speedup (8x Total)", color="#38A169", style="bold"]; + } + +--- + +.. _upstream_crossover_references: + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level implementation details, mathematical structures, and tuning scripts: + +* **Upstream Component Specifications**: + - `thesofproject/sof: src/audio/crossover/README.md `_ +* **Crossover Firmware Source Files**: + - ``src/audio/crossover/crossover.c``: Component initialization, multi-sink dispatch, and lifecycle. + - ``src/audio/crossover/crossover.h``: Crossover state definitions (``struct comp_data``) and function map prototypes. + - ``src/audio/crossover/crossover_user.h``: User parameter definitions (``struct sof_crossover_config``). + - ``src/include/module/crossover/crossover_common.h``: Common crossover state definitions (``struct crossover_state``) shared with Multi-Band DRC. + - ``src/audio/crossover/crossover_generic.c``: Portable scalar C splitting implementations (``split_2way``, ``split_3way``, ``split_4way``, and ``lr4_merge``). +* **Topology Definitions**: + - ``tools/topology/topology2/include/components/crossover.conf``: ALSA Topology 2 configuration class for Crossover widgets. +* **MATLAB / Octave Tuning Scripts**: + - ``src/audio/crossover/tune/sof_example_crossover.m``: Interactive script for generating Linkwitz-Riley crossover biquad coefficients across 2-way, 3-way, and 4-way configurations. + - ``src/audio/crossover/tune/sof_crossover_gen_coefs.m``: Low-level coefficient calculation and quantization functions. + +Related Subsystem Architecture Guides +===================================== + +* :ref:`drc_multiband_drc`: Single-band and multi-band dynamic range compression utilizing Linkwitz-Riley crossovers for spectral isolation. +* :ref:`eq_fir_iir`: Finite and Infinite Impulse Response equalizers, linear-phase filtering, and biquad cascades. +* :ref:`volume_module`: Per-channel gain scaling, smooth volume ramping, and zero-crossing muting. +* :ref:`src_asrc`: Sample rate conversion architecture handling fixed and drifting clocks across heterogeneous audio interfaces. +* :ref:`mixin_mixout`: Multi-stream audio mixing and distribution across post-crossover loudspeaker and headphone buses. +* :ref:`module_framework`: The standardized module interface, Source/Sink APIs, and memory sandboxing wrapping Crossover components. +* :ref:`pipeline_architecture`: How Crossover widgets are integrated into directed acyclic audio graphs (DAGs). diff --git a/developer_guides/firmware/pipeline_architecture.rst b/developer_guides/firmware/pipeline_architecture.rst index d4a6bded..769c8b70 100644 --- a/developer_guides/firmware/pipeline_architecture.rst +++ b/developer_guides/firmware/pipeline_architecture.rst @@ -466,6 +466,7 @@ Related Guides * :ref:`src_asrc`: Synchronous polyphase conversion, asynchronous Farrow drift tracking, push/pull topologies, and SIMD acceleration. * :ref:`eq_fir_iir`: Finite and Infinite Impulse Response equalizers, linear-phase FIR tap folding, Direct Form I biquad cascades, and dynamic IPC blob updates. * :ref:`drc_multiband_drc`: Single-band and multi-band dynamic range compression, lookahead pre-delay buffers, adaptive release ballistics, and Linkwitz-Riley crossover splitting. +* :ref:`crossover`: Linkwitz-Riley 4th-order (LR4) active multi-driver crossovers, 2-way/3-way/4-way splitting with all-pass phase alignment, and 1-to-N multi-sink buffer distribution. * :ref:`fw_init_boot`: Boot flow, hardware mailbox FW Ready handshake, and Zephyr initialization. * :ref:`topology2`: How pipelines and widgets are declared using ALSA Topology 2.0 configuration classes. * :ref:`sof_hostless_firmware`: How to create static pipelines compiled into ROM for standalone microcontrollers. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 39be4e36..41dfc66f 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -46,7 +46,7 @@ Audio Processing Modules & Algorithms * :ref:`src_asrc` (High-level architecture; also see upstream `SRC README `_ & `ASRC README `_) * :ref:`eq_fir_iir` (High-level architecture; also see upstream `FIR README `_ & `IIR README `_) * :ref:`drc_multiband_drc` (High-level architecture; also see upstream `DRC README `_ & `Multiband DRC README `_) -* `Crossover `_ +* :ref:`crossover` (High-level architecture; also see upstream `crossover README `_) * `DC Blocker `_ * `Time-Domain Fixed Beamformer (TDFB) `_ * `RTNR Noise Reduction `_ @@ -91,6 +91,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/src_asrc firmware/eq_fir_iir firmware/drc_multiband_drc + firmware/crossover rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 5377a96a1c92eb4a6b95c5766bbf87c30183466d Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Fri, 18 Sep 2026 17:05:20 +0100 Subject: [PATCH 14/64] docs: developer_guides: add high-level dc blocker architecture guide Add a comprehensive, high-level developer architecture guide for the DC Blocker subsystem (developer_guides/firmware/dcblock.rst). Topics covered: - Physical origins of DC bias (ADC preamplifier offset, PDM decimation leakage, thermal ground drift, synthetic non-linear algorithms) and system hazards (headroom penalty, asymmetric clipping, loudspeaker voice coil thermal destruction, cone excursion displacement, clicks/pops, and downstream DSP corruption in AEC, DRC, and VAD). - Digital filter theory and pole-zero mechanics of the first-order recursive high-pass DC blocker (H(z) = (1 - z^-1) / (1 - R z^-1)), exact 0 Hz transmission nulling, flat passband behavior, and cutoff frequency formulation. - Transient step response (y[n] = Delta_dc * R^n), exponential decay time constants (tau = 1 / (2*pi*fc)), settling times, and the engineering trade-off between sub-bass fidelity and transient recovery. - High-precision fixed-point arithmetic (Q2.30 coefficients, Q1.31 states, 64-bit Q3.61 intermediate accumulation), symmetric rounding, saturation clamping, and limit cycle oscillation elimination. - Multi-channel stream processing with independent per-channel state isolation, heterogeneous per-channel cutoff tuning, interleaved buffer traversal, and sample depth adaptability (S16, S24, S32). - Architecture-specific SIMD vector acceleration across Tensilica Xtensa HiFi 3, HiFi 4 (dual circular buffer registers for branchless processing), HiFi 5, and portable generic scalar C. - ALSA Topology 2 component configuration (dcblock.conf), Module Adapter lifecycle, LLEXT dynamic linking, IPC3/IPC4 control blobs, and pipeline deployment topologies in capture, playback, and post-effects. - GNU Octave / MATLAB tuning workflow and upstream code references. - Seven native vector Graphviz SVG architecture diagrams. - Clean Sphinx build with zero warnings under -W --keep-going and zero inclusive language violations. Signed-off-by: Liam Girdwood --- developer_guides/firmware/dcblock.rst | 822 ++++++++++++++++++ .../firmware/pipeline_architecture.rst | 1 + developer_guides/index.rst | 3 +- 3 files changed, 825 insertions(+), 1 deletion(-) create mode 100644 developer_guides/firmware/dcblock.rst diff --git a/developer_guides/firmware/dcblock.rst b/developer_guides/firmware/dcblock.rst new file mode 100644 index 00000000..83b60a35 --- /dev/null +++ b/developer_guides/firmware/dcblock.rst @@ -0,0 +1,822 @@ +.. _dcblock: + +DC Blocker Architecture +####################### + +The **DC Blocker** subsystem in Sound Open Firmware removes direct current (0 Hz DC) bias and infrasonic baseline drift from digital audio streams across microphone capture pipelines, loudspeaker playback paths, and non-linear audio processing blocks. + +In mixed-signal hardware and digital signal processing, DC offset is an insidious artifact: analog-to-digital converter (ADC) operational amplifier offsets, PDM microphone decimation leakage, grounding thermal drift, and synthetic non-linear processing algorithms all introduce static DC biases into audio signals. In digital audio pipelines, a DC bias robs signals of fixed-point dynamic range headroom, causes asymmetric waveform clipping, generates audible clicks and pops during stream transitions, threatens moving-coil loudspeaker voice coils with destructive resistive heating, and impairs downstream adaptive algorithms such as acoustic echo cancellers, beamformers, dynamic range compressors, and keyword spotters. + +Sound Open Firmware integrates a dedicated, highly optimized **first-order recursive high-pass DC blocking filter** (:math:`H(z) = \frac{1 - z^{-1}}{1 - R z^{-1}}`) providing complete 0 Hz transmission nulling, mathematically flat passband response across the human audible spectrum, configurable cutoff frequencies, 64-bit fixed-point accumulation, and architecture-specific SIMD vector acceleration across Cadence Tensilica Xtensa HiFi 3, HiFi 4, and HiFi 5 DSPs, alongside portable scalar implementations for ARM Cortex-M and RISC-V cores. + +This guide provides a comprehensive, high-level architectural walkthrough of the DC Blocker subsystem, analyzing the physical origins of DC bias, pole-zero digital filter mechanics, transient step responses, fixed-point precision and limit cycle avoidance, multi-channel stream processing, ALSA Topology 2 / IPC dynamic configuration, and SIMD hardware acceleration without delving into low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +.. _dcblock_origins_hazards: + +1. Physical Origins & Hazards of DC Offset in Audio Systems +*********************************************************** + +Direct current (DC) in audio refers to a constant, non-zero static voltage or digital baseline offset (:math:`0\text{ Hz}`) added to an alternating audio waveform. While humans cannot hear a static 0 Hz offset directly, its presence within digital audio pipelines creates severe acoustic, electrical, and algorithmic degradations. + +Physical and Algorithmic Sources of DC Bias +=========================================== + +DC offset enters digital audio pipelines through both hardware imperfections and non-linear digital algorithms: + +* **ADC Front-End Operational Amplifier Offset**: Real-world analog preamplifiers and delta-sigma ADCs exhibit slight differential transistor mismatches and input bias currents, producing a persistent analog DC voltage that digitizes into a non-zero digital mean value. +* **PDM Digital Microphone Decimation Leakage**: Digital MEMS microphones outputting Pulse Density Modulation (PDM) streams rely on internal sigma-delta modulators. Imperfections in internal integrator feedback loops and decimation sinc filters can pass residual DC offsets into the decimated PCM output. +* **Ground Drift & Thermal Asymmetry**: Single-ended analog inputs, long microphone cables, and uneven chassis heating introduce ground potential shifts and thermal gradients that appear as slow-moving DC wander. +* **Synthetic Non-Linear Audio Algorithms**: Non-linear signal processing operations—such as half-wave rectification in envelope detectors, asymmetric waveshapers, harmonic exciters, and non-linear dynamic bass synthesis—produce non-zero average DC components as an unavoidable mathematical byproduct of harmonic generation. + +Acoustic and Algorithmic Hazards +================================ + +Uncorrected DC offsets inflict severe degradation across both playback and capture pipelines: + +* **Dynamic Range Loss & Asymmetric Clipping**: In fixed-point PCM representation (:math:`Q1.15` or :math:`Q1.31`), signal amplitude is bounded within :math:`[-1.0, +1.0)`. A DC bias shifts the resting baseline away from zero, disproportionately reducing available headroom in one direction. For example, a :math:`+0.1` DC offset reduces positive headroom to :math:`+0.9` (a loss of nearly :math:`1\text{ dB}` of dynamic range). When loud peaks occur, the signal clips asymmetrically, introducing harsh even-order harmonic distortion. +* **Loudspeaker Voice Coil Thermal Destruction**: In playback pipelines, passing DC through a power amplifier into a moving-coil loudspeaker causes a continuous, unvarying electrical current (:math:`I_{dc} = V_{dc} / R_e`) to flow through the voice coil. Because the voice coil cannot radiate 0 Hz acoustic energy into the air, 100% of this electrical power dissipates as resistive heat (:math:`P = I^2 R`). In compact mobile speakers and headphones, continuous DC dissipation rapidly overheats voice coil adhesives, causing voice coil warping, bobbin rubbing, and permanent open-circuit burnout. +* **Permanent Speaker Cone Displacement & Intermodulation Distortion**: DC current generates a static Lorentz force (:math:`F = B \cdot l \cdot I_{dc}`), holding the speaker cone permanently displaced away from its neutral mechanical resting position (:math:`x_{dc} = F / k_s`). In this displaced state, the spider and surround suspensions operate in their non-linear mechanical compliance region. This restricts allowable linear excursion, produces premature bottoming-out, and generates severe intermodulation distortion (IMD) between low-frequency and high-frequency content. +* **Audible Clicks, Pops, and Thumps**: When starting, stopping, pausing, or gating an audio stream with a DC offset, the signal value abruptly steps between zero and the DC level. In the frequency domain, an instantaneous step function generates a wideband acoustic burst, perceived by the user as an annoying and unprofessional click, pop, or low-frequency thump. +* **Downstream DSP Algorithm Corruption**: Modern audio algorithms assume that input signals have zero mean (:math:`E[x] = 0`): + + - **Acoustic Echo Cancellation (AEC) & Beamforming (TDFB)**: Adaptive FIR filters adjust their weights via gradient descent (LMS/NLMS). A static DC offset skews gradient estimates, slows filter convergence, and causes adaptive cancellation filters to diverge. + - **Dynamic Range Compression (DRC)**: Envelope detectors compute signal energy via rectification or squaring. DC bias artificially elevates the measured signal energy, causing the compressor to continuously duck gain even during complete acoustic silence. + - **Voice Activity Detection (VAD) & Keyword Spotters (TFLM)**: Neural networks and energy-based detectors misinterpret DC energy as acoustic voice activity, preventing DSP power islands from entering low-power sleep states. + +.. graphviz:: + :caption: DC Offset Origins and Acoustic / DSP Hazards in Audio Pipelines + + digraph dc_hazards { + bgcolor="transparent"; + rankdir=LR; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_origins { + label="DC Offset Origins"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + adc_bias [label="ADC Preamplifier Offset\n& Transistor Mismatch", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + pdm_leak [label="PDM Digital MEMS\nDecimation Filter Leakage", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + nonlinear [label="Non-Linear Audio Effects\n(Waveshapers / Exciters)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + } + + sum_node [label="Audio Stream\nwith DC Bias\n(Non-Zero Mean)", fillcolor="#C53030", fontcolor="#FFFFFF", shape=ellipse]; + + subgraph cluster_hazards { + label="System Hazards Without DC Blocker"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + headroom [label="Headroom Loss &\nAsymmetric Clipping", fillcolor="#742A2A", fontcolor="#FFFFFF"]; + thermal [label="Voice Coil Thermal Burnout\n(Resistive Heating: P = I²R)", fillcolor="#742A2A", fontcolor="#FFFFFF"]; + excursion [label="Cone Offset Displacement\n& Intermodulation Distortion", fillcolor="#742A2A", fontcolor="#FFFFFF"]; + clicks [label="Audible Pops & Thumps\non Play/Pause/Mute", fillcolor="#742A2A", fontcolor="#FFFFFF"]; + dsp_error [label="AEC Divergence, DRC Ducking\n& False VAD Triggers", fillcolor="#742A2A", fontcolor="#FFFFFF"]; + } + + adc_bias -> sum_node; + pdm_leak -> sum_node; + nonlinear -> sum_node; + + sum_node -> headroom; + sum_node -> thermal; + sum_node -> excursion; + sum_node -> clicks; + sum_node -> dsp_error; + } + +--- + +.. _dcblock_filter_theory: + +2. Digital DC Blocker Filter Theory & Pole-Zero Mechanics +********************************************************* + +Sound Open Firmware eliminates DC bias using a classic, computationally efficient **first-order recursive digital high-pass filter**. + +Difference Equation & Z-Domain Transfer Function +================================================ + +The time-domain difference equation of the DC Blocker filter is expressed as: + +.. math:: + + y[n] = x[n] - x[n-1] + R \cdot y[n-1] + +where: + +* :math:`x[n]` is the current input audio sample. +* :math:`x[n-1]` is the previous input audio sample (feedforward delay). +* :math:`y[n-1]` is the previous filter output sample (feedback recursive delay). +* :math:`R` is the pole radius parameter (:math:`0 < R < 1`, typically :math:`0.98 \le R < 1.0`). +* :math:`y[n]` is the DC-free output audio sample. + +Taking the Z-transform of both sides: + +.. math:: + + Y(z) = X(z) - z^{-1} X(z) + R \cdot z^{-1} Y(z) + +.. math:: + + Y(z)(1 - R z^{-1}) = X(z)(1 - z^{-1}) + +yielding the discrete-time transfer function: + +.. math:: + + H(z) = \frac{Y(z)}{X(z)} = \frac{1 - z^{-1}}{1 - R z^{-1}} = \frac{z - 1}{z - R} + +Pole-Zero Geometry on the Complex Z-Plane +========================================= + +The transfer function reveals an exceptionally elegant geometric placement of poles and zeros: + +* **Transmission Zero at :math:`z = 1`**: The numerator :math:`(z - 1)` places an exact transmission zero on the unit circle at angle :math:`\omega = 0` (:math:`0\text{ Hz}` / DC). Evaluating the frequency response at DC (:math:`z = e^{j 0} = 1`): + + .. math:: + + H(1) = \frac{1 - 1}{1 - R} = 0 \quad (-\infty\text{ dB}) + + This mathematical null guarantees **100% complete rejection of any constant DC bias**. + +* **Stabilizing Pole at :math:`z = R`**: The denominator :math:`(z - R)` places a single pole on the positive real axis at radius :math:`R`. Because :math:`0 < R < 1`, the pole lies strictly inside the unit circle, guaranteeing **Bounded-Input Bounded-Output (BIBO) stability**. + + As frequency :math:`\omega` increases away from DC, the distance from the evaluation point :math:`e^{j \omega}` on the unit circle to the pole at :math:`z=R` rapidly approaches the distance to the zero at :math:`z=1`. The pole effectively cancels out the attenuation of the zero across higher frequencies, restoring the magnitude response back to unity (:math:`0\text{ dB}`). + +Frequency Response & Cutoff Frequency Formulation +================================================= + +At the Nyquist frequency (:math:`z = e^{j \pi} = -1`, corresponding to :math:`f_s / 2`): + +.. math:: + + H(-1) = \frac{1 - (-1)}{1 - R(-1)} = \frac{2}{1 + R} + +Since :math:`R` is very close to :math:`1.0` (for example, :math:`R = 0.995`), :math:`\frac{2}{1 + R} \approx \frac{2}{1.995} \approx 1.0025` (:math:`+0.02\text{ dB}`). Across the vast majority of the audible band (from :math:`\approx 100\text{ Hz}` to :math:`20\text{ kHz}`), the filter behaves as a virtually perfect flat wire with :math:`0\text{ dB}` gain and negligible phase distortion. + +The -3 dB cutoff frequency :math:`f_c` (the frequency at which :math:`|H(e^{j \omega_c})|^2 = \frac{1}{2}`) is derived analytically: + +.. math:: + + \cos\left(\frac{2\pi f_c}{f_s}\right) = \frac{2R}{1 + R^2} + +For values of :math:`R` close to :math:`1.0` and cutoff frequencies much lower than the sampling rate (:math:`f_c \ll f_s`), this relationship simplifies with high accuracy to the first-order approximation: + +.. math:: + + f_c \approx \frac{(1 - R) \cdot f_s}{2\pi} \quad \iff \quad R \approx 1 - \frac{2\pi f_c}{f_s} + +.. graphviz:: + :caption: Z-Domain Pole-Zero Constellation and Normalized Magnitude Frequency Response + + digraph dc_theory { + bgcolor="transparent"; + rankdir=LR; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_zplane { + label="Z-Domain Pole-Zero Constellation"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + unit_circle [label="Unit Circle (|z| = 1)\nStability Boundary", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + zero_dc [label="Transmission Zero at z = 1.0\n(Exact Rejection at 0 Hz / DC)", fillcolor="#C53030", fontcolor="#FFFFFF", shape=ellipse]; + pole_r [label="Stabilizing Pole at z = R\n(0 < R < 1, Real Axis)", fillcolor="#2B6CB0", fontcolor="#FFFFFF", shape=diamond]; + + unit_circle -> zero_dc [label="Placed on boundary"]; + unit_circle -> pole_r [label="Placed inside boundary"]; + } + + subgraph cluster_response { + label="Frequency Magnitude Response |H(f)|"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + dc_notch [label="0 Hz (DC):\n-∞ dB (Infinite Null)", fillcolor="#742A2A", fontcolor="#FFFFFF"]; + fc_point [label="Cutoff fc (-3.01 dB):\nfc ≈ (1 - R)·fs / (2π)", fillcolor="#D69E2E", fontcolor="#FFFFFF"]; + passband [label="Audible Passband (> fc):\nFlat 0.0 dB Unity Gain", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + dc_notch -> fc_point [label="Steep +6 dB/oct roll-off"]; + fc_point -> passband [label="Flattens to unity"]; + } + + zero_dc -> dc_notch [label="Enforces null", style="bold", color="#E53E3E"]; + pole_r -> passband [label="Restores passband gain", style="bold", color="#3182CE"]; + } + +--- + +.. _dcblock_step_response: + +3. Transient Step Response & The Cutoff Frequency Trade-Off +*********************************************************** + +While frequency-domain analysis shows how effectively the DC Blocker suppresses 0 Hz steady-state signals, time-domain transient analysis determines how fast the filter recovers from abrupt DC shifts. + +Time-Domain Step Response +========================= + +When an instantaneous DC offset step of magnitude :math:`\Delta_{dc}` enters the filter at sample :math:`n = 0` (such as during microphone power-on, stream unmuting, or an abrupt analog bias jump), the recursive difference equation produces: + +* At :math:`n = 0`: :math:`y[0] = \Delta_{dc} - 0 + 0 = \Delta_{dc}`. +* At :math:`n = 1`: :math:`y[1] = \Delta_{dc} - \Delta_{dc} + R \cdot y[0] = R \cdot \Delta_{dc}`. +* At :math:`n = 2`: :math:`y[2] = \Delta_{dc} - \Delta_{dc} + R \cdot y[1] = R^2 \cdot \Delta_{dc}`. +* At arbitrary sample :math:`n \ge 1`: + +.. math:: + + y[n] = \Delta_{dc} \cdot R^n + +The filter output decays toward zero along an exponential decay curve governed by the pole radius :math:`R`. + +Exponential Decay Envelope and Time Constant +============================================ + +Expressing the discrete decay in continuous time (:math:`t = n / f_s`): + +.. math:: + + R^n = e^{n \ln R} = e^{-t / \tau} + +The **decay time constant** :math:`\tau` (the duration required for the DC offset to decay to :math:`1/e \approx 36.8\%` of its initial amplitude) is: + +.. math:: + + \tau = -\frac{1}{f_s \ln R} \approx \frac{1}{f_s (1 - R)} \approx \frac{1}{2\pi f_c} + +The **settling time** :math:`t_s` required for the DC offset to decay to less than 1% (-40 dB) of its initial magnitude is approximately :math:`4.6 \cdot \tau`: + +.. math:: + + t_s \approx 4.6 \cdot \tau \approx \frac{4.6}{2\pi f_c} \approx \frac{0.73}{f_c} + +The Fundamental Engineering Dilemma +=================================== + +The relationship :math:`t_s \approx 0.73 / f_c` exposes a fundamental, inescapable engineering compromise in DC blocker design: + +1. **Ultralow Cutoff (:math:`f_c \le 20\text{ Hz}`, :math:`R \ge 0.997` at 48 kHz)**: + + - *Acoustic Advantage*: Preserves deep sub-bass musical reproduction (e.g. pipe organs, kick drums, 5-string bass guitars) with negligible amplitude attenuation and minimal low-frequency phase rotation. + - *Transient Penalty*: Settling time is long (:math:`t_s \approx 37\text{ ms}` at 20 Hz; :math:`t_s \approx 150\text{ ms}` at 5 Hz). When an abrupt DC transient or microphone handling thump occurs, a low-frequency damped transient tail lingers in the audio stream for hundreds of milliseconds. Furthermore, when :math:`R` is exceptionally close to :math:`1.0`, arithmetic truncation errors require 64-bit precision to prevent quantization hum. + +2. **Elevated Cutoff (:math:`f_c \ge 100\text{ Hz}`, :math:`R \le 0.987` at 48 kHz)**: + + - *Acoustic Advantage*: Blisteringly fast transient recovery (:math:`t_s < 7\text{ ms}`). DC offsets, microphone handling clicks, and ADC startup thumps are extinguished almost instantaneously. It also provides beneficial attenuation of infrasonic air conditioning rumble, wind noise, and physical mechanical vibrations. + - *Acoustic Penalty*: Audible roll-off of low-frequency musical bass. While unacceptable for full-range high-fidelity music playback, this response is **ideal for speech capture pipelines, teleconferencing, and voice trigger detection** where human vocal fundamentals lie above 80–100 Hz. + +Standard Configuration Presets +============================== + +Sound Open Firmware provides standard tuning presets configured for common sampling rates (16 kHz and 48 kHz): + +.. list-table:: + :widths: 15 15 15 20 35 + :header-rows: 1 + + * - Cutoff (:math:`f_c`) + - :math:`R` (@ 16 kHz) + - :math:`R` (@ 48 kHz) + - Settling Time (:math:`t_s`) + - Target Deployment Application + * - **20 Hz** + - 0.9922 + - 0.9974 + - ~37 ms + - High-fidelity music playback, studio monitors, mastering pipelines. + * - **40 Hz** + - 0.9844 + - 0.9948 + - ~18 ms + - Consumer multimedia playback, laptop speakers with limited bass extension. + * - **80 Hz** + - 0.9691 + - 0.9896 + - ~9 ms + - General communications capture, teleconferencing headsets. + * - **100 Hz** + - 0.9615 + - 0.9870 + - ~7 ms + - Voice assistant capture, keyword spotters, noisy mobile microphones. + * - **150 Hz** + - 0.9431 + - 0.9804 + - ~5 ms + - **SOF Default Preset**: Aggressive rumble suppression and ultra-fast DC settling. + +.. graphviz:: + :caption: Transient Step Response and DC Settling Times Across Filter Radius R + + digraph dc_step { + bgcolor="transparent"; + rankdir=TB; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + step_in [label="Input DC Transient Step (Δdc = +1.0 at n = 0)", fillcolor="#C53030", fontcolor="#FFFFFF"]; + + subgraph cluster_decay { + label="Decay Envelopes: y[n] = Δdc · Rⁿ"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + fast_decay [label="High Cutoff (R = 0.980, fc ≈ 150 Hz)\n• τ ≈ 1.0 ms\n• Settling time ts ≈ 5 ms\n• Rapid recovery; attenuates sub-bass\n• Ideal for Speech & Mic Capture", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + med_decay [label="Medium Cutoff (R = 0.990, fc ≈ 80 Hz)\n• τ ≈ 2.0 ms\n• Settling time ts ≈ 9 ms\n• Balanced voice & communications profile", fillcolor="#3182CE", fontcolor="#FFFFFF"]; + slow_decay [label="Low Cutoff (R = 0.997, fc ≈ 20 Hz)\n• τ ≈ 8.0 ms\n• Settling time ts ≈ 37 ms\n• Preserves full sub-bass musical fidelity\n• Ideal for Hi-Fi Playback", fillcolor="#805AD5", fontcolor="#FFFFFF"]; + } + + step_in -> fast_decay [label="R = 0.980"]; + step_in -> med_decay [label="R = 0.990"]; + step_in -> slow_decay [label="R = 0.997"]; + } + +--- + +.. _dcblock_fixed_point: + +4. Fixed-Point Arithmetic, Precision & Limit Cycle Elimination +************************************************************** + +In textbook floating-point arithmetic, evaluating :math:`y[n] = x[n] - x[n-1] + R \cdot y[n-1]` is straightforward. However, Sound Open Firmware operates predominantly on energy-efficient embedded digital signal processors utilizing fixed-point integer mathematics. Implementing recursive filters with poles close to the unit circle under fixed-point arithmetic introduces severe hazards that require rigorous numerical engineering. + +The Hazards of Fixed-Point Recursion +==================================== + +When the pole radius :math:`R` approaches :math:`1.0` (e.g. :math:`R = 0.997`): + +* **Limit Cycle Oscillations**: In a recursive filter, the product :math:`R \cdot y[n-1]` must be rounded to fit back into the state variable format. If naive truncation (floor) or rounding is applied, the state variable can become trapped in a non-zero repeating state even when the input signal has dropped to absolute zero (:math:`x[n] = 0`). These self-sustaining limit cycles manifest as an audible low-level whine, quantization hum, or persistent phantom DC drift. +* **Coefficient Quantization Drift**: If the coefficient :math:`R` lacks sufficient fractional bit depth, rounding :math:`R` can shift the pole position. Under coarse quantization, an intended :math:`R = 0.999` might round up to :math:`1.0` (turning the filter into a pure integrator that accumulates numerical overflow until saturation) or round down significantly (shifting :math:`f_c` from 20 Hz up to 150 Hz). + +High-Precision Data Path in SOF +=============================== + +To eliminate limit cycles and preserve mathematical precision, SOF implements the DC Blocker using a high-precision fixed-point architecture: + +* **Coefficient Representation (:math:`Q2.30`)**: The coefficient :math:`R` is stored as a 32-bit signed integer in :math:`Q2.30` format (2 integer bits including sign, 30 fractional bits). This yields a fractional quantization resolution of: + + .. math:: + + \Delta Q = 2^{-30} \approx 9.31 \times 10^{-10} + + Unity gain ($1.0$) is defined as `ONE_Q2_30` ($0x40000000 = 1073741824$). This immense fractional depth allows exact placement of poles arbitrarily close to the unit circle without quantization rounding error. + +* **State Variables (:math:`Q1.31`)**: The delay line states `x_prev` (:math:`x[n-1]`) and `y_prev` (:math:`y[n-1]`) are maintained as 32-bit signed integers in :math:`Q1.31` format, matching the DSP native audio sample depth. + +* **64-Bit Multiplication & Accumulation (:math:`Q3.61`)**: + + Multiplying the coefficient :math:`R` (:math:`Q2.30`) by the recursive state :math:`y[n-1]` (:math:`Q1.31`) yields a 64-bit product in :math:`Q3.61` format: + + .. math:: + + \text{Format}(R \cdot y[n-1]) = Q(2 + 1) . (30 + 31) = Q3.61 + +* **Symmetric Rounding and Shifting**: + + To recombine the recursive product with the feedforward difference :math:`(x[n] - x[n-1])`, the 64-bit product is scaled and rounded back to 32-bit resolution. SOF utilizes symmetric rounding (`Q_SHIFT_RND` or `AE_ROUND32F64SSYM`), adding a half-LSB rounding bias (:math:`2^{29}`) before arithmetic right-shifting. This completely eliminates DC bias accumulation and suppresses limit cycle oscillations into inaudibility below :math:`-140\text{ dB}`. + +* **Saturated Clamping**: + + The final output is passed through 32-bit saturation (`sat_int32()`). If transient numerical overshoot occurs, the output smoothly clamps to :math:`[-2^{31}, 2^{31}-1]` rather than wrapping around to the opposite polarity, preventing catastrophic full-scale crackles. + +Passthrough Bypass Mode +======================= + +When the DC Blocker is unconfigured or disabled via ALSA mixer controls, setting :math:`R = \text{ONE\_Q2\_30} = 1.0` transforms the transfer function into: + +.. math:: + + H(z) = \frac{1 - z^{-1}}{1 - 1 \cdot z^{-1}} = 1.0 + +In this state, the recursive pole perfectly cancels the feedforward zero, transforming the filter into a mathematically bit-exact, zero-attenuation passthrough. + +.. graphviz:: + :caption: Fixed-Point Arithmetic Data Path and 64-Bit Intermediate Accumulation + + digraph dc_arithmetic { + bgcolor="transparent"; + rankdir=LR; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + x_in [label="Input Sample x[n]\n(32-bit Q1.31)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + sub_diff [label="Feedforward Difference\nx[n] - x[n-1]\n(64-bit)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + z_x [label="Unit Delay\nx[n-1]\n(struct dcblock_state)", fillcolor="#718096", fontcolor="#FFFFFF"]; + + mul_r [label="64-Bit Multiplier\nR (Q2.30) × y[n-1] (Q1.31)\nProduct: Q3.61", fillcolor="#D69E2E", fontcolor="#FFFFFF"]; + r_coef [label="Pole Radius R\n(32-bit Q2.30)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + z_y [label="Recursive State\ny[n-1]\n(struct dcblock_state)", fillcolor="#718096", fontcolor="#FFFFFF"]; + + acc_sum [label="64-Bit Accumulator\n(Diff + R·y[n-1])", fillcolor="#D69E2E", fontcolor="#FFFFFF"]; + shift_rnd [label="Symmetric Rounding Shift\nQ_SHIFT_RND(61, 31)\n(Eliminates Limit Cycles)", fillcolor="#805AD5", fontcolor="#FFFFFF"]; + sat_out [label="32-Bit Saturation Clamp\nsat_int32()\nOutput y[n]", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + x_in -> sub_diff [label="Positive (+)", color="#3182CE"]; + x_in -> z_x [label="Store state"]; + z_x -> sub_diff [label="Negative (-)", color="#E53E3E"]; + + r_coef -> mul_r; + z_y -> mul_r; + + sub_diff -> acc_sum [label="64-bit diff"]; + mul_r -> acc_sum [label="64-bit prod"]; + + acc_sum -> shift_rnd; + shift_rnd -> sat_out; + sat_out -> z_y [label="Update y[n-1] feedback", style="dashed", color="#38A169"]; + } + +--- + +.. _dcblock_multichannel: + +5. Multi-Channel Processing & Buffer Stream Traversal +***************************************************** + +Audio streams in SOF frequently carry multi-channel audio—ranging from stereo playback (2 channels) up to dense microphone arrays (4, 6, or 8 channels for beamforming and speech recognition). The DC Blocker provides multi-channel stream processing with state isolation. + +Per-Channel Independent State Tracking +====================================== + +Because each physical microphone and audio channel possesses unique analog DC offsets and distinct signal histories, filter state variables must be strictly isolated. Cross-channel state contamination would destroy stereo imaging and introduce cross-channel phase distortion. + +SOF defines dedicated state tracking in private component data: + +.. code-block:: text + + struct comp_data { + struct dcblock_state state[PLATFORM_MAX_CHANNELS]; + int32_t R_coeffs[PLATFORM_MAX_CHANNELS]; + ... + }; + +* `state[ch].x_prev`: Tracks the prior input sample :math:`x[n-1]` independently for channel `ch`. +* `state[ch].y_prev`: Tracks the prior recursive output sample :math:`y[n-1]` independently for channel `ch`. +* `R_coeffs[ch]`: Stores the independent pole coefficient for channel `ch`. This enables **heterogeneous channel configurations**—for example, applying an aggressive :math:`150\text{ Hz}` cutoff on primary voice capture microphones while maintaining a gentle :math:`20\text{ Hz}` cutoff on an acoustic echo cancellation reference loopback channel. + +Interleaved Stream Traversal Mechanics +====================================== + +Audio buffers in SOF are formatted as interleaved PCM frames (:math:`L, R, L, R...` or :math:`C_0, C_1, C_2...`). Processing interleaved multi-channel buffers requires stepping through memory with a channel stride: + +1. **Outer Channel / Inner Frame Loop**: The processing routine iterates across channels :math:`ch \in [0, nch-1]`. For each channel, the filter loads `state[ch].x_prev`, `state[ch].y_prev`, and `R_coeffs[ch]`. +2. **Channel-Strided Pointer Stepping**: Pointers advance across interleaved frames using a stride increment: + + .. math:: + + \text{stride} = nch \times \text{sizeof}(\text{sample}) + +3. **Buffer Wrap Boundary Handling**: To prevent pointer corruption across circular ring buffers, the processing loop checks available non-wrapping frames using `audio_stream_samples_without_wrap()`, process chunks up to the buffer boundary, and then invokes `audio_stream_wrap()` to seamlessly loop pointers back to the buffer base. + +Format Adaptability Across Audio Depths +======================================= + +The DC Blocker supports all standard SOF PCM frame formats via dedicated inner processing routines: + +* **S16_LE (16-bit)**: Samples are loaded and sign-extended by 16 bits to :math:`Q1.31` for filtering, then scaled and saturated back to 16 bits via `sat_int16(Q_SHIFT_RND(y, 31, 15))`. +* **S24_4LE (24-bit in 32-bit container)**: Samples are shifted by 8 bits to :math:`Q1.31`, processed through the 64-bit accumulator, and rounded back to 24 bits with `sat_int24(Q_SHIFT_RND(y, 31, 23))`. +* **S32_LE (32-bit native)**: Samples undergo full 32-bit direct processing with zero bit-depth truncation. + +.. graphviz:: + :caption: Multi-Channel Interleaved Buffer Traversal and Independent State Isolation + + digraph dc_multichannel { + bgcolor="transparent"; + rankdir=TB; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_interleaved_in { + label="Source Stream Buffer (Interleaved Frames)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + in_c0 [label="Frame 0: Ch 0 (Left)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + in_c1 [label="Frame 0: Ch 1 (Right)", fillcolor="#805AD5", fontcolor="#FFFFFF"]; + in_c2 [label="Frame 1: Ch 0 (Left)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + in_c3 [label="Frame 1: Ch 1 (Right)", fillcolor="#805AD5", fontcolor="#FFFFFF"]; + + in_c0 -> in_c1 -> in_c2 -> in_c3 [style="invis"]; + } + + subgraph cluster_states { + label="Component Private Data: Isolated Channel States"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + state_c0 [label="Channel 0 State Structure:\n• x_prev[0], y_prev[0]\n• R_coeffs[0] (fc = 100 Hz)", fillcolor="#2C5282", fontcolor="#FFFFFF"]; + state_c1 [label="Channel 1 State Structure:\n• x_prev[1], y_prev[1]\n• R_coeffs[1] (fc = 100 Hz)", fillcolor="#553C9A", fontcolor="#FFFFFF"]; + } + + subgraph cluster_interleaved_out { + label="Sink Stream Buffer (DC-Free Audio)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + out_c0 [label="Frame 0: Ch 0 Clean", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + out_c1 [label="Frame 0: Ch 1 Clean", fillcolor="#38A169", fontcolor="#FFFFFF"]; + out_c2 [label="Frame 1: Ch 0 Clean", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + out_c3 [label="Frame 1: Ch 1 Clean", fillcolor="#38A169", fontcolor="#FFFFFF"]; + + out_c0 -> out_c1 -> out_c2 -> out_c3 [style="invis"]; + } + + in_c0 -> state_c0 [label="Stride load Ch 0", color="#3182CE"]; + in_c2 -> state_c0 [label="Stride load Ch 0", color="#3182CE"]; + + in_c1 -> state_c1 [label="Stride load Ch 1", color="#805AD5"]; + in_c3 -> state_c1 [label="Stride load Ch 1", color="#805AD5"]; + + state_c0 -> out_c0 [label="Write Ch 0", color="#38A169"]; + state_c0 -> out_c2 [label="Write Ch 0", color="#38A169"]; + + state_c1 -> out_c1 [label="Write Ch 1", color="#38A169"]; + state_c1 -> out_c3 [label="Write Ch 1", color="#38A169"]; + } + +--- + +.. _dcblock_simd: + +6. SIMD Vector Acceleration Across DSP Architectures +**************************************************** + +To achieve ultra-low power consumption and minimize DSP clock cycle consumption (MIPS), Sound Open Firmware implements specialized hardware vector optimizations across multiple DSP architectures. + +Cadence Tensilica Xtensa HiFi 3 Optimization +============================================ + +On Cadence Tensilica Xtensa HiFi 3 DSP cores: + +* **64-Bit Vector Accumulation (`AE_MULF32S_LL`)**: Multiplies the 32-bit :math:`Q2.30` coefficient :math:`R` by the 32-bit :math:`Q1.31` recursive state :math:`y[n-1]` using the lower 32 bits of 64-bit vector registers, generating a 64-bit product in :math:`Q2.62` representation. +* **Vector Subtraction & Addition (`AE_SUB64`, `AE_ADD64S`)**: Performs 64-bit subtraction :math:`(x[n] - x[n-1])` and 64-bit addition in single-cycle operations. +* **Symmetric Rounding (`AE_ROUND32F64SSYM`)**: Symmetrically rounds the 64-bit accumulated result back to 32 bits in a single hardware cycle. +* **Hardware Circular Buffer Addressing (`AE_SETCBEGIN0`, `AE_SETCEND0`)**: Programs the hardware circular address register `CBEGIN0` and `CEND0` with the source buffer boundary. The DSP automatically wraps input read pointers (`AE_L16_XC`, `AE_L32_XC`) in hardware with zero branching overhead. + +Cadence Tensilica Xtensa HiFi 4 Optimization: Dual Circular Registers +===================================================================== + +Cadence Tensilica Xtensa HiFi 4 cores introduce dual independent circular address registers, enabling a higher tier of throughput optimization: + +* **Simultaneous Source and Sink Circular Auto-Wrapping**: + + - Source buffer boundaries are bound to circular register 0 (`AE_SETCBEGIN0`, `AE_SETCEND0`). + - Sink buffer boundaries are bound to circular register 1 (`AE_SETCBEGIN1`, `AE_SETCEND1`). + +* **Branchless Inner Loop Execution**: + + In HiFi 3 or scalar C, the firmware must subdivide execution into chunks bounded by the closest wrap boundary between source and sink buffers. On HiFi 4, hardware automatically wraps both read pointers (`AE_L16_XC`, `AE_L32_XC`) and write pointers (`AE_S16_0_XC1`, `AE_S32_L_XC1`) simultaneously. As a result, the entire buffer of `frames` executes in a **single, unfragmented, branchless loop**, maximizing instruction cache efficiency and minimizing pipeline stalls. + +Xtensa HiFi 5 & Vector SIMD +=========================== + +On Cadence Tensilica Xtensa HiFi 5 cores, 256-bit SIMD registers execute 8 parallel 32-bit fixed-point operations concurrently. In multi-microphone array pipelines (such as 8-channel microphone arrays on smart speakers and conference room bars), HiFi 5 processes all 8 channels simultaneously across vector lanes. + +Portable Generic Scalar C +========================= + +For embedded microcontrollers lacking proprietary DSP extensions—such as the PJRC Teensy 4.1 (ARM Cortex-M7) and Espressif ESP32-P4 (RISC-V)—SOF provides a clean, portable scalar C implementation (`dcblock_generic.c`). The compiler maps the 64-bit accumulation and `Q_SHIFT_RND` macros to native hardware 32-bit multiplier pairs with zero precision loss. + +.. graphviz:: + :caption: SIMD Execution Pipelines on Xtensa HiFi 3, HiFi 4 (Dual Circular Buffers), and Scalar Architectures + + digraph dc_simd { + bgcolor="transparent"; + rankdir=LR; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_hifi3 { + label="Tensilica Xtensa HiFi 3"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + hifi3_circ [label="Circular Source Reg 0\nAE_SETCBEGIN0 / CEND0", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + hifi3_mac [label="Vector MAC Pipeline:\n• AE_MULF32S_LL (Q2.62)\n• AE_ADD64S / AE_SUB64\n• AE_ROUND32F64SSYM", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + hifi3_loop [label="Software Chunk Loop\n(Bounded by sink wrap)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + + hifi3_circ -> hifi3_mac -> hifi3_loop; + } + + subgraph cluster_hifi4 { + label="Tensilica Xtensa HiFi 4 (Dual Circular)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + hifi4_circ [label="Dual Hardware Circular Regs:\n• CBEGIN0: Source Read (AE_L32_XC)\n• CBEGIN1: Sink Write (AE_S32_L_XC1)", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + hifi4_loop [label="Flat Branchless Loop\n(Processes all frames in 1 pass\nwith zero wrap checks)", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + hifi4_circ -> hifi4_loop [label="Hardware auto-wrap"]; + } + + subgraph cluster_generic { + label="Generic Scalar C (ARM / RISC-V)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + scalar_code [label="Standard C Implementation:\n• int64_t 64-bit math\n• Q_SHIFT_RND rounding\n• audio_stream_wrap()", fillcolor="#718096", fontcolor="#FFFFFF"]; + } + } + +--- + +.. _dcblock_pipeline_ipc: + +7. Pipeline Integration, ALSA Topology 2 & IPC Interfaces +********************************************************* + +The DC Blocker component conforms to the standardized Sound Open Firmware **Module Adapter** interface and integrates into audio pipelines defined via ALSA Topology 2. + +ALSA Topology 2 Component Widget +================================ + +In ALSA Topology 2 (`tools/topology/topology2/include/components/dcblock.conf`), the DC Blocker is defined as a specialized processing effect widget: + +* **Widget Class**: `Class.Widget."dcblock"` +* **Widget Type**: `effect` +* **UUID**: `af:ef:09:b8:81:56:b1:42:9e:d6:04:bb:01:2d:d3:84` +* **Pin Configuration**: Exactly 1 input pin (`num_input_pins 1`) and 1 output pin (`num_output_pins 1`). +* **Power Management**: `no_pm "true"` (synchronous in-place audio stream processing without autonomous power gating). + +Topology instantiation is simple and declarative: + +.. code-block:: text + + Object.Widget.dcblock."1" { + index 1 + instance 0 + } + +Module Adapter & LLEXT Runtime Dynamic Linking +============================================== + +The DC Blocker implements the standard `struct module_interface` API: + +* `init`: Allocates private component data (`struct comp_data`), zeroes state delay lines, and creates a `comp_data_blob_handler` for dynamic control configuration. +* `prepare`: Validates that exactly one source buffer and one sink buffer are connected, negotiates frame formats (:math:`S16\_LE`, :math:`S24\_4LE`, or :math:`S32\_LE`), resolves the matching SIMD processing function from `dcblock_fnmap[]`, and extracts initial coefficients from the topology configuration blob. +* `process_audio_stream`: Calls the selected architecture-optimized processing function to transform input frames into DC-free sink audio. +* `reset`: Flushes internal delay line states (`x_prev = 0, y_prev = 0`) to prevent state discontinuities across stream restarts. +* `free`: Releases private memory and frees the blob handler. + +For platforms leveraging modular firmware packaging, the DC Blocker exports a standard Loadable Extension manifest (`SOF_LLEXT_MODULE_MANIFEST("DCBLOCK", ...)`), enabling dynamic loading into DSP SRAM on demand. + +Dynamic IPC Configuration Blobs (IPC3 & IPC4) +============================================= + +Cutoff frequencies can be updated dynamically at runtime without interrupting active audio playback or capture: + +* **IPC3**: Delivered via `SOF_IPC_COMP_SET_DATA` carrying a serialized binary configuration payload. +* **IPC4**: Delivered via `SET_LARGE_CONFIG` messages using the standard multi-fragment data blob protocol. The `comp_data_blob_handler` handles fragment reassembly, bounds validation, and atomic pointer assignment to `cd->config`. + +End-to-End Pipeline Deployments +=============================== + +The DC Blocker occupies critical strategic positions across SOF audio processing graphs: + +1. **Capture Pipeline (Microphone Ingestion)**: Positioned immediately after the hardware DAI Copier or PDM Receiver. Removing ADC DC offset before the signal reaches downstream processing prevents divergence in Acoustic Echo Cancellation (AEC), eliminates false energy triggers in Voice Activity Detectors (VAD), and stabilizes beamforming weights in the Time-Domain Fixed Beamformer (TDFB). +2. **Playback Pipeline (Amplifier & Driver Protection)**: Positioned before Volume Control, Dynamic Range Compression (DRC), and Smart Amp. Suppressing DC offsets protects speaker voice coils against thermal burning, prevents cone resting displacement, maximizes positive/negative dynamic headroom, and eliminates pops during play/pause transitions. +3. **Inter-Stage DC Decoupling**: Placed downstream of non-linear DSP algorithms (such as harmonic exciters, waveshapers, or soft clippers) to strip away artificial DC biases generated by non-linear distortion. + +.. graphviz:: + :caption: System Pipeline Topology: Capture Path Pre-Processing and Playback Protection Deployments + + digraph dc_pipeline { + bgcolor="transparent"; + rankdir=TB; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_capture { + label="Capture Pipeline (Microphone Ingestion & Pre-Processing)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + pdm_mic [label="PDM Digital Mics /\nAnalog ADC Front-End", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + copier_rx [label="DAI Copier (RX)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + dcb_cap [label="DC Blocker\n(fc = 100 Hz / 150 Hz)\n• Strips ADC offset\n• Rejects wind/handling", fillcolor="#C53030", fontcolor="#FFFFFF"]; + aec [label="Acoustic Echo Canceller\n(AEC)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + tdfb [label="Beamformer\n(TDFB)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + vad [label="Voice Activity Detector\n& Keyword Spotter (TFLM)", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + pdm_mic -> copier_rx -> dcb_cap; + dcb_cap -> aec [label="Zero-mean audio"]; + aec -> tdfb -> vad; + } + + subgraph cluster_playback { + label="Playback Pipeline (Amplifier & Transducer Protection)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + host_tx [label="Host Audio Stream\n(Decoder / Media Stream)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + dcb_play [label="DC Blocker\n(fc = 20 Hz / 40 Hz)\n• Preserves sub-bass\n• Prevents voice coil heat", fillcolor="#C53030", fontcolor="#FFFFFF"]; + eq [label="Equalizer\n(EQ FIR / IIR)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + drc [label="Dynamic Range\nCompressor (DRC)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + smart_amp [label="Smart Amp /\nDAI Copier (TX)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + speaker [label="Loudspeaker Driver\n(Zero DC Current / P=0W)", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + host_tx -> dcb_play; + dcb_play -> eq -> drc -> smart_amp -> speaker; + } + } + +--- + +.. _dcblock_tuning_references: + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level implementation details, mathematical tuning scripts, and topology configurations: + +* **Upstream Component Source Files**: + + - `thesofproject/sof: src/audio/dcblock/README.md `_: Component overview, directory layout, and architecture summary. + - `src/audio/dcblock/dcblock.c `_: Module lifecycle management (`init`, `prepare`, `process`, `reset`, `free`). + - `src/audio/dcblock/dcblock.h `_: Component private data structures (`struct dcblock_state`, `struct comp_data`), format map dispatch, and function declarations. + - `src/audio/dcblock/dcblock_generic.c `_: Portable scalar C fixed-point implementation with 64-bit accumulation and symmetric rounding. + - `src/audio/dcblock/dcblock_hifi3.c `_: Xtensa HiFi 3 SIMD vector optimizations and single circular source addressing. + - `src/audio/dcblock/dcblock_hifi4.c `_: Xtensa HiFi 4 optimizations featuring simultaneous dual circular buffer registers for branchless streaming. + - `src/audio/dcblock/dcblock_ipc3.c `_ & `dcblock_ipc4.c `_: Protocol-specific IPC handlers and stream parameter negotiation. + +* **Topology Definitions**: + + - `tools/topology/topology2/include/components/dcblock.conf `_: ALSA Topology 2 class definition for the DC Blocker widget. + +* **GNU Octave / MATLAB Tuning Scripts**: + + - `src/audio/dcblock/tune/sof_example_dcblock.m `_: Interactive script calculating optimal :math:`R` coefficients for target cutoff frequencies (20–200 Hz across 16 kHz and 48 kHz rates), exporting topology `.conf`, `.m4`, and binary `.bin` configuration blobs. + - `src/audio/dcblock/tune/sof_dcblock_plot_transferfn.m `_: Evaluates and plots the filter frequency magnitude transfer function :math:`H(z)`. + - `src/audio/dcblock/tune/sof_dcblock_plot_stepfn.m `_: Simulates and plots the transient time-domain step response to verify settling time and decay envelopes. + +Related Subsystem Architecture Guides +===================================== + +* :ref:`crossover`: Linkwitz-Riley 4th-order multi-way frequency division across active loudspeaker drivers. +* :ref:`drc_multiband_drc`: Dynamic range compression, soft knee limiting, lookahead delays, and multi-band dynamics control. +* :ref:`eq_fir_iir`: Finite and Infinite Impulse Response equalizers, cascaded biquads, and parametric speaker compensation. +* :ref:`volume_module`: Per-channel gain scaling, smooth volume ramping, and zero-crossing detection. +* :ref:`module_framework`: Standardized module lifecycle, Source/Sink APIs, and memory management. +* :ref:`pipeline_architecture`: How processing modules interconnect into directed acyclic audio graphs. diff --git a/developer_guides/firmware/pipeline_architecture.rst b/developer_guides/firmware/pipeline_architecture.rst index 769c8b70..2e1b69bd 100644 --- a/developer_guides/firmware/pipeline_architecture.rst +++ b/developer_guides/firmware/pipeline_architecture.rst @@ -467,6 +467,7 @@ Related Guides * :ref:`eq_fir_iir`: Finite and Infinite Impulse Response equalizers, linear-phase FIR tap folding, Direct Form I biquad cascades, and dynamic IPC blob updates. * :ref:`drc_multiband_drc`: Single-band and multi-band dynamic range compression, lookahead pre-delay buffers, adaptive release ballistics, and Linkwitz-Riley crossover splitting. * :ref:`crossover`: Linkwitz-Riley 4th-order (LR4) active multi-driver crossovers, 2-way/3-way/4-way splitting with all-pass phase alignment, and 1-to-N multi-sink buffer distribution. +* :ref:`dcblock`: First-order recursive high-pass DC blocking filter, 0 Hz transmission nulling, 64-bit fixed-point accumulation, and dual circular buffer SIMD acceleration. * :ref:`fw_init_boot`: Boot flow, hardware mailbox FW Ready handshake, and Zephyr initialization. * :ref:`topology2`: How pipelines and widgets are declared using ALSA Topology 2.0 configuration classes. * :ref:`sof_hostless_firmware`: How to create static pipelines compiled into ROM for standalone microcontrollers. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 41dfc66f..4fc84a74 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -47,7 +47,7 @@ Audio Processing Modules & Algorithms * :ref:`eq_fir_iir` (High-level architecture; also see upstream `FIR README `_ & `IIR README `_) * :ref:`drc_multiband_drc` (High-level architecture; also see upstream `DRC README `_ & `Multiband DRC README `_) * :ref:`crossover` (High-level architecture; also see upstream `crossover README `_) -* `DC Blocker `_ +* :ref:`dcblock` (High-level architecture; also see upstream `dcblock README `_) * `Time-Domain Fixed Beamformer (TDFB) `_ * `RTNR Noise Reduction `_ * `TensorFlow Lite Micro (TFLM) `_ @@ -92,6 +92,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/eq_fir_iir firmware/drc_multiband_drc firmware/crossover + firmware/dcblock rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 52390f743eb1a9396b548f7b80fc64ef3a34e9ea Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Fri, 18 Sep 2026 18:30:14 +0100 Subject: [PATCH 15/64] docs: developer_guides: add high-level tdfb architecture guide Add a comprehensive, high-level developer architecture guide for the Time-Domain Fixed Beamformer (TDFB) subsystem (developer_guides/firmware/tdfb.rst). Topics covered: - Spatial acoustics and microphone array principles: Far-field planar wavefront propagation, Time Difference of Arrival (TDOA), and advantages of Filter-and-Sum over Delay-and-Sum beamforming. - Filter-and-Sum FIR architecture and multi-channel mixing: Up to 16 FIR filter instances (<= 256 taps), input_channel_select, output_channel_mix, output_stream_mix, multi-beam simultaneous extraction (dual-beam stereo capture, speech + noise reference), and Q5.27 fixed-point headroom management. - Microphone array geometries: Uniform/non-uniform linear arrays (1D), circular ring arrays (2D 360-degree coverage), planar/rectangular arrays (azimuth + elevation 3D), and Q4.12 meter coordinates. - Direction of Arrival (DOA) tracking and acoustic localization: Pre-emphasis IIR filtering (500 Hz - 4 kHz speech band), ambient noise floor tracking with primitive VAD gate, pairwise cross-correlation lag extraction, 8-iteration geometric error minimization, and two-pole angle smoothing. - Host IPC control plane: ALSA mixer controls (process switch, direction tracking switch, steer azimuth enum, azimuth estimate enum), and asynchronous rate-limited host notifications (>= 200 ms). - SIMD vector acceleration across Tensilica Xtensa HiFi 3 (dual 32x16 MAC, circular buffer auto-wrapping), HiFi 2 EP, and portable scalar C. - System capture pipeline integration: Ingestion after DC Blocker, supplying high-SNR directional speech to AEC, RTNR, and VAD/TFLM. - Tuning workflows and upstream source code references. - Seven native vector Graphviz SVG architecture diagrams. - Clean Sphinx build under -W --keep-going and zero woke violations. Signed-off-by: Liam Girdwood --- .../firmware/pipeline_architecture.rst | 1 + developer_guides/firmware/tdfb.rst | 816 ++++++++++++++++++ developer_guides/index.rst | 3 +- 3 files changed, 819 insertions(+), 1 deletion(-) create mode 100644 developer_guides/firmware/tdfb.rst diff --git a/developer_guides/firmware/pipeline_architecture.rst b/developer_guides/firmware/pipeline_architecture.rst index 2e1b69bd..8b5a4ac7 100644 --- a/developer_guides/firmware/pipeline_architecture.rst +++ b/developer_guides/firmware/pipeline_architecture.rst @@ -468,6 +468,7 @@ Related Guides * :ref:`drc_multiband_drc`: Single-band and multi-band dynamic range compression, lookahead pre-delay buffers, adaptive release ballistics, and Linkwitz-Riley crossover splitting. * :ref:`crossover`: Linkwitz-Riley 4th-order (LR4) active multi-driver crossovers, 2-way/3-way/4-way splitting with all-pass phase alignment, and 1-to-N multi-sink buffer distribution. * :ref:`dcblock`: First-order recursive high-pass DC blocking filter, 0 Hz transmission nulling, 64-bit fixed-point accumulation, and dual circular buffer SIMD acceleration. +* :ref:`tdfb`: Spatial acoustic filtering, filter-and-sum FIR banks, multi-microphone array geometries, and autonomous Direction of Arrival (DOA) tracking. * :ref:`fw_init_boot`: Boot flow, hardware mailbox FW Ready handshake, and Zephyr initialization. * :ref:`topology2`: How pipelines and widgets are declared using ALSA Topology 2.0 configuration classes. * :ref:`sof_hostless_firmware`: How to create static pipelines compiled into ROM for standalone microcontrollers. diff --git a/developer_guides/firmware/tdfb.rst b/developer_guides/firmware/tdfb.rst new file mode 100644 index 00000000..7a7b3646 --- /dev/null +++ b/developer_guides/firmware/tdfb.rst @@ -0,0 +1,816 @@ +.. _tdfb: + +Time-Domain Fixed Beamformer (TDFB) Architecture +################################################ + +The **Time-Domain Fixed Beamformer (TDFB)** subsystem in Sound Open Firmware provides spatial acoustic filtering, directional sound capture, multi-microphone array processing, and real-time Direction of Arrival (DOA) tracking for voice user interfaces and telecommunication audio pipelines. + +In modern computing devices—including laptops, smart displays, conference systems, and mobile headsets—microphones must operate in acoustically hostile environments characterized by room reverberation, ambient diffuse noise, cooling fan hum, mechanical keyboard chatter, and competing background talkers. A single omnidirectional microphone captures sound equally from all directions, forcing downstream speech recognition engines and human listeners to contend with a low Signal-to-Noise Ratio (SNR). + +To overcome the physical limitations of single microphones, Sound Open Firmware implements a **Filter-and-Sum Time-Domain Beamformer**. By exploiting acoustic wave propagation delays across an array of physically separated microphones, the TDFB selectively amplifies acoustic wavefronts arriving from a configured look direction (the *acoustic beam*) while constructively canceling sound arriving from off-axis directions. Furthermore, the TDFB integrates an autonomous, low-latency **Direction of Arrival (DOA)** tracking engine that continuously estimates the spatial azimuth of an active talker and notifies the host operating system to dynamically steer the listening beam. + +This guide provides a comprehensive, high-level architectural walkthrough of the TDFB subsystem, analyzing planar wavefront propagation, filter-and-sum FIR topologies, microphone array geometry configurations, autonomous DOA tracking mechanics, ALSA Topology 2 / IPC dynamic steering controls, and SIMD hardware acceleration without delving into low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +.. _tdfb_spatial_principles: + +1. Spatial Acoustics & Microphone Array Principles +************************************************** + +Beamforming is the spatial analogue of spectral filtering: whereas a standard audio filter discriminates signals based on temporal frequency (Hertz), an acoustic beamformer discriminates signals based on spatial arrival angle (azimuth and elevation). + +Far-Field Wavefront Propagation and Time Difference of Arrival (TDOA) +===================================================================== + +Sound travels through air as acoustic pressure waves at a temperature-dependent speed of sound :math:`v \approx 340\text{ m/s}`. When an acoustic source (such as a person speaking) is positioned at a distance significantly greater than the physical dimensions of the microphone array, the sound waves arriving at the sensors approximate **planar wavefronts**: + +.. math:: + + \text{Distance } r \gg \frac{2 D^2}{\lambda} + +where :math:`D` is the array aperture (maximum distance between microphones) and :math:`\lambda = v / f` is the acoustic wavelength. + +Because each microphone occupies a distinct position in three-dimensional space, a planar wavefront striking the array reaches each sensor at a slightly different instant in time. For a simple two-microphone linear array with inter-sensor spacing :math:`d`, an acoustic wavefront arriving from azimuth angle :math:`\theta` (measured relative to the broadside perpendicular axis) travels an additional spatial distance :math:`\Delta x = d \sin\theta`. + +The resulting **Time Difference of Arrival (TDOA)** :math:`\tau` between the two microphones is: + +.. math:: + + \tau = \frac{\Delta x}{v} = \frac{d \sin\theta}{v} + +By delaying the signal from the first microphone by exactly :math:`\tau` before summing the two microphone channels together, the desired signals arriving from angle :math:`\theta` align perfectly in phase and sum constructively (+6 dB voltage boost). Conversely, sounds arriving from other angles arrive with phase discrepancies, causing destructive interference and spatial attenuation. + +Delay-and-Sum vs Filter-and-Sum Beamforming +=========================================== + +While elementary beamformers rely solely on pure time delays (*Delay-and-Sum*), practical broadband audio beamforming requires a **Filter-and-Sum** architecture: + +* **Shortcomings of Pure Delay-and-Sum**: + + - In a discrete digital system sampled at :math:`f_s` (e.g. 48 kHz), integer sample delays provide only coarse time quantization (:math:`1 / 48000 \approx 20.8\ \mu\text{s}` steps, corresponding to :math:`\approx 7\text{ mm}` spatial resolution). Steering an acoustic beam to arbitrary non-integer angles requires fractional delay interpolation. + - Delay-and-Sum beam patterns vary drastically with frequency. At low frequencies where wavelength is much larger than array aperture (:math:`\lambda \gg d`), phase differences across sensors are negligible, resulting in an excessively wide, omnidirectional beam. At high frequencies (:math:`\lambda < 2d`), spatial aliasing introduces undesirable grating lobes that pass off-axis noise. + +* **Advantages of Filter-and-Sum in SOF**: + + - Rather than applying a single scalar delay, each microphone channel passes through a dedicated Finite Impulse Response (FIR) filter before summation. + - The FIR filters execute arbitrary fractional delays, spectral shaping, and phase corrections simultaneously. + - FIR coefficients can be synthesized via optimization algorithms to deliver **frequency-invariant beamwidths**, equalize microphone chassis resonances, and place deep attenuation nulls in the direction of known stationary noise sources (such as laptop cooling vents or keyboard mechanisms). + +.. graphviz:: + :caption: Spatial Acoustic Wavefront Propagation and Planar Time Difference of Arrival (TDOA) + + digraph tdfb_wavefront { + bgcolor="transparent"; + rankdir=LR; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_sound { + label="Acoustic Sound Source"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + source [label="Active Talker\n(Far-field acoustic origin)\nAngle θ", fillcolor="#2B6CB0", fontcolor="#FFFFFF", shape=ellipse]; + } + + subgraph cluster_propagation { + label="Planar Acoustic Propagation (v ≈ 340 m/s)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + wave1 [label="Wavefront Crest (t0)\nStrikes Mic 0 first", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + wave2 [label="Wavefront Path Delay:\nΔx = d · sin(θ)\nΔt = Δx / v", fillcolor="#D69E2E", fontcolor="#FFFFFF"]; + wave3 [label="Wavefront Crest (t0 + Δt)\nStrikes Mic 1 later", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + + wave1 -> wave2 -> wave3 [style="invis"]; + } + + subgraph cluster_array { + label="Microphone Array Sensors"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + mic0 [label="Microphone 0 (Ch 0)\nPosition x0", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + mic1 [label="Microphone 1 (Ch 1)\nPosition x1 = x0 + d", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + } + + source -> wave1 [label="Acoustic travel"]; + wave1 -> mic0 [label="Direct arrival (t = 0)"]; + wave2 -> mic1 [label="Delayed arrival (t = Δt)"]; + } + +--- + +.. _tdfb_filter_sum_arch: + +2. Filter-and-Sum FIR Architecture & Multi-Channel Mixing +********************************************************* + +The SOF Time-Domain Fixed Beamformer component operates as a multi-input, multi-output filter bank. It processes up to 16 microphone input channels and feeds up to 16 parallel FIR filter instances, mixing filtered results into configured output channels and streams. + +Filter Bank Topology & Routing Matrix +===================================== + +The TDFB architecture is parameterized by configuration blobs generated offline and loaded at runtime: + +* **FIR Filter Bank (`fir[SOF_TDFB_FIR_MAX_COUNT]`)**: Accommodates up to 16 independent FIR filters. Each filter can feature an arbitrary length up to 256 taps (`SOF_TDFB_FIR_MAX_LENGTH = 256`), with tap counts aligned to multiples of 4 for SIMD vector execution. +* **Input Channel Selection (`input_channel_select[]`)**: An integer array assigning each FIR filter instance to a specific physical input channel :math:`ch_{in} \in [0, N_{in}-1]`. Multiple filters can tap the same microphone channel (e.g. feeding distinct beam angles or split frequency bands). +* **Output Channel Mixing Matrix (`output_channel_mix[]`)**: A bitmask vector for each filter instance determining which output channels receive the filtered audio. For instance, a bitmask of ``0x0001`` routes filter output to Channel 0, while ``0x0002`` routes to Channel 1. +* **Output Stream Mixing (`output_stream_mix[]`)**: Routes filter outputs to specific downstream sink streams in multi-stream configurations. + +Mathematical Signal Flow +======================== + +For output channel :math:`k`, the synthesized time-domain audio sample :math:`y_k[n]` is the linear summation of all FIR filters mapped to that channel: + +.. math:: + + y_k[n] = \sum_{i \in \mathcal{F}_k} \sum_{m=0}^{M_i-1} h_i[m] \cdot x_{s(i)}[n - m] + +where: + +* :math:`\mathcal{F}_k` is the set of filter indices configured to mix into output channel :math:`k` (via bitmask `output_channel_mix[i]`). +* :math:`s(i) = \text{input\_channel\_select}[i]` is the input microphone channel assigned to filter :math:`i`. +* :math:`h_i[m]` is the :math:`m`-th tap coefficient of FIR filter :math:`i`, stored in 16-bit fixed-point format. +* :math:`M_i` is the tap length of filter :math:`i`. +* :math:`x_{s(i)}[n - m]` is the historical input sample from microphone :math:`s(i)` retrieved from the circular delay line. + +Multi-Beam Simultaneous Extraction +================================== + +Because the TDFB executes an arbitrary filter bank matrix, a single component instance can extract multiple directional beams simultaneously: + +1. **Dual-Beam Stereo Capture**: In video recording and conference scenarios, the TDFB can synthesize a wide stereo image by steering Filter Bank A toward the left visual frame (:math:`-30^\circ` azimuth) routed to Output Channel 0 (Left), and Filter Bank B toward the right visual frame (:math:`+30^\circ` azimuth) routed to Output Channel 1 (Right). +2. **Speech + Noise Reference Extraction**: For advanced noise reduction pipelines, the TDFB can output a primary directional speech beam on Channel 0 steered directly at the user, alongside an orthogonal "anti-beam" (steered toward diffuse ambient noise or ceiling reflections) on Channel 1. Downstream speech AI modules and adaptive noise suppressors (RTNR) utilize this noise reference to cancel residual background interference without voice distortion. + +Fixed-Point Dynamic Headroom Management +======================================= + +When summing multiple coherent microphone signals, signal amplitude increases by up to :math:`N` times (:math:`+6\text{ dB}` per doubling of microphones). To prevent catastrophic integer overflow during summation: + +* FIR filtering computes products with 32-bit input samples and 16-bit coefficients. +* Intermediate filter outputs are accumulated in **Q5.27 fixed-point representation**. The 5 integer bits provide up to :math:`+30\text{ dB}` of headroom, allowing up to 16 filter outputs to sum into a single channel without overflow. +* The combined mix is subsequently scaled, symmetrically rounded, and saturated back to the sink buffer bit depth (:math:`S16\_LE`, :math:`S24\_4LE`, or :math:`S32\_LE`). + +.. graphviz:: + :caption: Filter-and-Sum Beamformer Architecture with Multi-Filter FIR Banks and Channel Routing + + digraph tdfb_filter_sum { + bgcolor="transparent"; + rankdir=LR; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_inputs { + label="Physical Microphone Inputs"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + mic_in0 [label="Mic In 0 (Left)\nx0[n]", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + mic_in1 [label="Mic In 1 (Center-Left)\nx1[n]", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + mic_in2 [label="Mic In 2 (Center-Right)\nx2[n]", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + mic_in3 [label="Mic In 3 (Right)\nx3[n]", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + } + + subgraph cluster_filters { + label="FIR Filter Bank (Up to 16 Filters, ≤ 256 Taps)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + fir0 [label="FIR 0 (h0[m])\ninput_channel_select = 0", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + fir1 [label="FIR 1 (h1[m])\ninput_channel_select = 1", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + fir2 [label="FIR 2 (h2[m])\ninput_channel_select = 2", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + fir3 [label="FIR 3 (h3[m])\ninput_channel_select = 3", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + } + + subgraph cluster_mix { + label="Accumulation & Headroom Matrix"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + sum0 [label="Summation Node (Ch 0)\nQ5.27 Accumulator\nHeadroom: +30 dB", fillcolor="#D69E2E", fontcolor="#FFFFFF", shape=ellipse]; + sat0 [label="Round & Saturate\n(sat_int16 / sat_int32)", fillcolor="#805AD5", fontcolor="#FFFFFF"]; + } + + subgraph cluster_output { + label="Output Beam Stream"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + beam_out [label="Focused Output Beam\ny[n] (High SNR)", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + } + + mic_in0 -> fir0; + mic_in1 -> fir1; + mic_in2 -> fir2; + mic_in3 -> fir3; + + fir0 -> sum0 [label="output_channel_mix = 1"]; + fir1 -> sum0 [label="output_channel_mix = 1"]; + fir2 -> sum0 [label="output_channel_mix = 1"]; + fir3 -> sum0 [label="output_channel_mix = 1"]; + + sum0 -> sat0; + sat0 -> beam_out; + } + +--- + +.. _tdfb_array_geometries: + +3. Microphone Array Geometries +****************************** + +The spatial performance, steering agility, and directivity index of a beamformer depend fundamentally on the physical arrangement of its microphones. Sound Open Firmware supports arbitrary 3D microphone coordinates configured via the `sof_tdfb_mic_location` structure, where each microphone position :math:`(x, y, z)` is represented in fixed-point :math:`Q4.12` meters. + +Standard Array Topologies +========================= + +* **Linear Arrays (1D)**: + + - *Geometry*: Microphones arranged along a single straight line (e.g. across the top bezel of a laptop display or along a soundbar). + - *Beam Characteristics*: Highly effective at steering across the horizontal azimuth plane (:math:`-90^\circ` to :math:`+90^\circ`). However, linear arrays possess cylindrical symmetry around the array axis, creating a front-back ambiguity (the array cannot distinguish sounds arriving from :math:`\theta` in front from :math:`180^\circ - \theta` behind). + - *End-Fire vs Broadside*: Sound arriving perpendicular to the array line is *broadside* (:math:`0^\circ`), yielding wide beams at low frequencies. Sound arriving parallel to the array line is *end-fire* (:math:`\pm 90^\circ`), providing the narrowest possible beamwidth for a given aperture. + +* **Circular / Ring Arrays (2D)**: + + - *Geometry*: Microphones distributed uniformly along the circumference of a circle (typically 4, 6, or 8 microphones on smart speakers, conference pucks, or IoT hubs). + - *Beam Characteristics*: Provides true :math:`360^\circ` uniform azimuth coverage without blind spots. The array can steer a symmetric acoustic cone in any horizontal direction with identical beamwidth and directivity index regardless of steering angle. + +* **L-Shaped & Planar Rectangular Arrays (2D / 3D)**: + + - *Geometry*: Microphones arranged across two perpendicular axes (e.g. corner placements on tablet frames or automotive dashboards). + - *Beam Characteristics*: Capable of resolving both **azimuth** (horizontal angle) and **elevation** (vertical angle) simultaneously. This enables the beamformer to isolate a standing talker from a seated talker and reject ceiling reflections. + +.. list-table:: + :widths: 20 25 25 30 + :header-rows: 1 + + * - Array Topology + - Typical Sensor Count + - Spatial Coverage + - Target Hardware Enclosure + * - **Uniform Linear** + - 2 to 4 microphones + - :math:`180^\circ` Azimuth (Frontal) + - Laptop lid bezels, television soundbars, video monitors. + * - **Non-Uniform Linear** + - 4 microphones (nested spacing) + - :math:`180^\circ` Broadside / Endfire + - Wideband capture (close pair for treble, outer pair for bass). + * - **Uniform Circular** + - 4 to 8 microphones + - Full :math:`360^\circ` Azimuth + - Tabletop conference pucks, smart home voice assistants. + * - **Planar / Rectangular** + - 4 microphones (2x2 grid) + - Azimuth + Elevation 3D + - In-vehicle telematics, high-end meeting room cameras. + +.. graphviz:: + :caption: Microphone Array Geometries: Linear, Circular, L-Shaped, and Rectangular Configurations + + digraph tdfb_geometries { + bgcolor="transparent"; + rankdir=TB; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_linear { + label="1D Uniform Linear Array (Laptop Bezel)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + lin_m0 [label="Mic 0\n(0, 0, 0)", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + lin_m1 [label="Mic 1\n(d, 0, 0)", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + lin_m2 [label="Mic 2\n(2d, 0, 0)", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + lin_m3 [label="Mic 3\n(3d, 0, 0)", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + lin_m0 -> lin_m1 -> lin_m2 -> lin_m3 [label="Spacing d", color="#38A169"]; + lin_prop [label="Steers -90° to +90° Azimuth\nFront/Back Symmetry", fillcolor="#4A5568", fontcolor="#FFFFFF", shape=note]; + lin_m3 -> lin_prop [style="dashed"]; + } + + subgraph cluster_circular { + label="2D Circular Ring Array (Smart Speaker)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + circ_m0 [label="Mic 0 (0°)", fillcolor="#3182CE", fontcolor="#FFFFFF"]; + circ_m1 [label="Mic 1 (90°)", fillcolor="#3182CE", fontcolor="#FFFFFF"]; + circ_m2 [label="Mic 2 (180°)", fillcolor="#3182CE", fontcolor="#FFFFFF"]; + circ_m3 [label="Mic 3 (270°)", fillcolor="#3182CE", fontcolor="#FFFFFF"]; + + circ_m0 -> circ_m1 -> circ_m2 -> circ_m3 -> circ_m0 [color="#3182CE"]; + circ_prop [label="Full 360° Omnidirectional Steering\nZero Azimuth Blind Spots", fillcolor="#4A5568", fontcolor="#FFFFFF", shape=note]; + circ_m2 -> circ_prop [style="dashed"]; + } + + subgraph cluster_planar { + label="3D Planar / Rectangular Array (Conference Display)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + rec_m0 [label="Mic 0 (Top-L)", fillcolor="#805AD5", fontcolor="#FFFFFF"]; + rec_m1 [label="Mic 1 (Top-R)", fillcolor="#805AD5", fontcolor="#FFFFFF"]; + rec_m2 [label="Mic 2 (Bot-L)", fillcolor="#805AD5", fontcolor="#FFFFFF"]; + rec_m3 [label="Mic 3 (Bot-R)", fillcolor="#805AD5", fontcolor="#FFFFFF"]; + + rec_m0 -> rec_m1 [color="#805AD5"]; + rec_m0 -> rec_m2 [color="#805AD5"]; + rec_m1 -> rec_m3 [color="#805AD5"]; + rec_m2 -> rec_m3 [color="#805AD5"]; + rec_prop [label="Azimuth + Elevation Steering\nIsolates Standing vs Seated Talkers", fillcolor="#4A5568", fontcolor="#FFFFFF", shape=note]; + rec_m3 -> rec_prop [style="dashed"]; + } + } + +--- + +.. _tdfb_doa_tracking: + +4. Direction of Arrival (DOA) Tracking & Acoustic Localization +************************************************************** + +In addition to static beamforming, the TDFB includes an autonomous, time-domain **Direction of Arrival (DOA)** estimation subsystem (`tdfb_direction.c`). The DOA engine continuously scans the acoustic sound field, pinpoints the angular coordinates of an active talker, and enables adaptive beam tracking. + +The DOA Estimation Processing Pipeline +====================================== + +The autonomous tracking loop runs concurrently with audio streaming through four sequential stages: + +1. **Pre-Emphasis IIR Filtering (`tdfb_direction_copy_emphasis`)**: + + Raw microphone audio contains heavy low-frequency acoustic energy (room reverberation, mechanical vibrations, HVAC hum) that exhibits weak spatial phase correlation and degrades TDOA estimation. To eliminate this interference, incoming samples pass through an IIR Direct Form I high-pass emphasis filter (:math:`16\text{ kHz}` and :math:`48\text{ kHz}` optimized filter tables) that suppresses frequencies below 500 Hz while boosting speech formants between 1 kHz and 4 kHz. + +2. **Energy Thresholding & Primitive VAD**: + + To avoid tracking background noise during pauses in human speech, the DOA engine tracks the long-term ambient noise floor (:math:`\text{level\_ambient}`) using an asymmetric leaky integrator. The tracking algorithm only executes when instantaneous signal energy exceeds the ambient noise estimate by a configurable power threshold (:math:`\text{POWER\_THRESHOLD} = 15.85 \approx +12\text{ dB}`). + +3. **Inter-Microphone Cross-Correlation**: + + The emphasized signals are stored in a circular delay buffer. The engine computes cross-correlation lag vectors across microphone pairs: + + .. math:: + + R_{ij}[\tau] = \sum_{n} x_i[n] \cdot x_j[n - \tau] + + The peak cross-correlation lag indicates the physical arrival time difference :math:`\tau_{ij}` between sensor :math:`i` and sensor :math:`j`. + +4. **Iterative Geometric Angle Search**: + + Using the known 3D physical coordinates of each microphone (:math:`x_i, y_i, z_i`), the algorithm simulates expected acoustic travel times for candidate sound source directions located on an evaluation sphere (source distance typically set to :math:`3.0\text{ meters}`). + + The search algorithm performs an 8-step iterative optimization loop (`AZ_ITERATIONS = 8`) over azimuth angle :math:`\theta`, evaluating theoretical versus observed time differences. The angle that minimizes total geometric squared error is selected as the instantaneous Direction of Arrival. + +Angle Smoothing & Jitter Suppression +==================================== + +Acoustic reflections and transient phonemes can cause momentary angle spikes. The raw estimated azimuth angle passes through a two-pole smoothing filter: + +.. math:: + + \theta_{smooth}[n] = 0.02 \cdot \theta_{raw}[n] + 0.98 \cdot \theta_{smooth}[n-1] + +This heavy exponential dampening ensures that the estimated beam angle glides smoothly across the acoustic scene without nervous, erratic jittering. + +.. graphviz:: + :caption: Direction of Arrival (DOA) Tracking Engine: Pre-Emphasis, Cross-Correlation, and Geometric Search + + digraph tdfb_doa { + bgcolor="transparent"; + rankdir=TB; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_input { + label="Microphone Stream Ingestion"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + pcm_in [label="Multi-Channel Microphone Audio\n(Channels 0 .. N-1)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + } + + subgraph cluster_conditioning { + label="Stage 1: Spectral Pre-Conditioning & VAD"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + emphasis [label="IIR Emphasis Filter (DF1)\n• High-pass > 500 Hz\n• Boosts Speech Formants (1-4 kHz)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + ambient [label="Ambient Noise Floor Estimator\n(Slow Asymmetric Leaky Integrator)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + vad_gate [label="Energy Threshold Gate\nIs (Power > Ambient + 12 dB)?", fillcolor="#D69E2E", fontcolor="#FFFFFF", shape=diamond]; + + pcm_in -> emphasis; + emphasis -> ambient; + emphasis -> vad_gate; + ambient -> vad_gate [label="Noise baseline"]; + } + + subgraph cluster_search { + label="Stage 2: Geometric Angle Estimation"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + xcorr [label="Pairwise Cross-Correlation\nR_ij[τ] Peak Lag Extraction", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + geom [label="3D Microphone Coordinates\nsof_tdfb_mic_location (Q4.12 meters)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + opt [label="Iterative Azimuth Search\n8-Iteration Error Minimization\nEvaluates Candidate Angles", fillcolor="#805AD5", fontcolor="#FFFFFF"]; + smooth [label="Angle Smoothing Filter\nθ_smooth = 0.02·θ + 0.98·θ_prev", fillcolor="#805AD5", fontcolor="#FFFFFF"]; + + vad_gate -> xcorr [label="Speech Detected (Pass)", color="#38A169"]; + geom -> opt [label="Sensor coordinates"]; + xcorr -> opt [label="Measured lags"]; + opt -> smooth; + } + + subgraph cluster_output { + label="Stage 3: Host Notification & Steering"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + notify [label="Host Asynchronous Notification\n(Throttled: Max 1 per 200 ms)", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + steer [label="Beam Steer Angle Index\n(Selects Target FIR Filter Set)", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + smooth -> notify [label="Azimuth estimate"]; + smooth -> steer [label="Updates active beam"]; + } + } + +--- + +.. _tdfb_control_plane: + +5. Host IPC Control Plane & Dynamic Steering +******************************************** + +The TDFB component exposes an interactive control plane to the host operating system via ALSA mixer controls, supporting both manual beam steering by user applications and autonomous steering reporting. + +ALSA Topology 2 Control Architecture +==================================== + +In ALSA Topology 2 (`tools/topology/topology2/include/components/tdfb.conf`), the TDFB declares four dedicated mixer controls: + +1. **Processing Switch (`SOF_TDFB_CTRL_INDEX_PROCESS = 0`)**: + + - Type: Binary toggle switch (0 = Bypass Passthrough, 1 = Beamforming Active). + - In bypass mode, input channels pass straight through to sink channels without FIR filtering overhead. + +2. **Direction Tracking Switch (`SOF_TDFB_CTRL_INDEX_DIRECTION = 1`)**: + + - Type: Binary toggle switch (0 = Static Fixed Beam, 1 = Dynamic DOA Tracking Active). + - When enabled, the DSP runs the background DOA cross-correlation engine to track talkers. + +3. **Steer Azimuth Enum (`SOF_TDFB_CTRL_INDEX_AZIMUTH = 0`)**: + + - Type: Enumerated control providing discrete steering angles. + - For a standard laptop linear array, the enum provides 13 selectable steering directions: + ``[-90°, -75°, -60°, -45°, -30°, -15°, 0°, +15°, +30°, +45°, +60°, +75°, +90°]``. + - Setting this control from user space (e.g. via `alsamixer`, `sof-ctl`, or an intelligent video conferencing app tracking face position) dynamically switches the active FIR filter bank to the corresponding angle. + +4. **Azimuth Estimate Readback Enum (`SOF_TDFB_CTRL_INDEX_AZIMUTH_ESTIMATE = 1`)**: + + - Type: Read-only enumerated control reflecting the DSP's autonomous real-time DOA estimate. + +Asynchronous Host IPC Notifications +=================================== + +When autonomous direction tracking is active, the DSP firmware detects shifts in talker position. Rather than forcing user space to poll the DSP continuously (which would waste host CPU cycles and prevent sleep states), the TDFB sends **unsolicited asynchronous IPC notification messages** to the Linux kernel driver: + +* **Event Throttling (`CONTROL_UPDATE_MIN_TIME = 0.2s`)**: To prevent flooding the IPC mailbox during rapid conversational speech, notification events are strictly rate-limited to fire no more frequently than once every 200 ms. +* **Hysteresis Masking (`CONTROL_UPDATE_MIN_MASK = 0x0F`)**: The talker's energy must exceed the ambient noise threshold across at least four consecutive audio periods before an angle change triggers an IPC notification. +* Upon receiving the notification, the Linux ASoC driver updates the corresponding ALSA control, which emits a standard `SNDRV_CTL_EVENT_MASK_VALUE` event to notify listening user space applications (such as camera tracking daemons). + +.. graphviz:: + :caption: Host-DSP Control Plane: ALSA Mixer Controls, Beam Angle Enumeration, and Asynchronous IPC Notifications + + digraph tdfb_controls { + bgcolor="transparent"; + rankdir=LR; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_host { + label="Linux Host User Space & Kernel Driver"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + app [label="Video Conference App /\nCamera Auto-Framing Daemon", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + alsa_kcontrol [label="ALSA Mixer Controls:\n• 'TDFB Process' (Switch)\n• 'TDFB Direction' (Switch)\n• 'TDFB Steer Azimuth' (Enum)\n• 'TDFB Estimate' (Enum)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + + app -> alsa_kcontrol [label="User angle selection\nor query"]; + } + + subgraph cluster_ipc { + label="IPC Mailbox Transport (IPC3 / IPC4)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + ipc_set [label="IPC SET_CONTROL\nUpdates active beam angle", fillcolor="#D69E2E", fontcolor="#FFFFFF"]; + ipc_notify [label="Asynchronous Notification\nDSP reports talker angle shift\n(Rate limited: ≥ 200 ms)", fillcolor="#D69E2E", fontcolor="#FFFFFF"]; + } + + subgraph cluster_dsp { + label="SOF DSP Firmware (TDFB Component)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + tdfb_ctrl [label="TDFB Control Handler\nSwaps active FIR filter set", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + doa_engine [label="DOA Tracking Engine\nDetects talker at +30°", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + tdfb_ctrl -> doa_engine [style="invis"]; + } + + alsa_kcontrol -> ipc_set [label="put command"]; + ipc_set -> tdfb_ctrl [label="Switches beam"]; + + doa_engine -> ipc_notify [label="New DOA estimated"]; + ipc_notify -> alsa_kcontrol [label="SNDRV_CTL_EVENT"]; + alsa_kcontrol -> app [label="Event notification"]; + } + +--- + +.. _tdfb_simd_acceleration: + +6. SIMD Vector Acceleration Across DSP Architectures +**************************************************** + +Evaluating up to 16 parallel FIR filters across multi-channel microphone streams imposes significant computational load. Sound Open Firmware leverages architecture-specific SIMD instruction sets to maximize energy efficiency. + +Dual-Sample Vector Processing Loop +================================== + +The TDFB inner loop processes audio **two samples at a time** (:math:`y0, y1`). This dual-sample structure matches the native execution width of DSP multiply-accumulate (MAC) pipelines and amortizes pointer updating overhead. + +Tensilica Xtensa HiFi 3 Optimization (`tdfb_hifi3.c`) +===================================================== + +On Cadence Tensilica Xtensa HiFi 3 DSP cores: + +* **Dual 32x16 Vector MAC (`fir_32x16_2x`)**: Computes two 32-bit audio sample convolutions against 16-bit packed filter coefficients in parallel using HiFi 3 MAC intrinsics. +* **Circular Buffer Pointer Auto-Wrapping**: Coefficients and delay line histories are mapped into hardware circular addressing registers via `fir_core_setup_circular()`, eliminating memory wrap branching instructions. +* **Vector Shifting and Packing (`AE_ROUND16X4F32SSYM`)**: The 32-bit accumulated filter results are shifted and packed into 16-bit sink buffers using symmetric rounding vector instructions. + +Tensilica Xtensa HiFi 2 EP Optimization (`tdfb_hifiep.c`) +========================================================= + +For platforms equipped with Tensilica HiFi 2 EP DSPs, specialized assembly optimizations accelerate multi-channel FIR tap accumulation with tailored register caching. + +Generic Portable Scalar C (`tdfb_generic.c`) +============================================ + +For embedded targets without proprietary DSP engines (such as the ARM Cortex-M7 on PJRC Teensy 4.1 or RISC-V on ESP32-P4), SOF provides a clean, portable scalar C implementation using 64-bit accumulators and standard `Q_SHIFT_RND` macros. + +.. graphviz:: + :caption: SIMD Vector Optimization across DSP Architectures (Xtensa HiFi 3 vs HiFi 2 EP vs Scalar C) + + digraph tdfb_simd { + bgcolor="transparent"; + rankdir=LR; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_hifi3 { + label="Cadence Tensilica Xtensa HiFi 3"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + hifi3_circ [label="Hardware Circular Buffers\nfir_core_setup_circular()", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + hifi3_mac [label="Dual 32x16 MAC Intrinsics\nfir_32x16_2x()\n(Processes 2 samples in parallel)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + hifi3_pack [label="Vector Symmetric Rounding\nAE_ROUND16X4F32SSYM", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + + hifi3_circ -> hifi3_mac -> hifi3_pack; + } + + subgraph cluster_hifiep { + label="Cadence Tensilica Xtensa HiFi 2 EP"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + ep_mac [label="Tailored HiFi 2 EP Assembly\nMulti-tap register caching", fillcolor="#805AD5", fontcolor="#FFFFFF"]; + } + + subgraph cluster_generic { + label="Portable Generic Scalar C (ARM / RISC-V)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + scalar_mac [label="Standard 64-Bit Math\nfir_32x16_2x generic\nQ_SHIFT_RND rounding", fillcolor="#718096", fontcolor="#FFFFFF"]; + } + } + +--- + +.. _tdfb_pipeline_integration: + +7. Microphone Array Capture Pipeline Integration +************************************************ + +The TDFB occupies a pivotal position within Sound Open Firmware capture pipelines, serving as the bridge between raw multi-channel hardware ingestion and downstream speech intelligence processing. + +End-to-End Microphone Pre-Processing Chain +========================================== + +In a representative modern laptop or smart speaker capture graph: + +1. **Hardware DAI Copier (PDM / I2S / SoundWire)**: Ingests raw digital microphone streams (e.g. 2, 4, or 8 channels). +2. **DC Blocker (:ref:`dcblock`)**: Removes ADC operational amplifier DC offsets and infrasonic rumble, establishing zero-mean signals (:math:`E[x] = 0`). +3. **Time-Domain Fixed Beamformer (TDFB)**: + + - Ingests the multi-channel DC-free microphone signals. + - Applies spatial filtering to attenuate off-axis ambient noise and room reverberation. + - Emits a high-SNR directional speech beam on Channel 0. + - Optionally emits a spatial ambient noise reference beam on Channel 1. + +4. **Acoustic Echo Cancellation (AEC)**: Cancels loudspeaker acoustic feedback picked up by the microphone beam, utilizing the clean spatial beam to accelerate adaptive filter convergence. +5. **Real-Time Noise Reduction (RTNR)**: Suppresses stationary and non-stationary diffuse background noise. +6. **Downstream Speech AI**: + + - **Voice Activity Detection (VAD)** and **Keyword Spotting (TensorFlow Lite Micro / TFLM)**: Detect wake words with high accuracy due to superior input SNR. + - **Host Audio Copier**: Transmits pristine speech to the operating system for recording, speech-to-text, or cellular transmission. + +.. graphviz:: + :caption: Microphone Array Capture Pipeline Architecture: Pre-Processing, Beamforming, and Downstream Speech AI + + digraph tdfb_pipeline { + bgcolor="transparent"; + rankdir=TB; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_hw { + label="Hardware Layer"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + pdm_mics [label="Digital PDM / SoundWire Mics\n(4-Channel Array: Ch 0..3)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + copier [label="DAI Copier (Capture Endpoint)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + + pdm_mics -> copier; + } + + subgraph cluster_dsp_pipe { + label="SOF Capture Audio Pipeline"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + dcb [label="DC Blocker Module\nStrips 0 Hz offset & rumble", fillcolor="#C53030", fontcolor="#FFFFFF"]; + tdfb_comp [label="Time-Domain Fixed Beamformer (TDFB)\n• Spatial filtering\n• Constructive target speech boost (+6 dB)\n• Destructive off-axis noise nulling", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + aec_comp [label="Acoustic Echo Canceller (AEC)\nRemoves speaker feedback", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + rtnr_comp [label="Noise Reduction (RTNR)\nSuppresses stationary noise", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + tflm_comp [label="Keyword Spotter (TFLM) / VAD\nWake word detection", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + copier -> dcb [label="4 channels"]; + dcb -> tdfb_comp [label="4 DC-free channels"]; + tdfb_comp -> aec_comp [label="High-SNR directional beam"]; + aec_comp -> rtnr_comp; + rtnr_comp -> tflm_comp; + } + + subgraph cluster_host { + label="Host Operating System"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + host_stream [label="ALSA Capture PCM Stream\n(Pristine Speech Audio)", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + tflm_comp -> host_stream; + } + } + +--- + +.. _tdfb_tuning_references: + +8. Upstream Code References & Related Guides +******************************************** + +For developers designing custom microphone arrays, calculating filter coefficients, or extending firmware capabilities: + +* **Upstream Component Source Files**: + + - `thesofproject/sof: src/audio/tdfb/README.md `_: Component overview and directory structure. + - `src/audio/tdfb/tdfb.c `_: Component initialization, format negotiation, and buffer traversal. + - `src/audio/tdfb/tdfb.h `_: User-space ABI headers (`sof_tdfb_config`, `sof_tdfb_angle`, `sof_tdfb_mic_location`). + - `src/audio/tdfb/tdfb_comp.h `_: Internal data structures (`struct tdfb_comp_data`, `struct tdfb_direction_data`). + - `src/audio/tdfb/tdfb_direction.c `_: Direction of Arrival (DOA) cross-correlation, emphasis filtering, and geometric search. + - `src/audio/tdfb/tdfb_generic.c `_: Portable generic scalar C filter-and-sum execution. + - `src/audio/tdfb/tdfb_hifi3.c `_: Tensilica Xtensa HiFi 3 SIMD vector acceleration. + - `src/audio/tdfb/tdfb_hifiep.c `_: Tensilica Xtensa HiFi 2 EP acceleration. + - `src/audio/tdfb/tdfb_ipc3.c `_ & `tdfb_ipc4.c `_: IPC protocol handlers and control event dispatchers. + +* **Topology Configuration**: + + - `tools/topology/topology2/include/components/tdfb.conf `_: ALSA Topology 2 widget definition (UUID `49:17:51:dd:fa:d9:5c:45:b3:a7:13:58:56:93:f1:af`). + +* **GNU Octave / MATLAB Tuning Suite (`src/audio/tdfb/tune/`)**: + + - `sof_example_all.sh `_: Top-level script generating topology blobs across all array types. + - `sof_bf_array_line.m `_ & `sof_bf_array_circ.m `_: Array coordinate generation for linear and circular geometries. + - `sof_bf_design.m `_: Core optimization engine synthesizing broadband FIR coefficients. + - `sof_example_two_beams.m `_: Dual-beam stereo capture generation. + +Related Subsystem Architecture Guides +===================================== + +* :ref:`time-domain-fixed-beamformer`: Algorithm tuning manual, array mathematical derivations, directivity index (DI), white noise gain (WNG), and polar plots. +* :ref:`dcblock`: First-order recursive high-pass filter stripping ADC DC offsets prior to beamforming. +* :ref:`eq_fir_iir`: Finite Impulse Response filter math, delay line management, and circular buffer mechanics. +* :ref:`module_framework`: Standardized module lifecycle, memory allocation, and IPC configuration handlers. +* :ref:`pipeline_architecture`: Graph scheduling, buffer management, and audio streaming topologies. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 4fc84a74..1bea61eb 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -48,7 +48,7 @@ Audio Processing Modules & Algorithms * :ref:`drc_multiband_drc` (High-level architecture; also see upstream `DRC README `_ & `Multiband DRC README `_) * :ref:`crossover` (High-level architecture; also see upstream `crossover README `_) * :ref:`dcblock` (High-level architecture; also see upstream `dcblock README `_) -* `Time-Domain Fixed Beamformer (TDFB) `_ +* :ref:`tdfb` (High-level architecture; also see upstream `tdfb README `_ & tuning guide :ref:`time-domain-fixed-beamformer`) * `RTNR Noise Reduction `_ * `TensorFlow Lite Micro (TFLM) `_ * `MFCC Feature Extraction `_ @@ -93,6 +93,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/drc_multiband_drc firmware/crossover firmware/dcblock + firmware/tdfb rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 6c5e063af6af9b6fe6c962101d5cbae430bf23d6 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Fri, 18 Sep 2026 20:26:03 +0100 Subject: [PATCH 16/64] docs: developer_guides: add high-level tflm architecture guide Add a comprehensive, high-level developer architecture guide for the TensorFlow Lite Micro (TFLM) embedded AI subsystem (developer_guides/firmware/tflm.rst). Topics covered: - Edge Audio AI Paradigm: Cloud offload vs. Host CPU processing vs. On-DSP microcontroller inference trade-offs (latency, privacy, power). - TensorFlow Lite Micro (TFLM) Component Architecture: Zero-heap static tensor arena (g_arena, kArenaSize), two-ended memory planning, zero-copy FlatBuffer model ingestion, and selective operator resolution via MicroMutableOpResolver. - End-to-End Audio Machine Learning Pipeline: Raw PCM capture, framing, windowing, FFT, Mel-scale filterbank, and 2D spectrogram feature matrix. - Asymmetric Int8 Affine Quantization: Mathematical formulation (r = S * (q - Z)), fixed-point matrix multiplication kernel mechanics, effective scaling factor decomposition (M = M_0 * 2^-n), and 32-bit integer accumulation without floating-point emulation. - Continuous Sliding Window Inference: 49-frame buffer maintenance and 20 ms temporal stride advancement for continuous 50 Hz evaluation. - Hardware Neural Network Acceleration: Tensilica NNLib (xa_nnlib) SIMD vector routines (depthwise convolution, matrix-vector dot products, vector softmax) and circular buffer addressing. - Host IPC Control Plane & Dynamic LLEXT Packaging: ALSA Topology 2 widget declaration, runtime model updates via IPC3/IPC4 blobs, and LLEXT modular packaging. - Microphone Voice AI Pipeline Integration: Capture chain from DMIC/I2S through DC Blocker, TDFB, MFCC, TFLM, and Host Wake trigger. - Upstream source references and tuning guide links. - Seven native vector Graphviz SVG architecture diagrams. - Clean Sphinx build under -W --keep-going and zero woke violations. Signed-off-by: Liam Girdwood --- .../firmware/pipeline_architecture.rst | 1 + developer_guides/firmware/tflm.rst | 826 ++++++++++++++++++ developer_guides/index.rst | 3 +- 3 files changed, 829 insertions(+), 1 deletion(-) create mode 100644 developer_guides/firmware/tflm.rst diff --git a/developer_guides/firmware/pipeline_architecture.rst b/developer_guides/firmware/pipeline_architecture.rst index 8b5a4ac7..e505bd76 100644 --- a/developer_guides/firmware/pipeline_architecture.rst +++ b/developer_guides/firmware/pipeline_architecture.rst @@ -469,6 +469,7 @@ Related Guides * :ref:`crossover`: Linkwitz-Riley 4th-order (LR4) active multi-driver crossovers, 2-way/3-way/4-way splitting with all-pass phase alignment, and 1-to-N multi-sink buffer distribution. * :ref:`dcblock`: First-order recursive high-pass DC blocking filter, 0 Hz transmission nulling, 64-bit fixed-point accumulation, and dual circular buffer SIMD acceleration. * :ref:`tdfb`: Spatial acoustic filtering, filter-and-sum FIR banks, multi-microphone array geometries, and autonomous Direction of Arrival (DOA) tracking. +* :ref:`tflm`: Embedded neural network inference, static tensor arena memory planning, 8-bit affine quantization, and Tensilica NNLib SIMD acceleration. * :ref:`fw_init_boot`: Boot flow, hardware mailbox FW Ready handshake, and Zephyr initialization. * :ref:`topology2`: How pipelines and widgets are declared using ALSA Topology 2.0 configuration classes. * :ref:`sof_hostless_firmware`: How to create static pipelines compiled into ROM for standalone microcontrollers. diff --git a/developer_guides/firmware/tflm.rst b/developer_guides/firmware/tflm.rst new file mode 100644 index 00000000..ccf5b2c3 --- /dev/null +++ b/developer_guides/firmware/tflm.rst @@ -0,0 +1,826 @@ +.. _tflm: + +TensorFlow Lite Micro (TFLM) Architecture +######################################### + +The **TensorFlow Lite Micro (TFLM)** subsystem in Sound Open Firmware provides on-device neural network inference, edge machine learning execution, and real-time audio event classification embedded directly within digital signal processor (DSP) audio pipelines. + +Historically, advanced speech recognition, voice biometric verification, keyword spotting, and acoustic scene analysis required streaming raw audio data across cloud networks to remote server farms. However, cloud-dependent machine learning introduces significant latency penalties, consumes substantial radio transmit power, fails entirely in offline environments, and creates sensitive user privacy and security liabilities. Conversely, running deep learning models on battery-powered edge computing devices requires overcoming severe physical constraints: audio DSPs possess limited static RAM (tens to hundreds of kilobytes), lack traditional hardware memory management units (MMUs), operate on fixed-point arithmetic units, and must adhere to strict milliwatt power envelopes. + +To solve this challenge, Sound Open Firmware integrates **TensorFlow Lite for Microcontrollers (TFLM)**—a bare-metal, C++17 machine learning runtime optimized by Google and customized for embedded DSP audio pipelines. Operating entirely without dynamic heap allocation, TFLM executes pre-trained, 8-bit quantized neural network models directly from a statically managed memory arena. When paired with SOF's spectral feature extraction modules (such as Mel-Frequency Cepstral Coefficients / MFCC), TFLM enables autonomous wake-word detection, acoustic event monitoring (e.g. glass break, smoke alarm sirens, baby cry), and intelligent voice activity detection directly on DSP audio hardware. + +This guide provides a comprehensive, high-level architectural walkthrough of the TFLM subsystem in SOF, analyzing static tensor arena memory planning, 8-bit integer affine quantization, end-to-end spectro-temporal feature ingestion, sliding window inference loops, Cadence Tensilica neural network library (NNLib) acceleration, ALSA Topology 2 / IPC dynamic model management, and loadable extension (LLEXT) integration without delving into low-level C++ source code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +.. _tflm_edge_ai_paradigm: + +1. Edge Audio AI & Microcontroller Machine Learning +*************************************************** + +Edge Artificial Intelligence represents a paradigm shift in audio processing: migrating machine learning inference from high-power central processors and remote cloud servers directly to the low-power DSP audio subsystem. + +Cloud vs Application Processor vs Audio DSP Inference +====================================================== + +Audio-driven computing systems deploy machine learning across three primary compute tiers: + +1. **Cloud Server Inference**: + + - *Characteristics*: Massive multi-billion parameter large language models and speech-to-text transformers running on GPU clusters. + - *Drawbacks*: Requires continuous high-bandwidth internet connectivity, introduces unpredictable network round-trip latency (100–500 ms), consumes substantial RF radio power, and exposes private ambient audio to cloud transmission risks. + +2. **Host Application Processor (Host CPU / NPU)**: + + - *Characteristics*: Multi-core mobile and desktop processors executing full-scale TensorFlow or ONNX runtimes in system DRAM. + - *Drawbacks*: Consumes watts of electrical power. Keeping the main application processor awake to continuously monitor microphones drains mobile device batteries within hours. + +3. **Embedded Audio DSP (SOF + TFLM)**: + + - *Characteristics*: Highly optimized, fixed-point neural networks executing on embedded DSP hardware islands. + - *Advantages*: Operates at milliwatt power consumption in low-power audio states, delivers instantaneous sub-20 ms local response times, ensures absolute data privacy (raw audio never leaves the DSP SRAM), and acts as an intelligent hardware gatekeeper that only wakes the host system when a valid trigger occurs. + +The Microcontroller ML Challenge: Severe Resource Constraints +============================================================= + +While modern deep learning frameworks assume gigabytes of virtual memory, multi-threaded operating systems, and floating-point vector hardware, embedded audio DSPs enforce stringent constraints: + +* **SRAM Scarcity**: Firmware memory is restricted to internal DSP static RAM (typically 64 KB to 512 KB) shared among RTOS stacks, audio stream buffers, filter states, and IPC mailboxes. +* **Prohibition of Dynamic Heap Allocation**: Standard `malloc()` and `new` operations are forbidden during steady-state audio processing. Dynamic allocation causes unpredictable runtime latency, non-deterministic execution times, and memory heap fragmentation that would inevitably crash long-running real-time audio streams. +* **Fixed-Point Arithmetic**: Many low-power microcontrollers and DSPs lack double-precision floating-point hardware. Efficient model execution requires mapping neural network weights and activations to 8-bit signed integers (`int8_t`). + +.. graphviz:: + :caption: Edge Audio AI Processing Paradigm: Cloud Offload vs On-DSP Microcontroller Inference + + digraph tflm_paradigm { + bgcolor="transparent"; + rankdir=LR; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_ambient { + label="Acoustic Environment"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + sound [label="Microphone Audio\n(Speech / Acoustic Events)", fillcolor="#2B6CB0", fontcolor="#FFFFFF", shape=ellipse]; + } + + subgraph cluster_dsp { + label="Low-Power DSP Tier (SOF + TFLM) - ALWAYS ON"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + dsp_dcb [label="Acoustic Clean:\nDC Blocker & Beamformer", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + dsp_mfcc [label="Feature Extraction:\nMFCC Spectrogram Engine", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + dsp_tflm [label="TFLM Edge Inference:\nQuantized Neural Network\nPower: < 5 mW\nLatency: < 20 ms", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + dsp_dcb -> dsp_mfcc -> dsp_tflm; + } + + subgraph cluster_host { + label="Host Tier (Application Processor) - ASLEEP"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + host_cpu [label="Host OS / Main CPU\nPower: 2 W - 15 W\n(Deep Sleep State)", fillcolor="#742A2A", fontcolor="#FFFFFF"]; + cloud [label="Cloud Server Infrastructure\n(High Latency / Privacy Risk)", fillcolor="#742A2A", fontcolor="#FFFFFF"]; + } + + sound -> dsp_dcb [label="Raw PCM"]; + dsp_tflm -> host_cpu [label="Wake Interrupt on Valid Trigger", color="#38A169", style="bold"]; + host_cpu -> cloud [label="Optional High-Level NLP", style="dashed"]; + } + +--- + +.. _tflm_runtime_architecture: + +2. TFLM Runtime Architecture & Memory Management +************************************************ + +Sound Open Firmware integrates the core TensorFlow Lite Micro runtime engine as a modular audio component (`tflm-classify.c`, `speech.cc`). TFLM differs fundamentally from standard TensorFlow Lite through its zero-heap, statically planned memory model. + +FlatBuffer Model Ingestion Without Deserialization +================================================== + +Neural network topologies and trained weights are exported from offline training environments as **FlatBuffers** (`.tflite` files). Unlike JSON, Protocol Buffers, or XML, FlatBuffers store structured hierarchical data in an internal binary layout that requires **zero unpacking, copying, or parsing**: + +* The firmware accesses model metadata, operator graphs, layer shapes, and quantized weight tensors directly from the binary buffer in Flash or DSP SRAM. +* Model representation is defined via `tflite::GetModel(g_micro_speech_quantized_model_data)`, providing an instantaneous, zero-allocation initialization path. + +The Static Tensor Arena (`g_arena`) +=================================== + +To guarantee deterministic real-time audio execution and prevent memory fragmentation, TFLM executes all tensor operations within a single, contiguous, pre-allocated memory pool known as the **Tensor Arena**: + +* In SOF, the arena is defined as a statically allocated, 16-byte aligned byte array (`alignas(16) static uint8_t g_arena[kArenaSize]`). +* For the standard speech classification network, `kArenaSize` is dimensioned to exactly 28,584 bytes (~28 KB). +* **Two-Phase Arena Allocation**: + + 1. *Head Allocation*: Contains persistent runtime objects, including the `tflite::MicroInterpreter`, tensor descriptor structures (`TfLiteTensor`), and node registration arrays. + 2. *Tail Allocation*: Contains scratch buffers and transient layer activations. TFLM's offline memory planner analyzes the neural network execution graph, calculating lifetime intervals for each layer's activations. Independent layers that do not execute concurrently reuse the exact same physical byte offsets in the arena, drastically shrinking total RAM consumption. + +Selective Operator Resolution (`MicroMutableOpResolver`) +======================================================== + +Standard machine learning runtimes link hundreds of mathematical kernels, swelling firmware binary footprints to multiple megabytes. TFLM resolves this through **selective operator registration**: + +* SOF declares a specialized operator resolver (`tflite::MicroMutableOpResolver<4>`). +* Only the exact mathematical operations utilized by the audio classifier are compiled and registered: + + 1. `AddReshape()`: Reshapes incoming multi-frame audio feature matrices into tensor dimensions expected by convolutional layers. + 2. `AddDepthwiseConv2D()`: Executes spatial-temporal convolutions with isolated per-channel kernels, drastically reducing multiply-accumulate operations. + 3. `AddFullyConnected()`: Computes dense inner-product projections between feature maps and output classification categories. + 4. `AddSoftmax()`: Normalizes output classification logits into a valid probability distribution summing to 1.0. + +* All unreferenced operators (e.g. RNN, LSTM, TransposeConv, MaxPool) are excluded by the linker, keeping the executable code footprint below 30 KB. + +.. graphviz:: + :caption: TensorFlow Lite Micro (TFLM) Component Architecture: Static Arena, Interpreter, and Op Resolver + + digraph tflm_runtime { + bgcolor="transparent"; + rankdir=TB; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_model { + label="Serialized Model Representation"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + flatbuffer [label="FlatBuffer Binary Model (.tflite)\n• Zero parsing / zero copy\n• Read-only weights in Flash/SRAM", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + } + + subgraph cluster_resolver { + label="Selective Operator Resolver (MicroMutableOpResolver)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + op_reshape [label="AddReshape()", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + op_dwconv [label="AddDepthwiseConv2D()", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + op_fc [label="AddFullyConnected()", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + op_softmax [label="AddSoftmax()", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + + op_reshape -> op_dwconv -> op_fc -> op_softmax [style="invis"]; + } + + subgraph cluster_arena { + label="Static Tensor Arena: g_arena[28584] (Zero Dynamic Heap)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + arena_head [label="Head Allocation:\n• MicroInterpreter instance\n• TfLiteTensor metadata headers\n• Node registration structures", fillcolor="#D69E2E", fontcolor="#FFFFFF"]; + arena_tail [label="Tail Allocation (Planned Lifetime Reuse):\n• Activation Buffer Layer N\n• Activation Buffer Layer N+1 (Reused)\n• Scratch workspace tensors", fillcolor="#805AD5", fontcolor="#FFFFFF"]; + + arena_head -> arena_tail [label="Contiguous single buffer", style="dashed"]; + } + + subgraph cluster_engine { + label="Inference Execution Engine"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + interpreter [label="MicroInterpreter Engine\n• AllocateTensors()\n• Invoke() pipeline execution", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + } + + flatbuffer -> interpreter [label="Model graph"]; + op_reshape -> interpreter [label="Registered ops"]; + arena_head -> interpreter [label="Binds static RAM"]; + } + +--- + +.. _tflm_feature_pipeline: + +3. Audio Feature Preprocessing & Spectrogram Ingestion +****************************************************** + +Deep neural networks cannot effectively process raw 16 kHz audio samples directly on low-power DSPs. A single second of audio contains 16,000 raw samples, demanding massive convolutional kernels and exorbitant memory bandwidth. Instead, audio streams pass through a **spectro-temporal feature extraction pipeline** prior to neural network evaluation. + +The MFCC / Filterbank Transformation Pipeline +============================================= + +In Sound Open Firmware, the audio feature extraction stage (typically handled by the upstream :ref:`module_framework` component `mfcc`) converts 1D temporal audio into a compact 2D time-frequency spectrogram: + +1. **Short-Time Windowing**: + + - Incoming 16 kHz audio is partitioned into overlapping frames of **30 ms duration** (480 samples). + - Frames advance with a **20 ms stride** (320 samples), producing 50 feature slices per second. + - A Hann or Hamming window is applied to each frame to eliminate edge discontinuities. + +2. **Spectral Transform (FFT)**: + + - A 512-point Fast Fourier Transform (FFT) converts each time-domain frame into a frequency-domain magnitude spectrum. + +3. **Mel-Scale Filterbank Integration**: + + - The linear frequency spectrum is filtered through **40 triangular bandpass filters** spaced logarithmically according to the human auditory Mel scale: + + .. math:: + + m = 2595 \log_{10}\left(1 + \frac{f}{700}\right) + + - Integrating spectral energy under each triangular filter condenses 257 complex frequency bins into exactly **40 energy coefficients** (`TFLM_FEATURE_SIZE = 40`). + +4. **Logarithmic Compression & Quantization**: + + - The dynamic range of the filterbank energies is logarithmically compressed (:math:`\log(E + \epsilon)`), emulating human perception of loudness. + - The resulting values are quantized into signed 8-bit integers (`int8_t`). + +The 2D Spectrogram Input Matrix +=============================== + +The TFLM classifier maintains a rolling temporal history of feature slices: + +* **Temporal Depth**: 49 consecutive time slices (`TFLM_FEATURE_COUNT = 49`). +* **Feature Width**: 40 Mel filterbank coefficients (`TFLM_FEATURE_SIZE = 40`). +* **Total Input Tensor Elements**: + + .. math:: + + N_{elements} = 40 \times 49 = 1,960 \text{ bytes} + +This :math:`40 \times 49` byte matrix forms a 2D spectro-temporal "acoustic fingerprint" spanning approximately **990 ms (~1 second)** of audio. The neural network evaluates this fingerprint to classify spoken words or acoustic events. + +.. graphviz:: + :caption: End-to-End Audio Machine Learning Pipeline: Raw PCM to MFCC Spectrogram to TFLM Classification + + digraph tflm_features { + bgcolor="transparent"; + rankdir=LR; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_pcm { + label="Time-Domain Audio Stream"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + pcm [label="16 kHz Mono PCM\n(480 samples / 30 ms frame\n320 samples / 20 ms stride)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + } + + subgraph cluster_mfcc { + label="Spectral Feature Extraction (MFCC / Filterbank)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + window [label="Hann Windowing\n& 512-point FFT", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + mel_fb [label="40 Triangular Mel Filters\n(Non-linear frequency warp)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + log_quant [label="Log Energy Compression\n& Int8 Quantization", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + + window -> mel_fb -> log_quant; + } + + subgraph cluster_matrix { + label="2D Spectrogram Rolling Buffer"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + matrix [label="Spectrogram Feature Matrix\n• 40 Frequency Bins\n• 49 Time Slices (~1 sec)\n• 1,960 Bytes (int8_t)", fillcolor="#D69E2E", fontcolor="#FFFFFF", shape=folder]; + } + + subgraph cluster_tflm_eval { + label="TFLM Neural Network Evaluation"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + infer [label="MicroInterpreter::Invoke()\nDepthwise Conv2D + FC", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + preds [label="Class Probabilities:\n• Silence: 0.02\n• Unknown: 0.05\n• Yes: 0.91\n• No: 0.02", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + infer -> preds; + } + + pcm -> window [label="Audio frames"]; + log_quant -> matrix [label="1 slice / 20 ms"]; + matrix -> infer [label="1,960 byte tensor"]; + } + +--- + +.. _tflm_quantization: + +4. Fixed-Point Arithmetic & Asymmetric Int8 Quantization +******************************************************** + +Deploying floating-point 32-bit (FP32) arithmetic on embedded DSPs requires excessive clock cycles and inflates memory footprints by 4x. TFLM resolves this by executing entirely in **quantized 8-bit integer (`int8_t`) representation**. + +Asymmetric Affine Quantization Formulation +========================================== + +TFLM implements standard asymmetric affine quantization mapping continuous floating-point real numbers :math:`r \in \mathbb{R}` to signed 8-bit integer values :math:`q \in [-128, +127]`: + +.. math:: + + r = S \cdot (q - Z) \quad \iff \quad q = \text{round}\left(\frac{r}{S}\right) + Z + +where: + +* :math:`S` is the positive floating-point **Scale factor**, representing the real-world delta between adjacent integer quantization steps. +* :math:`Z` is the integer **Zero-Point**, representing the exact quantized integer corresponding to real :math:`0.0`. +* Clamping enforces bounds: :math:`q \in [-128, +127]`. + +Integer Kernel Execution Without Floating-Point Math +==================================================== + +During neural network layer computation (such as matrix multiplication in Fully Connected or Depthwise Convolutional layers), input activations :math:`x` and weights :math:`w` are convolved to produce output activations :math:`y`: + +.. math:: + + r_y = \sum_i r_x^{(i)} \cdot r_w^{(i)} + +Substituting the quantization relations: + +.. math:: + + S_y (q_y - Z_y) = \sum_i S_x (q_x^{(i)} - Z_x) \cdot S_w^{(i)} (q_w^{(i)} - Z_w) + +Rearranging to isolate the output quantized integer :math:`q_y`: + +.. math:: + + q_y = \text{round}\left( M \cdot \sum_i (q_x^{(i)} - Z_x)(q_w^{(i)} - Z_w) \right) + Z_y + +where the multiplier constant :math:`M` is: + +.. math:: + + M = \frac{S_x \cdot S_w}{S_y} + +Crucially, :math:`M` is a fixed real scalar strictly between :math:`0` and :math:`1`. During model compilation, :math:`M` is decomposed into a **fixed-point 32-bit multiplier (:math:`M_0 \in [0.5, 1.0)`) and an arithmetic right-shift (:math:`2^{-n}`)**: + +.. math:: + + M \approx M_0 \cdot 2^{-n} + +As a result, the entire convolution and dense projection executes using **pure 32-bit integer multiply-accumulate operations and bit-shifts**, completely eliminating floating-point hardware requirements during model evaluation. + +Dequantization of Output Probabilities +====================================== + +After passing through the final Softmax activation layer, the raw integer outputs :math:`q_{out}[i]` must be converted to human-readable probability scores (:math:`0.0` to :math:`1.0`) for host notifications: + +.. math:: + + P[i] = (q_{out}[i] - Z_{out}) \cdot S_{out} + +This dequantization step executes once per classification invocation across the small number of output categories, imposing negligible computational overhead. + +.. graphviz:: + :caption: Asymmetric Int8 Affine Quantization Data Path and Fixed-Point Arithmetic Kernel + + digraph tflm_quant { + bgcolor="transparent"; + rankdir=LR; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_inputs { + label="Quantized Inputs (Int8)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + qx [label="Activation Sample\nq_x ∈ [-128, 127]", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + zx [label="Input Zero-Point\nZ_x", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + qw [label="Model Weight\nq_w ∈ [-128, 127]", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + zw [label="Weight Zero-Point\nZ_w (Typically 0)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + } + + subgraph cluster_core { + label="Fixed-Point Arithmetic Kernel (32-Bit Accumulator)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + sub_x [label="(q_x - Z_x)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + sub_w [label="(q_w - Z_w)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + acc [label="Integer MAC Accumulator\nΣ (q_x - Z_x)(q_w - Z_w)\n32-Bit Signed Int", fillcolor="#D69E2E", fontcolor="#FFFFFF", shape=ellipse]; + scale_m [label="Fixed-Point Scaling\nM = M0 · 2⁻ⁿ\n(Multiply + Right-Shift)", fillcolor="#805AD5", fontcolor="#FFFFFF"]; + add_zy [label="Add Output Zero-Point\n+ Z_y", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + clamp [label="Saturate to [-128, 127]\nOutput Activation q_y", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + sub_x -> acc; + sub_w -> acc; + acc -> scale_m -> add_zy -> clamp; + } + + qx -> sub_x; + zx -> sub_x; + qw -> sub_w; + zw -> sub_w; + } + +--- + +.. _tflm_sliding_window: + +5. Sliding Window Inference Mechanics +************************************* + +Audio event classification operates continuously over time. Rather than evaluating isolated, non-overlapping blocks of audio, the TDFB classifier implements a **continuous sliding window inference loop**. + +Temporal Striding & Frame Buffer Consumption +============================================ + +The component's audio processing routine (`tflm_process`) monitors available feature frames delivered by the upstream MFCC producer: + +1. **Buffer Readiness Gate**: + + - Inference requires a full temporal context of 49 feature slices (`TFLM_FEATURE_ELEM_COUNT = 1,960` bytes). + - As long as `features >= TFLM_FEATURE_ELEM_COUNT`, the module has sufficient data to invoke the neural network. + +2. **Model Invocation (`TF_ProcessClassify`)**: + + - The 1,960 bytes of contiguous feature data are copied into the model's input tensor. + - `interpreter->Invoke()` executes the neural network graph across all layers. + - Output category probabilities are dequantized into `cd->tfc.predictions[]`. + +3. **Window Advancement by One Stride**: + + - Instead of discarding all 49 frames, the component advances its read pointer by exactly **one temporal stride**: + + .. math:: + + \text{Advance} = \text{TFLM\_FEATURE\_SIZE} \times \text{frame\_bytes} = 40 \text{ bytes} + + - This corresponds to shifting the temporal window forward by exactly **20 ms**. + - The loop immediately re-checks available frames, allowing multiple overlapping evaluations if burst audio frames arrived during DSP scheduling delays. + +Classification Categories & Wake Detection +========================================== + +In the reference micro-speech implementation, the output layer computes probabilities across four distinct categories (`TFLM_CATEGORY_DATA`): + +* **"silence"**: Indicates complete acoustic silence or ambient background noise below speech threshold. +* **"unknown"**: Indicates human speech or audio activity that does not match configured target keywords. +* **"yes"**: Positive target keyword 1. +* **"no"**: Positive target keyword 2. + +A dedicated averaging and hysteresis module tracks prediction probabilities across consecutive windows. When a target keyword probability exceeds an activation threshold (e.g. :math:`P > 0.85`) consistently over multiple strides, a positive wake event is confirmed. + +.. graphviz:: + :caption: Continuous Sliding Window Inference Mechanics with 20 ms Temporal Strides + + digraph tflm_sliding { + bgcolor="transparent"; + rankdir=TB; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_stream { + label="Continuous Feature Stream (40 Mel Bins per Slice)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + s0 [label="Slice 0 (t=0ms)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + s1 [label="Slice 1 (t=20ms)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + s2 [label="Slice 2 (t=40ms)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + s48 [label="Slice 48 (t=960ms)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + s49 [label="Slice 49 (t=980ms)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + + s0 -> s1 -> s2 -> s48 -> s49 [style="invis"]; + } + + subgraph cluster_win1 { + label="Inference Window 1 (Evaluation at t = 960 ms)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + w1_eval [label="Window 1: Slices [0 .. 48] (1,960 bytes)\n• Invoke() Model\n• Result: P(yes) = 0.42 (Below threshold)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + } + + subgraph cluster_stride { + label="Advance Buffer by 1 Stride (20 ms / 40 bytes)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + release [label="source_release_data(40 bytes)\nDiscards Slice 0; Ingests Slice 49", fillcolor="#D69E2E", fontcolor="#FFFFFF"]; + } + + subgraph cluster_win2 { + label="Inference Window 2 (Evaluation at t = 980 ms)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + w2_eval [label="Window 2: Slices [1 .. 49] (1,960 bytes)\n• Invoke() Model\n• Result: P(yes) = 0.94 (WAKE TRIGGER!)", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + } + + s48 -> w1_eval; + w1_eval -> release; + release -> w2_eval; + } + +--- + +.. _tflm_nnlib_acceleration: + +6. Hardware Acceleration via Cadence Tensilica NNLib +**************************************************** + +Evaluating millions of multiply-accumulate operations in software loops would exhaust DSP battery budgets. Sound Open Firmware accelerates TFLM execution by replacing generic C++ kernel operators with hand-tuned assembly routines from the **Cadence Tensilica Neural Network Library (NNLib / `xa_nnlib`)**. + +Cadence Tensilica HiFi 4 & HiFi 5 NNLib Integration +=================================================== + +On Intel and NXP platforms powered by Tensilica Xtensa DSPs (e.g. Tiger Lake, Meteor Lake, Panther Lake, i.MX8), SOF's build system links specialized NNLib acceleration blocks (`CMakeLists.txt`): + +* **Vectorized Depthwise Convolution (`xa_nn_conv2d_depthwise_sym8sxasym8s`)**: + + Depthwise convolution processes each input channel with an independent 2D spatial filter. NNLib utilizes Xtensa SIMD vector registers (128-bit on HiFi 4, 256-bit on HiFi 5) to load multiple 8-bit activations and weights simultaneously, executing parallel multiply-accumulates with 32-bit internal saturation in single-cycle instructions. + +* **Pointwise Convolution & GEMM (`xa_nn_conv2d_pointwise`, `xa_nn_matXvec`)**: + + Pointwise :math:`1 \times 1` convolutions project channel representations into new dimensional spaces. NNLib implements high-throughput Matrix-Vector multiplications with circular buffer hardware pointers (`xa_nn_circ_buf`), achieving near-theoretical peak MAC utilization. + +* **Accelerated Non-Linear Activations (`xa_nn_softmax_asym8_asym8`)**: + + Softmax requires exponential operations (:math:`e^{z_i}`) that are computationally expensive on integer DSPs. NNLib implements vectorized fixed-point polynomial approximations that compute 8-bit Softmax distributions in a fraction of generic C++ execution cycles. + +Portable Generic Fallback +========================= + +For embedded microcontroller platforms without proprietary DSP vector extensions—such as the ARM Cortex-M7 on the PJRC Teensy 4.1 or RISC-V on the Espressif ESP32-P4—TFLM automatically falls back to optimized reference kernels utilizing standard integer arithmetic. + +.. graphviz:: + :caption: Hardware Neural Network Acceleration via Cadence Tensilica NNLib (xa_nnlib) and SIMD Vector Lanes + + digraph tflm_simd { + bgcolor="transparent"; + rankdir=LR; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_tflm_core { + label="TFLM Execution Graph"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + layer_conv [label="DepthwiseConv2D Layer", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + layer_fc [label="FullyConnected Layer", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + layer_sm [label="Softmax Layer", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + + layer_conv -> layer_fc -> layer_sm [style="invis"]; + } + + subgraph cluster_nnlib { + label="Cadence Tensilica NNLib Kernels (xa_nnlib)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + k_conv [label="xa_nn_conv2d_depthwise_sym8sxasym8s\n• 128-bit/256-bit SIMD vector MACs\n• 8-way parallel int8 math", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + k_fc [label="xa_nn_matXvec_asym8xasym8\n• Matrix-vector hardware looping\n• Circular buffer auto-wrap", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + k_sm [label="xa_nn_softmax_asym8_asym8\n• Fast fixed-point polynomial exp()", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + } + + subgraph cluster_hw { + label="Hardware Execution Unit"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + simd_alu [label="Tensilica Xtensa HiFi 4 / HiFi 5\nSIMD Vector ALUs & Register Files\nSingle-Cycle Vector Integer MACs", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + } + + layer_conv -> k_conv; + layer_fc -> k_fc; + layer_sm -> k_sm; + + k_conv -> simd_alu; + k_fc -> simd_alu; + k_sm -> simd_alu; + } + +--- + +.. _tflm_pipeline_integration: + +7. System Pipeline Integration & Dynamic Module Loading +******************************************************* + +The TFLM component bridges machine learning models into the Sound Open Firmware audio streaming graph, functioning as a standardized audio sink or inline analysis module. + +SOF Module Adapter & LLEXT Dynamic Linking +========================================== + +The TFLM classifier (`tflmcly`) is implemented as an SOF **Module Adapter**: + +* **Standard Module Interface**: Exports `init`, `process`, `set_configuration`, `reset`, and `free` entry points through `struct module_interface tflmcly_interface`. +* **UUID Registration**: Registered under unique identifier `UUIDREG_STR_TFLMCLY` (declared in `tflmcly.toml`). +* **Loadable Extension (LLEXT) Modular Packaging**: + + For modular firmware architectures, the entire TensorFlow Lite Micro engine, NNLib kernels, and classifier wrapper are packaged as a dynamically loadable ELF module: + + .. code-block:: text + + SOF_LLEXT_MODULE_MANIFEST("TFLMCLY", &tflmcly_interface, 1, SOF_REG_UUID(tflmcly), 40); + + This allows platforms to keep the TFLM machine learning engine offloaded in host storage, dynamically loading it into DSP SRAM only when the user enables voice trigger features. + +Dynamic Model Loading via IPC Blobs +=================================== + +Rather than hard-coding neural network weights into compiled firmware images, SOF supports **dynamic model configuration blobs**: + +* The component instantiates a `comp_data_blob_handler` (`cd->model_handler`). +* Host drivers transmit serialized `.tflite` FlatBuffer binaries via IPC4 `SET_LARGE_CONFIG` messages. +* The handler stages incoming fragments, validates the model FlatBuffer schema version (`model->version() == TFLITE_SCHEMA_VERSION`), and re-initializes the `MicroInterpreter` in place, enabling runtime updates of wake words or sound classification profiles without rebuilding firmware. + +End-to-End Voice AI Capture Pipeline +==================================== + +In a complete voice-enabled smart device, TFLM operates at the terminal stage of a multi-component capture graph: + +1. **Microphone Ingestion**: DAI Copier captures raw multi-channel audio from digital PDM or SoundWire microphones. +2. **DC Blocker (:ref:`dcblock`)**: Strips 0 Hz operational amplifier offsets and mechanical vibration rumble. +3. **Beamformer (:ref:`tdfb`)**: Isolates the primary user's voice and suppresses off-axis reverberation and room noise. +4. **Noise Reduction (RTNR)**: Attenuates stationary background hum. +5. **Feature Extraction (`mfcc`)**: Converts cleaned speech audio into 40-bin Mel spectrogram slices. +6. **Classifier (`tflm`)**: Continuously evaluates sliding spectrogram windows, detecting wake keywords and asserting an asynchronous host wakeup interrupt to initiate cloud speech processing. + +.. graphviz:: + :caption: Microphone Voice AI Pipeline Integration: Feature Extraction, Edge Model Evaluation, and Host Wake Trigger + + digraph tflm_system_pipe { + bgcolor="transparent"; + rankdir=TB; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_hw_in { + label="Audio Input Hardware"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + mics [label="Microphone Array Sensors\n(PDM / SoundWire)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + copier [label="DAI Copier (Endpoint)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + + mics -> copier; + } + + subgraph cluster_prep { + label="Acoustic Clean & Conditioning"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + dcb [label="DC Blocker\n(Strips 0 Hz offset)", fillcolor="#C53030", fontcolor="#FFFFFF"]; + tdfb [label="Time-Domain Fixed Beamformer\n(Enhances talker SNR by +6 dB)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + rtnr [label="Noise Reduction (RTNR)\n(Suppresses stationary noise)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + + copier -> dcb -> tdfb -> rtnr; + } + + subgraph cluster_ml { + label="Edge Machine Learning Intelligence"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + mfcc [label="MFCC Feature Extractor\nGenerates 40-bin Mel slices", fillcolor="#D69E2E", fontcolor="#FFFFFF"]; + tflm_mod [label="TFLM Classifier Module\n• Statically planned Tensor Arena\n• Quantized Int8 Inference\n• Tensilica NNLib Acceleration", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + rtnr -> mfcc [label="Clean 16 kHz Audio"]; + mfcc -> tflm_mod [label="Spectrogram slices"]; + } + + subgraph cluster_host_wake { + label="Host Power & OS Management"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + wake_irq [label="Host Wake Interrupt\n(P(keyword) > 0.85)", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + host_os [label="Host OS Awakes\n(Transcribes User Command)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + + tflm_mod -> wake_irq [label="Positive Wake Event", style="bold", color="#38A169"]; + wake_irq -> host_os; + } + } + +--- + +.. _tflm_tuning_references: + +8. Upstream Code References & Related Guides +******************************************** + +For developers designing custom machine learning models, quantizing neural networks, or integrating TFLM into custom topologies: + +* **Upstream Component Source Files**: + + - `thesofproject/sof: src/audio/tensorflow/README.md `_: Component overview and architecture summary. + - `src/audio/tensorflow/tflm-classify.c `_: SOF Module Adapter C implementation, buffer striding, and IPC configuration. + - `src/audio/tensorflow/speech.h `_: C-to-C++ bridging interface and tensor geometry definitions. + - `src/audio/tensorflow/speech.cc `_: TFLM `MicroInterpreter` initialization, tensor allocation, and inference execution. + - `src/audio/tensorflow/micro_speech_quantized_model_data.cc `_: Pre-compiled quantized model binary FlatBuffer byte array. + - `src/audio/tensorflow/tflmcly.toml `_: Topology metadata defining module type, UUID, and memory pins. + - `src/audio/tensorflow/CMakeLists.txt `_: Build specification linking `xa_nnlib`, `tflite-micro`, `flatbuffers`, and `gemmlowp`. + +* **Upstream Feature Extractor**: + + - `thesofproject/sof: src/audio/mfcc/README.md `_: Mel-Frequency Cepstral Coefficients feature generator. + +Related Subsystem Architecture Guides +===================================== + +* :ref:`tdfb`: Time-Domain Fixed Beamformer providing directional audio pre-processing and spatial noise nulling ahead of ML feature extraction. +* :ref:`dcblock`: First-order recursive high-pass filter stripping ADC DC offsets prior to spectral transformation. +* :ref:`module_framework`: Standardized module lifecycle, memory allocation, and IPC configuration handlers. +* :ref:`pipeline_architecture`: Graph scheduling, buffer management, and audio streaming topologies. +* :ref:`llext_modules`: Building dynamic loadable extensions (LLEXT) that integrate into SOF pipelines at runtime. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 1bea61eb..99257ae3 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -50,7 +50,7 @@ Audio Processing Modules & Algorithms * :ref:`dcblock` (High-level architecture; also see upstream `dcblock README `_) * :ref:`tdfb` (High-level architecture; also see upstream `tdfb README `_ & tuning guide :ref:`time-domain-fixed-beamformer`) * `RTNR Noise Reduction `_ -* `TensorFlow Lite Micro (TFLM) `_ +* :ref:`tflm` (High-level architecture; also see upstream `TFLM README `_) * `MFCC Feature Extraction `_ * `Smart Amp Protection `_ * `Sound Dose Evaluator `_ @@ -94,6 +94,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/crossover firmware/dcblock firmware/tdfb + firmware/tflm rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 880eb0b16691a46e77065daebf84cf3d16752924 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 09:05:05 +0100 Subject: [PATCH 17/64] docs: developer_guides: add high-level mfcc architecture guide Add comprehensive high-level Mel-Frequency Cepstral Coefficients (MFCC) feature extraction architecture developer guide, covering: - Psychoacoustic auditory perception & homomorphic source-filter separation - 5-stage transformation pipeline (pre-emphasis, framing/windowing, 32-bit real-to-complex FFT, auditory Mel filterbanks, DCT-II & sinusoidal cepstral lifter) - Slaney area normalization & uniform Mel filterbank synthesis - OpenAI Whisper feature extraction integration (80 Mel channels, dynamic m_max leaky tracking, top_db clamping, host offload) - Integrated Mel-domain VAD & Discontinuous Transmission (DTX) state machine - Fixed-point scratch overlay memory architecture (dual complex FFT buffers) & decoupled staging buffer - Cadence Tensilica HiFi 3/4/5 SIMD vector acceleration primitives - ALSA Topology 2 configuration (Class.Widget.mfcc) & Compress/Legacy PCM streaming modes - End-to-end edge AI audio pipeline with 7 native vector Graphviz SVG diagrams Signed-off-by: Liam Girdwood --- developer_guides/firmware/mfcc.rst | 840 ++++++++++++++++++ .../firmware/pipeline_architecture.rst | 1 + developer_guides/firmware/tflm.rst | 5 +- developer_guides/index.rst | 3 +- 4 files changed, 846 insertions(+), 3 deletions(-) create mode 100644 developer_guides/firmware/mfcc.rst diff --git a/developer_guides/firmware/mfcc.rst b/developer_guides/firmware/mfcc.rst new file mode 100644 index 00000000..897c0c33 --- /dev/null +++ b/developer_guides/firmware/mfcc.rst @@ -0,0 +1,840 @@ +.. _mfcc: + +Mel-Frequency Cepstral Coefficients (MFCC) Feature Extraction Architecture +########################################################################## + +The **Mel-Frequency Cepstral Coefficients (MFCC)** subsystem in Sound Open Firmware provides real-time, psychoacoustically motivated audio feature extraction and embedded spectral analysis directly on digital signal processor (DSP) hardware. + +In embedded speech recognition, keyword spotting, acoustic event detection, and generative speech-to-text models (such as OpenAI Whisper), feeding raw, uncompressed time-domain pulse-code modulation (PCM) audio samples directly into neural networks is computationally prohibitive. Raw audio waveforms exhibit immense data rates (e.g. 16,000 samples per second per channel), extreme temporal redundancy, sensitivity to room reverberation and phase shifts, and high computational dimensionality that overwhelms microcontrollers and DSP accelerators. + +To solve this challenge, Sound Open Firmware integrates an optimized, fixed-point **MFCC Feature Extraction Module**. Designed as an in-line audio processing component conforming to the SOF Module Adapter framework, the MFCC engine transforms continuous, raw acoustic waveforms into compact, decorrelated spectro-temporal feature representations. Operating with a zero-heap memory architecture, the module executes high-pass pre-emphasis filtering, analysis windowing, Fast Fourier Transform (FFT) analysis, triangular auditory Mel filterbank integration, dynamic range clamping, logarithmic compression, Discrete Cosine Transform (DCT Type-II) decorrelation, and sinusoidal cepstral liftering. + +Beyond classic cepstral coefficient extraction, the SOF MFCC subsystem incorporates modern deep learning features: an optimized **Mel-only mode** tailored for OpenAI Whisper and transformer speech models, an integrated **Mel-domain Voice Activity Detector (VAD)** utilizing IEC 61672-1 A-weighting formant curves, and **Discontinuous Transmission (DTX)** silence suppression that eliminates redundant DMA transfers to host CPUs and Neural Processing Units (NPUs). + +This guide provides a comprehensive, high-level architectural walkthrough of the MFCC feature extraction subsystem in SOF, examining psychoacoustic foundations, homomorphic source-filter separation, five-stage transformation pipelines, Whisper dynamic max tracking, Mel-domain VAD mechanics, fixed-point scratch overlay memory designs, Cadence Tensilica HiFi SIMD vector acceleration, and ALSA Topology 2 / IPC streaming modes without delving into low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +.. _mfcc_psychoacoustic_foundations: + +1. Psychoacoustic Foundations & Auditory Representation +******************************************************* + +The design of the MFCC feature extraction pipeline is grounded in empirical principles of human psychoacoustics and the homomorphic deconvolution of speech production. + +Biological Auditory Perception & The Mel Scale +============================================== + +The human auditory system does not perceive acoustic frequencies linearly. Sound waves entering the ear canal cause vibrations in the tympanic membrane and middle ear ossicles, which translate into traveling fluid waves within the cochlea. Along the length of the cochlear basilar membrane, different physical locations resonate at specific frequencies—a biological frequency decomposition known as *tonotopic organization* or the *place theory of pitch*. + +High-frequency sounds stimulate hair cells near the stiff, narrow base of the cochlea, whereas low-frequency sounds travel to the flexible, wide apex. Crucially, the density of auditory sensory receptors is non-linear: humans possess extraordinary frequency resolution at low frequencies (below 1,000 Hz) to discern fundamental pitch and vowel formants, but significantly coarser resolution at high frequencies (above 1,000 Hz) where broadband fricative noises and consonant transients reside. + +To model this non-linear perceptual sensitivity, Stevens, Volkmann, and Newman (1937) established the **Mel Scale**—a perceptual scale of pitches judged by human listeners to be equal in distance from one another. A frequency of 1,000 Hz at 40 dB above the listener's threshold is defined as 1,000 Mel. + +The conversion between physical acoustic frequency :math:`f` (in Hertz) and perceptual pitch :math:`m` (in Mel) is mathematically formulated using either the classic logarithmic approximation or the standard Auditory Toolbox formulation: + +.. math:: + + m = 2595 \cdot \log_{10}\left(1 + \frac{f}{700}\right) = 1127 \cdot \ln\left(1 + \frac{f}{700}\right) + +Conversely, the inverse transformation from Mel space back to physical frequency in Hertz is expressed as: + +.. math:: + + f = 700 \cdot \left(10^{\frac{m}{2595}} - 1\right) = 700 \cdot \left(e^{\frac{m}{1127}} - 1\right) + +In the Mel domain, equal distance corresponds to equal perceived musical interval and phonetic distinction. Below 1,000 Hz, the relationship between Hertz and Mel is approximately linear; above 1,000 Hz, the relationship becomes logarithmic, mirroring the human ear's critical auditory bandwidths. + +The Homomorphic Source-Filter Model of Speech +============================================= + +Human speech production is universally modeled as a linear time-invariant convolution of an acoustic excitation source :math:`e(t)` with the acoustic resonance of the vocal tract filter :math:`h(t)`: + +.. math:: + + s(t) = e(t) * h(t) + +* **The Excitation Source** :math:`e(t)`: Produced by airflow forced through the oscillating vocal cords (voiced speech, creating a periodic glottal pulse train with fundamental frequency :math:`F_0`) or turbulent airflow forced through a narrow constriction in the vocal tract (unvoiced speech, creating broadband white noise). +* **The Vocal Tract Filter** :math:`h(t)`: Formed by the pharyngeal, oral, and nasal cavities. The geometric shape of the tongue, lips, jaw, and velum acts as an acoustic resonator, amplifying specific resonant frequencies called **formants** (:math:`F_1, F_2, F_3`) that define phonetic vowels and consonants. + +In automatic speech recognition (ASR) and keyword spotting, the identity of spoken words is dictated almost entirely by the vocal tract filter :math:`h(t)` (the phonetic formants), whereas the excitation source :math:`e(t)` conveys speaker-dependent pitch, gender, emotion, and vocal fry. To recognize words accurately across different speakers, an audio feature extractor must **decouple the vocal tract resonance from the pitch excitation**. + +Because the source and filter are convolved in the time domain, their Fourier transforms are multiplied in the frequency domain: + +.. math:: + + S(f) = E(f) \cdot H(f) + +Taking the complex magnitude and applying the natural logarithm transforms multiplication into addition: + +.. math:: + + \log |S(f)| = \log |E(f)| + \log |H(f)| + +This mathematical operation is termed **homomorphic filtering**. In the log-magnitude spectrum, the slowly varying spectral envelope :math:`\log |H(f)|` (the vocal tract formants) is linearly superimposed upon the rapidly fluctuating harmonic ripples :math:`\log |E(f)|` (the glottal pitch harmonics). + +The Cepstrum & Quefrency Domain +=============================== + +To separate these additive components, the logarithm of the power spectrum is treated as an ordinary time-domain signal, and its inverse Fourier transform or Discrete Cosine Transform (DCT) is computed. The resulting mathematical domain is called the **Cepstrum** (an anagram of *spectrum*), and its horizontal axis is defined as **Quefrency** (an anagram of *frequency*), measured in units of time (seconds or samples): + +.. math:: + + c[n] = \text{DCT}\Big( \log |S(f)| \Big) = \text{DCT}\Big( \log |H(f)| \Big) + \text{DCT}\Big( \log |E(f)| \Big) + +* **Low-Quefrency Coefficients** (:math:`c_1` to :math:`c_{12}`): Represent the slowly varying spectral envelope, capturing the physical shape of the speaker's vocal tract and formant locations. These coefficients are virtually invariant to pitch and fundamental frequency. +* **High-Quefrency Coefficients**: Represent the rapid spectral variations corresponding to the glottal pitch period :math:`T_0 = 1 / F_0`. +* **Zero-th Coefficient** (:math:`c_0`): Represents the average log-energy of the entire frame. + +By retaining only the low-quefrency cepstral coefficients (typically the first 13 to 40 values) and discarding high-quefrency bins, the MFCC feature extractor effectively strips away speaker pitch and acoustic excitation, producing an invariant, robust representation of human speech. + +.. graphviz:: + :caption: Psychoacoustic Auditory Representation and Homomorphic Source-Filter Separation + + digraph mfcc_psychoacoustics { + bgcolor="transparent"; + node [fontname="Helvetica", fontsize=11, shape=box, style="filled,rounded", color="#0a7d91", fillcolor="#e0f4f7", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=10, color="#2c3e50", penwidth=1.2]; + + subgraph cluster_production { + label="Acoustic Speech Production Model"; + style="dashed,rounded"; + color="#7f8c8d"; + bgcolor="#f9fbfd"; + + glottal [label="Glottal Source e(t)\n(Vocal Cord Pitch Pulses)", fillcolor="#fff3cd", color="#f39c12"]; + vocal [label="Vocal Tract Filter h(t)\n(Pharynx, Tongue, Mouth Resonances)", fillcolor="#d1e7dd", color="#198754"]; + conv [label="Convolution\ns(t) = e(t) * h(t)", shape=circle, width=1.1, fillcolor="#cfe2ff", color="#0d6efd"]; + speech [label="Radiated Acoustic Waveform s(t)\n(Coupled Time-Domain Signal)", fillcolor="#e2e3e5", color="#6c757d"]; + + glottal -> conv [label="Pitch F0"]; + vocal -> conv [label="Formants F1-F3"]; + conv -> speech; + } + + subgraph cluster_homomorphic { + label="Homomorphic Deconvolution"; + style="dashed,rounded"; + color="#7f8c8d"; + bgcolor="#f9fbfd"; + + fft_log [label="Log Magnitude Spectrum\nln|S(f)| = ln|E(f)| + ln|H(f)|\n(Multiplication -> Addition)", fillcolor="#d1e7dd", color="#198754"]; + mel_fb [label="Auditory Mel Filterbank\n(Non-linear Cochlear Frequency Resolution)", fillcolor="#e0f4f7", color="#0a7d91"]; + dct [label="Discrete Cosine Transform (DCT-II)\n(Frequency Domain -> Quefrency Domain)", fillcolor="#d1e7dd", color="#198754"]; + } + + subgraph cluster_quefrency { + label="Quefrency Separation"; + style="dashed,rounded"; + color="#7f8c8d"; + bgcolor="#f9fbfd"; + + low_q [label="Low-Quefrency (c1 - c12)\nVocal Tract Formants / Phonetic Shape\n[Retained for Machine Learning]", fillcolor="#cfe2ff", color="#0d6efd", penwidth=2.0]; + high_q [label="High-Quefrency (> c13)\nGlottal Pitch Pulses / Speaker Harmonics\n[Discarded for Pitch Invariance]", fillcolor="#f8d7da", color="#dc3545", style="filled,dashed"]; + } + + speech -> fft_log [label="Fourier Transform"]; + fft_log -> mel_fb [label="Critical Bands"]; + mel_fb -> dct [label="Log Mel Energies"]; + dct -> low_q [label="Envelope"]; + dct -> high_q [label="Harmonics"]; + } + +--- + +.. _mfcc_component_architecture: + +2. Five-Stage MFCC Feature Extraction Engine +******************************************** + +The SOF MFCC subsystem structures feature extraction into a deterministic, five-stage mathematical processing pipeline executing within the module's `mfcc_process()` audio loop. + +.. graphviz:: + :caption: Five-Stage MFCC Feature Extraction Pipeline in Sound Open Firmware + + digraph mfcc_pipeline { + bgcolor="transparent"; + node [fontname="Helvetica", fontsize=11, shape=box, style="filled,rounded", color="#0a7d91", fillcolor="#e0f4f7", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=10, color="#2c3e50", penwidth=1.2]; + + pcm_in [label="Audio Source Pin\n(Interleaved S16, S24, or S32 PCM)", shape=parallelogram, fillcolor="#e2e3e5", color="#6c757d"]; + pre_emph [label="Stage 1: Pre-Emphasis Filter\ny[n] = x[n] - α·x[n-1]\n(Boosts high frequencies +6 dB/oct)", fillcolor="#fff3cd", color="#f39c12"]; + windowing [label="Stage 2: Overlapping Framing & Windowing\n(Circular Buffer Wrap + Hann/Hamming/Povey)\n[400 samples / 25 ms @ 16 kHz]", fillcolor="#d1e7dd", color="#198754"]; + fft [label="Stage 3: Fast Fourier Transform (FFT)\n32-bit Real-to-Complex (Radix-2/4)\nPower Spectrum |X[k]|²", fillcolor="#cfe2ff", color="#0d6efd"]; + mel_fb [label="Stage 4: Auditory Mel Filterbank\n(23 to 80 Triangular Bands + Slaney Norm)\nLogarithmic Compression (ln, log10, dB)", fillcolor="#e0f4f7", color="#0a7d91"]; + decision [label="Output Mode\nnum_ceps == 0 ?", shape=diamond, fillcolor="#fff3cd", color="#f39c12"]; + dct [label="Stage 5: DCT-II & Cepstral Lifter\nOrthonormal Decorrelation + Sinusoidal Lifter\n(Produces 13-40 Cepstral Coefficients)", fillcolor="#d1e7dd", color="#198754"]; + mel_out [label="Mel Log Spectrogram Output\n(e.g. 80 Mel bins for OpenAI Whisper)", shape=parallelogram, fillcolor="#cfe2ff", color="#0d6efd"]; + ceps_out [label="MFCC Feature Output\n(13 Cepstral Coefficients for KWS/ASR)", shape=parallelogram, fillcolor="#d1e7dd", color="#198754"]; + + pcm_in -> pre_emph [label="Channel Select"]; + pre_emph -> windowing [label="Q1.15 Stream"]; + windowing -> fft [label="Zero-Padded 512"]; + fft -> mel_fb [label="257 Power Bins"]; + mel_fb -> decision [label="Q9.23 Log Mel"]; + decision -> mel_out [label="Yes (Mel-Only)"]; + decision -> dct [label="No (num_ceps > 0)"]; + dct -> ceps_out; + } + +Stage 1: High-Pass Pre-Emphasis Filtering +========================================= + +During natural speech production, glottal airflow pulses radiating through the mouth opening experience acoustic impedance that causes a natural spectral roll-off of approximately **-6 dB per octave** across higher frequencies. Consequently, higher-frequency speech formants (above 1,000 Hz) possess significantly lower energy than low-frequency vowel fundamentals, despite conveying crucial phonetic information (such as dental and sibilant consonants like /s/, /t/, /f/). + +The SOF MFCC module applies a first-order high-pass finite impulse response (FIR) pre-emphasis filter to flatten the speech spectrum and balance dynamic range: + +.. math:: + + y[n] = x[n] - \alpha \cdot x[n-1] + +Where :math:`\alpha` is the pre-emphasis coefficient, configured in fixed-point :math:`Q1.15` format (typically :math:`\alpha = 0.97`, represented as `31785`). This high-pass filter provides a :math:`+6\text{ dB/octave}` boost, equalizing the dynamic range of formant peaks across the entire Nyquist bandwidth and improving the numerical conditioning of subsequent fixed-point FFT stages. + +Stage 2: Overlapping Framing & Analysis Windowing +================================================= + +Speech is a non-stationary signal whose spectral characteristics evolve continuously over time. However, across short temporal intervals of **20 to 30 milliseconds**, the vocal tract geometry remains physically quasi-stationary. + +The MFCC module segments the continuous audio stream into overlapping frames: + +* **Frame Length** (:math:`N`): Duration of each analysis window, typically 25 ms (400 samples at 16,000 Hz). +* **Frame Shift / Hop Size** (:math:`H`): Time interval between successive analysis frames, typically 10 ms (160 samples at 16,000 Hz). +* **Frame Overlap**: The consecutive frames overlap by :math:`N - H = 240\text{ samples}` (15 ms), ensuring smooth temporal continuity and preventing data loss at frame boundaries. + +To avoid abrupt truncation at frame edges (which introduces severe spectral leakage and artificial high-frequency sidelobes in the frequency domain), the module applies a tapered analysis window function :math:`w[n]`: + +.. math:: + + x_w[n] = x[n] \cdot w[n], \quad 0 \le n < N + +The SOF MFCC engine supports five distinct window geometries: + +1. **Hann Window**: :math:`w[n] = 0.5 - 0.5 \cos\left(\frac{2\pi n}{N - 1}\right)`, delivering -32 dB sidelobe suppression. +2. **Hamming Window**: :math:`w[n] = 0.54 - 0.46 \cos\left(\frac{2\pi n}{N - 1}\right)`, optimizing first sidelobe attenuation to -43 dB. +3. **Blackman Window**: Three-term cosine window with :math:`\alpha_0 = 0.42`, delivering -58 dB sidelobe suppression. +4. **Povey Window**: :math:`w[n] = \left(0.5 - 0.5 \cos\left(\frac{2\pi n}{N - 1}\right)\right)^{0.85}`, the standard window used in the Kaldi speech recognition toolkit. +5. **Rectangular Window**: Uniform weighting (:math:`w[n] = 1.0`), used for baseline acoustic benchmarking. + +Stage 3: Fast Fourier Transform (FFT) & Power Spectrum +====================================================== + +To convert the windowed time-domain frames into the frequency domain, the module zero-pads the frame length :math:`N` up to the next power of two (e.g. 400 samples zero-padded to 512 samples) and executes a 32-bit complex Fast Fourier Transform: + +.. math:: + + X[k] = \sum_{n=0}^{N_{FFT}-1} x_w[n] \cdot e^{-j \frac{2\pi k n}{N_{FFT}}}, \quad 0 \le k < N_{FFT} + +Because the input audio is strictly real-valued, the resulting complex spectrum is conjugate-symmetric (:math:`X[N_{FFT} - k] = X^*[k]`). The engine only needs to compute and retain the non-redundant positive frequencies: + +.. math:: + + K_{bins} = \frac{N_{FFT}}{2} + 1 + +For a 512-point FFT, exactly 257 complex frequency bins are produced. The power spectrum :math:`P[k]` is subsequently computed as the squared magnitude of each bin: + +.. math:: + + P[k] = \frac{1}{N_{FFT}} |X[k]|^2 = \frac{1}{N_{FFT}} \Big( \text{Re}\{X[k]\}^2 + \text{Im}\{X[k]\}^2 \Big) + +Stage 4: Auditory Mel Filterbank Integration +============================================ + +The linear frequency power spectrum :math:`P[k]` is mapped into the auditory Mel domain by passing it through a bank of :math:`M` overlapping triangular bandpass filters: + +.. math:: + + S_m = \sum_{k=0}^{K_{bins}-1} P[k] \cdot H_m[k], \quad 0 \le m < M + +Each triangular filter :math:`H_m[k]` is parameterized by three boundary frequencies in Hertz: lower edge :math:`f_{m-1}`, center peak :math:`f_m`, and upper edge :math:`f_{m+1}`: + +.. math:: + + H_m[k] = \begin{cases} + 0 & f[k] < f_{m-1} \\ + \frac{f[k] - f_{m-1}}{f_m - f_{m-1}} & f_{m-1} \le f[k] \le f_m \\ + \frac{f_{m+1} - f[k]}{f_{m+1} - f_m} & f_m \le f[k] \le f_{m+1} \\ + 0 & f[k] > f_{m+1} + \end{cases} + +Following filterbank summation, logarithmic compression models human non-linear loudness perception (the Weber-Fechner law): + +.. math:: + + E_m = \log(S_m) + +The engine supports three logarithmic bases via configuration: natural logarithm (`MEL_LOG_IS_LOG`), base-10 logarithm (`MEL_LOG_IS_LOG10`), and decibels (`MEL_LOG_IS_DB` where :math:`E_m = 10 \log_{10}(S_m)`). A minimum power floor parameter (`pmin`, typically :math:`10^{-9}`) prevents numerical underflow or infinite negative logarithms during absolute digital silence. + +Stage 5: Discrete Cosine Transform (DCT-II) & Cepstral Liftering +================================================================ + +In classic MFCC extraction (when `num_ceps > 0`), the log Mel filterbank energies :math:`E_m` are highly correlated with one another due to spectral overlap between adjacent triangular filters. To compact the energy and produce uncorrelated features, the module applies an **orthonormal Discrete Cosine Transform of Type II (DCT-II)**: + +.. math:: + + c_n = \sum_{m=0}^{M-1} E_m \cdot \cos\left( \frac{\pi n (m + 0.5)}{M} \right), \quad 0 \le n < N_{ceps} + +Because the DCT decomposes the spectrum into orthogonal cosine basis functions, it acts as an optimal Karhunen-Loève transform (KLT) approximation for speech signals, concentrating the vast majority of phonetic information into the first 13 coefficients (:math:`c_0` to :math:`c_{12}`). + +Finally, to equalize the numerical variance between lower-order coefficients (which have very high amplitudes) and higher-order coefficients (which have smaller amplitudes), the module applies a **sinusoidal cepstral lifter**: + +.. math:: + + \hat{c}_n = c_n \cdot w_{lifter}[n] = c_n \cdot \left( 1 + \frac{L}{2} \sin\left(\frac{\pi n}{L}\right) \right) + +Where :math:`L` is the cepstral lifter parameter (configured in :math:`Q7.9` format, typically :math:`L = 22.0`). This sinusoidal weighting scales up higher-order coefficients, balancing their contribution in downstream Euclidean distance metrics and neural network classifiers. + +--- + +.. _mfcc_mel_filterbank_slaney: + +3. Auditory Mel Filterbank & Slaney Area Normalization +****************************************************** + +A critical architectural feature of the SOF MFCC filterbank is the distinction between standard triangular filters and Malcolm Slaney's area-normalized filterbank (`norm = MFCC_MEL_NORM_SLANEY`), widely adopted in modern machine learning libraries (such as Librosa and Kaldi). + +.. graphviz:: + :caption: Auditory Mel Filterbank Construction: Linear Spacing in Mel vs Expanding Bandwidths in Hertz + + digraph mel_filterbank_geometry { + bgcolor="transparent"; + node [fontname="Helvetica", fontsize=11, shape=box, style="filled,rounded", color="#0a7d91", fillcolor="#e0f4f7", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=10, color="#2c3e50", penwidth=1.2]; + + subgraph cluster_mel_axis { + label="Uniform Mel Domain (0 to 2840 Mel)"; + style="dashed,rounded"; + color="#7f8c8d"; + bgcolor="#f9fbfd"; + + m0 [label="Mel Bin 0\n(m = 0)", fillcolor="#d1e7dd", color="#198754"]; + m1 [label="Mel Bin 10\n(m = 355)", fillcolor="#d1e7dd", color="#198754"]; + m2 [label="Mel Bin 40\n(m = 1420)", fillcolor="#d1e7dd", color="#198754"]; + m3 [label="Mel Bin 80\n(m = 2840)", fillcolor="#d1e7dd", color="#198754"]; + + m0 -> m1 [label="Equal Δm\n(35.5 Mel)"]; + m1 -> m2 [label="Equal Δm\n(35.5 Mel)"]; + m2 -> m3 [label="Equal Δm\n(35.5 Mel)"]; + } + + subgraph cluster_hz_axis { + label="Physical Frequency Domain (0 to 8,000 Hz)"; + style="dashed,rounded"; + color="#7f8c8d"; + bgcolor="#f9fbfd"; + + f0 [label="f0 = 0 Hz\n(Narrow Bandwidth Δf = 32 Hz)", fillcolor="#fff3cd", color="#f39c12"]; + f1 [label="f10 = 250 Hz\n(Narrow Bandwidth Δf = 45 Hz)", fillcolor="#fff3cd", color="#f39c12"]; + f2 [label="f40 = 1,480 Hz\n(Moderate Bandwidth Δf = 175 Hz)", fillcolor="#cfe2ff", color="#0d6efd"]; + f3 [label="f80 = 8,000 Hz\n(Broad Bandwidth Δf = 980 Hz)", fillcolor="#f8d7da", color="#dc3545"]; + + f0 -> f1 [label="Dense Filters\n(High Resolution)"]; + f1 -> f2 [label="Logarithmic Spacing"]; + f2 -> f3 [label="Broad Filters\n(Coarse Resolution)"]; + } + + m0 -> f0 [style="dotted", label="Inverse Mel"]; + m1 -> f1 [style="dotted", label="Inverse Mel"]; + m2 -> f2 [style="dotted", label="Inverse Mel"]; + m3 -> f3 [style="dotted", label="Inverse Mel"]; + } + +The Problem with Unnormalized Triangular Filters +================================================ + +In a standard triangular filterbank, every triangular filter has a peak amplitude of :math:`1.0`. However, because the filters are uniformly spaced in the Mel domain, their physical bandwidth in Hertz expands dramatically as frequency increases: + +* At 200 Hz, a filter may span a bandwidth of only 40 Hz. +* At 6,000 Hz, a filter spans a bandwidth exceeding 1,000 Hz. + +If all filters have a peak height of 1.0, the area under each triangle is proportional to its bandwidth (:math:`\text{Area} = 0.5 \times \Delta f`). Consequently, high-frequency filters integrate power over a much wider frequency span, artificially inflating high-frequency energies and tilting the spectral balance upward. + +Slaney Area Normalization Principle +=================================== + +To ensure that flat white noise produces equal energy across all filterbank channels, Malcolm Slaney's Auditory Toolbox normalizes each triangular filter by dividing its coefficients by the filter's acoustic bandwidth in Hertz: + +.. math:: + + H_{m, \text{Slaney}}[k] = H_m[k] \cdot \left( \frac{2}{f_{m+1} - f_{m-1}} \right) + +This normalization ensures that the integral of each triangular filter equals unity (:math:`\int H_{m,\text{Slaney}}(f) df = 1`). In the SOF firmware, this calculation is executed during `mfcc_setup()` using high-precision integer division, scaling the filter weights so that each bin measures true **power spectral density** rather than total integrated bandwidth power. + +--- + +.. _mfcc_whisper_integration: + +4. OpenAI Whisper & Modern Deep Learning Feature Integration +************************************************************ + +While traditional speech pipelines require cepstral coefficients (:math:`c_1` to :math:`c_{12}`), modern deep neural networks—such as OpenAI Whisper, Conformer, and wav2vec 2.0—bypass the DCT entirely. These architectures ingest high-resolution **80-channel log Mel spectrograms** directly, using multi-head self-attention mechanisms to learn optimal representations. + +The SOF MFCC module provides first-class support for OpenAI Whisper front-end feature generation embedded entirely within the audio DSP. + +.. graphviz:: + :caption: OpenAI Whisper Preprocessing Data Path: Dynamic Max Tracking, Clamping, and Scaling + + digraph whisper_preprocessing { + bgcolor="transparent"; + node [fontname="Helvetica", fontsize=11, shape=box, style="filled,rounded", color="#0a7d91", fillcolor="#e0f4f7", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=10, color="#2c3e50", penwidth=1.2]; + + raw_mel [label="Raw Log Mel Spectrum\n(80 Mel Bins @ 16 kHz in Q9.23)", shape=parallelogram, fillcolor="#e2e3e5", color="#6c757d"]; + peak_detect [label="Peak Search\nFind Maximum Bin\npeak = max(mel_log_32[j])", fillcolor="#fff3cd", color="#f39c12"]; + dyn_mmax [label="Dynamic mmax Tracker\n(Instant Rise on Higher Peak,\nSlow Exponential Leaky Decay on Lower)", fillcolor="#d1e7dd", color="#198754"]; + clamp [label="Top-dB Noise Gate Clamp\nclamp_val = mmax - (top_db << 16)\nmel[j] = max(mel[j], clamp_val)\n[Clamps 80 dB below active peak]", fillcolor="#cfe2ff", color="#0d6efd"]; + scale_offset [label="Whisper Affine Normalization\nval = mel[j] + mel_offset (default +4.0)\nout[j] = val * mel_scale (default 0.25)\n[Q9.23 output in [-1.0, 1.0] range]", fillcolor="#e0f4f7", color="#0a7d91"]; + whisper_npu [label="OpenAI Whisper Model\n(Host CPU / GPU / Intel NPU via OpenVINO)", shape=parallelogram, fillcolor="#d1e7dd", color="#198754", penwidth=2.0]; + + raw_mel -> peak_detect; + peak_detect -> dyn_mmax [label="Frame Peak"]; + dyn_mmax -> clamp [label="Adaptive Ceiling mmax"]; + raw_mel -> clamp [label="80 Mel Bins"]; + clamp -> scale_offset [label="Dynamic Range Bounded"]; + scale_offset -> whisper_npu [label="Normalized Features"]; + } + +Whisper Architectural Requirements +================================== + +OpenAI Whisper specifies precise mathematical constraints for audio ingestion: + +1. **80 Mel Filterbank Channels**: Spanning 0 Hz to 8,000 Hz on 16 kHz audio. +2. **25 ms Window with 10 ms Stride**: Exactly 400 samples framing with 160 samples hop. +3. **Dynamic Range Clamping**: Clamping minimum Mel values to :math:`m_{max} - 8.0` in natural log units (equivalent to -80 dB below the frame peak). +4. **Affine Scaling and Normalization**: Shifting by :math:`+4.0` and scaling by :math:`0.25` to map values into the normalized dynamic range :math:`[-1.0, 1.0]` expected by Whisper transformer encoders: + +.. math:: + + M_{\text{Whisper}}[j] = \frac{\max\Big(M[j], \; m_{max} - 8.0\Big) + 4.0}{4.0} = 0.25 \cdot \Big(\text{clamped}[j] + 4.0\Big) + +On-DSP Dynamic Max Tracking & Clamping +====================================== + +In offline Python implementations, :math:`m_{max}` is computed globally across an entire 30-second audio buffer. In real-time streaming DSP firmware, future audio is unknown. The SOF MFCC module solves this by implementing **dynamic peak Mel tracking** with asymmetric exponential decay: + +* **Instantaneous Attack**: When the current frame peak exceeds the tracked maximum (:math:`\text{peak} > m_{max}`), the tracker immediately snaps upward: + + .. math:: + + m_{max} = \text{peak} + +* **Leaky Decay**: During quieter intervals, :math:`m_{max}` decays slowly according to an exponential decay coefficient :math:`\alpha_{mmax}` (`config->mmax_coef`): + + .. math:: + + m_{max} \gets m_{max} + \alpha_{mmax} \cdot (\text{peak} - m_{max}) + +This dynamic tracking ensures that speech signals remain perfectly clamped against local acoustic volume levels without clipping sudden loud utterances or dropping faint whispering. + +Zero Host Wakeup & NPU Streaming +================================= + +By executing pre-emphasis, windowing, FFT, Mel filterbanks, dynamic max tracking, and affine normalization directly on the low-power DSP, the host application processor remains in deep low-power sleep (D3 / S0ix). When speech occurs, the DSP streams normalized Mel frames directly to the Intel NPU or GPU via OpenVINO, bypassing all CPU-side feature extraction and eliminating host cache thrashing. + +--- + +.. _mfcc_vad_dtx_architecture: + +5. Integrated Mel-Domain VAD & Discontinuous Transmission +********************************************************* + +A major innovation of the SOF MFCC module is its integrated **Mel-Domain Voice Activity Detector (VAD)** and **Discontinuous Transmission (DTX)** engine. Rather than running a separate, computationally redundant time-domain VAD component, the MFCC module evaluates voice activity directly on the 80-channel Mel log spectrum already computed in SRAM. + +.. graphviz:: + :caption: Integrated Mel-Domain VAD and Discontinuous Transmission (DTX) State Machine + + digraph mfcc_vad_dtx { + bgcolor="transparent"; + node [fontname="Helvetica", fontsize=11, shape=box, style="filled,rounded", color="#0a7d91", fillcolor="#e0f4f7", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=10, color="#2c3e50", penwidth=1.2]; + + mel_in [label="Q9.23 Mel Log Spectrum\n(From Stage 4 Filterbank)", shape=parallelogram, fillcolor="#e2e3e5", color="#6c757d"]; + + subgraph cluster_vad { + label="Mel-Domain Voice Activity Detection"; + style="dashed,rounded"; + color="#7f8c8d"; + bgcolor="#f9fbfd"; + + a_weight [label="IEC 61672 A-Weighting\n(Emphasizes 1-4 kHz Formants,\nAttenuates Sub-Bass & Infrasound)", fillcolor="#fff3cd", color="#f39c12"]; + noise_track [label="Per-Bin Noise Floor Tracking\nInstant Follow-Down on Drop\nSlow Exponential Rise: α_slow = 0.003", fillcolor="#d1e7dd", color="#198754"]; + delta [label="Energy Delta Calculation\nΔE = E_signal - E_noise\n(Weighted Q9.23 Formant Energy)", fillcolor="#cfe2ff", color="#0d6efd"]; + threshold [label="Threshold Comparator\nΔE > 0.30 (2,516,582 in Q9.23)?", shape=diamond, fillcolor="#fff3cd", color="#f39c12"]; + hangover [label="Hangover Counter Gate\n(Maintains VAD=1 for 20 Frames / 200 ms\nPrevents Word Ending Chopping)", fillcolor="#d1e7dd", color="#198754"]; + } + + subgraph cluster_dtx { + label="Discontinuous Transmission (DTX)"; + style="dashed,rounded"; + color="#7f8c8d"; + bgcolor="#f9fbfd"; + + dtx_decision [label="VAD == 0 && DTX Enabled?", shape=diamond, fillcolor="#fff3cd", color="#f39c12"]; + trailing [label="Trailing Silence Counter\nSend 10 Silence Frames\n(Preserves Acoustic Context)", fillcolor="#cfe2ff", color="#0d6efd"]; + suppress [label="Frame Suppressed!\n(0 Bytes Committed to Sink,\nHost & DMA Sleep)", fillcolor="#f8d7da", color="#dc3545", penwidth=2.0]; + keepalive [label="Periodic Keep-Alive Update\n(Send 1 Silence Frame every N hops)", fillcolor="#d1e7dd", color="#198754"]; + transmit [label="Transmit Frame to Sink\n(Header + Mel / Cepstral Payload)", shape=parallelogram, fillcolor="#cfe2ff", color="#0d6efd", penwidth=2.0]; + } + + mel_in -> a_weight; + mel_in -> noise_track; + a_weight -> delta; + noise_track -> delta; + delta -> threshold; + threshold -> hangover [label="Speech Detected"]; + threshold -> hangover [label="Below Threshold"]; + hangover -> dtx_decision [label="VAD Decision"]; + + dtx_decision -> transmit [label="VAD == 1 (Active Speech)"]; + dtx_decision -> trailing [label="VAD == 0 (Silence)"]; + trailing -> transmit [label="count <= dtx_trailing"]; + trailing -> keepalive [label="count > dtx_trailing"]; + keepalive -> transmit [label="Counter Reached Interval"]; + keepalive -> suppress [label="Between Intervals"]; + } + +IEC 61672-1:2013 A-Weighting Formant Emphasis +============================================= + +Ambient acoustic environments contain significant low-frequency energy—HVAC rumble, vehicle road noise, wind buffeting—that possesses high physical energy but zero speech relevance. Conversely, human speech formants are concentrated in the **1,000 Hz to 4,000 Hz** band. + +To maximize detection sensitivity, the SOF VAD computes speech-frequency emphasis weights by linearly interpolating the international standard **IEC 61672-1:2013 A-weighting table** across the center frequencies of all Mel bins. The weights :math:`W[m]` are normalized to sum to :math:`1.0` in :math:`Q1.15` format, prioritizing frequencies between 1 kHz and 4 kHz while heavily attenuating infrasound below 100 Hz. + +Per-Bin Adaptive Noise Floor Tracking +===================================== + +Rather than applying a scalar noise floor, the VAD maintains an independent noise floor :math:`N[m]` for **every individual Mel bin**: + +1. **Instant Downward Tracking**: If the current Mel bin energy :math:`M[m]` drops below the current noise floor estimate, speech is impossible. The noise floor instantly drops to match the new minimum: + + .. math:: + + N[m] = M[m] + +2. **Slow Upward Rise**: When the energy is higher than the floor, the noise floor rises very slowly with an exponential smoothing factor :math:`\alpha`: + + .. math:: + + N[m] \gets N[m] + \alpha \cdot (M[m] - N[m]) + +To ensure rapid convergence upon stream initialization, the engine uses a dual-rate scheme: + +* **Fast Initialization Phase (`init_frames = 100`)**: During the first 100 frames (~1.0 second), :math:`\alpha_{fast} = 0.020` (`655` in :math:`Q1.15`), rapidly acquiring the ambient room noise profile. +* **Steady-State Tracking Phase**: After 100 frames, :math:`\alpha_{slow} = 0.003` (`98` in :math:`Q1.15`), preventing the noise floor from rising during sustained spoken sentences. + +Energy Delta & Hangover Mechanics +================================= + +The VAD computes the A-weighted total signal energy :math:`E_{signal}` and noise floor energy :math:`E_{noise}`: + +.. math:: + + E_{signal} = \sum_{m=0}^{M-1} W[m] \cdot M[m], \quad E_{noise} = \sum_{m=0}^{M-1} W[m] \cdot N[m] + +The energy delta is evaluated: + +.. math:: + + \Delta E = E_{signal} - E_{noise} + +If :math:`\Delta E` exceeds the detection threshold (`MFCC_VAD_ENERGY_THRESHOLD = 2516582` in :math:`Q9.23`, corresponding to 0.30 natural log units or ~2.6 dB SNR), speech is declared (`vad_flag = 1`), and the **hangover counter** is reset to its maximum (`hangover_max = 20` frames, or 200 ms). + +During brief pauses between syllables or trailing stop-consonant releases, :math:`\Delta E` may drop below the threshold. The hangover counter decrements frame by frame, keeping :math:`vad\_flag = 1` active. This prevents stuttering or chopping at the ends of words. + +Discontinuous Transmission (DTX) Silence Suppression +===================================================== + +Streaming continuous silence frames across the host PCIe or SoundWire bus wastes power and memory bandwidth. When DTX is enabled (`enable_dtx = true`): + +1. **Trailing Silence Preservation**: When speech concludes, the module continues transmitting a configurable number of trailing silence hops (`dtx_trailing_silence_hops`, e.g. 10 frames = 100 ms). This ensures downstream speech recognizers and VAD models observe natural sentence termination and acoustic decay. +2. **Complete Frame Suppression**: After trailing frames expire, the module completely suppresses output: zero bytes are committed to the sink, and the audio pipeline produces no DMA interrupts. +3. **Periodic Keep-Alive Frames**: During prolonged silence, the module optionally transmits a single silence frame every :math:`N` hops (`dtx_silence_hops_interval`), maintaining pipeline keep-alive status and updating host ambient noise trackers without continuous streaming. + +--- + +.. _mfcc_memory_design: + +6. Fixed-Point Arithmetic & Scratch Overlay Memory Design +********************************************************* + +Audio DSPs operate under severe internal static RAM (SRAM) constraints. Allocating independent buffers for every mathematical stage (FFT input, FFT output, power spectrum, Mel log spectrum, DCT matrix) would exhaust memory and cause cache evictions. + +The SOF MFCC subsystem solves this through an advanced **SRAM Scratch Overlay Architecture** combined with a decoupled output staging buffer. + +.. graphviz:: + :caption: Dual FFT Buffer Scratch Overlays and Decoupled Staging Memory Map + + digraph mfcc_memory_overlay { + bgcolor="transparent"; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#0a7d91", fillcolor="#e0f4f7", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=10, color="#2c3e50", penwidth=1.2]; + + subgraph cluster_buf1 { + label="Buffer 1: fft_buf (512 Complex32 = 4,096 Bytes)"; + style="dashed,rounded"; + color="#7f8c8d"; + bgcolor="#f9fbfd"; + + fft_in [label="Phase 1: FFT Input Data\n(512 x 32-bit Real + 512 x 32-bit Imag = 4,096 B)", fillcolor="#cfe2ff", color="#0d6efd"]; + pwr_spec [label="Phase 2 Scratch: Power Spectrum\n(257 x 32-bit = 1,028 B)", fillcolor="#d1e7dd", color="#198754"]; + mel_log [label="Phase 2 Scratch: Mel Log 32\n(80 x 32-bit = 320 B in Q9.23)", fillcolor="#fff3cd", color="#f39c12"]; + unused1 [label="Unused Scratch Reserve\n(2,748 Bytes)", fillcolor="#e2e3e5", color="#6c757d", style="filled,dashed"]; + + fft_in -> pwr_spec [style="invis"]; + pwr_spec -> mel_log [style="invis"]; + mel_log -> unused1 [style="invis"]; + } + + subgraph cluster_buf2 { + label="Buffer 2: fft_out (512 Complex32 = 4,096 Bytes)"; + style="dashed,rounded"; + color="#7f8c8d"; + bgcolor="#f9fbfd"; + + fft_out_blk [label="Phase 1: FFT Output Data\n(512 x 32-bit Complex Frequency Bins = 4,096 B)", fillcolor="#cfe2ff", color="#0d6efd"]; + mel_16 [label="Phase 2 Scratch: Mel Spectra 16b\n(80 x 16-bit = 160 B in Q9.7)", fillcolor="#d1e7dd", color="#198754"]; + ceps_16 [label="Phase 2 Scratch: Cepstral Coefs\n(13 x 16-bit = 26 B in Q9.7)", fillcolor="#fff3cd", color="#f39c12"]; + unused2 [label="Unused Scratch Reserve\n(3,910 Bytes)", fillcolor="#e2e3e5", color="#6c757d", style="filled,dashed"]; + + fft_out_blk -> mel_16 [style="invis"]; + mel_16 -> ceps_16 [style="invis"]; + ceps_16 -> unused2 [style="invis"]; + } + + subgraph cluster_stage { + label="Decoupled Output Staging Buffer (out_stage)"; + style="dashed,rounded"; + color="#7f8c8d"; + bgcolor="#f9fbfd"; + + stage_buf [label="Dedicated Output Staging Buffer\n(80 x 32-bit = 320 B)\nDecouples Sink Drain from STFT Scratch", fillcolor="#e0f4f7", color="#0a7d91", penwidth=2.0]; + } + + mel_log -> stage_buf [label="Copy Mel Output"]; + ceps_16 -> stage_buf [label="Widen to Q9.23"]; + } + +The Two-Buffer Scratch Overlay Map +================================== + +The module allocates exactly two 32-bit complex buffers sized for the padded FFT (:math:`N_{FFT} = 512 \implies 4,096\text{ bytes}` each): + +1. **Buffer 1 (`fft->fft_buf`, 4,096 bytes)**: + * *Phase 1*: Receives windowed time-domain audio samples in the real part with zeroed imaginary components. + * *Phase 2 (Post-FFT Overlay)*: Overlaid to hold the computed 32-bit power spectrum (`power_spectra`, 257 bins = 1,028 bytes) and the 32-bit Mel log spectrum (`mel_log_32`, 80 bins = 320 bytes in :math:`Q9.23`). +2. **Buffer 2 (`fft->fft_out`, 4,096 bytes)**: + * *Phase 1*: Receives raw complex frequency bins from the FFT engine. + * *Phase 2 (Post-FFT Overlay)*: Overlaid to hold 16-bit Mel log spectra (`mel_spectra`, 80 bins in :math:`Q9.7` = 160 bytes) and 16-bit cepstral coefficients (`cepstral_coef`, 13 bins in :math:`Q9.7` = 26 bytes) for DCT matrix multiplication. + +Decoupled Output Staging Buffer +=============================== + +In multi-period audio pipelines, a sink buffer may not drain all features within a single 10 ms period. If output data were held directly inside the FFT scratch space, the next STFT hop could not execute without corrupting pending data. + +To solve this, the SOF MFCC subsystem allocates a dedicated **output staging buffer (`out_stage`)**: + +* Upon completion of the STFT hop, the prepared features (Mel or widened cepstral coefficients) are immediately copied into `out_stage`. +* The STFT scratch buffers (`fft_buf` and `fft_out`) are instantly freed to process the next incoming audio frame. +* Sink drainage proceeds asynchronously across multiple periods using read pointer `out_data_ptr` and remaining count `out_remain`. + +Fixed-Point Number Representations Across Stages +================================================ + +To maintain maximum numerical precision without floating-point emulation, data types transition systematically across fixed-point formats: + +.. table:: Fixed-Point Number Format Transitions in SOF MFCC Pipeline + :widths: 22 18 20 40 + + +--------------------------+--------------------+------------------------+-------------------------------------------------------+ + | Processing Stage | Data Type | Fixed-Point Format | Dynamic Range / Description | + +==========================+====================+========================+=======================================================+ + | Input PCM Audio | `int16_t` | :math:`Q1.15` | Full scale audio waveform [-1.0, +0.999] | + +--------------------------+--------------------+------------------------+-------------------------------------------------------+ + | Windowed FFT Input | `icomplex32` | :math:`Q1.31` | Windowed audio samples in real container | + +--------------------------+--------------------+------------------------+-------------------------------------------------------+ + | FFT Power Spectrum | `int32_t` | :math:`Q1.31` | Normalized power spectral magnitude squared | + +--------------------------+--------------------+------------------------+-------------------------------------------------------+ + | Mel Log Energy (32-bit) | `int32_t` | :math:`Q9.23` | 9 integer bits, 23 fractional bits (Mel log values) | + +--------------------------+--------------------+------------------------+-------------------------------------------------------+ + | Mel Energy for DCT | `int16_t` | :math:`Q9.7` | Scaled 16-bit Mel log values for matrix multiply | + +--------------------------+--------------------+------------------------+-------------------------------------------------------+ + | DCT Cepstral Output | `int16_t` | :math:`Q9.7` | 16-bit orthogonal cepstral coefficients | + +--------------------------+--------------------+------------------------+-------------------------------------------------------+ + | Output Stream Payload | `int32_t` | :math:`Q9.23` | Widened 32-bit features committed to sink | + +--------------------------+--------------------+------------------------+-------------------------------------------------------+ + +--- + +.. _mfcc_simd_acceleration: + +7. SIMD Vector Acceleration Across DSP Architectures +**************************************************** + +The SOF MFCC module leverages Cadence Tensilica Xtensa HiFi 3 and HiFi 4/5 DSP instruction set architectures (ISAs) to execute overlap buffer shifting, windowing, and FFT execution with minimal cycle counts. + +Cadence Tensilica HiFi 3 & HiFi 4 Optimization +============================================== + +On Intel and NXP audio DSP cores featuring Xtensa HiFi 3 or HiFi 4 engines (`mfcc_hifi3.c`, `mfcc_hifi4.c`), key operations are vectorized: + +1. **Hardware Circular Buffer Addressing (`AE_SETCBEGIN0` / `AE_SETCEND0`)**: + When extracting overlap samples from the circular input buffer (`mfcc_fill_prev_samples()`), the DSP hardware circular addressing registers auto-wrap read pointers without software boundary comparison branches. +2. **Vectorized 32-bit Load & Store**: + Samples are fetched using 32-bit circular loads (`AE_L32_XC`) and packed using auto-incrementing 32-bit stores (`AE_S32_L_IP`), halving memory bus transactions. +3. **SIMD Fractional Windowing Multiplication**: + In `mfcc_apply_window()`, time-domain samples and window coefficients are multiplied using fractional multiply-with-rounding vector instructions: + + * `sample = AE_SLAI32S(sample, 16)`: Shifts 16-bit audio into 32-bit :math:`Q1.31` container. + * `temp = AE_MULFP32X16X2RS_L(sample, win)`: Multiplies 32-bit sample by 16-bit window with rounding and saturation. + * `temp = AE_SLAA32S(temp, input_shift)`: Applies dynamic scaling shift. + * `AE_S32_L_XP(temp, fft_in, fft_inc)`: Writes directly into FFT real scratch with auto-increment. + +Generic Portable Fallback +========================= + +On microcontrollers and processors lacking Tensilica HiFi extensions—such as ARM Cortex-M7 (PJRC Teensy 4.1) or RISC-V RV32 (Espressif ESP32-P4 / ESP32-C6)—the subsystem compiles clean, highly portable scalar C implementations (`mfcc_generic.c`), guaranteeing full bit-exact feature equivalence across simulation testbenches and silicon targets. + +--- + +.. _mfcc_topology_streaming: + +8. ALSA Topology 2, Host Streaming Modes & AI Pipeline Integration +****************************************************************** + +The SOF MFCC module is declared declaratively in ALSA Topology 2.0 configuration files and integrated into end-to-end edge-to-host artificial intelligence capture pipelines. + +ALSA Topology 2 Widget Declaration +================================== + +In `tools/topology/topology2/include/components/mfcc.conf`, the module is defined with its unique cryptographic UUID and dedicated VAD notification control: + +.. code-block:: text + + Class.Widget."mfcc" { + uuid "73:a7:10:db:a4:1a:ea:4c:a2:1f:2d:57:a5:c9:82:eb" + type "effect" + no_pm "true" + num_input_pins 1 + num_output_pins 1 + + # Switch control notifying user space of VAD state transitions + Object.Control { + mixer."1" { + Object.Base.channel.1 { name "fc"; shift 0; } + Object.Base.ops.1 { name "ctl"; info "volsw"; get 259; put 259; } + max 1 + } + } + } + +Dual Output Streaming Modes +=========================== + +The module provides two distinct mechanisms for delivering feature data to downstream consumers: + +1. **Compress Output Mode (`compress_output = true`)**: + + * Designed for edge AI models (such as Whisper or TFLM). + * Feature frames are packed contiguously into the sink without zero-padding, matching the exact byte length of the features. + * Every frame is prepended with a 24-byte **Data Header (`struct mfcc_data_header`)**: + + * `magic`: Fixed identifier `0x6d666363` (ASCII `"mfcc"`). + * `frame_number`: Monotonically increasing hop counter. + * `energy`: Speech-weighted signal energy (:math:`Q9.23`). + * `noise_energy`: Estimated background noise energy (:math:`Q9.23`). + * `vad_flag`: Voice activity decision (`1` = speech, `0` = silence). + + * Integrates seamlessly with DTX to suppress frames during silence. + +2. **Legacy PCM Output Mode (`compress_output = false`)**: + + * Treats the sink buffer as an opaque byte container sized to match standard PCM period boundaries (e.g. S16_LE, S24_4LE, S32_LE). + * Unfilled samples in the period are zero-padded, allowing standard ALSA tools (`arecord`) and host testbench scripts to capture feature streams over conventional PCM audio nodes. + +End-to-End Edge Speech Pipeline Integration +=========================================== + +In production edge-to-host AI systems, the MFCC module sits at the center of the acoustic capture graph: + +.. graphviz:: + :caption: Complete Edge-to-Host AI Speech Pipeline: From Microphone Array to Whisper and TFLM + + digraph speech_pipeline { + bgcolor="transparent"; + node [fontname="Helvetica", fontsize=11, shape=box, style="filled,rounded", color="#0a7d91", fillcolor="#e0f4f7", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=10, color="#2c3e50", penwidth=1.2]; + + subgraph cluster_dsp { + label="Low-Power Audio DSP Island (SOF Firmware)"; + style="dashed,rounded"; + color="#7f8c8d"; + bgcolor="#f9fbfd"; + + mics [label="Digital Mic Array\n(4x DMIC / PDM @ 16 kHz)", shape=parallelogram, fillcolor="#e2e3e5", color="#6c757d"]; + dcblock [label="DC Blocker Filter\n(0 Hz Nulling, Rumble Rejection)", fillcolor="#d1e7dd", color="#198754"]; + tdfb [label="TDFB Beamformer\n(Spatial Acoustic Directivity & DOA)", fillcolor="#cfe2ff", color="#0d6efd"]; + mfcc_comp [label="MFCC Feature Extractor\n(Mel Log Energies + VAD + DTX)", fillcolor="#e0f4f7", color="#0a7d91", penwidth=2.0]; + tflm [label="TFLM Neural Classifier\n(On-DSP Wake Word / KWS)", fillcolor="#fff3cd", color="#f39c12"]; + + mics -> dcblock [label="Multi-Channel PCM"]; + dcblock -> tdfb [label="Cleaned PCM"]; + tdfb -> mfcc_comp [label="Directional Speech"]; + mfcc_comp -> tflm [label="Internal Spectrogram\n(Zero Host Wakeup)"]; + } + + subgraph cluster_host { + label="Host Application Processor / NPU (Linux Kernel & User Space)"; + style="dashed,rounded"; + color="#7f8c8d"; + bgcolor="#f9fbfd"; + + pcm_node [label="ALSA Compress PCM Node\n(e.g. hw:0,48 Audio Features)", shape=parallelogram, fillcolor="#e2e3e5", color="#6c757d"]; + openvino [label="OpenVINO Runtime\n(Intel NPU / GPU / CPU)", fillcolor="#d1e7dd", color="#198754"]; + whisper [label="OpenAI Whisper Model\n(High-Accuracy Speech-to-Text)", fillcolor="#cfe2ff", color="#0d6efd", penwidth=2.0]; + + pcm_node -> openvino [label="Zero-Copy Features"]; + openvino -> whisper [label="Normalized 80 Mel"]; + } + + mfcc_comp -> pcm_node [label="DTX Streamed\nMel Features", color="#0a7d91", penwidth=2.0]; + tflm -> pcm_node [label="Wake Trigger IPC\n(Interrupts Host)", color="#e74c3c", style="dashed"]; + } + +--- + +.. _mfcc_tuning_workflow: + +9. Tuning Workflow & Upstream Source References +*********************************************** + +Sound Open Firmware provides a complete software ecosystem for tuning, simulating, and validating MFCC feature extraction. + +MATLAB & GNU Octave Tuning Tools +================================ + +Under `src/audio/mfcc/tune/`: + +* `setup_mfcc.m`: Generates binary configuration blobs (`sof_mfcc_config`) from user-defined parameters (sample rate, frame length, hop size, window type, Mel bins, Slaney normalization, Whisper scaling). +* `run_mfcc.sh`: Shell script executing the SOF testbench (`testbench`) on raw audio files across S16, S24, and S32 bit depths, with optional Xtensa simulator (`xt-run`) execution. +* `decode_all.m`, `decode_mel.m`, `decode_ceps.m`: Decodes and plots generated binary feature files, visualizing 80-bin Mel spectrograms and 13-coefficient cepstral trajectories. +* `sof_mel_to_text_live_dsp_vad.py`: Live streaming Python application connecting the DSP audio features device (`hw:0,48`) directly to OpenVINO Whisper models running on the Intel NPU. +* `sof_mel_spectrogram_compress.py`: Real-time GTK 4 live spectrogram viewer displaying Mel energy waterfalls and VAD flags. + +Upstream Source Code References +=============================== + +* `src/audio/mfcc/README.md `_: Upstream module overview. +* `src/include/user/mfcc.h `_: Configuration ABI, window types, and Whisper parameters. +* `src/include/sof/audio/mfcc/mfcc_comp.h `_: Module private data, buffers, and STFT declarations. +* `src/include/sof/audio/mfcc/mfcc_vad.h `_: Mel-domain VAD state and A-weighting tables. +* `src/audio/mfcc/mfcc.c `_: Module lifecycle, init, prepare, and process callbacks. +* `src/audio/mfcc/mfcc_setup.c `_: Parameter validation, FFT plan, Mel filterbank, and DCT initialization. +* `src/audio/mfcc/mfcc_common.c `_: STFT pipeline execution, windowing, Mel calculation, and compress output. +* `src/audio/mfcc/mfcc_vad.c `_: Mel-domain VAD update and noise floor tracking. +* `src/audio/mfcc/mfcc_hifi3.c `_: Tensilica Xtensa HiFi 3 SIMD vectorization. +* `src/audio/mfcc/mfcc_hifi4.c `_: Tensilica Xtensa HiFi 4/5 SIMD vectorization. +* `tools/topology/topology2/include/components/mfcc.conf `_: ALSA Topology 2 widget definition. diff --git a/developer_guides/firmware/pipeline_architecture.rst b/developer_guides/firmware/pipeline_architecture.rst index e505bd76..c2be58c5 100644 --- a/developer_guides/firmware/pipeline_architecture.rst +++ b/developer_guides/firmware/pipeline_architecture.rst @@ -470,6 +470,7 @@ Related Guides * :ref:`dcblock`: First-order recursive high-pass DC blocking filter, 0 Hz transmission nulling, 64-bit fixed-point accumulation, and dual circular buffer SIMD acceleration. * :ref:`tdfb`: Spatial acoustic filtering, filter-and-sum FIR banks, multi-microphone array geometries, and autonomous Direction of Arrival (DOA) tracking. * :ref:`tflm`: Embedded neural network inference, static tensor arena memory planning, 8-bit affine quantization, and Tensilica NNLib SIMD acceleration. +* :ref:`mfcc`: Real-time Mel-Frequency Cepstral Coefficients feature extraction, auditory filterbanks, OpenAI Whisper preprocessing, and Mel-domain VAD. * :ref:`fw_init_boot`: Boot flow, hardware mailbox FW Ready handshake, and Zephyr initialization. * :ref:`topology2`: How pipelines and widgets are declared using ALSA Topology 2.0 configuration classes. * :ref:`sof_hostless_firmware`: How to create static pipelines compiled into ROM for standalone microcontrollers. diff --git a/developer_guides/firmware/tflm.rst b/developer_guides/firmware/tflm.rst index ccf5b2c3..ef0bcce2 100644 --- a/developer_guides/firmware/tflm.rst +++ b/developer_guides/firmware/tflm.rst @@ -234,7 +234,7 @@ Deep neural networks cannot effectively process raw 16 kHz audio samples directl The MFCC / Filterbank Transformation Pipeline ============================================= -In Sound Open Firmware, the audio feature extraction stage (typically handled by the upstream :ref:`module_framework` component `mfcc`) converts 1D temporal audio into a compact 2D time-frequency spectrogram: +In Sound Open Firmware, the audio feature extraction stage (typically handled by the upstream :ref:`module_framework` component :ref:`mfcc`) converts 1D temporal audio into a compact 2D time-frequency spectrogram: 1. **Short-Time Windowing**: @@ -814,11 +814,12 @@ For developers designing custom machine learning models, quantizing neural netwo * **Upstream Feature Extractor**: - - `thesofproject/sof: src/audio/mfcc/README.md `_: Mel-Frequency Cepstral Coefficients feature generator. + - :ref:`mfcc`: Mel-Frequency Cepstral Coefficients feature generator and auditory filterbank engine (also see `src/audio/mfcc/README.md `_). Related Subsystem Architecture Guides ===================================== +* :ref:`mfcc`: Mel-Frequency Cepstral Coefficients feature extraction, auditory filterbanks, OpenAI Whisper preprocessing, and Mel-domain VAD. * :ref:`tdfb`: Time-Domain Fixed Beamformer providing directional audio pre-processing and spatial noise nulling ahead of ML feature extraction. * :ref:`dcblock`: First-order recursive high-pass filter stripping ADC DC offsets prior to spectral transformation. * :ref:`module_framework`: Standardized module lifecycle, memory allocation, and IPC configuration handlers. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 99257ae3..b5b80df4 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -51,7 +51,7 @@ Audio Processing Modules & Algorithms * :ref:`tdfb` (High-level architecture; also see upstream `tdfb README `_ & tuning guide :ref:`time-domain-fixed-beamformer`) * `RTNR Noise Reduction `_ * :ref:`tflm` (High-level architecture; also see upstream `TFLM README `_) -* `MFCC Feature Extraction `_ +* :ref:`mfcc` (High-level architecture; also see upstream `MFCC README `_) * `Smart Amp Protection `_ * `Sound Dose Evaluator `_ * `Copier `_, `Mux `_ & `Selector `_ @@ -95,6 +95,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/dcblock firmware/tdfb firmware/tflm + firmware/mfcc rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 4cb5cb8df0e7479375abbe3d07f5cc23ff7619e3 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 09:31:58 +0100 Subject: [PATCH 18/64] docs: developer_guides: add high-level smart amp architecture guide Add comprehensive high-level Smart Amplifier Protection & Physics developer guide, covering: - Physical electro-acoustic foundations & speaker damage mechanisms (thermal burnout vs excursion bottoming) - Current & voltage (I/V) sense telemetry, back-EMF isolation & continuous voice coil temperature tracking - Two-layer architecture (generic component middleware vs solution-specific inner models: PASSTHRU_AMP, MAXIM_DSM) - Three-block structured memory hierarchy (MOD_MEMBLK_PRIVATE, MOD_MEMBLK_FRAME, MOD_MEMBLK_PARAM) & double-buffering - Dual-pipeline asynchronous scheduling (playback feed-forward vs capture I/V feedback stream synchronization) - Multi-band excursion limiting, slow broadband thermal limiting, dynamic bass extension & psychoacoustic harmonic synthesis - IPC3/IPC4 configuration blobs, volatile telemetry readback & factory assembly line calibration - ALSA Topology 2 declaration (Class.Widget.smart_amp) & end-to-end audio graph with 7 native vector Graphviz SVG diagrams Signed-off-by: Liam Girdwood --- .../firmware/drc_multiband_drc.rst | 1 + .../firmware/pipeline_architecture.rst | 1 + developer_guides/firmware/smart_amp.rst | 880 ++++++++++++++++++ developer_guides/index.rst | 3 +- 4 files changed, 884 insertions(+), 1 deletion(-) create mode 100644 developer_guides/firmware/smart_amp.rst diff --git a/developer_guides/firmware/drc_multiband_drc.rst b/developer_guides/firmware/drc_multiband_drc.rst index 241e5fc4..06616da4 100644 --- a/developer_guides/firmware/drc_multiband_drc.rst +++ b/developer_guides/firmware/drc_multiband_drc.rst @@ -567,6 +567,7 @@ For developers seeking low-level implementation details, mathematical structures Related Subsystem Architecture Guides ===================================== +* :ref:`smart_amp`: Adaptive speaker protection, real-time current/voltage (I/V) sense telemetry, and excursion/thermal limiters. * :ref:`volume_module`: Per-channel gain scaling, smooth volume ramping, and zero-crossing muting. * :ref:`eq_fir_iir`: Finite and Infinite Impulse Response equalizers, linear-phase filtering, and biquad cascades. * :ref:`src_asrc`: Sample rate conversion architecture handling fixed and drifting clocks across heterogeneous audio interfaces. diff --git a/developer_guides/firmware/pipeline_architecture.rst b/developer_guides/firmware/pipeline_architecture.rst index c2be58c5..580b7d09 100644 --- a/developer_guides/firmware/pipeline_architecture.rst +++ b/developer_guides/firmware/pipeline_architecture.rst @@ -471,6 +471,7 @@ Related Guides * :ref:`tdfb`: Spatial acoustic filtering, filter-and-sum FIR banks, multi-microphone array geometries, and autonomous Direction of Arrival (DOA) tracking. * :ref:`tflm`: Embedded neural network inference, static tensor arena memory planning, 8-bit affine quantization, and Tensilica NNLib SIMD acceleration. * :ref:`mfcc`: Real-time Mel-Frequency Cepstral Coefficients feature extraction, auditory filterbanks, OpenAI Whisper preprocessing, and Mel-domain VAD. +* :ref:`smart_amp`: Adaptive speaker protection, real-time current/voltage (I/V) sense telemetry, thermal/excursion limiters, and two-layer generic/inner model architecture. * :ref:`fw_init_boot`: Boot flow, hardware mailbox FW Ready handshake, and Zephyr initialization. * :ref:`topology2`: How pipelines and widgets are declared using ALSA Topology 2.0 configuration classes. * :ref:`sof_hostless_firmware`: How to create static pipelines compiled into ROM for standalone microcontrollers. diff --git a/developer_guides/firmware/smart_amp.rst b/developer_guides/firmware/smart_amp.rst new file mode 100644 index 00000000..9456090a --- /dev/null +++ b/developer_guides/firmware/smart_amp.rst @@ -0,0 +1,880 @@ +.. _smart_amp: + +==================================== +Smart Amplifier Protection & Physics +==================================== + +.. contents:: + :local: + :depth: 3 + +Overview +======== + +Modern laptops, tablets, and mobile devices operate under aggressive physical and industrial design constraints. Thin chassis profiles require miniature micro-speakers with tiny voice coils, compact neodymium magnetic assemblies, and lightweight polymer diaphragms housed within sub-optimal acoustic back-cavities. + +While these transducers are engineered for compact integration, their physical limits severely constrain maximum acoustic loudness and low-frequency bass reproduction: + +* **Thermal Limits**: Sustained high-voltage audio signals dissipate electrical energy as resistive heat in the voice coil winding. Excessive temperatures degrade wire insulation, soften structural bobbins, melt coil adhesives, and ultimately cause open-circuit thermal burnout. +* **Mechanical Excursion Limits**: Low-frequency signals near or below the loudspeaker mechanical resonance frequency drive large peak-to-peak diaphragm excursions. If the cone displacement exceeds physical suspension bounds, the voice coil former violently strikes the back plate (*bottoming out*), suspensions tear, and severe acoustic distortion occurs. + +Conventional audio architectures protect transducers using static brickwall limiters or aggressive fixed Dynamic Range Compression (DRC). Because static limiters must accommodate worst-case ambient temperatures, component manufacturing variances, and crest factors, they force engineers to set gain thresholds 6 dB to 12 dB below what the speaker could safely reproduce. + +**Smart Amplifier Protection** (*Smart Amp*) bridges this acoustic gap. By coupling a feed-forward audio processing pipeline with a real-time current (:math:`I`) and voltage (:math:`V`) feedback sense stream digitized directly at the amplifier terminals, the DSP continuously tracks the physical state of the voice coil (instantaneous resistance, temperature, and cone excursion). This adaptive closed loop enables the firmware to drive micro-speakers right to their true physical boundaries—extracting up to +6 dB to +10 dB of clean acoustic loudness, deep dynamic bass extension, and high vocal intelligibility without risk of transducer damage. + +.. graphviz:: + :caption: Figure 160: Electro-Acoustic Transducer Physics, Thermal Heating, and Excursion Boundaries + :alt: Electro-Acoustic Transducer Physics, Thermal Heating, and Excursion Boundaries + + digraph smart_amp_physics { + graph [rankdir=TB, bgcolor="transparent", fontsize=11, fontname="Bitstream Vera Sans"]; + node [shape=box, style="filled,rounded", fontname="Bitstream Vera Sans", fontsize=10, penwidth=1.5]; + edge [fontname="Bitstream Vera Sans", fontsize=9, penwidth=1.2]; + + subgraph cluster_elec { + label = "Electrical Domain (Amplifier Terminals & Voice Coil)"; + style = "filled,rounded"; + color = "#2B6CB0"; + fillcolor = "#EBF8FF"; + + p_drive [label="Drive Voltage V(t)\nClass-D Power Stage", fillcolor="#BEE3F8", color="#2B6CB0"]; + p_coil [label="Voice Coil Impedance\nZ(s) = Re(T) + s*Le + Zem(s)", fillcolor="#FFFFFF", color="#2B6CB0"]; + p_curr [label="Coil Current I(t)\nSense Resistor / ADC", fillcolor="#BEE3F8", color="#2B6CB0"]; + p_heat [label="Joule Dissipation\nP = I(t)^2 * Re(T)", fillcolor="#FED7D7", color="#C53030"]; + } + + subgraph cluster_mech { + label = "Electro-Mechanical Domain (Motor & Suspension)"; + style = "filled,rounded"; + color = "#2C7A7B"; + fillcolor = "#E6FFFA"; + + p_lorentz [label="Lorentz Force\nF(t) = Bl * I(t)", fillcolor="#B2F5EA", color="#2C7A7B"]; + p_backemf [label="Back-EMF Feedback\ne_emf(t) = Bl * v(t)", fillcolor="#B2F5EA", color="#2C7A7B"]; + p_disp [label="Diaphragm Kinematics\nx(t) = integral(v(t) dt)", fillcolor="#FFFFFF", color="#2C7A7B"]; + p_res [label="Mechanical Resonance\nfs = 1 / (2*pi*sqrt(Mms*Cms))", fillcolor="#E2E8F0", color="#4A5568"]; + } + + subgraph cluster_limits { + label = "Physical Transducer Damage Bounds"; + style = "filled,rounded"; + color = "#C53030"; + fillcolor = "#FFF5F5"; + + p_tlimit [label="Thermal Boundary T_max\nAdhesive breakdown (100°C - 130°C)\nBurnout / open circuit", fillcolor="#FED7D7", color="#C53030"]; + p_xlimit [label="Mechanical Boundary X_mech\nVoice coil bottoming\nSuspension tearing / clipping", fillcolor="#FED7D7", color="#C53030"]; + } + + p_drive -> p_coil [label="Applies V(t)"]; + p_coil -> p_curr [label="Produces I(t)"]; + p_curr -> p_heat [label="Resistive loss"]; + p_curr -> p_lorentz [label="Bl coupling"]; + p_heat -> p_tlimit [label="Delta T >= 80°C\nExceeds limits", color="#C53030", style="dashed"]; + + p_lorentz -> p_disp [label="Drives mass Mms"]; + p_disp -> p_backemf [label="Velocity v(t)"]; + p_backemf -> p_coil [label="Counter-voltage", color="#2B6CB0"]; + p_disp -> p_res [label="Peaks near fs"]; + p_disp -> p_xlimit [label="|x(t)| > X_max\nBottoming", color="#C53030", style="dashed"]; + } + +Electro-Acoustic Foundations & Speaker Physics +============================================== + +To safely control micro-speakers, the firmware requires an accurate mathematical model of the electro-dynamic transducer. A conventional moving-coil loudspeaker functions as an electro-mechanical energy converter characterized by Thiele-Small parameters. + +The Lumped-Parameter Loudspeaker Model +-------------------------------------- + +The electrical behavior of the loudspeaker voice coil is governed by Kirchhoff's voltage law: + +.. math:: + + V(t) = I(t) \cdot R_e(T) + L_e \frac{d I(t)}{dt} + e_{\text{emf}}(t) + +Where: + +* :math:`V(t)` is the instantaneous voltage across the voice coil terminals (in Volts). +* :math:`I(t)` is the instantaneous current through the voice coil (in Amperes). +* :math:`R_e(T)` is the temperature-dependent DC electrical resistance of the voice coil (in Ohms). +* :math:`L_e` is the voice coil inductance (in Henrys). +* :math:`e_{\text{emf}}(t)` is the back-electromotive force generated by the coil moving through the magnetic field: + +.. math:: + + e_{\text{emf}}(t) = B \cdot l \cdot \dot{x}(t) = B \cdot l \cdot v(t) + +Where: + +* :math:`B` is the magnetic flux density in the voice coil air gap (in Tesla). +* :math:`l` is the length of the voice coil wire within the magnetic field (in meters). +* :math:`B \cdot l` is the electro-mechanical force factor (in Newton/Ampere or Tesla-meters). +* :math:`v(t) = \dot{x}(t)` is the instantaneous velocity of the cone diaphragm (in meters/second). + +Newton's second law governs the mechanical domain: + +.. math:: + + F(t) = B \cdot l \cdot I(t) = M_{ms} \ddot{x}(t) + R_{ms} \dot{x}(t) + \frac{1}{C_{ms}} x(t) + +Where: + +* :math:`M_{ms}` is the total moving mass of the diaphragm and voice coil assembly (in kilograms). +* :math:`R_{ms}` is the mechanical damping resistance of the suspension (in Newton-seconds/meter). +* :math:`C_{ms}` is the mechanical compliance of the suspension (in meters/Newton). +* :math:`x(t)` is the cone displacement (excursion) relative to its resting position (in meters). + +The mechanical resonance frequency :math:`f_s` is defined by: + +.. math:: + + f_s = \frac{1}{2\pi \sqrt{M_{ms} C_{ms}}} + +Near :math:`f_s`, the mechanical impedance reaches a minimum, causing cone excursion to peak sharply for a given drive voltage. + +Thermal Dissipation Mechanics +----------------------------- + +Typical micro-speakers exhibit an electrical-to-acoustic power conversion efficiency of less than 1%. More than 99% of the delivered electrical power is converted into heat: + +.. math:: + + P_{\text{diss}}(t) = I^2(t) \cdot R_e(T) + +The heat generated in the voice coil conducts across the narrow air gap into the magnet pole piece and frame, eventually radiating into the enclosure air. This thermodynamic system is modeled as a two-stage thermal RC network: + +1. **Voice Coil Thermal Node**: Small thermal capacitance :math:`C_{th,coil}` with a fast thermal time constant :math:`\tau_{th,coil} = R_{th,coil} \cdot C_{th,coil}` (typically 0.5 to 2.0 seconds). +2. **Magnet/Frame Thermal Node**: Large thermal capacitance :math:`C_{th,magnet}` with a slow thermal time constant :math:`\tau_{th,magnet}` (typically 30 to 120 seconds). + +As voice coil temperature rises, the electrical resistance of the copper winding increases linearly according to the temperature coefficient of copper (:math:`\alpha_{Cu} \approx 0.00393 / ^\circ\text{C}`): + +.. math:: + + R_e(T) = R_0 \cdot \left[ 1 + \alpha_{Cu} (T - T_0) \right] + +Where :math:`R_0` is the cold voice coil resistance measured at baseline ambient temperature :math:`T_0` (typically 20°C). + +Real-Time Current and Voltage (I/V) Sense +========================================= + +Hardware Architecture & Telemetry Capture +----------------------------------------- + +In modern smart amplifier hardware (such as Maxim DSM, Cirrus Logic, Texas Instruments, or Realtek smart power amplifiers), the integrated circuit incorporates dedicated on-chip Current-Sense and Voltage-Sense Analog-to-Digital Converters (ADCs). + +* **Voltage Sense (V-Sense)**: Measures the actual differential voltage applied across the speaker voice coil terminals after Class-D output filtering, capturing battery voltage drops and amplifier clipping. +* **Current Sense (I-Sense)**: Measures the current flowing through the voice coil using an ultra-low-value series resistor or an integrated current mirror. + +The digitized :math:`I` and :math:`V` samples are multiplexed into a high-speed digital audio bus (e.g. SoundWire or TDM / I2S) and streamed upstream into the DSP as an asynchronous capture stream. + +.. graphviz:: + :caption: Figure 161: Real-Time I/V Sense Feedback Loop and Parameter Estimation + :alt: Real-Time I/V Sense Feedback Loop and Parameter Estimation + + digraph smart_amp_iv { + graph [rankdir=LR, bgcolor="transparent", fontsize=11, fontname="Bitstream Vera Sans"]; + node [shape=box, style="filled,rounded", fontname="Bitstream Vera Sans", fontsize=10, penwidth=1.5]; + edge [fontname="Bitstream Vera Sans", fontsize=9, penwidth=1.2]; + + subgraph cluster_hw { + label = "Hardware Smart Amplifier IC"; + style = "filled,rounded"; + color = "#4A5568"; + fillcolor = "#EDF2F7"; + + hw_dac [label="Class-D Power Amp\nBridge-Tied Load (BTL)", fillcolor="#CBD5E0", color="#4A5568"]; + hw_spk [label="Micro-Speaker\nVoice Coil & Cone", fillcolor="#FEB2B2", color="#C53030"]; + hw_vsense [label="V-Sense ADC\nTerminal Voltage V(t)", fillcolor="#BEE3F8", color="#2B6CB0"]; + hw_isense [label="I-Sense ADC\nCoil Current I(t)", fillcolor="#BEE3F8", color="#2B6CB0"]; + hw_tdm [label="Upstream Transmitter\nSoundWire / TDM DAI", fillcolor="#CBD5E0", color="#4A5568"]; + } + + subgraph cluster_dsp { + label = "Sound Open Firmware (SOF) Smart Amp Engine"; + style = "filled,rounded"; + color = "#2B6CB0"; + fillcolor = "#EBF8FF"; + + dsp_rx [label="Capture DAI Copier\nI/V Ingestion Buffer", fillcolor="#FFFFFF", color="#2B6CB0"]; + dsp_est_r [label="Resistance Tracking\nRe(t) = LowFreq(V) / LowFreq(I)", fillcolor="#BEE3F8", color="#2B6CB0"]; + dsp_est_t [label="Thermal Estimator\nDelta T = (Re - R0) / (alpha * R0)", fillcolor="#FED7D7", color="#C53030"]; + dsp_est_x [label="Back-EMF & Excursion\ne_emf = V - I*Re - Le(dI/dt)\nx(t) = integral(e_emf / Bl)", fillcolor="#B2F5EA", color="#2C7A7B"]; + dsp_ctrl [label="Dynamic Protection Core\nExcursion Limiter + Thermal Limiter", fillcolor="#FEFCBF", color="#B7791F"]; + } + + hw_dac -> hw_spk [label="Drives audio"]; + hw_spk -> hw_vsense [label="Terminal tap"]; + hw_spk -> hw_isense [label="Current sense"]; + hw_vsense -> hw_tdm [label="V samples"]; + hw_isense -> hw_tdm [label="I samples"]; + + hw_tdm -> dsp_rx [label="SoundWire / I2S", color="#2B6CB0", penwidth=1.8]; + dsp_rx -> dsp_est_r [label="Synchronous I/V"]; + dsp_est_r -> dsp_est_t [label="Voice coil Re(t)"]; + dsp_rx -> dsp_est_x [label="High-speed I/V"]; + dsp_est_r -> dsp_est_x [label="Compensated Re(t)"]; + + dsp_est_t -> dsp_ctrl [label="Temperature T_coil", color="#C53030"]; + dsp_est_x -> dsp_ctrl [label="Displacement x(t)", color="#2C7A7B"]; + } + +Continuous Resistance and Temperature Tracking +---------------------------------------------- + +Because the coil resistance varies with temperature, the firmware isolates the low-frequency electrical impedance. By computing the ratio of voltage to current over a low-pass filtered band: + +.. math:: + + R_e(t) = \frac{\langle V_{\text{low}}(t) \cdot I_{\text{low}}(t) \rangle}{\langle I_{\text{low}}^2(t) \rangle} + +Once the instantaneous resistance :math:`R_e(t)` is derived, the absolute voice coil temperature is determined: + +.. math:: + + T_{\text{coil}}(t) = T_0 + \frac{R_e(t) - R_0}{\alpha_{Cu} \cdot R_0} + +This measurement operates continuously without requiring offline calibration breaks, allowing the DSP to detect overheating in real time. + +Back-EMF Isolation and Excursion Prediction +------------------------------------------- + +Predicting mechanical excursion purely from the input audio signal requires assuming static, idealized parameters. However, mechanical compliance :math:`C_{ms}` shifts by up to 50% across temperature and aging, and enclosure acoustic back-cavities can suffer air leaks. + +By measuring both :math:`V(t)` and :math:`I(t)`, the firmware computes the true back-EMF: + +.. math:: + + e_{\text{emf}}(t) = V(t) - I(t) \cdot R_e(t) - L_e \frac{d I(t)}{dt} + +Because :math:`e_{\text{emf}}(t) = B \cdot l \cdot v(t)`, the diaphragm velocity is directly extracted: + +.. math:: + + v(t) = \frac{e_{\text{emf}}(t)}{B \cdot l} + +Integrating velocity yields the instantaneous physical cone displacement :math:`x(t)`: + +.. math:: + + x(t) = \int_0^t v(\tau) d\tau = \int_0^t \frac{V(\tau) - I(\tau) R_e(\tau) - L_e \frac{d I(\tau)}{d\tau}}{B \cdot l} d\tau + +This closed-loop displacement measurement reflects true transducer behavior in real time, automatically compensating for enclosure leaks, barometric pressure changes, and component aging. + +The Sound Open Firmware Two-Layer Architecture +============================================== + +To accommodate diverse amplifier vendor silicon while maintaining a unified, testable codebase, Sound Open Firmware structures the Smart Amplifier component into two distinct layers: + +1. **Generic Component Layer** (`smart_amp.c`, `smart_amp_generic.c`): Open-source middleware interfacing with the SOF pipeline, scheduler, memory allocator, and ALSA topology. +2. **Inner Model Layer** (`smart_amp_mod_data_base`, `struct inner_model_ops`): Solution-specific algorithm implementation. + +.. graphviz:: + :caption: Figure 162: Architectural Decoupling: Generic SOF Layer vs Solution-Specific Inner Model + :alt: Architectural Decoupling: Generic SOF Layer vs Solution-Specific Inner Model + + digraph smart_amp_twolayer { + graph [rankdir=TB, bgcolor="transparent", fontsize=11, fontname="Bitstream Vera Sans"]; + node [shape=box, style="filled,rounded", fontname="Bitstream Vera Sans", fontsize=10, penwidth=1.5]; + edge [fontname="Bitstream Vera Sans", fontsize=9, penwidth=1.2]; + + subgraph cluster_sof { + label = "Sound Open Firmware Infrastructure"; + style = "filled,rounded"; + color = "#4A5568"; + fillcolor = "#F7FAFC"; + + sof_pipe [label="Playback Pipeline\nFeed-Forward Source (FF)", fillcolor="#EDF2F7", color="#4A5568"]; + sof_cap [label="Capture Pipeline\nFeedback Source (FB)", fillcolor="#EDF2F7", color="#4A5568"]; + sof_sink [label="Downstream DAI Sink\nProtected Audio Output", fillcolor="#EDF2F7", color="#4A5568"]; + sof_ipc [label="IPC Control Plane\nALSA byte controls / SET_LARGE_CONFIG", fillcolor="#EDF2F7", color="#4A5568"]; + } + + subgraph cluster_generic { + label = "Generic Smart Amp Layer (smart_amp.c & smart_amp_generic.c)"; + style = "filled,rounded"; + color = "#2B6CB0"; + fillcolor = "#EBF8FF"; + + g_adapter [label="SOF Processing Module Adapter\nInterface lifecycle & triggers", fillcolor="#BEE3F8", color="#2B6CB0"]; + g_memmgr [label="Runtime Memory Manager\nAllocates & owns all memory blocks", fillcolor="#BEE3F8", color="#2B6CB0"]; + g_remap [label="Channel Remapping Engine\nsource_ch_map & feedback_ch_map", fillcolor="#BEE3F8", color="#2B6CB0"]; + g_conv [label="Format Conversion\nS16_LE / S24_4LE -> S32_LE inner format", fillcolor="#BEE3F8", color="#2B6CB0"]; + g_buffers [label="Intermediate Stream Buffers\nff_mod, fb_mod, out_mod", fillcolor="#FFFFFF", color="#2B6CB0"]; + } + + subgraph cluster_inner { + label = "Inner Model Layer (smart_amp_mod_data_base & inner_model_ops)"; + style = "filled,rounded"; + color = "#2C7A7B"; + fillcolor = "#E6FFFA"; + + i_ops [label="Inner Model Operations Table\ninit(), query_memblk_size(), set_memblk(),\nset_fmt(), ff_proc(), fb_proc(), get/set_config()", fillcolor="#B2F5EA", color="#2C7A7B"]; + + subgraph cluster_impls { + label = "Supported Inner Model Implementations (Kconfig Selectable)"; + style = "dotted"; + color = "#2C7A7B"; + + impl_pass [label="PASSTHRU_AMP\nZero latency, bypass audio,\nconsumes/drops FB frames", fillcolor="#E2E8F0", color="#4A5568"]; + impl_dsm [label="MAXIM_DSM\nDynamic Speaker Management\n(Statically linked libdsm.a)", fillcolor="#FEFCBF", color="#B7791F"]; + impl_test [label="Test Smart Amp Module\nSynthetic model for unit tests", fillcolor="#E2E8F0", color="#4A5568"]; + } + } + + sof_pipe -> g_remap [label="FF Frames"]; + sof_cap -> g_remap [label="FB Frames (I/V)"]; + g_remap -> g_conv -> g_buffers; + g_buffers -> sof_sink [label="Safe Out"]; + + sof_ipc -> g_adapter [label="Config & Model Blobs"]; + g_adapter -> g_memmgr; + g_adapter -> i_ops [label="Dispatches mod_ops", color="#2B6CB0", penwidth=1.5]; + + i_ops -> impl_pass [style="dashed"]; + i_ops -> impl_dsm [style="dashed"]; + i_ops -> impl_test [style="dashed"]; + + g_buffers -> i_ops [label="Pointers to data buffers", color="#2C7A7B"]; + } + +Generic Layer Responsibilities +------------------------------ + +The generic layer acts as the standard SOF processing component: + +* **Lifecycle and Pipeline Orchestration**: Implements `init()`, `prepare()`, `process()`, `trigger()`, `reset()`, and `free()` matching the unified SOF module adapter interface. +* **Full Memory Ownership**: To ensure deterministic isolation, the generic layer completely manages memory allocation. It queries the inner model for required buffer sizes across distinct lifecycle stages and allocates all buffers using SOF core memory allocators. +* **Channel Remapping**: The physical arrangement of audio channels in the capture pipeline often varies across hardware designs (e.g., Left-V, Left-I, Right-V, Right-I vs. interleaved I/V pairs). The generic layer applies runtime channel maps (`source_ch_map` and `feedback_ch_map`) to rearrange multi-channel streams into standard formats before feeding the inner model. +* **Format Conversion**: Allows the inner model to operate at a fixed, high-precision bit depth (such as 32-bit fixed point) regardless of whether external pipelines stream in `S16_LE`, `S24_4LE`, or `S32_LE`. + +Inner Model Interface (`inner_model_ops`) +----------------------------------------- + +The inner model interacts with the generic layer strictly through an operations table: + +.. list-table:: Sound Open Firmware Inner Model Operations Interface + :widths: 25 20 55 + :header-rows: 1 + + * - Operation Name + - Invocation Stage + - Functional Responsibility + * - ``init()`` + - Component Creation + - Initializes internal model state variables and coefficients. + * - ``query_memblk_size()`` + - Creation & Prepare + - Returns the memory size in bytes required for a specific memory block category. + * - ``set_memblk()`` + - Creation & Prepare + - Receives allocated memory buffer pointers from the generic layer. + * - ``get_supported_fmts()`` + - Component Prepare + - Reports the array of PCM sample formats supported by the inner algorithm. + * - ``set_fmt()`` + - Component Prepare + - Sets the negotiated operating sample format for feed-forward and feedback streams. + * - ``ff_proc()`` + - Audio Stream Processing + - Executes protection algorithms (excursion/thermal limiting) on playback frames. + * - ``fb_proc()`` + - Audio Stream Processing + - Ingests and processes current/voltage sense feedback frames to update physical models. + * - ``get_config()`` / ``set_config()`` + - Control Plane / IPC + - Reads runtime telemetry or updates static speaker model and calibration data. + * - ``reset()`` + - Stream Stop / Reset + - Flushes internal delay lines, clears history buffers, and resets filter states. + +Supported Inner Model Implementations +------------------------------------- + +SOF supports multiple implementations selected at firmware build time via Kconfig: + +1. **Passthrough Smart Amp (`PASSTHRU_AMP`)**: + + * UUID: `64a794f0-55d3-4bca-9d5b-7b588badd037`. + * Open-source reference implementation requiring no proprietary libraries. + * Audio frames passing through feed-forward are copied directly to output without modification or latency. + * Feedback frames arriving on the capture input are consumed and discarded. + * Serves as the default baseline for pipeline bring-up, open-hardware testing, and continuous integration. + +2. **Maxim Dynamic Speaker Management (`MAXIM_DSM`)**: + + * UUID: `0cd84e80-ebd3-11ea-adc1-0242ac120002`. + * Integrates Maxim's proprietary DSM algorithm via a pre-compiled static library (`libdsm.a`). + * Provides real-time voice coil temperature estimation, non-linear excursion prediction, thermal limiting, and dynamic bass boost. + * Includes a stub implementation (`MAXIM_DSM_STUB`) for CI building and unit testing without proprietary source code. + +Three-Block Structured Memory Architecture +========================================== + +Hard real-time embedded audio DSPs forbid dynamic heap allocations (`malloc`, `free`) during streaming to avoid non-deterministic latency and memory fragmentation. The Smart Amp component organizes all runtime memory into three strictly classified blocks: + +.. graphviz:: + :caption: Figure 163: Memory Classification and Double-Buffer Rate Matching + :alt: Memory Classification and Double-Buffer Rate Matching + + digraph smart_amp_mem { + graph [rankdir=TB, bgcolor="transparent", fontsize=11, fontname="Bitstream Vera Sans"]; + node [shape=box, style="filled,rounded", fontname="Bitstream Vera Sans", fontsize=10, penwidth=1.5]; + edge [fontname="Bitstream Vera Sans", fontsize=9, penwidth=1.2]; + + subgraph cluster_mems { + label = "Three-Block Memory Hierarchy (enum smart_amp_mod_memblk)"; + style = "filled,rounded"; + color = "#4A5568"; + fillcolor = "#F7FAFC"; + + subgraph cluster_priv { + label = "MOD_MEMBLK_PRIVATE (Allocated BEFORE model init)"; + style = "filled,rounded"; + color = "#2B6CB0"; + fillcolor = "#EBF8FF"; + + m_handle [label="Algorithm State Handle\ne.g. dsmhandle (Persistent context)", fillcolor="#BEE3F8", color="#2B6CB0"]; + m_filter [label="Persistent Filter History\nBiquad delays & state variables", fillcolor="#BEE3F8", color="#2B6CB0"]; + } + + subgraph cluster_frame { + label = "MOD_MEMBLK_FRAME (Allocated AFTER model init)"; + style = "filled,rounded"; + color = "#2C7A7B"; + fillcolor = "#E6FFFA"; + + m_work [label="Scratch Processing Arrays\nInput, Output, Voltage, Current working arrays", fillcolor="#B2F5EA", color="#2C7A7B"]; + m_db_ff [label="Feed-Forward Double-Buffer\nSMART_AMP_FF_BUF_DB_SZ\nDecouples SOF ticks from fixed block size", fillcolor="#B2F5EA", color="#2C7A7B"]; + m_db_fb [label="Feedback Double-Buffer\nSMART_AMP_FB_BUF_DB_SZ\nBuffers async I/V capture frames", fillcolor="#B2F5EA", color="#2C7A7B"]; + } + + subgraph cluster_param { + label = "MOD_MEMBLK_PARAM (Allocated AFTER model init)"; + style = "filled,rounded"; + color = "#B7791F"; + fillcolor = "#FEFCBF"; + + m_caldata [label="Static Model Calibration Table\nThiele-Small parameters, limits, EQ presets", fillcolor="#FFFFF0", color="#B7791F"]; + m_vol [label="Volatile Telemetry Mirror\nReal-time temperature, excursion, Rdc readback", fillcolor="#FFFFF0", color="#B7791F"]; + } + } + + subgraph cluster_flow { + label = "Buffer Decoupling Mechanics (Variable SOF Period <-> Fixed Algorithm Frame)"; + style = "filled,rounded"; + color = "#4A5568"; + fillcolor = "#EDF2F7"; + + f_in [label="SOF Variable Input (e.g. 1ms / 48 frames)"]; + f_ring [label="Accumulation Double-Buffer\n(ff.avail, ff_out.avail)"]; + f_block [label="Fixed Algorithm Block Execution\n(e.g. DSM_FRM_SZ = 48 frames)"]; + f_out [label="SOF Variable Output Drain"]; + } + + f_in -> f_ring [label="Appends new samples"]; + f_ring -> f_block [label="Triggers when avail >= block_size"]; + f_block -> f_ring [label="Stores processed frames"]; + f_ring -> f_out [label="Drains to sink"]; + + m_work -> f_block [style="dashed", color="#2C7A7B"]; + m_db_ff -> f_ring [style="dashed", color="#2C7A7B"]; + } + +1. **Private Memory Block (`MOD_MEMBLK_PRIVATE`)**: + + * Queried and allocated **before** the inner model is initialized. + * Holds the algorithm context handle (e.g., `dsmhandle`), static mathematical tables, and circular delay lines. + * Persists across pipeline run, pause, and resume cycles. + +2. **Frame Buffer Memory Block (`MOD_MEMBLK_FRAME`)**: + + * Queried and allocated **after** model initialization. + * Contains working sample arrays for feed-forward input, processed output, feedback voltage, and feedback current. + * Allocates dedicated accumulation double-buffers (`SMART_AMP_FF_BUF_DB_SZ` and `SMART_AMP_FB_BUF_DB_SZ`). These buffers decouple SOF's variable-period scheduling pipeline ticks from the fixed-frame blocks required by the speaker protection library. + +3. **Parameter Memory Block (`MOD_MEMBLK_PARAM`)**: + + * Queried and allocated **after** model initialization. + * Houses static speaker model parameters, Thiele-Small characteristics, thermal dissipation thresholds, and multi-band equalizer coefficients. + * Maintains volatile telemetry mirrors for dynamic host status readbacks. + +Dual-Pipeline Topology & Asynchronous Buffer Scheduling +======================================================== + +A unique architectural feature of the Smart Amplifier component is its role as a multi-endpoint bridge connecting two asynchronous SOF pipelines: + +1. **Feed-Forward (FF) Playback Pipeline**: Originates from host audio streams (via Mixin/Mixout, Volume, and Equalizer) and delivers safe audio to the speaker amplifier DAC. +2. **Feedback (FB) Capture Pipeline**: Originates from the amplifier I/V sense capture DAI and delivers digitized current and voltage frames into the smart amp. + +.. graphviz:: + :caption: Figure 164: Dual-Pipeline Topology: Playback and Capture Synchronization + :alt: Dual-Pipeline Topology: Playback and Capture Synchronization + + digraph smart_amp_topology { + graph [rankdir=LR, bgcolor="transparent", fontsize=11, fontname="Bitstream Vera Sans"]; + node [shape=box, style="filled,rounded", fontname="Bitstream Vera Sans", fontsize=10, penwidth=1.5]; + edge [fontname="Bitstream Vera Sans", fontsize=9, penwidth=1.2]; + + subgraph cluster_playback { + label = "Playback Pipeline (timer domain 1)"; + style = "filled,rounded"; + color = "#2B6CB0"; + fillcolor = "#EBF8FF"; + + pb_host [label="Host Playback Stream\nPCM Capture", fillcolor="#FFFFFF", color="#2B6CB0"]; + pb_vol [label="Volume / Gain\nModule", fillcolor="#BEE3F8", color="#2B6CB0"]; + pb_buf [label="source_buf\n(Pin 0 Input)", fillcolor="#BEE3F8", color="#2B6CB0"]; + pb_sink [label="sink_buf\n(Pin 0 Output)", fillcolor="#BEE3F8", color="#2B6CB0"]; + pb_dai [label="DAI Copier (Tx)\nTo Amplifier DAC", fillcolor="#FFFFFF", color="#2B6CB0"]; + } + + subgraph cluster_smart { + label = "Smart Amp Component"; + style = "filled,rounded"; + color = "#B7791F"; + fillcolor = "#FEFCBF"; + + sa_core [label="smart_amp_process()\nDual-stream coordinator", fillcolor="#FFFFF0", color="#B7791F"]; + sa_ff [label="smart_amp_ff_process()\nExcursion/Thermal Limiter", fillcolor="#FFFFF0", color="#B7791F"]; + sa_fb [label="smart_amp_fb_process()\nI/V Telemetry Ingestion", fillcolor="#FFFFF0", color="#B7791F"]; + } + + subgraph cluster_capture { + label = "Capture Pipeline (timer domain 2)"; + style = "filled,rounded"; + color = "#2C7A7B"; + fillcolor = "#E6FFFA"; + + cap_dai [label="DAI Copier (Rx)\nFrom Amplifier I/V ADCs", fillcolor="#FFFFFF", color="#2C7A7B"]; + cap_demux [label="Demux / Routing\nSeparates I/V channels", fillcolor="#B2F5EA", color="#2C7A7B"]; + cap_buf [label="feedback_buf\n(Pin 1 Input)", fillcolor="#B2F5EA", color="#2C7A7B"]; + } + + pb_host -> pb_vol -> pb_buf -> sa_core; + cap_dai -> cap_demux -> cap_buf -> sa_core; + + sa_core -> sa_fb [label="Consumes FB frames"]; + sa_core -> sa_ff [label="Processes FF frames"]; + + sa_ff -> pb_sink -> pb_dai; + } + +The Asynchronous Scheduling Challenge +------------------------------------- + +In practical hardware, the playback DAI and capture DAI operate on separate hardware FIFOs and may trigger on different DMA interrupt boundaries or timer ticks. The smart amp component handles this divergence: + +1. **Available Frame Computation**: + + The component queries available frames in the playback path: + + .. math:: + + \text{avail\_passthrough\_frames} = \min(\text{source\_avail}, \text{sink\_free}) + +2. **Feedback Stream Synchronization**: + + If a valid feedback stream is attached and active, the component calculates available feedback frames: + + .. math:: + + \text{avail\_feedback\_frames} = \min(\text{avail\_passthrough\_frames}, \text{feedback\_avail}) + +3. **Decoupled Processing Sequence**: + + * **Cache Invalidation**: Feedback buffer memory ranges are invalidated (`buffer_stream_invalidate()`) to synchronize DSP L1 data cache with incoming DMA writes. + * **Feedback Consumption**: The feedback process (`fb_proc()`) ingests up to `avail_feedback_frames`, re-orders I/V channels into planar buffers, and updates the physical models. The feedback buffer read pointer is immediately advanced. + * **Feed-Forward Protection**: Playback audio is processed through `ff_proc()`, applying instantaneous gain reduction and excursion notches informed by the newly updated speaker state. + * **Cache Writeback**: Processed sink audio is written back to L1 cache (`buffer_stream_writeback()`), and sink produce pointers are updated. + +If the capture pipeline experiences a momentary glitch or underflow, the feed-forward audio continues rolling using recent model estimates, ensuring audio playback never stutters or drops out. + +Dynamic Protection & Acoustic Enhancement Engines +================================================= + +The Smart Amplifier processing core incorporates three coordinated algorithms operating across distinct frequency and temporal domains: + +.. graphviz:: + :caption: Figure 165: Dynamic Protection Architecture: Multi-Band Excursion Limiter and Thermal Controller + :alt: Dynamic Protection Architecture: Multi-Band Excursion Limiter and Thermal Controller + + digraph smart_amp_engines { + graph [rankdir=LR, bgcolor="transparent", fontsize=11, fontname="Bitstream Vera Sans"]; + node [shape=box, style="filled,rounded", fontname="Bitstream Vera Sans", fontsize=10, penwidth=1.5]; + edge [fontname="Bitstream Vera Sans", fontsize=9, penwidth=1.2]; + + in_audio [label="Audio Input\nUnprocessed Playback", fillcolor="#EDF2F7", color="#4A5568"]; + + subgraph cluster_dbe { + label = "Acoustic Enhancement"; + style = "filled,rounded"; + color = "#2B6CB0"; + fillcolor = "#EBF8FF"; + + e_dbe [label="Dynamic Bass Extension (DBE)\nAdaptive resonant boost\nRolls off at high drive", fillcolor="#BEE3F8", color="#2B6CB0"]; + e_harm [label="Harmonic Bass Synthesizer\nNon-linear 2nd/3rd harmonics\n(Missing Fundamental)", fillcolor="#BEE3F8", color="#2B6CB0"]; + } + + subgraph cluster_xlimit { + label = "Fast Excursion Limiting Loop (Mechanical)"; + style = "filled,rounded"; + color = "#2C7A7B"; + fillcolor = "#E6FFFA"; + + x_pred [label="Excursion Predictor\nx_hat(t) = H_disp(s) * V_in", fillcolor="#B2F5EA", color="#2C7A7B"]; + x_peak [label="Peak Detector\nTracks margin to X_max", fillcolor="#B2F5EA", color="#2C7A7B"]; + x_filter [label="Dynamic High-Pass / Notch\nShifts cutoff upward near fs\nwhen |x| -> X_max", fillcolor="#FFFFFF", color="#2C7A7B"]; + } + + subgraph cluster_tlimit { + label = "Slow Thermal Limiting Loop (Electrical)"; + style = "filled,rounded"; + color = "#C53030"; + fillcolor = "#FFF5F5"; + + t_meas [label="Measured Temperature\nT_coil from Re tracking", fillcolor="#FED7D7", color="#C53030"]; + t_gain [label="Slow RMS Gain Attenuator\nSmooth broadband ducking\nwhen T_coil -> T_max", fillcolor="#FFFFFF", color="#C53030"]; + } + + out_audio [label="Safe Audio Output\nTo Amplifier DAC", fillcolor="#EDF2F7", color="#4A5568"]; + + in_audio -> e_dbe; + e_dbe -> e_harm -> x_filter; + + x_pred -> x_peak [label="Predicted cone motion"]; + x_peak -> x_filter [label="Frequency-selective cut", color="#2C7A7B", penwidth=1.5]; + x_peak -> e_dbe [label="Throttle bass boost", color="#2C7A7B", style="dashed"]; + + x_filter -> t_gain -> out_audio; + t_meas -> t_gain [label="Thermal reduction", color="#C53030", penwidth=1.5]; + } + +1. Excursion Limiter (Sub-Millisecond Mechanical Protection) +------------------------------------------------------------ + +Loudspeaker displacement is heavily concentrated in the low-frequency band near resonance :math:`f_s`. The Excursion Limiter enforces physical bounds: + +* **Displacement Prediction**: Filters the input signal through a linear/non-linear displacement transfer function :math:`H_x(s) = \frac{X(s)}{V(s)}` modeling the speaker suspension and acoustic enclosure. +* **Frequency-Selective Clamping**: When predicted excursion approaches the linear limit :math:`X_{max}`, the limiter does *not* apply crude wideband attenuation. Instead, it dynamically shifts a high-pass filter corner frequency upward, or engages a sharp parametric notch filter centered at :math:`f_s`. +* **Perceptual Benefit**: Midrange dialogue and high frequencies pass through unaffected at full volume, maintaining punch and speech clarity while eliminating bass bottoming. + +2. Thermal Limiter (Multi-Second Electrical Protection) +------------------------------------------------------- + +Thermal damage results from long-term integrated power dissipation. The Thermal Limiter protects against coil burnout: + +* **Temperature Thresholding**: Compares the real-time measured temperature :math:`T_{\text{coil}}(t)` against two configured thresholds: :math:`T_{\text{warn}}` (e.g. 95°C) and :math:`T_{\text{max}}` (e.g. 115°C). +* **Smooth Broadband Compression**: When :math:`T_{\text{coil}} > T_{\text{warn}}`, the limiter applies a slow-acting gain reduction (attack time constant 1 to 5 seconds, release time constant 10 to 30 seconds). +* **Perceptual Benefit**: The user perceives no sudden pumping, clicks, or dynamic modulation. The overall volume level smoothly scales back to an equilibrium level where heat generation matches ambient thermal dissipation. + +3. Dynamic Bass Extension (DBE) & Harmonic Synthesis +---------------------------------------------------- + +Micro-speakers typically have a natural low-frequency roll-off starting around 300 Hz to 500 Hz. At low to moderate listening levels, the speaker operates with substantial excursion and thermal headroom. + +* **Dynamic Bass Boost**: The DBE module applies an equalization shelf boosting frequencies between 100 Hz and 300 Hz. As the overall playback volume increases and the excursion limiter detects that :math:`x(t)` is approaching :math:`X_{max}`, the bass boost automatically rolls off. +* **Psychoacoustic Missing Fundamental**: When bass frequencies below 150 Hz cannot be physically reproduced without tearing the speaker diaphragm, the algorithm synthesizes 2nd and 3rd harmonics (e.g. 200 Hz and 300 Hz for a 100 Hz bass note). The human auditory cortex interprets these harmonics as the original low-frequency fundamental without requiring large cone excursions. + +Control Plane, Calibration & Production Workflow +================================================ + +IPC Configuration Structures +---------------------------- + +The Smart Amplifier component exposes two standard binary configuration payloads via IPC3 (`SOF_IPC_COMP_SET_DATA`) and IPC4 (`SET_LARGE_CONFIG` / `GET_LARGE_CONFIG`): + +.. list-table:: Smart Amplifier IPC Configuration Payloads + :widths: 20 25 55 + :header-rows: 1 + + * - Config Type + - Identifier + - Structure & Contents + * - **Type 0** + - ``SOF_SMART_AMP_CONFIG`` + - ``struct sof_smart_amp_config``: Topology channel mapping, feedback channel count, `source_ch_map[PLATFORM_MAX_CHANNELS]`, and `feedback_ch_map[PLATFORM_MAX_CHANNELS]`. + * - **Type 1** + - ``SOF_SMART_AMP_MODEL`` + - Vendor-specific speaker calibration blob: Thiele-Small parameters (:math:`R_0, Bl, M_{ms}, C_{ms}, R_{ms}`), thermal limits (:math:`T_{max}, \tau_{th}`), excursion thresholds (:math:`X_{max}`), and tuning EQ curves. + +Volatile Telemetry Readback +--------------------------- + +The host operating system or manufacturing test utility can query real-time operating metrics by issuing a `GET_LARGE_CONFIG` request for `SOF_SMART_AMP_MODEL`. + +When the command arrives with `msg_index = 0`, the firmware executes `maxim_dsm_get_volatile_param()`, reading live internal state from the DSP algorithm and packing the data into the response payload: + +* Instantaneous voice coil resistance :math:`R_e(t)`. +* Real-time voice coil temperature :math:`T_{\text{coil}}(t)`. +* Maximum instantaneous cone excursion :math:`x_{\text{peak}}`. +* Thermal limiter gain reduction (dB). +* Excursion limiter attenuation (dB). + +Factory Assembly Line Calibration +--------------------------------- + +Loudspeaker voice coil resistance :math:`R_0` varies by up to :math:`\pm 10\%` during factory manufacturing due to copper wire draw tolerances and winding tension. If an algorithm assumed a nominal :math:`R_0 = 6.0\,\Omega` on a speaker whose actual cold resistance is :math:`6.6\,\Omega`, the thermal estimator would calculate a persistent +25°C error, triggering premature thermal limiting. + +To prevent this, production lines execute a calibration routine: + +1. The device is stabilized in a controlled temperature environment (e.g., :math:`T_0 = 20^\circ\text{C}`). +2. A factory diagnostic utility plays a low-level, inaudible test tone or pilot sequence. +3. The DSP measures the baseline cold resistance :math:`R_0` via I/V feedback. +4. The calculated :math:`R_0` value is flashed into non-volatile device storage (ACPI tables or factory calibration partitions). +5. At boot, the Linux sound card initialization script loads the calibrated :math:`R_0` into the Smart Amp component via ALSA byte controls. + +ALSA Topology 2 Declaration & System Graph +========================================== + +Topology 2 Widget Declaration +----------------------------- + +The Smart Amplifier component is declared in ALSA Topology 2.0 configuration files via `Class.Widget."smart_amp"` (defined in `tools/topology/topology2/include/components/smart_amp.conf`): + +.. code-block:: none + + Class.Widget."smart_amp" { + DefineAttribute."index" {} + + + # Declares 2 input pins (FF audio and FB sense) and 1 output pin + num_input_pins 2 + num_output_pins 1 + type "effect" + no_pm "true" + cpc 5000 + is_pages 1 + + # ALSA byte control for runtime configuration and model delivery + Object.Control.bytes."1" { + !access [ + tlv_read + tlv_callback + ] + Object.Base.extops.1 { + name "extctl" + get 258 + put 0 + } + max 4096 + } + } + +End-to-End System Hardware and Firmware Audio Graph +--------------------------------------------------- + +The complete signal routing connects host audio applications, firmware processing modules, hardware DAI links, and the physical loudspeaker transducer: + +.. graphviz:: + :caption: Figure 166: Complete End-to-End Hardware and Firmware Audio Graph + :alt: Complete End-to-End Hardware and Firmware Audio Graph + + digraph smart_amp_e2e { + graph [rankdir=TB, bgcolor="transparent", fontsize=11, fontname="Bitstream Vera Sans"]; + node [shape=box, style="filled,rounded", fontname="Bitstream Vera Sans", fontsize=10, penwidth=1.5]; + edge [fontname="Bitstream Vera Sans", fontsize=9, penwidth=1.2]; + + subgraph cluster_host { + label = "Linux Host OS (ALSA / SoundWire Driver)"; + style = "filled,rounded"; + color = "#4A5568"; + fillcolor = "#EDF2F7"; + + h_app [label="Audio Application\n(Music / Video / Call)", fillcolor="#CBD5E0", color="#4A5568"]; + h_alsa [label="ALSA UCM & Topology 2\nLoads Calibration Blob & Channel Map", fillcolor="#CBD5E0", color="#4A5568"]; + } + + subgraph cluster_dsp_fw { + label = "Audio DSP Firmware (Sound Open Firmware)"; + style = "filled,rounded"; + color = "#2B6CB0"; + fillcolor = "#EBF8FF"; + + subgraph cluster_playback_pipe { + label = "Playback Pipeline (ID: 1)"; + style = "filled,rounded"; + color = "#2B6CB0"; + fillcolor = "#FFFFFF"; + + fw_mixin [label="Mixin / Mixer", fillcolor="#BEE3F8", color="#2B6CB0"]; + fw_dcb [label="DC Blocker", fillcolor="#BEE3F8", color="#2B6CB0"]; + fw_eq [label="Equalizer (FIR/IIR)", fillcolor="#BEE3F8", color="#2B6CB0"]; + fw_drc [label="Dynamic Range Compressor", fillcolor="#BEE3F8", color="#2B6CB0"]; + fw_sa [label="Smart Amplifier\n(smart_amp.c)\nPin 0: Feedforward\nPin 1: Feedback\nPin 0 Out: Protected Audio", fillcolor="#FEFCBF", color="#B7791F", penwidth=2.0]; + fw_copier_tx [label="DAI Copier (Tx)\nSoundWire / I2S Output", fillcolor="#BEE3F8", color="#2B6CB0"]; + } + + subgraph cluster_capture_pipe { + label = "Feedback Capture Pipeline (ID: 2)"; + style = "filled,rounded"; + color = "#2C7A7B"; + fillcolor = "#FFFFFF"; + + fw_copier_rx [label="DAI Copier (Rx)\nSoundWire / I2S Input", fillcolor="#B2F5EA", color="#2C7A7B"]; + fw_demux [label="Channel Demux\nIsolates I and V Streams", fillcolor="#B2F5EA", color="#2C7A7B"]; + } + } + + subgraph cluster_hardware { + label = "Hardware Platform"; + style = "filled,rounded"; + color = "#C53030"; + fillcolor = "#FFF5F5"; + + hw_amp [label="Smart Amplifier IC (e.g. MAX98373 / CS35L41)\nClass-D BTL Power Stage + I/V Sense ADCs", fillcolor="#FED7D7", color="#C53030", penwidth=1.8]; + hw_speaker [label="Micro-Speaker Transducer\nVoice Coil + Neodymium Magnet + Diaphragm", fillcolor="#FED7D7", color="#C53030", penwidth=1.8]; + } + + h_app -> fw_mixin [label="PCM Audio Stream"]; + h_alsa -> fw_sa [label="Model Blob & Config", style="dashed", color="#B7791F"]; + + fw_mixin -> fw_dcb -> fw_eq -> fw_drc -> fw_sa [label="Feedforward Playback"]; + fw_sa -> fw_copier_tx [label="Protected Audio"]; + fw_copier_tx -> hw_amp [label="Digital Audio (Tx)", color="#2B6CB0", penwidth=1.5]; + + hw_amp -> hw_speaker [label="High-Power Analog Drive", color="#C53030", penwidth=2.0]; + hw_speaker -> hw_amp [label="I/V Sense Terminal Feedback", color="#C53030", style="dotted", penwidth=1.5]; + + hw_amp -> fw_copier_rx [label="Digitized I/V Sense (Rx)", color="#2C7A7B", penwidth=1.5]; + fw_copier_rx -> fw_demux -> fw_sa [label="Feedback Stream (Pin 1)", color="#2C7A7B", penwidth=1.5]; + } + +Developer Tuning & Diagnostic Workflow +====================================== + +When integrating a new micro-speaker into a Sound Open Firmware platform: + +1. **Speaker Characterization**: Measure baseline Thiele-Small parameters (:math:`R_0, Bl, M_{ms}, C_{ms}, R_{ms}`) and determine destructive thermal (:math:`T_{max}`) and mechanical (:math:`X_{max}`) limits using laser vibrometry and thermal imaging. +2. **Topology Channel Mapping**: Configure `source_ch_map` and `feedback_ch_map` in ALSA Topology 2 to ensure the V-sense and I-sense capture channels align with the physical amplifier pinout. +3. **Model Generation & Injection**: Compile Characterization parameters into a binary model blob (`SOF_SMART_AMP_MODEL`) and deliver it to the component via ALSA mixer controls (`amixer cset`). +4. **Telemetry Verification**: Stream high-volume audio while monitoring volatile parameters (:math:`R_e(t), T_{\text{coil}}(t), x(t)`) to verify that the thermal and excursion limiters activate smoothly before reaching physical boundaries. + +Upstream Source Code References +=============================== + +* **Component Core & Generic Layer**: + + - `src/audio/smart_amp/smart_amp.c `_: Processing module adapter, multi-endpoint binding, memory manager, and process loop. + - `src/audio/smart_amp/smart_amp_generic.c `_: Channel remapping, format conversion routines, and pointer helpers. + - `src/include/sof/audio/smart_amp/smart_amp.h `_: Internal interfaces, buffer definitions, and memory block enums. + - `src/include/user/smart_amp.h `_: User-space and IPC configuration structures. + +* **Inner Model Implementations**: + + - `src/audio/smart_amp/smart_amp_passthru.c `_: Open-source passthrough inner model. + - `src/audio/smart_amp/smart_amp_maxim_dsm.c `_: Maxim DSM adapter, volatile telemetry readback, and frame reordering. + - `src/audio/smart_amp/include/dsm_api/inc/dsm_api_public.h `_: Public interface declarations for DSM. + +* **ALSA Topology 2 Configurations**: + + - `tools/topology/topology2/include/components/smart_amp.conf `_: Topology 2 smart amp widget class. + - `tools/topology/topology2/include/pipelines/cavs/mixout-gain-smart-amp-dai-copier-playback.conf `_: Playback pipeline integrating Smart Amp with gain and DAI copiers. + +Related Architecture Guides +=========================== + +* :ref:`drc_multiband_drc`: Wideband and multi-band dynamic range compression, lookahead pre-delay buffers, and speaker protection leveling. +* :ref:`dcblock`: First-order recursive high-pass filter eliminating DC offsets before speaker power amplification. +* :ref:`crossover`: Multi-way active digital frequency division splitting audio across woofers and tweeters. +* :ref:`volume_module`: High-precision volume scaling, zero-crossing smooth ramping, and soft mute mechanics. +* :ref:`module_framework`: Standardized lifecycle, memory management, and IPC configuration handlers. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index b5b80df4..4962b664 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -52,7 +52,7 @@ Audio Processing Modules & Algorithms * `RTNR Noise Reduction `_ * :ref:`tflm` (High-level architecture; also see upstream `TFLM README `_) * :ref:`mfcc` (High-level architecture; also see upstream `MFCC README `_) -* `Smart Amp Protection `_ +* :ref:`smart_amp` (High-level architecture; also see upstream `Smart Amp README `_) * `Sound Dose Evaluator `_ * `Copier `_, `Mux `_ & `Selector `_ * `PCM Format Converter `_ @@ -96,6 +96,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/tdfb firmware/tflm firmware/mfcc + firmware/smart_amp rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 0efc31ecfa956e972cf96481810cb4577cd862c3 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 10:53:27 +0100 Subject: [PATCH 19/64] doc: add sound dose evaluator architecture guide Create a comprehensive developer guide for the Sound Dose Evaluator subsystem in Sound Open Firmware. Key topics covered: - Auditory health physiology, Temporary/Permanent Threshold Shift (TTS/PTS), and international regulatory frameworks (IEC 62368-1 Clause 10.6, WHO-ITU H.870, and Calculated Sound Dose / 3 dB exchange rule). - IEC 61672-1 Class 1 A-weighting acoustic transfer function and cascaded Direct Form I (DF1) biquad realization in SOF. - Fixed-point real-time energy accumulation in 64-bit precision. - 1-second interval periodic trigger and logarithmic mean dBFS conversion. - Momentary Exposure Level (MEL) calculation and acoustic sensitivity mapping. - Unsolicited IPC4 asynchronous event notifications to host exposure daemons. - Closed-loop dynamic protection via smooth exponential per-frame gain ramping. - ALSA Topology 2 widget configuration, byte controls, and end-to-end graph. - 7 native vector Graphviz SVG diagrams (Figures 167-173). Signed-off-by: Liam Girdwood --- .../firmware/drc_multiband_drc.rst | 1 + .../firmware/pipeline_architecture.rst | 1 + developer_guides/firmware/smart_amp.rst | 1 + developer_guides/firmware/sound_dose.rst | 794 ++++++++++++++++++ developer_guides/index.rst | 3 +- 5 files changed, 799 insertions(+), 1 deletion(-) create mode 100644 developer_guides/firmware/sound_dose.rst diff --git a/developer_guides/firmware/drc_multiband_drc.rst b/developer_guides/firmware/drc_multiband_drc.rst index 06616da4..96df83ea 100644 --- a/developer_guides/firmware/drc_multiband_drc.rst +++ b/developer_guides/firmware/drc_multiband_drc.rst @@ -568,6 +568,7 @@ Related Subsystem Architecture Guides ===================================== * :ref:`smart_amp`: Adaptive speaker protection, real-time current/voltage (I/V) sense telemetry, and excursion/thermal limiters. +* :ref:`sound_dose`: Auditory safety evaluation, IEC 61672-1 A-weighting integration, and dynamic gain limiting. * :ref:`volume_module`: Per-channel gain scaling, smooth volume ramping, and zero-crossing muting. * :ref:`eq_fir_iir`: Finite and Infinite Impulse Response equalizers, linear-phase filtering, and biquad cascades. * :ref:`src_asrc`: Sample rate conversion architecture handling fixed and drifting clocks across heterogeneous audio interfaces. diff --git a/developer_guides/firmware/pipeline_architecture.rst b/developer_guides/firmware/pipeline_architecture.rst index 580b7d09..2830a2ab 100644 --- a/developer_guides/firmware/pipeline_architecture.rst +++ b/developer_guides/firmware/pipeline_architecture.rst @@ -472,6 +472,7 @@ Related Guides * :ref:`tflm`: Embedded neural network inference, static tensor arena memory planning, 8-bit affine quantization, and Tensilica NNLib SIMD acceleration. * :ref:`mfcc`: Real-time Mel-Frequency Cepstral Coefficients feature extraction, auditory filterbanks, OpenAI Whisper preprocessing, and Mel-domain VAD. * :ref:`smart_amp`: Adaptive speaker protection, real-time current/voltage (I/V) sense telemetry, thermal/excursion limiters, and two-layer generic/inner model architecture. +* :ref:`sound_dose`: Auditory health protection (IEC 62368-1 / WHO-ITU H.870), IEC 61672-1 A-weighting filtering, 64-bit energy accumulation, and dynamic gain limiting. * :ref:`fw_init_boot`: Boot flow, hardware mailbox FW Ready handshake, and Zephyr initialization. * :ref:`topology2`: How pipelines and widgets are declared using ALSA Topology 2.0 configuration classes. * :ref:`sof_hostless_firmware`: How to create static pipelines compiled into ROM for standalone microcontrollers. diff --git a/developer_guides/firmware/smart_amp.rst b/developer_guides/firmware/smart_amp.rst index 9456090a..0e798da1 100644 --- a/developer_guides/firmware/smart_amp.rst +++ b/developer_guides/firmware/smart_amp.rst @@ -874,6 +874,7 @@ Related Architecture Guides =========================== * :ref:`drc_multiband_drc`: Wideband and multi-band dynamic range compression, lookahead pre-delay buffers, and speaker protection leveling. +* :ref:`sound_dose`: Auditory health protection, IEC 61672-1 A-weighting acoustic integration, and autonomous headphone volume limiting. * :ref:`dcblock`: First-order recursive high-pass filter eliminating DC offsets before speaker power amplification. * :ref:`crossover`: Multi-way active digital frequency division splitting audio across woofers and tweeters. * :ref:`volume_module`: High-precision volume scaling, zero-crossing smooth ramping, and soft mute mechanics. diff --git a/developer_guides/firmware/sound_dose.rst b/developer_guides/firmware/sound_dose.rst new file mode 100644 index 00000000..e3448b46 --- /dev/null +++ b/developer_guides/firmware/sound_dose.rst @@ -0,0 +1,794 @@ +.. _sound_dose: + +Sound Dose Evaluator Architecture +################################# + +The **Sound Dose Evaluator** is an autonomous, real-time auditory safety subsystem in Sound Open Firmware (SOF). Designed to comply with international consumer audio health regulations—specifically **IEC 62368-1** and **WHO-ITU H.870**—the Sound Dose module continuously analyzes audio streams routed to headphones and headsets. It computes spectral energy exposure in real time, translates digital audio levels into physical sound pressure levels (:math:`\text{dBSPL}`), tracks cumulative exposure across rolling temporal windows, and autonomously reports exposure metrics to the host operating system while providing artifact-free dynamic attenuation when safe exposure limits are exceeded. + +Auditory Health Physiology & International Regulatory Mandates +============================================================== + +Prolonged exposure to high sound pressure levels induces irreversible physiological damage to the human auditory system. The human inner ear contains the **cochlea**, a fluid-filled, spiral-shaped cavity lined with the basilar membrane. Transduction of acoustic vibrations into neural impulses is performed by approximately 15,000 hair cells: + +* **Inner Hair Cells (IHCs)**: Primary sensory transducers that release neurotransmitters to auditory nerve fibers in response to stereocilia deflection. +* **Outer Hair Cells (OHCs)**: Electromotile amplifiers that actively alter their length via the motor protein prestin, providing up to 50 dB of mechanical amplification for quiet sounds and sharpening frequency selectivity. + +When exposed to excessive acoustic energy, outer hair cells undergo intense metabolic overload. This causes severe oxidative stress, marked accumulation of reactive oxygen species (ROS), intracellular calcium excitotoxicity, mitochondrial swelling, and structural rupture of stereocilia tip-links. While moderate over-exposure leads to a **Temporary Threshold Shift (TTS)** that recovers over several hours as cellular homeostasis is restored, repeated or severe acoustic trauma results in permanent hair cell apoptosis and spiral ganglion synaptic decoupling—causing irreversible **Permanent Threshold Shift (PTS)**, high-frequency sensorineural hearing loss, and chronic tinnitus. + +.. graphviz:: + :caption: Figure 167: Auditory Perception & Hearing Damage Risk Curve: Sound Pressure Level vs Maximum Safe Exposure Time (IEC 62368-1 / WHO-ITU H.870) + :alt: Auditory perception and hearing damage risk curve showing permissible weekly exposure time as a function of sound pressure level according to IEC 62368-1 and WHO-ITU H.870. + + digraph sound_dose_damage_curve { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica", fontsize=10]; + + subgraph cluster_legend { + label = "Regulatory Sound Exposure Classifications (IEC 62368-1 Clause 10.6 / WHO-ITU H.870)"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + zone_safe [label="Safe Acoustic Zone\n< 80 dBA\nIndefinite listening without risk of hearing impairment\nWeekly Dose: < 100% CSD", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + zone_advisory [label="Advisory Warning Zone\n80 dBA to 89 dBA\n80% to 100% of Weekly Sound Dose (CSD)\nSystem issues user notification; continuous exposure tracking", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + zone_hazard [label="Hazardous Acoustic Zone\n>= 90 dBA or > 100% CSD\nImmediate risk of Permanent Threshold Shift (PTS)\nMandatory gain attenuation & user acknowledgment required", fillcolor="#FED7D7", color="#C53030", penwidth=2.0]; + } + + subgraph cluster_curve { + label = "3 dB Exchange Rate: Sound Pressure Level vs Permissible Exposure Time per Week"; + style = "filled,rounded"; + color = "#4A5568"; + fillcolor = "#FFFFFF"; + + pt80 [label="80 dBA\n40 Hours / Week\nBaseline 100% CSD (1.6 Pa²h)", fillcolor="#E2E8F0", color="#4A5568"]; + pt83 [label="83 dBA\n20 Hours / Week\nExposure halving rule", fillcolor="#E2E8F0", color="#4A5568"]; + pt86 [label="86 dBA\n10 Hours / Week\n2x energy density", fillcolor="#E2E8F0", color="#4A5568"]; + pt89 [label="89 dBA\n5 Hours / Week\nHigh exposure risk", fillcolor="#FEFCBF", color="#B7791F"]; + pt92 [label="92 dBA\n2.5 Hours / Week\n(150 Minutes)", fillcolor="#FEFCBF", color="#B7791F"]; + pt95 [label="95 dBA\n1.25 Hours / Week\n(75 Minutes)", fillcolor="#FED7D7", color="#C53030"]; + pt100 [label="100 dBA\n23.7 Minutes / Week\nMandatory Cap Threshold", fillcolor="#FED7D7", color="#C53030", penwidth=2.0]; + pt105 [label=">= 105 dBA\n< 7.5 Minutes / Week\nInstantaneous Acoustic Trauma", fillcolor="#FED7D7", color="#9B2C2C", penwidth=2.2]; + + pt80 -> pt83 [label="+3 dB (time / 2)", color="#4A5568"]; + pt83 -> pt86 [label="+3 dB (time / 2)", color="#4A5568"]; + pt86 -> pt89 [label="+3 dB (time / 2)", color="#B7791F"]; + pt89 -> pt92 [label="+3 dB (time / 2)", color="#B7791F"]; + pt92 -> pt95 [label="+3 dB (time / 2)", color="#C53030"]; + pt95 -> pt100 [label="+5 dB (time / 3.16)", color="#C53030"]; + pt100 -> pt105 [label="+5 dB (severe risk)", color="#9B2C2C"]; + } + + subgraph cluster_pathology { + label = "Cellular Pathology in Organ of Corti"; + style = "filled,rounded"; + color = "#718096"; + fillcolor = "#F8FAFC"; + + cell_normal [label="Normal Outer Hair Cells\nIntact stereocilia bundles\nHealthy cochlear amplification", fillcolor="#C6F6D5", color="#276749"]; + cell_metabolic [label="Metabolic Exhaustion\nReactive Oxygen Species (ROS) accumulation\nTip-link decoupling & Temporary Threshold Shift (TTS)", fillcolor="#FEFCBF", color="#B7791F"]; + cell_permanent [label="Cell Apoptosis & Synaptic Decoupling\nIrreversible stereocilia loss\nPermanent Threshold Shift (PTS) & Tinnitus", fillcolor="#FED7D7", color="#C53030"]; + + cell_normal -> cell_metabolic [label="Exceeds 80 dBA for > 40h", color="#B7791F"]; + cell_metabolic -> cell_permanent [label="Sustained overload without recovery", color="#C53030"]; + } + + zone_safe -> pt80 [style="dashed", color="#276749"]; + zone_advisory -> pt89 [style="dashed", color="#B7791F"]; + zone_hazard -> pt100 [style="dashed", color="#C53030"]; + pt100 -> cell_permanent [style="dotted", color="#C53030"]; + } + +Calculated Sound Dose (CSD) and the 3 dB Exchange Rule +------------------------------------------------------ + +To protect consumers against premature hearing loss, the International Electrotechnical Commission (**IEC 62368-1 Clause 10.6**) and the World Health Organization together with the International Telecommunication Union (**WHO-ITU H.870**) established standardized personal audio safety requirements: + +1. **Calculated Sound Dose (CSD)**: The total acoustic energy absorbed by the human ear, integrated over a rolling 7-day window. A reference weekly dose of **100% CSD** corresponds to continuous exposure of **80 dBA for 40 hours per week**, representing an acoustic energy dosage of: + + .. math:: + + \text{Dose}_{\text{ref}} = (20\,\mu\text{Pa} \cdot 10^{80/20})^2 \cdot 40\,\text{hours} \approx 1.6\,\text{Pa}^2\text{h} + +2. **The 3 dB Equal Energy Exchange Principle**: Acoustic sound intensity doubles with every :math:`+3\,\text{dB}` increase. Therefore, the permissible exposure duration before reaching 100% CSD is halved for every 3 dB increase in sound pressure level: + + .. math:: + + T_{\text{safe}}(\text{MEL}) = 40\,\text{hours} \cdot 10^{\frac{80 - \text{MEL}}{10}} + +3. **Mandatory Protective Actions**: + * **Advisory Warning (80% CSD)**: The system alerts the user that they are approaching their maximum weekly sound exposure budget. + * **Mandatory Attenuation (100% CSD)**: The audio framework automatically engages a dynamic volume limiter, attenuating playback down to a safe exposure level (< 80 dBA). The user cannot override this attenuation without acknowledging a formal hearing hazard prompt. + * **Instantaneous Exposure Cap**: Listening levels exceeding **100 dBA** are strictly restricted in continuous duration to prevent acute acoustic trauma. + +IEC 61672-1 Class 1 A-Weighting Filter Cascade +============================================== + +The human ear does not perceive all acoustic frequencies with equal sensitivity. As demonstrated by the Robinson-Dadson and ISO 226 equal-loudness contours, human hearing is significantly less sensitive at low frequencies (< 500 Hz) and ultra-high frequencies (> 10 kHz), while exhibiting peak resonance between 2 kHz and 4 kHz due to the acoustic dimensions of the ear canal. + +To ensure that the calculated sound energy reflects true physiological hearing hazard, raw digital PCM samples must be processed through an **A-weighting frequency curve** defined by **IEC 61672-1**. + +Continuous-Domain Transfer Function +----------------------------------- + +The standardized continuous-time A-weighting frequency response :math:`R_A(f)` is defined analytically as: + +.. math:: + + R_A(f) = \frac{12194^2 \cdot f^4}{(f^2 + 20.6^2) \cdot \sqrt{(f^2 + 107.7^2)(f^2 + 737.9^2)} \cdot (f^2 + 12194^2)} + +The weighting in decibels :math:`A(f)` is normalized to 0 dB at 1000 Hz: + +.. math:: + + A(f) = 20 \log_{10}(R_A(f)) - 20 \log_{10}(R_A(1000)) = 20 \log_{10}(R_A(f)) + 2.00\,\text{dB} + +.. graphviz:: + :caption: Figure 168: IEC 61672-1 Class 1 A-Weighting Acoustic Filter Frequency Response Curve & Direct Form I Biquad Cascade + :alt: IEC 61672-1 Class 1 A-weighting frequency response curve and Direct Form I biquad cascade implementation in SOF. + + digraph a_weighting_cascade { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica", fontsize=9]; + + subgraph cluster_response { + label = "IEC 61672-1 Class 1 A-Weighting Target Magnitude Profile (Relative to 1 kHz)"; + style = "filled,rounded"; + color = "#2B6CB0"; + fillcolor = "#EBF8FF"; + + subgraph cluster_sub_bass { + label = "Bass Attenuation (< 500 Hz)"; + style = "filled,rounded"; + color = "#3182CE"; + fillcolor = "#FFFFFF"; + f_20 [label="20 Hz: -50.5 dB\nSub-audible cutoff", fillcolor="#BEE3F8", color="#2B6CB0"]; + f_100 [label="100 Hz: -19.1 dB\nBass rolloff", fillcolor="#BEE3F8", color="#2B6CB0"]; + f_500 [label="500 Hz: -3.2 dB\nTransition band", fillcolor="#BEE3F8", color="#2B6CB0"]; + f_20 -> f_100 -> f_500; + } + + subgraph cluster_sub_critical { + label = "Critical Sensitivity Band (1–4 kHz)"; + style = "filled,rounded"; + color = "#D69E2E"; + fillcolor = "#FFFFFF"; + f_1k [label="1 kHz: 0.0 dB\nStandard Reference", fillcolor="#C6F6D5", color="#276749", penwidth=2.0]; + f_3k [label="3 kHz: +1.2 dB\nEar canal resonance", fillcolor="#FEFCBF", color="#B7791F", penwidth=2.0]; + f_1k -> f_3k; + } + + subgraph cluster_sub_treble { + label = "Treble Rolloff (> 5 kHz)"; + style = "filled,rounded"; + color = "#4A5568"; + fillcolor = "#FFFFFF"; + f_10k [label="10 kHz: -2.5 dB\nHigh frequency drop", fillcolor="#BEE3F8", color="#2B6CB0"]; + f_20k [label="20 kHz: -9.3 dB\nNyquist threshold", fillcolor="#BEE3F8", color="#2B6CB0"]; + f_10k -> f_20k; + } + + f_500 -> f_1k -> f_10k [style="dashed", color="#2B6CB0"]; + } + + subgraph cluster_biquad_cascade { + label = "Cascaded Direct Form I (DF1) Biquad IIR Architecture (sound_dose.c)"; + style = "filled,rounded"; + color = "#2F855A"; + fillcolor = "#F0FFF4"; + + in_pcm [label="Linear PCM Audio x[n]\n(Scaled by Gain g in Q2.30)", fillcolor="#ED8936", fontcolor="#FFFFFF", shape=ellipse, penwidth=1.5]; + + bq1 [label="Biquad Stage 1 (High-Pass / Low-Shelf)\nb0, b1, b2 feedforward | a1, a2 feedback\nAttenuates low rumble and sub-bass", fillcolor="#FFFFFF", color="#38A169", penwidth=1.5]; + bq2 [label="Biquad Stage 2 (Mid-Frequency Resonator)\nb0, b1, b2 feedforward | a1, a2 feedback\nModels human ear canal 3-4 kHz boost", fillcolor="#FFFFFF", color="#38A169", penwidth=1.5]; + bq3 [label="Biquad Stage 3 (Treble Shaper / High-Cut)\nb0, b1, b2 feedforward | a1, a2 feedback\nModels high-frequency psychoacoustic rolloff", fillcolor="#FFFFFF", color="#38A169", penwidth=1.5]; + + out_weighted [label="A-Weighted Stream y_A[n]\n(Q1.15 / Q1.31 Format)", fillcolor="#48BB78", fontcolor="#FFFFFF", shape=ellipse, penwidth=1.8]; + + in_pcm -> bq1 [label="x[n]", color="#2F855A", penwidth=1.5]; + bq1 -> bq2 [label="Stage 1 Output", color="#2F855A", penwidth=1.5]; + bq2 -> bq3 [label="Stage 2 Output", color="#2F855A", penwidth=1.5]; + bq3 -> out_weighted [label="Weighted Samples", color="#2F855A", penwidth=1.8]; + } + + f_3k -> bq2 [style="dotted", label="Matches physical curve", color="#B7791F"]; + } + +Discrete Direct Form I Realization in SOF +----------------------------------------- + +In SOF, the continuous A-weighting curve is bilinear-transformed and mapped into a sequence of cascaded second-order IIR biquad sections. SOF adopts the **Direct Form I (DF1)** structure because it exhibits superior numerical immunity against coefficient quantization and limit cycles in fixed-point DSP arithmetic: + +.. math:: + + y[n] = b_0 x[n] + b_1 x[n-1] + b_2 x[n-2] - a_1 y[n-1] - a_2 y[n-2] + +Key structural characteristics: + +* **Pre-Computed Header Blobs**: To avoid runtime transcendental math on the DSP, filter coefficient sets are pre-computed for standard audio rates and packaged into firmware headers: + + - ``sound_dose_iir_48k.h``: Optimized for 48 kHz operation. + - ``sound_dose_iir_44k.h``: Optimized for 44.1 kHz operation. + +* **Per-Channel State Isolation**: Dedicated delay registers and history buffers (``cd->delay_lines``) are dynamically allocated for each audio channel (up to ``PLATFORM_MAX_CHANNELS``), preventing cross-channel phase contamination. + +Fixed-Point Real-Time Energy Accumulation Architecture +====================================================== + +The Sound Dose module processes audio frames in fixed-point representation. Depending on the pipeline configuration, samples arrive in either signed 16-bit (``S16_LE`` in :math:`Q1.15` format) or signed 32-bit (``S32_LE`` in :math:`Q1.31` format). + +Sample Processing and Squaring +------------------------------ + +For each incoming audio frame: + +1. **Protective Gain Application**: The input sample :math:`x[n]` is scaled by the module's active internal gain :math:`g \in Q2.30`: + + .. math:: + + x_{\text{scaled}}[n] = \text{sat}_{16}\left( \frac{g \cdot x[n]}{2^{30}} \right) + + This scaled sample is written directly to the output sink buffer, ensuring that protective gain adjustments apply to the listening path without latency. + +2. **A-Weighting Filtering**: In parallel, :math:`x_{\text{scaled}}[n]` is passed through the channel's Direct Form I biquad cascade, yielding the frequency-weighted sample :math:`y_A[n]`. + +3. **Instantaneous Power Calculation**: The weighted sample is squared to determine instantaneous acoustic power: + + .. math:: + + P_i[n] = y_A[n] \cdot y_A[n] + + In fixed-point math, multiplying two :math:`Q1.15` numbers produces a :math:`Q2.30` result. For 32-bit audio, multiplying two :math:`Q1.31` numbers yields a :math:`Q2.62` intermediate result. + +64-Bit Energy Accumulation +-------------------------- + +To eliminate numerical overflow during prolonged listening, the instantaneous power values are accumulated across time in a dedicated 64-bit signed integer for each channel: + +.. math:: + + E_{\text{ch}}[k] = \sum_{n=0}^{N-1} y_A[n]^2 + +A 64-bit integer provides massive dynamic headroom. Even if full-scale white noise or square waves are played continuously at 48 kHz, accumulating :math:`2^{30}` power units per sample over a full 1-second period (48,000 samples) consumes only: + +.. math:: + + 48000 \cdot 2^{30} \approx 5.15 \times 10^{13} \ll 2^{63} - 1 \approx 9.22 \times 10^{18} + +This mathematical headroom guarantees that accumulator overflow is physically impossible. + +1-Second Periodic Trigger & Logarithmic Mean Conversion +======================================================= + +The Sound Dose Evaluator operates on a synchronized **1-second reporting window**. The DSP tracks total processed frames within the current accumulation epoch (``cd->frames_count``). When ``cd->frames_count >= cd->report_count`` (e.g. 48,000 frames at 48 kHz), the 1-second conversion routine is triggered. + +.. graphviz:: + :caption: Figure 169: Fixed-Point 64-Bit Energy Integration and Logarithmic dBFS/MEL Conversion Flowchart + :alt: Flowchart showing fixed-point 64-bit energy integration, bit shifting, base-2 logarithm conversion, and decibel calibration in SOF. + + digraph energy_to_mel_flow { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica", fontsize=9]; + + subgraph cluster_sample_loop { + label = "Per-Sample High-Speed Processing Loop (Every Audio Frame)"; + style = "filled,rounded"; + color = "#2B6CB0"; + fillcolor = "#EBF8FF"; + + s1 [label="Input PCM Sample x[n]\n(Q1.15 / Q1.31)", fillcolor="#BEE3F8", color="#2B6CB0"]; + s2 [label="Apply Dynamic Gain\nx_scaled = sat(x[n] * gain)", fillcolor="#BEE3F8", color="#2B6CB0"]; + s3 [label="IEC 61672-1 DF1 Filter\ny_A[n] = iir_df1(x_scaled)", fillcolor="#C6F6D5", color="#276749"]; + s4 [label="Square Weighted Sample\nP = y_A[n] * y_A[n] (Q2.30)", fillcolor="#FEFCBF", color="#B7791F"]; + s5 [label="64-Bit Accumulator\ncd->energy[ch] += P", fillcolor="#FED7D7", color="#C53030", penwidth=1.5]; + + s1 -> s2 -> s3 -> s4 -> s5; + } + + subgraph cluster_1s_trigger { + label = "1-Second Periodic Trigger (cd->frames_count >= cd->report_count)"; + style = "filled,rounded"; + color = "#702459"; + fillcolor = "#FDF2F8"; + + t1 [label="Channel Energy Summation\nenergy_sum = sum(energy[ch])", fillcolor="#FBB6CE", color="#97266D"]; + t2 [label="Scale Down Energy by 19 Bits\nlog_arg = (uint32_t)(energy_sum >> 19)", fillcolor="#FBB6CE", color="#97266D"]; + t3 [label="Fast Integer Base-2 Logarithm\ntmp = base2_logarithm(log_arg) (Q16.16)", fillcolor="#D6BCFA", color="#6B46C1", penwidth=1.8]; + t4 [label="Compensate Q2.30 & Shift\ntmp += 65536 * (19 - 30)", fillcolor="#E2E8F0", color="#4A5568"]; + t5 [label="Normalize for Temporal Mean\ntmp += log2(1 / Fs) * 2^16", fillcolor="#E2E8F0", color="#4A5568"]; + t6 [label="Convert Base-2 to Base-10 Decibels\ntmp = tmp * (10 / log2(10)) * 2^29", fillcolor="#C6F6D5", color="#276749", penwidth=1.5]; + t7 [label="Apply Calibration Offsets\n+3.0 dB (Filter) + 3.01 dB (Sine Full-Scale)\n-1.5 dB * channels (Stereo Binaural Correction)", fillcolor="#FEFCBF", color="#B7791F"]; + t8 [label="Result: Mean dBFS Exposure\ncd->level_dbfs (Q16.16)", fillcolor="#B2F5EA", color="#234E52", penwidth=2.0]; + + t1 -> t2 -> t3 -> t4 -> t5 -> t6 -> t7 -> t8; + } + + subgraph cluster_mel_out { + label = "Momentary Exposure Level (MEL) Derivation"; + style = "filled,rounded"; + color = "#2F855A"; + fillcolor = "#F0FFF4"; + + m1 [label="Convert dBFS to Centi-Decibels\ndbfs_value = (level_dbfs * 100) >> 16", fillcolor="#C6F6D5", color="#276749"]; + m2 [label="Inject Sensitivity & Volume Offsets\nMEL = dbfs_value + sens_dbfs_dbspl + volume_offset", fillcolor="#C6F6D5", color="#276749", penwidth=2.0]; + m3 [label="Calculate Exact Stream Time (us)\nfrom cd->total_frames_count * rate_to_us", fillcolor="#E2E8F0", color="#4A5568"]; + m4 [label="Dispatch Unsolicited IPC4 Notification\nSOF_AUDIO_FEATURE_SOUND_DOSE_MEL", fillcolor="#ED8936", fontcolor="#FFFFFF", shape=ellipse, penwidth=2.0]; + + m1 -> m2 -> m4; + m3 -> m4; + } + + s5 -> t1 [label="Every 48,000 frames", color="#97266D", style="dashed", penwidth=1.5]; + t8 -> m1 [color="#276749"]; + } + +The Logarithmic Math Pipeline +----------------------------- + +Converting a 64-bit integer energy sum into standardized decibels relative to full scale (:math:`\text{dBFS}`) requires careful mathematical scaling in fixed-point arithmetic: + +1. **Downscaling for 32-Bit Logarithm**: The total accumulated energy :math:`E_{\text{sum}} = \sum_{\text{ch}} E_{\text{ch}}` is right-shifted by 19 bits (``SOUND_DOSE_ENERGY_SHIFT``) to ensure it fits comfortably within an unsigned 32-bit integer: + + .. math:: + + \text{arg}_{\text{log}} = \max\left( 1, \left\lfloor \frac{E_{\text{sum}}}{2^{19}} \right\rfloor \right) + +2. **Base-2 Logarithm**: The DSP invokes ``base2_logarithm(arg_log)``, which utilizes binary leading-zero count and polynomial approximation to produce :math:`\log_2(\text{arg}_{\text{log}})` in signed :math:`Q16.16` fixed-point format. + +3. **Fixed Offset Compensation**: Because the original samples were in :math:`Q1.15` (squared to :math:`Q2.30`) and downshifted by 19 bits, the logarithm must be corrected by: + + .. math:: + + \Delta_{\text{scale}} = 65536 \cdot (19 - 30) = -11 \cdot 65536 + +4. **Mean Power Normalization**: To convert accumulated total energy over the 1-second epoch into mean continuous power per sample, the logarithm of the reciprocal frame count is added: + + .. math:: + + \Delta_{\text{mean}} = \log_2\left( \frac{1}{f_s} \right) \cdot 2^{16} + + where :math:`\Delta_{\text{mean}} = -1019134` for 48 kHz and :math:`-1011122` for 44.1 kHz. + +5. **Base-2 to Decibel Transformation**: Decibels are base-10 logarithmic measures (:math:`10 \log_{10}(P)`). The base-2 logarithm is converted via multiplication with :math:`\frac{10}{\log_2(10)}` in :math:`Q29` fixed-point format: + + .. math:: + + \text{multiplier} = \left\lfloor \frac{10}{\log_2(10)} \cdot 2^{29} \right\rfloor = 1616142483 + +6. **Filter and Sine Reference Offsets**: + * **Filter Offset**: :math:`+3.00\,\text{dB}` (``SOUND_DOSE_WEIGHT_FILTERS_OFFS_Q16``) accounts for insertion gain characteristics of the discrete A-weighting filter cascade. + * **Full-Scale Offset**: :math:`+3.01\,\text{dB}` (``SOUND_DOSE_DFBS_OFFS_Q16``) ensures that a full-scale digital sine wave (:math:`0\,\text{dBFS}`) computes precisely to :math:`0.00\,\text{dBFS}` RMS power. + +7. **Binaural Multichannel Correction**: Summing power across multiple channels inflates total acoustic energy. In a headphone listening scenario, each ear receives acoustic power from its corresponding channel. To model binaural loudness perception accurately, a correction factor of :math:`-1.5\,\text{dB}` per channel (``SOUND_DOSE_MEL_CHANNELS_SUM_FIX``) is applied, subtracting :math:`-3.0\,\text{dB}` for standard stereo streams. + +Momentary Exposure Level (MEL) Derivation & Acoustic Calibration +================================================================ + +While digital decibels relative to full scale (:math:`\text{dBFS}`) quantify the electrical signal inside the DSP, auditory health is dictated by physical sound pressure level in air (:math:`\text{dBSPL}`) at the user's eardrum. + +The **Momentary Exposure Level (MEL)** is the 1-second A-weighted sound pressure level (:math:`\text{dBA}`) delivered by the headphones. In SOF, all exposure levels are stored as centi-decibels (:math:`0.01\,\text{dB}` precision, where :math:`85.00\,\text{dB} = 8500`). + +The MEL is derived directly from three distinct variables: + +.. math:: + + \text{MEL} = \text{dBFS} + \text{sens\_dbfs\_dbspl} + \text{volume\_offset} + +.. list-table:: Sound Dose Acoustic Calibration Parameters + :widths: 25 20 20 35 + :header-rows: 1 + + * - Parameter + - Topology ID / Control + - Unit + - Functional Description + * - ``dbfs_value`` + - Telemetry Payload + - centi-dBFS + - 1-second RMS A-weighted digital signal level (:math:`-100.00` to :math:`0.00\,\text{dBFS}`). + * - ``sens_dbfs_dbspl`` + - Setup Parameter (0) + - centi-dB + - Electro-acoustic sensitivity of DAC, power amplifier, and target headphone transducer (e.g. :math:`0\,\text{dBFS} = 100\,\text{dBSPL}` at maximum volume). + * - ``volume_offset`` + - Volume Parameter (1) + - centi-dB + - Dynamic attenuation introduced by user-facing volume sliders relative to maximum gain (e.g. :math:`-12.00\,\text{dB} = -1200`). + * - ``current_gain`` + - Gain Parameter (2) + - centi-dB + - Autonomous protective attenuation commanded by the host dose daemon (e.g. :math:`-6.00\,\text{dB}`). + +High-Precision Stream Time Tracking +----------------------------------- + +The Sound Dose module generates microsecond-accurate stream timestamps to allow host exposure daemons to correlate sound exposure with real-world clocks: + +.. math:: + + t_{\text{stream}} = \text{total\_frames\_count} \cdot \left( \frac{1\,000\,000}{f_s} \right) + +To prevent 64-bit integer division in the DSP's high-priority execution context, the rate reciprocal is pre-calculated in :math:`Q26` fixed-point format (``SOUND_DOSE_1M_OVER_48K_Q26``). Multiplication is performed in split 32x32-to-64-bit arithmetic to provide 96-bit internal precision, eliminating timestamp jitter or drift across weeks of continuous playback. + +.. graphviz:: + :caption: Figure 170: Calculated Sound Dose (CSD) Accumulation & 7-Day Rolling Weekly Exposure Dose Budgeting + :alt: Diagram showing weekly sound dose accumulation over time, rolling 7-day exposure integration, and regulatory threshold warnings. + + digraph csd_timeline { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica", fontsize=9]; + + subgraph cluster_timeline { + label = "Rolling 7-Day Calculated Sound Dose (CSD) Budget"; + style = "filled,rounded"; + color = "#4A5568"; + fillcolor = "#FFFFFF"; + + d1 [label="Day 1 (Mon)\nListening: 4h @ 80 dBA\nDaily Dose: 10%\nWeekly Total: 10%", fillcolor="#C6F6D5", color="#276749"]; + d2 [label="Day 2 (Tue)\nListening: 2h @ 86 dBA\n(4x Energy Density)\nDaily Dose: 20%\nWeekly Total: 30%", fillcolor="#C6F6D5", color="#276749"]; + d3 [label="Day 3 (Wed)\nListening: 1.5h @ 89 dBA\nDaily Dose: 30%\nWeekly Total: 60%", fillcolor="#C6F6D5", color="#276749"]; + d4 [label="Day 4 (Thu)\nListening: 1h @ 92 dBA\nDaily Dose: 25%\nWeekly Total: 85%\n[ADVISORY WARNING ISSUED]", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + d5 [label="Day 5 (Fri)\nListening: 45m @ 95 dBA\nDaily Dose: 20%\nWeekly Total: 105%\n[MANDATORY ATTENUATION TRIGGERED]", fillcolor="#FED7D7", color="#C53030", penwidth=2.2]; + d6 [label="Day 6 (Sat)\nAttenuated Listening: < 75 dBA\nAutonomous gain clamp (-6 dB)\nWeekly Total: Decays rolling window", fillcolor="#FEFCBF", color="#B7791F"]; + d7 [label="Day 7 (Sun)\nRecovery Period\nOld Day 1 drops off window\nWeekly Total: Drops below 80%", fillcolor="#C6F6D5", color="#276749"]; + + d1 -> d2 -> d3 -> d4 -> d5 -> d6 -> d7 [color="#4A5568", penwidth=1.5]; + } + + subgraph cluster_host_actions { + label = "Host Exposure Management Actions"; + style = "filled,rounded"; + color = "#2B6CB0"; + fillcolor = "#EBF8FF"; + + h_warn [label="OS System Alert: 'High Volume Warning'\nUser notified of 80% weekly dose budget exhaustion", fillcolor="#FEFCBF", color="#B7791F"]; + h_clamp [label="Host Issues SOF_SOUND_DOSE_GAIN_PARAM_ID (-6 dB)\nForced DSP attenuation prevents permanent hearing loss", fillcolor="#FED7D7", color="#C53030", penwidth=2.0]; + h_recover [label="Host Restores 0 dB Gain\nAfter rolling exposure returns to safe regulatory envelope", fillcolor="#C6F6D5", color="#276749"]; + } + + d4 -> h_warn [color="#B7791F", style="dashed"]; + d5 -> h_clamp [color="#C53030", style="dashed", penwidth=1.8]; + d7 -> h_recover [color="#276749", style="dashed"]; + } + +Asynchronous IPC4 Event Notification & Host Exposure Management +=============================================================== + +Traditional audio telemetry frameworks rely on periodic host polling, forcing the host CPU to wake up frequently and query hardware registers over memory buses. This wastes power and degrades battery life on mobile devices. + +The Sound Dose module utilizes an **unsolicited event notification model** under the SOF IPC4 framework: + +1. **Autonomous Periodic Notification**: Exactly once every second, the DSP constructs an unsolicited IPC message: + + * Notification Type: ``SOF_IPC4_MODULE_NOTIFICATION`` + * Global Classification: ``SOF_IPC4_GLB_NOTIFICATION`` + * Event ID: ``SOF_AUDIO_FEATURE_SOUND_DOSE_MEL`` + * Target: Firmware-generated message directed to host mailbox + +2. **Payload Encapsulation**: The message encapsulates the ``struct sof_audio_feature`` container holding the active ``struct sof_sound_dose`` record: + + * ``stream_time_us``: 64-bit microsecond timestamp + * ``mel_value``: 1-second Momentary Exposure Level (centi-dBA) + * ``dbfs_value``: 1-second digital audio level (centi-dBFS) + * ``current_sens_dbfs_dbspl``: Configured acoustic sensitivity + * ``current_volume_offset``: Active volume attenuation + * ``current_gain``: Current autonomous protection gain + +.. graphviz:: + :caption: Figure 171: Sound Open Firmware Sound Dose Processing Pipeline & Periodic Asynchronous Notification State Machine + :alt: State machine and lifecycle of the Sound Dose module in Sound Open Firmware. + + digraph sound_dose_lifecycle { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica", fontsize=9]; + + s_uninit [label="STATE_UNREGISTERED\nModule library loaded in DRAM", fillcolor="#E2E8F0", color="#4A5568"]; + s_init [label="STATE_INIT (sound_dose_init)\nAllocate private component data (cd)\nSet default calibration: Sens=100dB, Vol=0dB, Gain=0dB\nInitialize IPC4 Notification Proto Message", fillcolor="#BEE3F8", color="#2B6CB0"]; + s_prep [label="STATE_PREPARE (sound_dose_prepare)\nVerify Sample Rate (44.1 kHz / 48 kHz)\nAllocate DF1 Filter Delay Lines (cd->delay_lines)\nInitialize Biquad Coefficients & Clear 64-Bit Energy", fillcolor="#C6F6D5", color="#276749"]; + s_proc [label="STATE_PROCESS (sound_dose_process)\nApply Gain Ramping (0.05 dB / frame)\nFilter Samples through A-Weighting DF1\nAccumulate Energy into cd->energy[ch]", fillcolor="#FEFCBF", color="#B7791F", penwidth=2.0]; + s_report [label="REPORT_MEL (sound_dose_report_mel)\nCalculate dBFS & MEL\nUpdate Stream Timestamp\nSend IPC4 Unsolicited Message to Host", fillcolor="#FED7D7", color="#C53030", penwidth=2.0]; + s_reset [label="STATE_RESET (sound_dose_reset)\nRestore baseline calibration parameters\nZero energy accumulators", fillcolor="#E2E8F0", color="#4A5568"]; + s_free [label="STATE_FREE (sound_dose_free)\nFree delay lines, IPC message, and component memory", fillcolor="#E2E8F0", color="#4A5568"]; + + s_uninit -> s_init [label="Pipeline Creation"]; + s_init -> s_prep [label="IPC Set Format / Prepare"]; + s_prep -> s_proc [label="Pipeline Trigger START"]; + s_proc -> s_report [label="frames >= report_count (1 sec)", color="#C53030", penwidth=1.5]; + s_report -> s_proc [label="Reset frames counter", color="#276749"]; + s_proc -> s_reset [label="Pipeline Trigger STOP"]; + s_reset -> s_prep [label="Restart Stream"]; + s_reset -> s_free [label="Pipeline Deletion"]; + } + +Host Sound Dose Daemon Integration +---------------------------------- + +In modern operating systems (such as Linux with PipeWire, ChromeOS, or Android Audio HAL), an unprivileged user-space **Sound Dose Daemon** listens on the ALSA control event interface for ``SOF_AUDIO_FEATURE_SOUND_DOSE_MEL`` notifications: + +* **Exposure Integration**: When a notification arrives, the daemon reads ``mel_value`` and adds the 1-second energy slice to rolling daily and weekly exposure databases. +* **Persistent Tracking**: Because headphones may be unplugged and plugged back in, or the device rebooted, the host daemon maintains persistent logs across sessions to guarantee continuous 7-day CSD tracking. +* **Closed-Loop Feedback**: If the weekly dose reaches 100%, the daemon issues a hardware control command back to the DSP to clamp playback loudness. + +Closed-Loop Dynamic Protection: Smooth Attenuation Ramping +========================================================== + +When the user exceeds their safe exposure threshold, the system must reduce listening volume. However, abruptly clamping digital gain creates audible clicks, pops, and sudden discontinuities that severely degrade user experience. Furthermore, if volume reduction is implemented merely by moving the standard user mixer slider, the user can easily drag the slider back up, defeating auditory safety safeguards. + +The Sound Dose module resolves both challenges through **autonomous internal attenuation** paired with **smooth exponential gain ramping**. + +Decoupled Protection Gain Control +--------------------------------- + +Sound Dose provides a dedicated byte control (``SOF_SOUND_DOSE_GAIN_PARAM_ID``) that accepts attenuation requests between :math:`-100.00\,\text{dB}` and :math:`0.00\,\text{dB}`. This control is intentionally hidden from standard user-accessible ALSA mixer volume controls. When the host exposure daemon detects dangerous exposure levels, it issues an attenuation command (e.g. :math:`-6.00\,\text{dB}`) directly to this parameter. The user volume slider remains intact, but the effective output is clamped. + +Exponential Gain Slew Ramping +----------------------------- + +To transition smoothly between gain targets without audio artifacts, the DSP applies an exponential slew rate of **0.05 dB per frame** in :math:`Q2.30` fixed-point arithmetic: + +* **Ramping Down (Attenuation)**: + When a lower target gain is commanded (``new_gain < gain``), the gain is scaled downward each frame: + + .. math:: + + g[n] = \max\left( g_{\text{target}},\, \frac{g[n-1] \cdot \text{GAIN\_DOWN}}{2^{30}} \right) + + where: + + .. math:: + + \text{GAIN\_DOWN} = \left\lfloor 10^{-0.05 / 20} \cdot 2^{30} \right\rfloor = 1067578625 + +* **Ramping Up (Recovery)**: + When the user or daemon restores gain (``new_gain > gain``), the gain is scaled upward each frame: + + .. math:: + + g[n] = \min\left( g_{\text{target}},\, \frac{g[n-1] \cdot \text{GAIN\_UP}}{2^{30}} \right) + + where: + + .. math:: + + \text{GAIN\_UP} = \left\lfloor 10^{+0.05 / 20} \cdot 2^{30} \right\rfloor = 1079940603 + +At a 48 kHz sampling rate, ramping gain down by 6 dB requires 120 frames, completing in just **2.5 milliseconds**—fast enough to protect the user's ears immediately, yet completely free of audible clicks or pops. + +.. graphviz:: + :caption: Figure 172: Closed-Loop Host-DSP Protective Feedback: Dynamic Gain Attenuation and Volume Limiting + :alt: Closed-loop feedback architecture between DSP Sound Dose module and Host Exposure Management Daemon. + + digraph closed_loop_feedback { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", margin="0.14,0.08"]; + edge [fontname="Helvetica", fontsize=9]; + + subgraph cluster_host { + label = "Host Operating System (Linux / Android / ChromeOS)"; + style = "filled,rounded"; + color = "#2F855A"; + fillcolor = "#F0FFF4"; + + h_driver [label="ALSA SOF Kernel Driver\nRoutes IPC events to kctl", fillcolor="#E2E8F0", color="#4A5568"]; + h_daemon [label="User-Space Sound Dose Daemon\n(PipeWire / Audio HAL)\nTracks 7-Day Rolling CSD", fillcolor="#C6F6D5", color="#276749", penwidth=2.0]; + h_policy [label="Regulatory Threshold Policy\nCSD >= 100% or MEL > 100 dBA", fillcolor="#FED7D7", color="#C53030", penwidth=1.8]; + + h_driver -> h_daemon [label="Notify MEL", color="#276749"]; + h_daemon -> h_policy [label="Evaluate Exposure", color="#276749"]; + } + + subgraph cluster_ipc { + label = "IPC4 Mailbox Interface (Cross-Domain Bridge)"; + style = "filled,rounded"; + color = "#718096"; + fillcolor = "#F7FAFC"; + + ipc_notif [label="Unsolicited Notification (Every 1.0s)\nSOF_AUDIO_FEATURE_SOUND_DOSE_MEL\nPayload: stream_time, dBFS, MEL", fillcolor="#FED7D7", color="#C53030", penwidth=1.8]; + ipc_cmd [label="Set Config Command (Large Config)\nSOF_SOUND_DOSE_GAIN_PARAM_ID\nCommanded Attenuation (e.g. -6 dB)", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + } + + subgraph cluster_dsp { + label = "Sound Open Firmware DSP Engine"; + style = "filled,rounded"; + color = "#2B6CB0"; + fillcolor = "#EBF8FF"; + + dsp_audio [label="Audio Processing Stream\n(Playback Pipeline)", fillcolor="#BEE3F8", color="#2B6CB0"]; + dsp_sd [label="Sound Dose Module\n(sound_dose.c)\nIEC 61672-1 Filtering & Energy Accumulation", fillcolor="#C6F6D5", color="#276749", penwidth=2.0]; + dsp_ramp [label="Exponential Slew Engine\nGAIN_DOWN: -0.05 dB/frame\nGAIN_UP: +0.05 dB/frame", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.5]; + dsp_out [label="DAI Copier Output\n(To Headphone DAC)", fillcolor="#BEE3F8", color="#2B6CB0"]; + + dsp_audio -> dsp_sd -> dsp_out [color="#2B6CB0", penwidth=1.5]; + dsp_ramp -> dsp_sd [label="Smooth Gain g", color="#B7791F", penwidth=1.5]; + } + + dsp_sd -> ipc_notif [label="1-Sec Telemetry Event", constraint=false, color="#C53030", penwidth=1.8]; + ipc_notif -> h_driver [label="Kernel Mailbox IRQ", constraint=false, color="#C53030", penwidth=1.8]; + h_policy -> ipc_cmd [label="Trigger Dynamic Clamp", color="#B7791F", penwidth=1.8]; + ipc_cmd -> dsp_ramp [label="Inject Target Gain", color="#B7791F", penwidth=1.8]; + } + +ALSA Topology 2 Integration & End-to-End Headphone Protection Audio Graph +========================================================================= + +In ALSA Topology 2, the Sound Dose module is defined under ``Class.Widget."sound_dose"`` with its unique UUID. It is typically positioned as the final processing block in the playback pipeline immediately prior to the DAI Copier, ensuring that all upstream volume adjustments, software equalizers, and dynamic range compressors are accounted for in the dose evaluation. + +Widget Definition +----------------- + +The widget class is defined in ``topology2/include/components/sound_dose.conf``: + +.. code-block:: text + + Class.Widget."sound_dose" { + DefineAttribute."index" { type "integer" } + DefineAttribute."instance" { type "integer" } + + + + attributes { + !constructor [ "index", "instance" ] + !mandatory [ + "num_input_pins", + "num_output_pins", + "num_input_audio_formats", + "num_output_audio_formats" + ] + !immutable [ "uuid", "type" ] + unique "instance" + } + + uuid "7c:9d:3f:a4:75:ea:d5:44:94:2d:96:79:91:a3:38:09" + type "effect" + no_pm "true" + num_input_pins 1 + num_output_pins 1 + } + +Topology Controls Architecture +------------------------------ + +The Sound Dose widget exposes four specialized byte controls defined in ``sound_dose_controls_playback.conf``: + +1. **Bytes Control 1 (Setup)**: Configures ``sens_dbfs_dbspl`` (:math:`-10.00` to :math:`+130.00\,\text{dB}`). Usually loaded at boot via ``setup_sens_100db.conf``. +2. **Bytes Control 2 (Volume)**: Reports user volume slider changes (:math:`-100.00` to :math:`+40.00\,\text{dB}`). +3. **Bytes Control 3 (Gain)**: Dynamic attenuation control (:math:`-100.00` to :math:`0.00\,\text{dB}`). Used by host daemons to enforce safe listening levels. +4. **Bytes Control 4 (Payload)**: Telemetry channel carrying the active ``struct sof_sound_dose`` record. + +.. graphviz:: + :caption: Figure 173: End-to-End Headphone Protection Audio Graph: From Host Media Stream and Sound Dose Widget to Headphone Output + :alt: End-to-end audio pipeline graph showing host stream, mixer, volume, DRC, sound dose widget, and headphone hardware. + + digraph end_to_end_headphone_pipeline { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", margin="0.14,0.08"]; + edge [fontname="Helvetica", fontsize=9]; + + subgraph cluster_host { + label = "Host Operating System (Audio Client Layer)"; + style = "filled,rounded"; + color = "#4A5568"; + fillcolor = "#F7FAFC"; + + h_media [label="Media Player / Browser\nAudio Stream (PCM S16/S32)", fillcolor="#E2E8F0", color="#4A5568"]; + h_daemon [label="Sound Dose Daemon (PipeWire / Audio HAL)\nMaintains 7-Day Rolling CSD Database", fillcolor="#C6F6D5", color="#276749", penwidth=2.0]; + } + + subgraph cluster_sof_dsp { + label = "Sound Open Firmware (DSP Headphone Playback Pipeline)"; + style = "filled,rounded"; + color = "#2B6CB0"; + fillcolor = "#EBF8FF"; + + fw_copier_in [label="Host Copier (Rx)\nBuffer DMA Input", fillcolor="#BEE3F8", color="#2B6CB0"]; + fw_vol [label="Volume Module\nUser Volume Ramping", fillcolor="#BEE3F8", color="#2B6CB0"]; + fw_eq [label="Headphone Equalizer\nFIR/IIR Acoustic Target", fillcolor="#BEE3F8", color="#2B6CB0"]; + fw_drc [label="Dynamic Range\nCompressor (DRC)", fillcolor="#BEE3F8", color="#2B6CB0"]; + fw_sd [label="Sound Dose Widget (sound_dose.c)\nIEC 61672-1 A-Weighting & MEL Integration\nDynamic Protection Slew (-0.05 dB/frame)", fillcolor="#FED7D7", color="#C53030", penwidth=2.2]; + fw_copier_out [label="DAI Copier (Tx)\nSoundWire / I2S Output", fillcolor="#BEE3F8", color="#2B6CB0"]; + + fw_copier_in -> fw_vol -> fw_eq -> fw_drc -> fw_sd -> fw_copier_out [label="Processed Audio Stream", color="#2B6CB0", penwidth=1.8]; + } + + subgraph cluster_hardware { + label = "Audio Hardware Platform & Acoustic Output"; + style = "filled,rounded"; + color = "#C53030"; + fillcolor = "#FFF5F5"; + + hw_bus [label="SoundWire / I2S Bus\nDigital Serial Interface", fillcolor="#E2E8F0", color="#4A5568"]; + hw_codec [label="Audio Codec / DAC\nHeadphone Power Amp Stage", fillcolor="#FED7D7", color="#C53030", penwidth=1.5]; + hw_phones [label="Headphones / Headset\nCalibrated Transducer\n(e.g. 100 dBSPL @ 0 dBFS)", fillcolor="#FED7D7", color="#C53030", penwidth=1.8]; + hw_ear [label="Human Ear Canal & Tympanic Membrane\nSafe Listening Envelope Preserved (< 100% CSD)", fillcolor="#C6F6D5", color="#276749", penwidth=2.0]; + + hw_bus -> hw_codec -> hw_phones -> hw_ear [label="Acoustic Waves", color="#C53030", penwidth=2.0]; + } + + h_media -> fw_copier_in [label="ALSA Playback Stream", color="#2B6CB0", penwidth=1.8]; + fw_sd -> h_daemon [label="1-Sec MEL Notifications (IPC4)", constraint=false, color="#C53030", style="dashed", penwidth=1.8]; + h_daemon -> fw_sd [label="Dynamic Gain Clamp (SOF_SOUND_DOSE_GAIN_PARAM_ID)", color="#B7791F", style="dashed", penwidth=1.8]; + fw_copier_out -> hw_bus [label="Digital Serial Tx", color="#2B6CB0", penwidth=1.8]; + } + +Developer Calibration & Diagnostic Runbook +========================================== + +When bringing up Sound Dose on a new hardware platform or validating compliance with IEC 62368-1: + +1. **Acoustic Sensitivity Calibration**: + Connect the target reference headphones to a calibrated artificial ear fixture (e.g. an **IEC 60318-4 ear simulator** or Head and Torso Simulator (HATS)). Play a 1 kHz sinusoidal test tone at :math:`0\,\text{dBFS}` with the system volume set to 100%. Record the measured acoustic output in :math:`\text{dBSPL}` (for example, :math:`102.5\,\text{dBSPL}`). Configure this baseline sensitivity in the topology or via ALSA controls: + + .. code-block:: bash + + # Set sensitivity to 102.5 dB (10250 centi-dB) + amixer -c 0 cset name='Headphone Sound Dose setup bytes' 0x72,0x28,0x00,0x00 + +2. **Telemetry Verification**: + Monitor the 1-second asynchronous IPC4 notification stream using DSP logger utilities (see :ref:`dbg-traces`): + + .. code-block:: bash + + # Stream real-time DSP trace logs + sof-logger -t -f 1 | grep -i "sound_dose" + + Verify that ``Time``, ``dBFS``, and ``MEL`` increment predictably every 1 second: + + .. code-block:: text + + comp_info: sound_dose: Time 42 dBFS -1800 MEL 8450 + +3. **Dynamic Limiter Validation**: + Inject an attenuation command via ``amixer`` while monitoring audio playback: + + .. code-block:: bash + + # Force dynamic attenuation of -10 dB (-1000 centi-dB) + amixer -c 0 cset name='Headphone Sound Dose gain bytes' 0x18,0xfc,0x00,0x00 + + Listen for clean, artifact-free attenuation without clicks or pops, verifying that the slew rate ramps smoothly at 0.05 dB per frame. + +Upstream Source Code References +=============================== + +* **Component Core & Generic Processing**: + + - `src/audio/sound_dose/sound_dose.c `_: Module adapter lifecycle (``init``, ``prepare``, ``process``, ``reset``, ``free``), gain ramping, and frame scheduling. + - `src/audio/sound_dose/sound_dose-generic.c `_: Real-time fixed-point sample processing, Direct Form I filtering, 64-bit energy accumulation, and logarithmic conversion. + - `src/audio/sound_dose/sound_dose.h `_: Component private structures, filter constants, fixed-point shifts, and math coefficients. + - `src/include/user/sound_dose.h `_: ABI definitions, control parameter IDs, and telemetry structures. + +* **IPC4 Notification & Control Interface**: + + - `src/audio/sound_dose/sound_dose-ipc4.c `_: Unsolicited IPC4 notification constructor and large config handlers. + - `src/include/user/audio_feature.h `_: Standard audio feature container definitions. + +* **Pre-Computed Filter Coefficients**: + + - `src/audio/sound_dose/sound_dose_iir_48k.h `_: Pre-calculated IEC 61672-1 Class 1 A-weighting coefficients for 48 kHz. + - `src/audio/sound_dose/sound_dose_iir_44k.h `_: Pre-calculated IEC 61672-1 Class 1 A-weighting coefficients for 44.1 kHz. + +* **ALSA Topology 2 Configurations**: + + - `tools/topology/topology2/include/components/sound_dose.conf `_: Topology 2 sound dose widget declaration. + - `tools/topology/topology2/include/bench/sound_dose_controls_playback.conf `_: Benchmark topology control definitions for setup, volume, gain, and data payload. + +Related Architecture Guides +=========================== + +* :ref:`volume_module`: High-precision volume scaling, zero-crossing smooth ramping, and soft mute mechanics. +* :ref:`drc_multiband_drc`: Wideband dynamic range compression, speaker protection leveling, and lookahead pre-delay buffers. +* :ref:`smart_amp`: Adaptive loudspeaker protection, real-time current/voltage (I/V) sense telemetry, and excursion/thermal limiters. +* :ref:`dcblock`: High-pass filtering to remove unwanted DC offsets prior to digital power amplification. +* :ref:`ipc_infrastructure`: SOF asynchronous messaging, mailbox management, and event notification architecture. +* :ref:`module_framework`: Standardized lifecycle, memory allocation flags, and processing module adapters. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 4962b664..c77d555c 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -53,7 +53,7 @@ Audio Processing Modules & Algorithms * :ref:`tflm` (High-level architecture; also see upstream `TFLM README `_) * :ref:`mfcc` (High-level architecture; also see upstream `MFCC README `_) * :ref:`smart_amp` (High-level architecture; also see upstream `Smart Amp README `_) -* `Sound Dose Evaluator `_ +* :ref:`sound_dose` (High-level architecture; also see upstream `Sound Dose README `_) * `Copier `_, `Mux `_ & `Selector `_ * `PCM Format Converter `_ @@ -97,6 +97,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/tflm firmware/mfcc firmware/smart_amp + firmware/sound_dose rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 4d6ef54a93edf393f142a225c322a37b2a414c71 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 11:06:50 +0100 Subject: [PATCH 20/64] docs: developer_guides: add high-level copier, mux, and selector architecture guide Add comprehensive high-level architecture documentation for Copier, Multiplexer/Demultiplexer, and Channel Selector components under developer_guides/firmware/copier_mux_selector.rst. Key topics covered: - Separation of concerns between boundary movers (Copier), multi-stream crossbars (Mux/Demux), and intra-stream mixers (Selector). - Copier subsystem deep dive: Host Copier, DAI Copier, IPC Gateway Copier, four binding topologies, and Fast Mode. - 4-way stream fan-out and dynamic per-sink format conversion pipeline. - Linear Link Position (LLP) telemetry and DSP wall-clock hardware timestamping synchronizer. - Integrated Copier Gain and static bit-shift attenuation. - Multiplexer & Demultiplexer binary bitmask routing matrices and pre-compiled lookup table optimizations. - IPC4 Echo Cancellation (AEC) reference stream aggregation with autonomous zero-padding fallback for missing reference streams. - Channel Selector intra-stream channel extraction and 8x8 Q10 matrix mixing (downmixing, upmixing, and channel swapping). - ALSA Topology 2 widget declarations (UUIDs, node types, and controls). - End-to-end system audio graph walkthrough integrating simultaneous media playback, speaker loopback, microphone capture, and AEC. - 7 native Graphviz vector diagrams (Figures 174 through 180). Signed-off-by: Liam Girdwood --- .../firmware/copier_mux_selector.rst | 832 ++++++++++++++++++ developer_guides/index.rst | 3 +- 2 files changed, 834 insertions(+), 1 deletion(-) create mode 100644 developer_guides/firmware/copier_mux_selector.rst diff --git a/developer_guides/firmware/copier_mux_selector.rst b/developer_guides/firmware/copier_mux_selector.rst new file mode 100644 index 00000000..97440ded --- /dev/null +++ b/developer_guides/firmware/copier_mux_selector.rst @@ -0,0 +1,832 @@ +.. _copier_mux_selector: + +Data Routing, Multiplexing & Selection Architecture: Copier, Multiplexer & Selector +#################################################################################### + +In Sound Open Firmware (SOF), audio processing pipelines are decoupled from raw hardware transport and stream topology management. The subsystem responsible for moving audio data across execution boundaries, translating stream formats, routing multiple audio channels, and synchronizing hardware streams consists of three foundational components: + +* **Copier**: The universal boundary data mover and hardware endpoint abstraction module. In IPC4 architectures, the Copier interfaces directly with DMA engines (Host DMA, Digital Audio Interfaces, and Inter-Core IPC Gateways), provides 1-to-N multi-pin stream fan-out, executes dynamic per-sink PCM format conversions, tracks Linear Link Position (LLP) telemetry, and latches DSP wall-clock hardware timestamps. +* **Multiplexer & Demultiplexer (Mux/Demux)**: The multi-stream channel routing crossbar. In IPC3, the Mux/Demux dynamically cross-connects audio channels between :math:`N` inputs and :math:`M` outputs via bitmask routing matrices. In IPC4, the Multiplexer serves as the standardized multi-pin stream aggregator for Echo Cancellation (AEC), fusing primary microphone capture audio with reference playback streams into a synchronized multi-channel stream. +* **Selector**: The intra-stream channel extraction, permutation, and linear downmixing engine. Operating within a single stream, the Selector extracts designated channel subsets (e.g. isolating active microphones from a high-density microphone array), swaps channel assignments, and executes arbitrary :math:`8 \times 8` matrix mixing in :math:`Q10` fixed-point arithmetic. + +Together, these three components establish the complete routing, fan-out, aggregation, and isolation infrastructure required by modern multi-stream audio architectures. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +Executive Architecture Overview: The SOF Data Routing & Endpoint Ecosystem +========================================================================== + +Audio data routing within a modern digital signal processor must reconcile two divergent architectural requirements: + +1. **Hardware Transport Decoupling**: Hardware peripherals (PCIe Host DMA, High Definition Audio links, Serial Synchronous Ports, SoundWire Audio Link Hubs, and PDM digital microphones) operate with rigid FIFO layouts, burst alignments, and hardware frame rates. Internal DSP algorithms, conversely, require uniform circular buffers, predictable frame block sizes, and arbitrary bit depths. +2. **Dynamic Stream Topologies**: Operating systems and audio middleware demand complex routing topologies—including simultaneous media playback, voice assistant capture, acoustic echo cancellation loopback taps, multi-mic spatial beamforming, and offload processing—all sharing concurrent access to shared audio streams without mutual interference. + +SOF resolves these demands through a strict separation of concerns among the Copier, Multiplexer, and Selector components: + +.. list-table:: SOF Routing Component Capability Matrix + :widths: 22 26 26 26 + :header-rows: 1 + + * - Capability / Feature + - Copier Subsystem + - Multiplexer / Demux + - Selector Component + * - **Primary Purpose** + - Hardware endpoint bridging, stream splitting & format conversion. + - Multi-stream channel crossbar & IPC4 AEC stream aggregation. + - Intra-stream channel selection, permutation, and matrix downmixing. + * - **Pin Topology** + - 1 Input Pin, up to 4 Output Pins (Fan-out). + - IPC3: N-in / 1-out (Mux) or 1-in / N-out (Demux). IPC4: 2-in (Mic/Ref), 1-out. + - 1 Input Pin, 1 Output Pin. + * - **Hardware Gateways** + - Direct interface to Host, DAI, & IPC gateways on Pin 0. + - None (Internal DSP stream routing only). + - None (Internal DSP stream processing only). + * - **Format Adaptation** + - Dynamic per-sink format conversion on all sinks. + - Matches input/output stream channel counts & formats. + - Operates on native audio formats with matrix math. + * - **Mathematical Model** + - Bit-depth conversion, frame shift attenuation (:math:`x \gg k`). + - Bitmask matrix cross-wiring (:math:`\text{mask}[\text{ch}]`). + - :math:`8 \times 8` :math:`Q10` fixed-point coefficient matrix. + * - **Timing & Telemetry** + - Linear Link Position (LLP) & DSP Wall-Clock Timestamps. + - Zero-latency sample pass-through with reference sync. + - Frame-synchronized sample selection & mixing. + +--- + +Copier Subsystem Deep Dive: Hardware Endpoint Abstraction +========================================================= + +The **Copier** (UUID ``9ba00c83-ca12-4a83-943c-1fa2e82f9dda``) is the mandatory endpoint and pipeline boundary module in SOF IPC4 architectures. Every pipeline that exchanges audio with the host operating system or external audio codecs begins or terminates with a Copier instance. + +Binding Configurations +---------------------- + +A Copier instance can be instantiated and bound within a pipeline in four distinct topological configurations: + +1. **Input Gateway Ingestion (Case 1)**: + Connects an input hardware gateway to downstream DSP processing modules: + + .. math:: + + \text{InputGateway} \longrightarrow \text{Copier} \longrightarrow \text{DestinationModule} + + Used for host playback pipelines (where the gateway is a Host DMA stream) and audio capture pipelines (where the gateway is a DAI interface receiving from microphones or line-in). + +2. **Output Gateway Transmission (Case 2)**: + Connects upstream DSP processing modules to an output hardware gateway: + + .. math:: + + \text{SourceModule} \longrightarrow \text{Copier} \longrightarrow \text{OutputGateway} + + Used for speaker playback pipelines (delivering processed audio to DAI hardware) and host recording pipelines (delivering captured audio to Host DMA ring buffers). + +3. **Inter-Module Format Bridging (Case 3)**: + Connects two internal DSP modules without a hardware gateway: + + .. math:: + + \text{SourceModule} \longrightarrow \text{Copier} \longrightarrow \text{DestinationModule} + + Used when splitting pipelines across distinct scheduling domains, core boundaries, or when executing complex format adaptations between incompatible processing modules. + +4. **Gateway Transmission with Local Tap (Case 4)**: + Connects upstream DSP modules simultaneously to an output gateway and one or more internal destination modules: + + .. math:: + + \text{SourceModule} \longrightarrow \text{Copier} \begin{cases} \longrightarrow \text{OutputGateway} \\ \longrightarrow \text{DestinationModule} \end{cases} + + Used for hardware loopback taps, where speaker playback audio is transmitted to the physical amplifier while simultaneously being tapped and fed into an Echo Cancellation reference pipeline. + +.. important:: + **The Gateway Pin 0 Invariant**: + In all Copier configurations interfacing with hardware, the gateway is strictly connected to **Pin 0** (Input Pin 0 for input gateways, Output Pin 0 for output gateways). Auxiliary destination modules and loopback taps are bound exclusively to Output Pins 1, 2, or 3. + +.. graphviz:: + :caption: Figure 174: SOF Data Movement and Gateway Interconnect Topology (Host Copier, DAI Copier, and Gateway Copier) + :alt: Architecture of SOF data movement showing Host Copier, DAI Copier, and Gateway Copier binding cases with circular buffers and DMA engines. + + digraph sof_copier_gateway_interconnect { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_host_domain { + label = "Host Operating System & Shared Memory Space"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + host_playback_ring [label="Host Playback Ring Buffer\n(ALSA / AudioFlinger PCM DMA Space)\nCircular Ring Pointer Tracking", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + host_capture_ring [label="Host Capture Ring Buffer\n(ALSA Capture PCM DMA Space)\nUser-Space Ingestion Ring", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + fpi_sync [label="FPI Stream Synchronization Group\n(Period Elapsed & Position Synchronizer)\nSynchronous Multi-Stream Latency Alignment", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.4]; + } + + subgraph cluster_dsp_pipeline { + label = "DSP Firmware Pipeline Architecture"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + host_copier_rx [label="Host Copier (Input Gateway)\nUUID: 9BA00C83-CA12-4A83-943C...\nInput Pin 0: Host DMA FIFO\nManages Host Ring Pointers & Wrap", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + dsp_processing [label="DSP Audio Processing Pipeline\nVolume / Equalizer / DRC / Beamforming\nUniform Periodic Block Processing", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + dai_copier_tx [label="DAI Copier (Output Gateway)\nOutput Pin 0: Hardware DAI Link\nOutput Pin 1: Loopback Reference Tap\nMultichannel Hardware Dispatcher", fillcolor="#FED7D7", color="#C53030", penwidth=1.8]; + gateway_copier [label="IPC Gateway Copier\nInter-Core / Inter-Pipeline DMA\nZero Host Overhead Gateway", fillcolor="#E9D8FD", color="#6B46C1", penwidth=1.6]; + } + + subgraph cluster_hardware_domain { + label = "Hardware Audio Interfaces (DAI & Interconnects)"; + style = "filled,rounded"; + color = "#E2E8F0"; + fillcolor = "#F7FAFC"; + + hw_ssp [label="Intel SSP / I2S Engine\nStereo / TDM Serial Framing", fillcolor="#EDF2F7", color="#4A5568"]; + hw_sndw [label="SoundWire Audio Link Hub (ALH)\nMulti-PDI Aggregation Gateway", fillcolor="#EDF2F7", color="#4A5568"]; + hw_dmic [label="Digital Microphone (DMIC)\nPdm Decimation & Multichannel DMA", fillcolor="#EDF2F7", color="#4A5568"]; + hw_hda [label="High Definition Audio (HDA) Bus\nHD-A Link DMA Tag Controller", fillcolor="#EDF2F7", color="#4A5568"]; + } + + host_playback_ring -> host_copier_rx [label="Host DMA Read", color="#3182CE", penwidth=1.6]; + fpi_sync -> host_copier_rx [label="FPI Sync Signal", style="dashed", color="#4A5568"]; + host_copier_rx -> dsp_processing [label="Pin 0 Audio Stream", color="#276749", penwidth=1.6]; + dsp_processing -> dai_copier_tx [label="Processed Frames", color="#B7791F", penwidth=1.6]; + dai_copier_tx -> hw_ssp [label="Pin 0 (SSP Link)", color="#C53030", penwidth=1.6]; + dai_copier_tx -> hw_sndw [label="Pin 0 (SoundWire ALH)", color="#C53030", penwidth=1.6]; + dai_copier_tx -> hw_hda [label="Pin 0 (HDA Bus)", color="#C53030", penwidth=1.6]; + hw_dmic -> gateway_copier [label="PDM Capture DMA", color="#6B46C1", penwidth=1.6]; + gateway_copier -> host_capture_ring [label="Host DMA Write", color="#3182CE", penwidth=1.6]; + } + +Host Copier Engine +------------------ + +The **Host Copier** connects the DSP memory space to the host operating system's cyclic DMA buffers. In playback mode, it pulls audio data from host memory into DSP local memory; in capture mode, it pushes processed DSP frames to host memory. + +* **Circular Buffer Pointer Tracking**: The Host Copier continuously tracks host read/write pointers. It calculates available space and data counts, handles circular buffer wrap-around, and notifies the host driver when period elapsed events occur. +* **Frame Position Index (FPI) Synchronization Groups**: To prevent phase drift across multi-stream presentations (such as multichannel audio where front, rear, and center/subwoofer channels are split across multiple ALSA substreams), SOF provides FPI update groups (``CONFIG_HOST_DMA_STREAM_SYNCHRONIZATION``). Multiple Host Copiers can be assigned to a common ``fpi_sync_group`` with a shared update period in microseconds. All copiers within the group latch and update their host FIFO position indices synchronously, ensuring perfect phase alignment. + +DAI Copier Engine +----------------- + +The **DAI Copier** bridges DSP audio buffers to external digital audio serial buses: + +* **High Definition Audio (HDA)**: Direct connection to Intel HDA link DMA streams. +* **Serial Synchronous Port (SSP / I2S)**: Interfaces with standard I2S, left-justified, right-justified, or multichannel TDM serial codecs. +* **Digital Microphone (DMIC)**: Interfaces with hardware PDM decimation filters, capturing up to 8 digital microphone channels. +* **SoundWire / Audio Link Hub (ALH)**: Implements multi-gateway aggregation (``is_multi_gateway(node_id)``). When high-channel-count audio (e.g. 4-channel surround or multi-speaker smart amps) is distributed across multiple SoundWire Data Port Interfaces (PDIs), the DAI Copier inspects the ``sof_alh_configuration_blob``, instantiates multiple DAI sub-indices, and automatically multiplexes or demultiplexes the multichannel stream across physical SoundWire data lines using nibble-encoded channel bitmasks. + +IPC Gateway Copier +------------------ + +When audio must traverse pipeline boundaries across heterogeneous DSP cores (such as passing decoded media frames from Primary Core 0 to Secondary Core 1 for post-processing), the **IPC Gateway Copier** uses hardware Inter-Processor Communication (IPC) gateways or shared SRAM FIFO windows. It decouples the scheduling loops of the two pipelines without engaging host DMA channels or triggering host interrupts. + +Copier Fast Mode +---------------- + +Under normal scheduling, a Copier transfers exactly its configured Input Block Size (IBS) or Output Block Size (OBS) per scheduling period. When ``IPC4_COPIER_FAST_MODE`` is enabled in the copier feature mask, the Copier is permitted to burst-transfer multiples of the block size in a single execution tick. Fast Mode is activated during pipeline pre-filling and deep-sleep playback buffer draining, provided all downstream sinks are bound to data-processing queues rather than fixed real-time DAIs. + +--- + +Multi-Pin Fan-Out & Dynamic Per-Sink Format Conversion +====================================================== + +In modern audio architectures, a single audio source must frequently be distributed to multiple consumers operating with distinct sample rates, bit depths, or channel layouts. The Copier natively provides a 1-to-N stream splitter with independent format conversion per output pin. + +Stream Fan-Out Topologies +------------------------- + +The Copier supports up to 4 simultaneous output pins (:math:`\text{Pin}_0, \text{Pin}_1, \text{Pin}_2, \text{Pin}_3`). Each output pin operates with its own circular buffer sink and independently configured audio format: + +.. math:: + + x_{\text{in}}[n] \in \mathcal{F}_{\text{in}} \xrightarrow{\text{Copier}} \begin{cases} + y_0[n] \in \mathcal{F}_{\text{out}, 0} & (\text{Pin 0: Hardware Gateway or Primary Pipeline}) \\ + y_1[n] \in \mathcal{F}_{\text{out}, 1} & (\text{Pin 1: Acoustic Echo Cancellation Reference Tap}) \\ + y_2[n] \in \mathcal{F}_{\text{out}, 2} & (\text{Pin 2: Speech Recognition / Hotword Detector}) \\ + y_3[n] \in \mathcal{F}_{\text{out}, 3} & (\text{Pin 3: Telemetry / Loopback Monitor}) + \end{cases} + +Runtime Per-Sink Format Setup +----------------------------- + +While Pin 0's format is established during initial module instantiation, auxiliary output pins (Pins 1 through 3) can be dynamically configured at runtime via the IPC4 command ``IPC4_COPIER_MODULE_CFG_PARAM_SET_SINK_FORMAT``. The host driver supplies a configuration structure specifying: + +* Target Sink Identifier (Pin Index). +* Upstream Source Audio Format (validating that the input stream format matches expected characteristics). +* Downstream Sink Audio Format (specifying target container bit depth, valid bit resolution, channel count, sample rate, and interleaving scheme). + +Dedicated PCM Converter Execution +--------------------------------- + +When an output pin's target format differs from the input stream, the Copier dynamically binds a specialized PCM converter routine (``pcm_converter_func``) for that specific pin. During every processing period, the Copier reads input audio frames once, pushes un-converted samples directly to sinks with matching formats, and passes the input frames through the dedicated converter routines for sinks requiring transformation: + +* **Container Width Conversion**: 16-bit packed (:math:`S16\_LE`), 24-bit in 32-bit container (:math:`S24\_4LE`), and 32-bit full scale (:math:`S32\_LE`). +* **Bit Depth Formatting**: Arithmetic sign extension, arithmetic left/right shifting, and bit truncation. +* **Channel Layout Adaptation**: Selective channel stripping, channel duplication, or channel remapping according to the runtime channel mask. + +.. graphviz:: + :caption: Figure 175: Copier 4-Way Stream Splitting & Dynamic Per-Sink Format Conversion Pipeline + :alt: Diagram of Copier 4-way stream splitting showing input pin and 4 output pins with independent PCM format conversion engines. + + digraph copier_fanout_format_pipeline { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + input_stream [label="Primary Input Stream\nPin 0 Input\nFormat: 48 kHz / 2-Ch / 32-bit (S32_LE)\nBase Format Reference", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + + subgraph cluster_copier_core { + label = "Copier Multi-Pin Fan-Out Engine (UUID: 9BA00C83...)"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#FFFFFF"; + + copier_rx [label="Stream Ingestion &\nCircular Buffer Dispatcher", fillcolor="#EDF2F7", color="#4A5568"]; + sink0_conv [label="Sink 0 Converter:\nPass-Through Engine\nNo Conversion Required", fillcolor="#E2E8F0", color="#4A5568"]; + sink1_conv [label="Sink 1 Converter:\n32-bit -> 16-bit S16_LE\nDownscale with Rounding", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + sink2_conv [label="Sink 2 Converter:\nChannel Remap & Mask\nIsolate Channel 0 (Mono)", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + sink3_conv [label="Sink 3 Converter:\n32-bit -> 24-bit S24_4LE\nBit Mask & Sign Extend", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + } + + subgraph cluster_sinks { + label = "Output Sink Endpoints"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + sink0_out [label="Output Pin 0 (Hardware Gateway)\nFormat: 48 kHz / 2-Ch / S32_LE\nDestination: Physical Speaker DAI", fillcolor="#FED7D7", color="#C53030", penwidth=1.8]; + sink1_out [label="Output Pin 1 (AEC Reference Tap)\nFormat: 48 kHz / 2-Ch / S16_LE\nDestination: AEC Mux Input Pin 1", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + sink2_out [label="Output Pin 2 (Voice Trigger Tap)\nFormat: 48 kHz / 1-Ch / S16_LE\nDestination: Hotword / Wake Engine", fillcolor="#E9D8FD", color="#6B46C1", penwidth=1.6]; + sink3_out [label="Output Pin 3 (Diagnostic Loopback)\nFormat: 48 kHz / 2-Ch / S24_4LE\nDestination: Host Logging Stream", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.4]; + } + + input_stream -> copier_rx [label="Input Frames", color="#276749", penwidth=1.8]; + copier_rx -> sink0_conv [label="Pin 0 Dispatch", color="#4A5568"]; + copier_rx -> sink1_conv [label="Pin 1 Dispatch", color="#B7791F"]; + copier_rx -> sink2_conv [label="Pin 2 Dispatch", color="#B7791F"]; + copier_rx -> sink3_conv [label="Pin 3 Dispatch", color="#B7791F"]; + + sink0_conv -> sink0_out [label="Unchanged 32-bit", color="#C53030", penwidth=1.8]; + sink1_conv -> sink1_out [label="Converted 16-bit", color="#3182CE", penwidth=1.6]; + sink2_conv -> sink2_out [label="Extracted Mono", color="#6B46C1", penwidth=1.6]; + sink3_conv -> sink3_out [label="Packed 24-in-32", color="#4A5568", penwidth=1.4]; + } + +--- + +Linear Link Position (LLP) Telemetry & DSP Hardware Timestamping Synchronizer +============================================================================= + +In multimedia playback and interactive communications, audio-video synchronization (lip-sync) and low-latency device pairing require precise knowledge of the exact hardware time an audio sample crosses the digital-to-analog boundary. + +Linear Link Position (LLP) Reporting +------------------------------------ + +For High Definition Audio (HDA) links, hardware DMA controllers maintain continuous link position counters accessible to the host controller via standard PCI registers. For non-HDA digital interfaces (such as Serial Synchronous Ports, SoundWire links, and PDM digital microphones), standard hardware counters are unavailable to host software. + +The Copier bridges this architectural gap through the **Linear Link Position (LLP)** telemetry interface: + +* **Telemetry Query Commands**: The host driver sends ``IPC4_COPIER_MODULE_CFG_PARAM_LLP_READING`` or ``IPC4_COPIER_MODULE_CFG_PARAM_LLP_READING_EXTENDED`` via a Large Config Get operation. +* **Cumulative Frame Accumulation**: The Copier maintains 64-bit continuous frame counters tracking the exact number of samples pushed to or pulled from the hardware FIFO: + + .. math:: + + \text{LLP}_{\text{extended}} = \left\{ \text{LLP}_{\text{bytes}}, \text{TotalDataProcessed}_{\text{bytes}}, \text{WallClockTimestamp}_{\mu\text{s}} \right\} + +* **Drift & Jitter Elimination**: By correlating total processed bytes against the hardware interface's sample clock, host drivers calculate link FIFO depth and compensate for clock drift between host system time and the audio crystal oscillator without physical hardware probes. + +DSP Hardware Timestamping Synchronizer +-------------------------------------- + +To eliminate software latency and interrupt jitter during timestamp acquisition, the Copier interfaces directly with dedicated DSP timestamping hardware registers: + +* **Hardware Initialization**: The host initializes timestamping using the parameter ``IPC4_COPIER_MODULE_CFG_PARAM_TIMESTAMP_INIT``, passing the low-level configuration register value ``tsctrl_reg``. +* **Hardware Register Pass-Through**: The Copier programs ``tsctrl_reg`` directly into the local timestamp control register of the physical interface (e.g. SSP local timestamp register). +* **Clock Latching**: Upon the arrival of a hardware frame sync pulse (e.g. I2S word select transition or SoundWire synchronization frame), the hardware automatically latches the current 64-bit DSP wall-clock counter into a shadow register. Software queries read this latched value directly, yielding sub-microsecond timestamp precision completely free of RTOS task scheduling jitter. + +.. graphviz:: + :caption: Figure 176: Linear Link Position (LLP) Telemetry & DSP Wall-Clock Hardware Timestamping Synchronizer + :alt: Architectural diagram of Linear Link Position reporting and hardware wall-clock timestamp latching in the Copier. + + digraph copier_llp_timestamp_telemetry { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_host_query { + label = "Host Operating System Audio Subsystem"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + host_alsa [label="ALSA / PulseAudio / PipeWire Engine\nLip-Sync & Clock Drift Estimator\nIssues Large Config Get (Param ID 4 / 5)", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + } + + subgraph cluster_copier_runtime { + label = "DSP Copier Subsystem Runtime"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + copier_telemetry [label="Copier Telemetry Handler\nEvaluates LLP & Extracted Processed Bytes\nReturns struct ipc4_llp_reading_extended", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + accumulator_64bit [label="64-Bit Continuous Frame Accumulators\nInput Processed: input_total_data_processed\nOutput Processed: output_total_data_processed", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + tsctrl_driver [label="Hardware Timestamp Controller\nProgrammed via tsctrl_reg\nArms Hardware Latching Shadow Registers", fillcolor="#E9D8FD", color="#6B46C1", penwidth=1.6]; + } + + subgraph cluster_hw_registers { + label = "Hardware Interface & Wall-Clock Peripheral Registers"; + style = "filled,rounded"; + color = "#E2E8F0"; + fillcolor = "#F7FAFC"; + + dsp_wall_clock [label="DSP Free-Running Wall Clock\nHigh-Resolution 64-Bit Cycle Counter", fillcolor="#EDF2F7", color="#4A5568"]; + hw_latch_reg [label="Hardware Local Timestamp Register\nAtomic Hardware Latch Register\nLatched on Physical Frame Sync Edge", fillcolor="#FED7D7", color="#C53030", penwidth=1.8]; + dai_fifo [label="Hardware DAI FIFO / Link Serializer\nPhysical Audio Bit Stream Interface", fillcolor="#EDF2F7", color="#4A5568"]; + } + + host_alsa -> copier_telemetry [label="Large Config Get (LLP)", color="#3182CE", penwidth=1.6]; + copier_telemetry -> host_alsa [label="64-Bit LLP Payload", color="#3182CE", penwidth=1.6, constraint=false]; + accumulator_64bit -> copier_telemetry [label="Accumulated Bytes", color="#B7791F"]; + tsctrl_driver -> hw_latch_reg [label="tsctrl_reg Config", color="#6B46C1", penwidth=1.6]; + dsp_wall_clock -> hw_latch_reg [label="Continuous Clock Feed", style="dotted", color="#4A5568"]; + dai_fifo -> hw_latch_reg [label="Frame Sync Pulse Latch", color="#C53030", penwidth=1.8]; + hw_latch_reg -> copier_telemetry [label="Latched Hardware Timestamp", color="#C53030", penwidth=1.6]; + } + +--- + +Integrated Copier Gain & Attenuation Architecture +================================================= + +In addition to routing and format adaptation, the Copier provides integrated sample attenuation and gain management. This capability allows topologies to control audio levels and prevent clipping at boundary interfaces without the memory and scheduling overhead of dedicating an independent Volume processing widget. + +Static Bit-Shift Attenuation +---------------------------- + +For high-bit-depth audio streams, the Copier supports direct hardware-style attenuation via arithmetic bit shifting: + +* **Configuration**: Commanded via ``IPC4_COPIER_MODULE_CFG_ATTENUATION``. +* **Application Scope**: Permitted when the output pin is configured for 32-bit sample containers and the source is bound to a hardware gateway. +* **Mathematical Operation**: For an attenuation parameter :math:`A \in [1..31]`, every output sample is arithmetically right-shifted: + + .. math:: + + y[n] = x[n] \gg A + + This provides rapid, zero-multiplication step attenuation in :math:`6 \text{ dB}` increments (:math:`-6 \text{ dB}, -12 \text{ dB}, -18 \text{ dB}, \dots`), ideal for safeguarding high-power digital amplifier stages during link bring-up. + +Copier Gain Engine +------------------ + +When configured with ``CONFIG_COPIER_GAIN``, the Copier incorporates a dedicated gain sub-engine: + +* **Static Volume Gain**: Applies linear channel-specific scaling factors. +* **Mute Control**: Instantly forces sample values to digital zero without disrupting stream framing or tearing down DMA descriptors. +* **Smooth Transition Ramping**: When changing volume levels or toggling mute, the Copier Gain engine applies smooth linear or exponential sample ramps across configurable millisecond durations. This completely suppresses audible pops, clicks, or zipper noise during stream transitions. + +--- + +Multiplexer & Demultiplexer Architecture: Matrix Bitmask Crossbar +================================================================= + +The **Multiplexer / Demultiplexer** component (UUID ``68:68:b2:c4:30:14:0e:47:a0:89:15:d1:c7:7f:85:1a``) is the channel crossbar router of Sound Open Firmware. Unlike audio mixers (such as Mixin/Mixout), the Multiplexer performs pure channel routing and stream aggregation: it copies, redistributes, or splits individual audio channels without summing or arithmetic scaling. + +Matrix Bitmask Routing Model +---------------------------- + +In IPC3 topologies, routing between input and output streams is defined by an :math:`8 \times 8` binary routing matrix encoded into an array of 8-bit masks: + +.. math:: + + \mathbf{M} \in \{0, 1\}^{8 \times 8} + +* **Multiplexer Mode** (:math:`N` Inputs :math:`\to` 1 Output): + Each stream maintains an array ``mask[PLATFORM_MAX_CHANNELS]``, where each element corresponds to an **input channel**. The bit positions set within ``mask[ch]`` indicate the designated **output channels** to which that input channel must be copied: + + .. math:: + + y[\text{out\_ch}] = x[\text{in\_ch}] \quad \Longleftrightarrow \quad \left( \mathbf{M}_{\text{in\_ch}} \;\&\; (1 \ll \text{out\_ch}) \right) \neq 0 + +* **Demultiplexer Mode** (1 Input :math:`\to` :math:`N` Outputs): + In demultiplexer mode, the mapping is inverted: each element of ``mask[ch]`` corresponds to an **output channel**, and the bit positions indicate which **input channel** provides the source sample. + +.. note:: + **Zero Mixing Invariant**: + The Multiplexer/Demultiplexer component strictly forbids audio mixing. If a configuration specifies multiple input channels mapped to the same output channel bit, the component rejects the configuration during initialization with an error. + +Pre-Computed Lookup Tables +-------------------------- + +To achieve zero-overhead execution during real-time processing, the component compiles the binary bitmask matrix into a pre-computed lookup table (``mux_look_up``) during the pipeline ``prepare`` phase. The lookup table resolves source and destination memory pointers, buffer offsets, channel stride increments (``src_inc``, ``dest_inc``), and element counts. During inner processing loops, the DSP executes direct assembly copy operations without evaluating conditional branches or computing bit shifts. + +.. graphviz:: + :caption: Figure 177: Multiplexer (Mux) & Demultiplexer (Demux) Channel Routing Matrix & Bitmask Architecture + :alt: Diagram of Mux and Demux channel routing showing 8x8 binary bitmask matrices and zero-overhead lookup table dispatch. + + digraph mux_demux_routing_matrix { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_inputs { + label = "Input Audio Channels"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + in_s0_c0 [label="Stream 0: Channel 0\n(Left Channel)", fillcolor="#EBF8FF", color="#3182CE"]; + in_s0_c1 [label="Stream 0: Channel 1\n(Right Channel)", fillcolor="#EBF8FF", color="#3182CE"]; + in_s1_c0 [label="Stream 1: Channel 0\n(Auxiliary Mic / Ref 0)", fillcolor="#FEFCBF", color="#B7791F"]; + in_s1_c1 [label="Stream 1: Channel 1\n(Auxiliary Mic / Ref 1)", fillcolor="#FEFCBF", color="#B7791F"]; + } + + subgraph cluster_matrix_core { + label = "8x8 Channel Routing Matrix & Compiled Lookup Table"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + matrix_eval [label="Matrix Bitmask Mapping\nStream 0: mask[0]=0x01, mask[1]=0x02\nStream 1: mask[0]=0x04, mask[1]=0x08\nStrict No-Summing Invariant", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + lookup_tbl [label="Compiled Lookup Table (mux_look_up)\nDirect Stride & Pointer Offsets\nZero Conditional Branching Inner Loop", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.6]; + } + + subgraph cluster_outputs { + label = "Aggregated Output Stream"; + style = "filled,rounded"; + color = "#FED7D7"; + fillcolor = "#F7FAFC"; + + out_c0 [label="Output Slot 0 (Left)", fillcolor="#EBF8FF", color="#3182CE"]; + out_c1 [label="Output Slot 1 (Right)", fillcolor="#EBF8FF", color="#3182CE"]; + out_c2 [label="Output Slot 2 (Ref 0)", fillcolor="#FEFCBF", color="#B7791F"]; + out_c3 [label="Output Slot 3 (Ref 1)", fillcolor="#FEFCBF", color="#B7791F"]; + } + + in_s0_c0 -> matrix_eval [label="Map to Bit 0 (0x01)", color="#3182CE"]; + in_s0_c1 -> matrix_eval [label="Map to Bit 1 (0x02)", color="#3182CE"]; + in_s1_c0 -> matrix_eval [label="Map to Bit 2 (0x04)", color="#B7791F"]; + in_s1_c1 -> matrix_eval [label="Map to Bit 3 (0x08)", color="#B7791F"]; + + matrix_eval -> lookup_tbl [label="Compile Table", color="#276749", penwidth=1.6]; + + lookup_tbl -> out_c0 [label="Copy Slot 0", color="#3182CE", penwidth=1.6]; + lookup_tbl -> out_c1 [label="Copy Slot 1", color="#3182CE", penwidth=1.6]; + lookup_tbl -> out_c2 [label="Copy Slot 2", color="#B7791F", penwidth=1.6]; + lookup_tbl -> out_c3 [label="Copy Slot 3", color="#B7791F", penwidth=1.6]; + } + +--- + +IPC4 Echo Cancellation (AEC) Reference Stream Aggregator +======================================================== + +In SOF IPC4 topologies, the Multiplexer component assumes a critical, standardized role: the **Acoustic Echo Cancellation (AEC) Reference Stream Aggregator**. + +Speech processing algorithms, beamformers, and voice recognition engines require two synchronized audio inputs: + +1. The acoustic capture stream picked up by physical microphones (containing the user's speech plus echo from the device's loudspeakers). +2. The reference playback stream sent to the loudspeakers (the pure echo source). + +To pass both streams into a single processing algorithm via standard single-input module adapters, the Multiplexer aggregates them into a composite multi-channel stream. + +Deterministic Channel Allocation +-------------------------------- + +In IPC4, the Multiplexer defines a deterministic pin mapping: + +* **Input Pin 0 (Primary Capture Stream)**: + Contains :math:`M` channels (:math:`\text{Ch}_0 \dots \text{Ch}_{M-1}`, where :math:`M \le 4`) representing the physical microphone signals. These channels are mapped directly to output channels :math:`0 \dots M-1`: + + .. math:: + + y[\text{ch}] = x_0[\text{ch}], \quad \forall \; \text{ch} \in [0, M-1] + +* **Input Pin 1 (Reference Stream)**: + Contains :math:`N` channels (:math:`\text{Ch}_0 \dots \text{Ch}_{N-1}`, where :math:`N \le 2`) representing the loudspeaker playback signals tapped from the output Copier. These channels are appended immediately following the capture channels: + + .. math:: + + y[M + \text{ch}] = x_1[\text{ch}], \quad \forall \; \text{ch} \in [0, N-1] + +Total output channel count is therefore exactly :math:`M + N`. For example, a 2-channel microphone array combined with a 2-channel speaker reference yields a 4-channel output stream where channels 0 and 1 represent microphones and channels 2 and 3 represent reference audio. + +Fault-Tolerant Zero-Padding Mechanics +------------------------------------- + +In real-time operating systems, playback streams can start, stop, or pause independently of microphone capture. If the loudspeaker playback pipeline stops, Input Pin 1 ceases delivering data. + +To prevent pipeline stalling or algorithmic crashes in downstream AEC algorithms, the IPC4 Multiplexer implements autonomous fault tolerance: + +* **Primary Stream Invariant**: If Input Pin 0 (microphone capture) is disconnected or starving, the Multiplexer produces no output. Capture pipelines only execute when microphone data is actively present. +* **Reference Stream Zero-Padding**: If Input Pin 1 (echo reference) is disconnected, paused, or starving, the Multiplexer does **not** stall. Instead, it processes microphone frames normally and automatically pads the reference output slots (:math:`M \dots M+N-1`) with digital zeros: + + .. math:: + + y[M + \text{ch}] = 0, \quad \forall \; \text{ch} \in [0, N-1] + +This zero-padding ensures that downstream AEC algorithms maintain continuous frame synchronization without experiencing pipeline underflow, allowing transparent adaptation when media playback starts and stops. + +.. graphviz:: + :caption: Figure 178: IPC4 Echo Cancellation (AEC) Reference Stream Aggregation via Multiplexer + :alt: Architecture of IPC4 Echo Cancellation stream aggregation showing Pin 0 mic capture, Pin 1 speaker reference tap, and zero-padding fallback. + + digraph ipc4_aec_mux_aggregation { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_sources { + label = "Input Stream Sources"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + mic_stream [label="Microphone Capture Stream\nInput Pin 0 (M Channels)\nM = 2 Channels (Mic Left, Mic Right)\nContinuous Capture Source", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + ref_stream [label="Speaker Playback Reference Tap\nInput Pin 1 (N Channels)\nN = 2 Channels (Spk Left, Spk Right)\nDynamic / Intermittent Stream", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.8]; + } + + subgraph cluster_mux_core { + label = "IPC4 Multiplexer Core (UUID: MUX4_UUID)"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + pin_eval [label="Input Pin Monitor\nCheck Pin 0 & Pin 1 Status", fillcolor="#EDF2F7", color="#4A5568"]; + channel_align [label="Channel Aggregator\nSlot 0..1: Mic Channels\nSlot 2..3: Reference Channels", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + zero_pad [label="Autonomous Zero-Padding Engine\nFills Slots 2..3 with 0x00000000\nif Reference Pin is Disconnected", fillcolor="#FED7D7", color="#C53030", penwidth=1.6]; + } + + subgraph cluster_downstream { + label = "Composite Multi-Channel Destination"; + style = "filled,rounded"; + color = "#E2E8F0"; + fillcolor = "#F7FAFC"; + + aec_input [label="Acoustic Echo Cancellation / TDFB Module\n4-Channel Composite Stream Input\n[Mic L, Mic R, Ref L, Ref R]\nContinuous Real-Time Processing", fillcolor="#E9D8FD", color="#6B46C1", penwidth=1.8]; + } + + mic_stream -> pin_eval [label="Pin 0 Frames", color="#276749", penwidth=1.8]; + ref_stream -> pin_eval [label="Pin 1 Frames", color="#3182CE", penwidth=1.8]; + + pin_eval -> channel_align [label="Reference Active", color="#276749", penwidth=1.6]; + pin_eval -> zero_pad [label="Reference Inactive / Stalled", color="#C53030", style="dashed", penwidth=1.6]; + + channel_align -> aec_input [label="Composite 4-Ch Output", color="#6B46C1", penwidth=1.8]; + zero_pad -> aec_input [label="Zero-Padded 4-Ch Output", color="#C53030", style="dashed", penwidth=1.6]; + } + +--- + +Selector Component: Dynamic Channel Extraction, Permutation & Matrix Swapping +============================================================================= + +While the Multiplexer routes audio channels across multiple streams, the **Selector** component (UUID ``c1:92:fe:32:17:1e:c2:4f:97:58:c7:f3:54:2e:98:0a``) operates inside a single stream to isolate, rearrange, or downmix channels. + +Channel Extraction and Dropping +------------------------------- + +High-density audio interfaces frequently deliver more channels than required by downstream processing. For example, a digital microphone controller may provide an 8-channel TDM capture stream, whereas a voice assistant module requires only 2 primary microphone signals. + +The Selector extracts the designated channels and drops the remainder: + +.. math:: + + \mathbf{y}[n] = \begin{bmatrix} x_{\text{sel}[0]}[n] \\ x_{\text{sel}[1]}[n] \end{bmatrix}, \quad \text{where } \mathbf{x}[n] \in \mathbb{R}^8, \; \mathbf{y}[n] \in \mathbb{R}^2 + +In IPC3 mode, this is controlled by the configuration parameters ``in_channels_count``, ``out_channels_count``, and ``sel_channel``. + +IPC4 Fixed-Point Matrix Mixing Model +------------------------------------ + +In IPC4 architectures, the Selector evolves into a general-purpose linear matrix mixer. Channel routing, permutation, and downmixing are defined by an :math:`8 \times 8` matrix of 16-bit signed coefficients in :math:`Q10` fixed-point format (``struct ipc4_selector_coeffs_config``): + +.. math:: + + y_i[n] = \sum_{j=0}^{M-1} c_{i,j} \cdot x_j[n], \quad i \in [0, N-1] + +where :math:`M` is the input channel count, :math:`N` is the output channel count, and :math:`c_{i,j}` are the :math:`Q10` mixing coefficients. In :math:`Q10` arithmetic: + +* Unity gain (:math:`1.0`) is represented by :math:`1024` (``SEL_COEF_ONE_Q10``). +* Complete attenuation (:math:`0.0`) is represented by :math:`0`. +* Half gain (:math:`-6.02 \text{ dB}`) is represented by :math:`512`. + +This matrix formulation enables diverse audio transformations: + +* **Channel Permutation & Swapping**: Setting off-diagonal coefficients to 1024 swaps channels (e.g. reversing Left and Right channels): + + .. math:: + + \mathbf{C}_{\text{swap}} = \begin{bmatrix} 0 & 1024 \\ 1024 & 0 \end{bmatrix} + +* **Stereo-to-Mono Downmixing**: Summing Left and Right channels with equal weighting (:math:`-6 \text{ dB}` per channel) prevents arithmetic overflow: + + .. math:: + + \mathbf{C}_{\text{downmix}} = \begin{bmatrix} 512 & 512 \end{bmatrix} + +* **5.1 Surround Downmixing**: Converting 6-channel surround sound to 2-channel stereo with standard psychoacoustic ITU coefficients: + + .. math:: + + \begin{aligned} + L_{\text{out}} &= L + 0.707 C + 0.707 L_s \\ + R_{\text{out}} &= R + 0.707 C + 0.707 R_s + \end{aligned} + +Multi-Profile Configuration Caching +----------------------------------- + +A single Selector widget can store up to 8 distinct configuration profiles in memory (``SEL_MAX_NUM_CONFIGS = 8``). When stream parameters change dynamically (such as switching from stereo to quad-channel microphone capture), the Selector matches the active stream's channel count and channel configuration against its cached profiles, applying the corresponding mixing coefficients instantly without issuing new IPC round-trips to the host driver. + +.. graphviz:: + :caption: Figure 179: Selector Component: Dynamic Channel Extraction, Permutation & Matrix Swapping + :alt: Diagram of Selector component demonstrating 8x8 Q10 matrix mixing, channel extraction, channel swapping, and downmixing. + + digraph selector_matrix_permutation { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_source_channels { + label = "Multi-Channel Input Stream (e.g. 8-Ch DMIC)"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + ch0 [label="Ch 0: Mic 1 (Front Left)", fillcolor="#EBF8FF", color="#3182CE"]; + ch1 [label="Ch 1: Mic 2 (Front Right)", fillcolor="#EBF8FF", color="#3182CE"]; + ch2 [label="Ch 2: Mic 3 (Rear Left)", fillcolor="#EDF2F7", color="#4A5568"]; + ch3 [label="Ch 3: Mic 4 (Rear Right)", fillcolor="#EDF2F7", color="#4A5568"]; + ch_unused [label="Ch 4..7: Unused Sensors\n(To Be Dropped)", fillcolor="#FED7D7", color="#C53030"]; + } + + subgraph cluster_selector_core { + label = "Selector Core (UUID: MICSEL_UUID)"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + matrix_q10 [label="8x8 Q10 Coefficient Matrix\nc[0][0] = 1024 (1.0x)\nc[1][1] = 1024 (1.0x)\nc[i][j] = 0 (Unused/Dropped)", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + profile_cache [label="Configuration Cache\nStores up to 8 Profiles\nDynamic Topology Matching", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + } + + subgraph cluster_sink_channels { + label = "Selected Output Stream (Stereo Clean)"; + style = "filled,rounded"; + color = "#E2E8F0"; + fillcolor = "#F7FAFC"; + + out_left [label="Out Ch 0: Primary Mic L", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + out_right [label="Out Ch 1: Primary Mic R", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + } + + ch0 -> matrix_q10 [label="Gain 1024 (Unity)", color="#3182CE", penwidth=1.6]; + ch1 -> matrix_q10 [label="Gain 1024 (Unity)", color="#3182CE", penwidth=1.6]; + ch2 -> matrix_q10 [label="Gain 0 (Drop)", color="#4A5568", style="dotted"]; + ch3 -> matrix_q10 [label="Gain 0 (Drop)", color="#4A5568", style="dotted"]; + ch_unused -> matrix_q10 [label="Gain 0 (Drop)", color="#C53030", style="dotted"]; + + matrix_q10 -> out_left [label="Channel 0 Stream", color="#3182CE", penwidth=1.6]; + matrix_q10 -> out_right [label="Channel 1 Stream", color="#3182CE", penwidth=1.6]; + profile_cache -> matrix_q10 [label="Active Profile", style="dashed", color="#B7791F"]; + } + +--- + +ALSA Topology 2 Integration & Widget Declarations +================================================= + +In ALSA Topology 2 (``topology2``), the Copier, Multiplexer, and Selector are instantiated as declarative widget objects. + +Copier Widget Declarations +-------------------------- + +Copiers are defined using dedicated configuration templates in ``tools/topology/topology2/include/components/``: + +* ``dai-copier.conf``: Declares hardware interface copiers (HDA, SSP, DMIC, ALH) bound to physical DAIs. Attributes include ``copier_type``, ``direction``, ``node_type``, and ``cpc`` (cycles per chunk). +* ``host-copier.conf``: Declares host PCM endpoint copiers interfacing with host DMA streams. +* ``module-copier.conf``: Declares inter-pipeline or inter-core boundary copiers. + +All Copier widgets share the standardized UUID: + +.. code-block:: text + + UUID: 83:0c:a0:9b:12:ca:83:4a:94:3c:1f:a2:e8:2f:9d:da + +Multiplexer / Demultiplexer Widget Declarations +----------------------------------------------- + +Multiplexers and Demultiplexers are declared using ``muxdemux.conf`` with widget type ``effect``: + +.. code-block:: text + + Class.Widget."muxdemux" { + UUID: "68:68:b2:c4:30:14:0e:47:a0:89:15:d1:c7:7f:85:1a" + type: "effect" + num_input_pins: 2 + num_output_pins: 1 + } + +The widget includes an ALSA byte control used to upload runtime routing matrices or AEC reference mappings. + +Selector Widget Declarations +---------------------------- + +The Selector is declared using ``micsel.conf`` with widget type ``effect``: + +.. code-block:: text + + Class.Widget."micsel" { + UUID: "c1:92:fe:32:17:1e:c2:4f:97:58:c7:f3:54:2e:98:0a" + type: "effect" + num_input_pins: 1 + num_output_pins: 1 + } + +Its configuration blob carries the :math:`8 \times 8` :math:`Q10` coefficient tables and channel selection masks. + +--- + +End-to-End System Audio Graph: Component Synergy +================================================ + +In production systems, Copier, Multiplexer, and Selector do not operate in isolation; they interact seamlessly across concurrent playback, capture, and voice assistant pipelines. + +The following architectural graph illustrates how these components interlock in a complete PC audio topology featuring simultaneous media playback, acoustic echo cancellation, and beamformed voice capture: + +.. graphviz:: + :caption: Figure 180: End-to-End System Audio Graph: Media Playback, Voice Capture, AEC Muxing, and Loopback Monitoring + :alt: Complete end-to-end audio graph showing Host Copier, Volume, DRC, DAI Copier, Selector, AEC Multiplexer, and Voice Pipeline. + + digraph end_to_end_system_audio_graph { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_playback_pipeline { + label = "Media Playback Pipeline (Core 0)"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + host_play_copier [label="Host Copier (Playback)\nIngests Stereo Media from OS\nTracks Host Ring Pointers", fillcolor="#C6F6D5", color="#276749", penwidth=1.6]; + pb_vol [label="Volume / EQ / DRC\nDynamic Processing & Protection", fillcolor="#FEFCBF", color="#B7791F"]; + dai_play_copier [label="DAI Copier (Speaker Output)\nPin 0: Hardware Speaker Link\nPin 1: AEC Loopback Tap (48 kHz)", fillcolor="#FED7D7", color="#C53030", penwidth=1.8]; + hw_speakers [label="Physical Speakers / Codec\nStereo Acoustic Output", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.4]; + } + + subgraph cluster_capture_pipeline { + label = "Microphone Capture & Voice Pre-Processing Pipeline (Core 0)"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#FFFFFF"; + + hw_dmic_in [label="Hardware DMIC Array\n4-Channel Raw PDM Capture", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.4]; + dai_cap_copier [label="DAI Copier (DMIC Capture)\nIngests 4-Channel PDM Stream\nProduces 48 kHz / 32-bit Audio", fillcolor="#C6F6D5", color="#276749", penwidth=1.6]; + mic_selector [label="Selector Widget (Channel Isolation)\nExtracts Primary 2 Voice Mics\nDrops 2 Auxiliary Channels", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + aec_mux [label="IPC4 Multiplexer Widget (AEC Aggregator)\nPin 0: 2-Ch Clean Voice Mics\nPin 1: 2-Ch Speaker Reference Tap\nOutputs 4-Ch Synchronized Stream", fillcolor="#E9D8FD", color="#6B46C1", penwidth=2.0]; + aec_tdfb [label="Acoustic Echo Cancellation &\nTDFB Beamforming Engine\nCancels Echo & Enhances Target Voice", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + host_cap_copier [label="Host Copier (Voice Capture)\nPushes Clean Enhanced Voice to OS\n(PipeWire / Google Meet / Teams)", fillcolor="#C6F6D5", color="#276749", penwidth=1.6]; + } + + host_play_copier -> pb_vol [label="Stereo Audio", color="#276749", penwidth=1.6]; + pb_vol -> dai_play_copier [label="Processed Frames", color="#B7791F", penwidth=1.6]; + dai_play_copier -> hw_speakers [label="Pin 0 (DAI Link)", color="#C53030", penwidth=1.8]; + + hw_dmic_in -> dai_cap_copier [label="4-Ch PDM DMA", color="#4A5568", penwidth=1.6]; + dai_cap_copier -> mic_selector [label="4-Ch Raw Audio", color="#276749", penwidth=1.6]; + mic_selector -> aec_mux [label="Pin 0: 2-Ch Selected Mics", color="#B7791F", penwidth=1.8]; + + dai_play_copier -> aec_mux [label="Pin 1: 2-Ch Speaker Echo Reference", color="#3182CE", penwidth=1.8, style="dashed"]; + + aec_mux -> aec_tdfb [label="4-Ch Composite Stream\n[Mics + Ref]", color="#6B46C1", penwidth=2.0]; + aec_tdfb -> host_cap_copier [label="Clean Enhanced Voice", color="#276749", penwidth=1.8]; + } + +Workflow Walkthrough +-------------------- + +1. **Host Ingestion**: The Host Copier pulls stereo audio from user space and feeds the volume, equalizer, and DRC modules. +2. **Playback Delivery & Loopback Tapping**: The DAI Copier transmits audio to physical speakers via Pin 0 while simultaneously tapping the identical signal onto Output Pin 1. +3. **Microphone Capture & Selection**: The DAI Capture Copier ingests 4 channels from the digital microphone array. The Selector isolates the two primary front-facing microphones and drops the auxiliary background channels. +4. **Echo Reference Aggregation**: The Multiplexer fuses the 2-channel microphone audio on Pin 0 with the 2-channel speaker loopback reference on Pin 1 into a synchronized 4-channel composite stream. +5. **Speech Enhancement & Delivery**: Downstream Acoustic Echo Cancellation (AEC) and Time-Domain Fixed Beamforming (TDFB) cancel the speaker echo and beamform the user's speech. The final clean audio stream is written into host memory by the Host Capture Copier. + +Through this coordinated division of labor, Sound Open Firmware delivers modular, high-performance, and mathematically robust audio graphs across desktop, mobile, and embedded platforms. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index c77d555c..b48f6978 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -54,7 +54,7 @@ Audio Processing Modules & Algorithms * :ref:`mfcc` (High-level architecture; also see upstream `MFCC README `_) * :ref:`smart_amp` (High-level architecture; also see upstream `Smart Amp README `_) * :ref:`sound_dose` (High-level architecture; also see upstream `Sound Dose README `_) -* `Copier `_, `Mux `_ & `Selector `_ +* :ref:`copier_mux_selector` (High-level architecture; also see upstream `Copier README `_, `Mux README `_ & `Selector README `_) * `PCM Format Converter `_ .. _algorithm-specific-information: @@ -98,6 +98,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/mfcc firmware/smart_amp firmware/sound_dose + firmware/copier_mux_selector rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 36cc1ac3cdf1c406d9c479090ec108c9510f5443 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 11:16:40 +0100 Subject: [PATCH 21/64] doc: developer_guides: add comprehensive pcm format converter architecture guide Author a comprehensive, modern architectural guide for the Sound Open Firmware (SOF) PCM Format Converter subsystem. Key architectural concepts and sections: - Supported formats matrix covering U8, G.711 A-law/mu-law, S16_LE, S24_3LE, S24_4LE, S24_4LE_MSB, S32_LE, and IEEE-754 single-precision float. - Container geometry vs valid bit depth resolution, sign-extension, and dual dispatch tables (pcm_func_map and pcm_func_vc_map). - Linear fragmentation engine (pcm_convert_as_linear) and circular buffer boundary resolution. - Tensilica HiFi3/HiFi4 vector SIMD acceleration using AE_LA16X4_IP, AE_CVT32X2F16, AE_SRAI32R, AE_SLAI32S, and AE_SA32X2_IP. - G.711 logarithmic companding curves (13-segment A-law, 15-segment mu-law), inversion masks, and bit packing. - Channel remapping and selective zero-fill muting architecture (0xF mute nibble and out-of-bounds protection). - Fixed-to-float normalization and float-to-fixed denormalization bridge. - Seven native vector Graphviz SVG diagrams (Figures 181 through 187). - Integrated into developer_guides/index.rst toctree and modules list. Signed-off-by: Liam Girdwood --- developer_guides/firmware/pcm_converter.rst | 754 ++++++++++++++++++++ developer_guides/index.rst | 3 +- 2 files changed, 756 insertions(+), 1 deletion(-) create mode 100644 developer_guides/firmware/pcm_converter.rst diff --git a/developer_guides/firmware/pcm_converter.rst b/developer_guides/firmware/pcm_converter.rst new file mode 100644 index 00000000..fe7aac07 --- /dev/null +++ b/developer_guides/firmware/pcm_converter.rst @@ -0,0 +1,754 @@ +.. _pcm_converter: + +PCM Format Converter Architecture +################################# + +In Sound Open Firmware (SOF), digital audio streams traversing processing pipelines must frequently adapt between heterogeneous sample representations. Different audio peripherals, host operating system interfaces, hardware accelerators, and DSP algorithms enforce distinct word lengths, container alignments, channel arrangements, and numerical representations: + +* **Hardware Codecs & DAIs**: Serial synchronous interfaces (SSP/I2S), SoundWire links, and High Definition Audio (HDA) busses often require 24-bit samples packed into 32-bit containers (:math:`S24\_4LE`), 3-byte packed words (:math:`S24\_3LE`), or legacy 16-bit frames (:math:`S16\_LE`). +* **DSP Processing Engines**: Fixed-point audio algorithms (such as Volume, Equalizers, Dynamic Range Compressors, and Beamformers) typically compute with 32-bit headroom (:math:`S32\_LE`) to prevent intermediate arithmetic overflow. +* **Machine Learning & Neural Networks**: Keyword spotters and acoustic classifiers (such as TensorFlow Lite for Microcontrollers) often ingest 16-bit integer or single-precision 32-bit floating-point (:math:`FLOAT`) tensors. +* **Telephony & Bluetooth Subsystems**: Hands-Free Profile (HFP) and legacy voice communications operate with non-uniform logarithmic companded speech (:math:`\text{G.711 A-law}` and :math:`\mu\text{-law}`). + +The **PCM Format Converter** is the dedicated, high-throughput subsystem in SOF that performs real-time translations across this format spectrum. It provides bit-exact precision, prevents arithmetic overflow through saturation, resolves circular buffer wrapping boundaries without intermediate memory copies, and leverages Tensilica HiFi3/HiFi4 SIMD vectorization to achieve near-zero CPU cycle overhead. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +Executive Overview & The PCM Format Landscape +============================================= + +The PCM Format Converter operates in two distinct execution contexts within the SOF firmware architecture: + +1. **Embedded Inline Helper (Copier Sink Engine)**: + When attached to a Copier module, the converter executes inline on secondary output pins (:math:`\text{Pin}_1 \dots \text{Pin}_3`) via ``pcm_converter_func``. This enables a single primary 32-bit pipeline to fan out simultaneously to a 16-bit Acoustic Echo Cancellation reference tap, a 24-bit speaker amplifier, and a mono diagnostic stream without dedicating intermediate buffer memory or spawning additional pipeline tasks. + +2. **Standalone Pipeline Component**: + When declared as an independent processing widget in ALSA Topology, the format converter is instantiated between two incompatible modules (e.g. bridging a 16-bit host decoder to a 32-bit post-processing pipeline, or bridging a 32-bit beamformer to a single-precision floating-point neural network). + +Supported Format Spectrum +------------------------- + +The subsystem provides complete, bi-directional conversion coverage across eight primary digital audio representations: + +.. list-table:: SOF Supported PCM Sample Representations + :widths: 18 16 16 22 28 + :header-rows: 1 + + * - Format Identifier + - Container Size + - Valid Resolution + - Numeric Representation + - Primary Domain / Application + * - ``U8`` + - 8 bits + - 8 bits + - Unsigned integer (:math:`[0 \dots 255]`) + - Legacy audio & low-bandwidth telemetry. + * - ``A_LAW`` + - 8 bits + - 8 bits (companded) + - ITU-T G.711 A-law logarithmic + - European telephony & Bluetooth HFP voice. + * - ``MU_LAW`` + - 8 bits + - 8 bits (companded) + - ITU-T G.711 :math:`\mu`-law logarithmic + - North American telephony & cellular speech. + * - ``S16_LE`` + - 16 bits + - 16 bits + - Signed two's complement integer + - Standard CD audio, Voice Wakeup & TFLM. + * - ``S24_3LE`` + - 24 bits (3 bytes) + - 24 bits + - Signed two's complement integer + - Packed serial DAIs & compact capture. + * - ``S24_4LE`` + - 32 bits (4 bytes) + - 24 bits (LSB-aligned) + - Signed two's complement integer + - High Definition Audio & SoundWire ALH. + * - ``S24_4LE_MSB`` + - 32 bits (4 bytes) + - 24 bits (MSB-aligned) + - Signed two's complement integer + - Specialized I2S DACs & DSP DMA engines. + * - ``S32_LE`` + - 32 bits + - 32 bits + - Signed two's complement integer + - Internal SOF processing pipeline backbone. + * - ``FLOAT`` + - 32 bits + - 24-bit mantissa + - IEEE-754 single-precision float + - Neural inference, ML models & Steam Audio. + +.. graphviz:: + :caption: Figure 181: SOF PCM Format Conversion Matrix & Supported Sample Representations + :alt: Diagram showing the full format conversion matrix interconnecting U8, A-law, mu-law, S16, S24, S32, and IEEE-754 float. + + digraph pcm_format_matrix { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_telephony { + label = "Telephony & Legacy Formats (8-bit)"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + fmt_u8 [label="U8\nUnsigned 8-bit Integer\nBias: +128", fillcolor="#EDF2F7", color="#4A5568"]; + fmt_alaw [label="G.711 A-Law\n8-bit Logarithmic Companded\n13-Segment Piecewise Curve", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + fmt_mulaw [label="G.711 μ-Law\n8-bit Logarithmic Companded\n15-Segment Piecewise Curve", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + } + + subgraph cluster_standard { + label = "Standard Digital Audio (16-bit & 24-bit)"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + fmt_s16 [label="S16_LE\nSigned 16-bit Two's Complement\nQ1.15 / Full Scale ±32767", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + fmt_s24_3 [label="S24_3LE\nPacked 24-bit (3 Bytes / Sample)\nDense Memory Layout", fillcolor="#EBF8FF", color="#3182CE"]; + fmt_s24_4 [label="S24_4LE\n24-bit in 32-bit Container (LSB)\nQ1.23 in 32-bit Word", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + fmt_s24_msb [label="S24_4LE_MSB\n24-bit in 32-bit Container (MSB)\nHigh 24 bits active, Low 8 bits 0", fillcolor="#EBF8FF", color="#3182CE"]; + } + + subgraph cluster_dsp_core { + label = "High-Resolution DSP & Machine Learning Core (32-bit)"; + style = "filled,rounded"; + color = "#E9D8FD"; + fillcolor = "#F7FAFC"; + + fmt_s32 [label="S32_LE (SOF Core Processing Backbone)\nSigned 32-bit Two's Complement Integer\nQ1.31 / Full Dynamic Headroom", fillcolor="#FED7D7", color="#C53030", penwidth=2.0]; + fmt_float [label="FLOAT (IEEE-754 Single Precision)\n32-bit Normalized Float [-1.0, +1.0]\nTFLM / Valve Steam Audio / ML Inference", fillcolor="#E9D8FD", color="#6B46C1", penwidth=1.8]; + } + + fmt_u8 -> fmt_s32 [label="Offset Binary to Q1.31", color="#4A5568"]; + fmt_s32 -> fmt_u8 [label="Rounding & Unsigned Shift", color="#4A5568"]; + + fmt_alaw -> fmt_s32 [label="A-Law Expansion", color="#B7791F", penwidth=1.6]; + fmt_s32 -> fmt_alaw [label="A-Law Compression", color="#B7791F", penwidth=1.6]; + + fmt_mulaw -> fmt_s32 [label="μ-Law Expansion", color="#B7791F", penwidth=1.6]; + fmt_s32 -> fmt_mulaw [label="μ-Law Compression", color="#B7791F", penwidth=1.6]; + + fmt_s16 -> fmt_s32 [label="Arithmetic Left Shift << 16", color="#276749", penwidth=1.8]; + fmt_s32 -> fmt_s16 [label="Saturating Right Shift >> 16", color="#276749", penwidth=1.8]; + + fmt_s16 -> fmt_s24_4 [label="Shift << 8", color="#3182CE"]; + fmt_s24_4 -> fmt_s16 [label="Shift >> 8 with Rounding", color="#3182CE"]; + + fmt_s24_3 -> fmt_s24_4 [label="Byte Unpack & Sign Extend", color="#3182CE"]; + fmt_s24_4 -> fmt_s24_3 [label="Byte Pack (Drop Byte 3)", color="#3182CE"]; + + fmt_s24_4 -> fmt_s32 [label="Arithmetic Left Shift << 8", color="#3182CE", penwidth=1.6]; + fmt_s32 -> fmt_s24_4 [label="Saturating Right Shift >> 8", color="#3182CE", penwidth=1.6]; + + fmt_s24_4 -> fmt_s24_msb [label="Shift << 8", color="#3182CE"]; + fmt_s24_msb -> fmt_s24_4 [label="Shift >> 8", color="#3182CE"]; + + fmt_s32 -> fmt_float [label="Float Divide / 2^31", color="#6B46C1", penwidth=1.8]; + fmt_float -> fmt_s32 [label="Multiply * 2^31 & Sat Clamp", color="#6B46C1", penwidth=1.8]; + + fmt_s16 -> fmt_float [label="Float Divide / 32768.0", color="#6B46C1"]; + fmt_float -> fmt_s16 [label="Multiply * 32768.0 & Clamp", color="#6B46C1"]; + } + +--- + +Container Geometry & Valid Bit Formatting +========================================= + +In audio memory architectures, sample representation involves two distinct orthogonal dimensions: + +1. **Container Size** (:math:`C`): The physical number of bytes allocated in memory for each sample (e.g. 2 bytes for 16-bit, 3 bytes for packed 24-bit, 4 bytes for 32-bit). +2. **Valid Bit Depth** (:math:`V`): The actual number of information-carrying bits produced by an ADC or consumed by a DAC (e.g. 16, 20, 24, or 32 bits). + +Alignment Paradigms +------------------- + +When the valid bit depth is smaller than the physical container size (:math:`V < C`), the sample can be aligned within the container in multiple ways: + +* **LSB Alignment with Sign Extension** (Standard :math:`S24\_4LE`): + The 24 valid bits reside in the least significant bit positions (bits 0 to 23). Bit 23 is arithmetically sign-extended across bits 24 through 31. This representation allows direct arithmetic operations in standard integer ALUs without pre-shifting: + + .. math:: + + \text{Word}_{32} = \left( \text{Sample}_{24} \;\&\; \text{0x00FFFFFF} \right) \;|\; \left( \text{Sample}_{24}[23] \times \text{0xFF000000} \right) + +* **MSB Alignment** (:math:`S24\_4LE\_MSB`): + The 24 valid bits reside in the most significant bit positions (bits 8 to 31). The lower 8 bits (bits 0 to 7) are padded with digital zeros. This layout is standard for audio DAIs (such as I2S and HDA) where serial bit transmitters shift out the most significant bit first: + + .. math:: + + \text{Word}_{32} = \text{Sample}_{24} \ll 8 + +* **Packed 3-Byte Representation** (:math:`S24\_3LE`): + Three consecutive bytes store each 24-bit sample without padding. While minimizing DMA memory bandwidth across PCIe or memory busses, 3-byte packing causes memory accesses to cross 32-bit word and cache line boundaries, requiring specialized byte-assembly logic. + +Dual Function Dispatch Architecture +----------------------------------- + +To resolve the exact conversion kernel required for any pipeline connection, SOF maintains two complementary static lookup tables: + +1. **Flat Format Mapping** (``pcm_func_map``): + Matches standard source and sink frame format enums (e.g. ``SOF_IPC_FRAME_S16_LE`` to ``SOF_IPC_FRAME_S32_LE``). Used when container size and valid bit depth are identical. + +2. **Container and Valid-Bit Mapping** (``pcm_func_vc_map``): + Matches multi-dimensional triples ``(valid_bits, container_size, frame_fmt)``, handling asymmetric valid-bit packings (such as 24-in-32 LSB vs MSB). This allows instant dispatch of specialized routines like ``pcm_convert_s24_c32_to_s16_c16``. + +.. graphviz:: + :caption: Figure 182: Container vs Valid Bit Formatting (16-in-16, 24-in-32, and 32-in-32 Alignment) + :alt: Bit layout diagram showing 16-bit in 16-bit container, 24-bit LSB in 32-bit container, 24-bit MSB in 32-bit container, and full 32-bit. + + digraph container_bit_formatting { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=record, style="filled,rounded"]; + + c16 [label="{ S16_LE (16-bit Container) | { Bit 15 (Sign) | Bits 14..0 (15 Valid Bits) } }", fillcolor="#C6F6D5", color="#276749", penwidth=1.6]; + + c24_3 [label="{ S24_3LE (24-bit Packed Container - 3 Bytes) | { Byte 0 (Bits 0..7) | Byte 1 (Bits 8..15) | Byte 2: Bit 23 (Sign) + Bits 16..22 } }", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + + c24_4_lsb [label="{ S24_4LE (24-bit Valid in 32-bit Container - LSB Aligned) | { Bits 31..24 (Sign Extension: 8x Bit 23) | Bit 23 (Sign) | Bits 22..0 (23 Valid Audio Bits) } }", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + + c24_4_msb [label="{ S24_4LE_MSB (24-bit Valid in 32-bit Container - MSB Aligned) | { Bit 31 (Sign) | Bits 30..8 (23 Valid Audio Bits) | Bits 7..0 (Zero Padded: 0x00) } }", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + + c32 [label="{ S32_LE (Full 32-bit Container) | { Bit 31 (Sign) | Bits 30..0 (31 Valid Information Bits) } }", fillcolor="#FED7D7", color="#C53030", penwidth=2.0]; + } + +--- + +Circular Buffer Boundary Resolution & Linear Fragmentation Engine +================================================================= + +A central challenge in real-time embedded DSP audio is that audio frames reside in **circular ring buffers**. When an audio processing period executes, the requested block of samples frequently wraps around the boundary between the buffer's physical end and its beginning. + +If a format converter had to check for circular wrap-around on every individual sample within its inner loop, performance would plummet due to pipeline branch mispredictions and register stalls. + +The Two-Step Linear Chunking Quantum +------------------------------------ + +SOF solves this problem through the **Linear Fragmentation Engine** (``pcm_convert_as_linear``). Instead of processing samples individually or allocating intermediate scratch memory, the engine decomposes circular buffer processing into at most two contiguous linear operations: + +1. **Calculate Available Contiguous Samples in Source**: + Let :math:`\text{r\_ptr}` be the current read pointer in the source buffer, and :math:`\text{end}_{\text{src}}` be the physical end address. The maximum number of linear samples before wrapping is: + + .. math:: + + N_1 = \frac{\text{end}_{\text{src}} - \text{r\_ptr}}{S_{\text{in}}} + + where :math:`S_{\text{in}}` is the source sample size in bytes (:math:`\text{bytes\_per\_sample}`). + +2. **Calculate Available Contiguous Space in Sink**: + Similarly, for write pointer :math:`\text{w\_ptr}` and sink buffer end :math:`\text{end}_{\text{sink}}`: + + .. math:: + + N_2 = \frac{\text{end}_{\text{sink}} - \text{w\_ptr}}{S_{\text{out}}} + +3. **Determine the Maximum Linear Chunk Quantum**: + The engine computes the largest contiguous slice that can be processed without wrapping in *either* the source or sink buffer: + + .. math:: + + \text{chunk} = \min \Big( N_1, \; N_2, \; \text{remaining\_samples} \Big) + +Zero-Copy Vector Execution +-------------------------- + +Once :math:`\text{chunk}` is established, the engine dispatches a high-speed linear conversion kernel (``pcm_converter_lin_func``) directly on the linear memory pointers: + +.. math:: + + \text{converter}\Big(\text{r\_ptr}, \; \text{w\_ptr}, \; \text{chunk}\Big) + +Because the slice is guaranteed to be completely contiguous in physical memory, the conversion kernel executes at full SIMD vector memory bandwidth with zero boundary checks. + +Upon completion of the chunk: + +* The source read pointer is advanced by :math:`\text{chunk} \times S_{\text{in}}` and wrapped modulo buffer size: + + .. math:: + + \text{r\_ptr} = \text{audio\_stream\_wrap}\big(\text{source}, \; \text{r\_ptr} + \text{chunk} \cdot S_{\text{in}}\big) + +* The sink write pointer is advanced by :math:`\text{chunk} \times S_{\text{out}}` and wrapped modulo buffer size: + + .. math:: + + \text{w\_ptr} = \text{audio\_stream\_wrap}\big(\text{sink}, \; \text{w\_ptr} + \text{chunk} \cdot S_{\text{out}}\big) + +* The remaining sample count is decremented. + +Any remaining samples wrap to the buffer start address and are processed in a second contiguous linear pass. Thus, arbitrary circular buffer transfers are completed in at most two kernel invocations with zero memory copying. + +.. graphviz:: + :caption: Figure 183: Linear Fragmentation & Circular Buffer Boundary Resolution Engine + :alt: Architectural diagram illustrating two circular buffers, read/write pointers, linear chunk calculation, and wrap-around handling. + + digraph linear_fragmentation_engine { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_source_ring { + label = "Source Circular Buffer (e.g. S16_LE / 2 Bytes)"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + src_start [label="Buffer Start", fillcolor="#EDF2F7", color="#4A5568"]; + src_head [label="Wrapped Head Slice\n(Remaining Samples - Chunk 1)", fillcolor="#FEFCBF", color="#B7791F"]; + src_rptr [label="Current Read Pointer (r_ptr)\nUnprocessed Source Data", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + src_tail [label="Contiguous Linear Slice N1\n(Distance to Buffer End)", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + src_end [label="Buffer End Boundary", fillcolor="#EDF2F7", color="#4A5568"]; + + src_start -> src_head -> src_rptr -> src_tail -> src_end [style="invis"]; + } + + subgraph cluster_eval { + label = "Boundary Resolution Evaluator (pcm_convert_as_linear)"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#FFFFFF"; + + calc_chunk [label="Chunk Quantum Evaluator\nN1 = (end_src - r_ptr) / S_in\nN2 = (end_sink - w_ptr) / S_out\nChunk 1 = min(N1, N2, TotalSamples)", fillcolor="#E9D8FD", color="#6B46C1", penwidth=1.8]; + kernel_exec [label="Direct Vector Kernel Execution\nconverter(r_ptr, w_ptr, Chunk 1)\nZero Inner-Loop Branching", fillcolor="#FED7D7", color="#C53030", penwidth=2.0]; + wrap_step [label="Modulo Pointer Advancement\nr_ptr = wrap(r_ptr + Chunk1 * S_in)\nw_ptr = wrap(w_ptr + Chunk1 * S_out)", fillcolor="#EDF2F7", color="#4A5568"]; + } + + subgraph cluster_sink_ring { + label = "Sink Circular Buffer (e.g. S32_LE / 4 Bytes)"; + style = "filled,rounded"; + color = "#FED7D7"; + fillcolor = "#F7FAFC"; + + sink_start [label="Buffer Start", fillcolor="#EDF2F7", color="#4A5568"]; + sink_head [label="Wrapped Head Slice (Pass 2)", fillcolor="#FEFCBF", color="#B7791F"]; + sink_wptr [label="Current Write Pointer (w_ptr)\nTarget Insertion Address", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + sink_tail [label="Contiguous Linear Slice N2\n(Distance to Buffer End)", fillcolor="#FED7D7", color="#C53030", penwidth=1.6]; + sink_end [label="Buffer End Boundary", fillcolor="#EDF2F7", color="#4A5568"]; + + sink_start -> sink_head -> sink_wptr -> sink_tail -> sink_end [style="invis"]; + } + + src_rptr -> calc_chunk [label="Source Distance N1", color="#3182CE"]; + sink_wptr -> calc_chunk [label="Sink Distance N2", color="#C53030"]; + calc_chunk -> kernel_exec [label="Chunk 1 Size", color="#6B46C1", penwidth=1.8]; + kernel_exec -> wrap_step [label="Pass 1 Complete", color="#C53030"]; + wrap_step -> calc_chunk [label="Execute Pass 2 for Wrapped Head", color="#B7791F", style="dashed", constraint=false]; + } + +--- + +Vectorized SIMD Acceleration: Tensilica HiFi3/HiFi4 & Modern DSP Engines +======================================================================== + +PCM format conversion is an inherently parallel, vectorizable workload. In fixed-point architectures, converting sixteen 16-bit samples to sixteen 32-bit samples requires identical arithmetic shifting and sign extension across every sample. + +SOF incorporates dedicated SIMD vector implementations (``pcm_converter_hifi3.c``) for Cadence Tensilica HiFi3 and HiFi4 DSP architectures. + +The HiFi3/HiFi4 Processing Pipeline +----------------------------------- + +The vectorized engine leverages 64-bit and 128-bit vector registers and specialized Tensilica Instruction Extension (TIE) intrinsics: + +1. **Alignment Initialization**: + Memory streams are primed using vector alignment pointers: + + .. code-block:: text + + ae_valign inu = AE_ZALIGN64(); + ae_valign outu = AE_ZALIGN64(); + inu = AE_LA64_PP(in); + +2. **Quad-Sample Vector Load (16-bit to 32-bit)**: + Loads four 16-bit signed samples (:math:`x_0, x_1, x_2, x_3`) in a single cycle into a 64-bit vector register ``ae_int16x4``: + + .. code-block:: text + + AE_LA16X4_IP(sample, inu, in); + +3. **Parallel Vector Unpack & Shift**: + The four 16-bit samples are expanded into two pairs of 32-bit vector registers (``ae_int32x2``), sign-extended, and shifted: + + .. code-block:: text + + /* High two samples (x0, x1) shifted to 24-bit valid */ + AE_SA32X2_IP(AE_SRAI32(AE_CVT32X2F16_32(sample), 8), outu, out); + + /* Low two samples (x2, x3) shifted to 24-bit valid */ + AE_SA32X2_IP(AE_SRAI32(AE_CVT32X2F16_10(sample), 8), outu, out); + +4. **Flushing and Tail Handling**: + The output vector alignment buffer is flushed to memory using ``AE_SA64POS_FP``. If the total sample count is not an exact multiple of 4, the remaining residue samples (1, 2, or 3 samples) are processed in an unrolled scalar tail loop using ``AE_L16_IP`` and ``AE_S32_L_IP`` to prevent memory access overruns past the buffer boundary. + +Rounding and Saturation Mechanics +--------------------------------- + +When down-converting from higher precision to lower precision (e.g. :math:`S32\_LE \to S16\_LE` or :math:`S24\_4LE \to S16\_LE`), simple truncation introduces negative DC bias and harmonic distortion. The HiFi3 engine applies **convergent rounding** and **symmetric saturation**: + +.. math:: + + y[n] = \text{sat}_{16}\left( \left\lfloor \frac{x[n] + 2^{B-1}}{2^B} \right\rfloor \right) + +In HiFi3 intrinsics, this is executed using ``AE_SRAI32R`` (arithmetic shift right with rounding) followed by ``AE_SLAI32S`` (arithmetic shift left with saturation). If an audio peak exceeds the dynamic range of 16-bit audio (:math:`+32767` or :math:`-32768`), the sample is clamped to the rail without arithmetic wrap-around inversion. + +.. graphviz:: + :caption: Figure 184: Tensilica HiFi3/HiFi4 SIMD Vectorized Conversion Pipeline (AE_LA16X4 & AE_SA32X2) + :alt: Dataflow diagram of Tensilica HiFi3/HiFi4 SIMD vectorization showing 64-bit load, unpack, shift, rounding, and dual 32-bit vector store. + + digraph hifi3_vector_pipeline { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_input_mem { + label = "Linear Input Stream in Memory"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + raw_samples [label="Memory Address [in]\nFour 16-bit Samples: [ s0 | s1 | s2 | s3 ]\nTotal: 64 Bits", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + } + + subgraph cluster_simd_core { + label = "Tensilica HiFi3 / HiFi4 SIMD Execution Core"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#FFFFFF"; + + vec_load [label="AE_LA16X4_IP\nAtomic 64-bit Vector Register Load\nae_int16x4 sample", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + vec_unpack_hi [label="AE_CVT32X2F16_32\nUnpack High Pair [s0, s1]\nExpand 16-bit -> 32-bit", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + vec_unpack_lo [label="AE_CVT32X2F16_10\nUnpack Low Pair [s2, s3]\nExpand 16-bit -> 32-bit", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + vec_shift_hi [label="AE_SRAI32 / Rounding\nArithmetic Shift & Align\nae_int32x2 out_hi", fillcolor="#FED7D7", color="#C53030", penwidth=1.6]; + vec_shift_lo [label="AE_SRAI32 / Rounding\nArithmetic Shift & Align\nae_int32x2 out_lo", fillcolor="#FED7D7", color="#C53030", penwidth=1.6]; + vec_store_hi [label="AE_SA32X2_IP\nStore Vector Pair [s0, s1]\n64-bit Aligned Write", fillcolor="#E9D8FD", color="#6B46C1", penwidth=1.8]; + vec_store_lo [label="AE_SA32X2_IP\nStore Vector Pair [s2, s3]\n64-bit Aligned Write", fillcolor="#E9D8FD", color="#6B46C1", penwidth=1.8]; + } + + subgraph cluster_output_mem { + label = "Linear Output Stream in Memory"; + style = "filled,rounded"; + color = "#FED7D7"; + fillcolor = "#F7FAFC"; + + out_samples [label="Memory Address [out]\nFour 32-bit Converted Words:\n[ Word(s0) | Word(s1) | Word(s2) | Word(s3) ]\nTotal: 128 Bits", fillcolor="#FED7D7", color="#C53030", penwidth=1.8]; + } + + raw_samples -> vec_load [label="64-Bit Load", color="#276749", penwidth=1.8]; + vec_load -> vec_unpack_hi [label="High 32 Bits", color="#3182CE"]; + vec_load -> vec_unpack_lo [label="Low 32 Bits", color="#3182CE"]; + + vec_unpack_hi -> vec_shift_hi [label="Sign Extend", color="#B7791F"]; + vec_unpack_lo -> vec_shift_lo [label="Sign Extend", color="#B7791F"]; + + vec_shift_hi -> vec_store_hi [label="32-bit Vector", color="#C53030"]; + vec_shift_lo -> vec_store_lo [label="32-bit Vector", color="#C53030"]; + + vec_store_hi -> out_samples [label="Write 64 Bits", color="#6B46C1", penwidth=1.6]; + vec_store_lo -> out_samples [label="Write 64 Bits", color="#6B46C1", penwidth=1.6]; + } + +--- + +G.711 Logarithmic Companding Mathematics (A-Law & μ-Law) +======================================================== + +Standard linear PCM allocates quantization levels uniformly across the entire dynamic range. In voice communications, human hearing sensitivity is logarithmic: quiet consonants carry vital phonetic information, whereas loud vowels mask quantization distortion. + +The **ITU-T G.711** standard defines non-uniform logarithmic companding (compressing/expanding), achieving the perceptual speech quality of 12-bit to 14-bit linear PCM within a compact **8-bit word** at an :math:`8 \text{ kHz}` sampling rate (:math:`64 \text{ kbps}`). + +A-Law Companding (European Telephony & Bluetooth HFP) +----------------------------------------------------- + +The continuous analytical A-law compression characteristic is defined as: + +.. math:: + + F(x) = \text{sgn}(x) \cdot \begin{cases} + \dfrac{A |x|}{1 + \ln(A)}, & 0 \le |x| < \dfrac{1}{A} \\[8pt] + \dfrac{1 + \ln(A |x|)}{1 + \ln(A)}, & \dfrac{1}{A} \le |x| \le 1 + \end{cases} + +where :math:`A = 87.6`. In digital systems, this continuous function is approximated by a **13-segment piecewise linear curve** (4 segments in positive quadrant, 4 in negative, with the central segment through zero counting as one linear slope). + +An 8-bit A-law byte is structured as: + +* Bit 7: Sign bit (:math:`1 = \text{positive}, 0 = \text{negative}`). +* Bits 6..4: Chord (Segment exponent :math:`0 \dots 7`). +* Bits 3..0: Step (Position along segment mantissa :math:`0 \dots 15`). + +.. note:: + **Transmission Inversion Mask**: + To prevent long runs of digital zeros on physical telecommunication trunks (which would cause clock recovery failure in phase-locked loops), standard G.711 A-law inverts every even bit (mask ``0x55``). SOF automatically applies this inversion mask during encoding and decoding. + +μ-Law Companding (North American Telephony) +------------------------------------------- + +The continuous analytical :math:`\mu`-law compression characteristic is defined as: + +.. math:: + + F(x) = \text{sgn}(x) \cdot \frac{\ln(1 + \mu |x|)}{\ln(1 + \mu)}, \quad \text{where } \mu = 255 + +Digital :math:`\mu`-law uses a **15-segment piecewise linear approximation**. Unlike A-law, :math:`\mu`-law incorporates a bias offset of :math:`+33` before segment quantization to avoid a flat central step at zero. + +* All 8 bits of the encoded :math:`\mu`-law byte are inverted (mask ``0xFF``). + +SOF Expansion and Compression Kernels +------------------------------------- + +SOF implements optimized bit-manipulation tables: + +* **Expansion** (:math:`\text{G.711} \to S32\_LE`): + Extracts chord and step fields, reconstructs the 13-bit/14-bit linear integer value, applies the sign bit, and arithmetically left-shifts into full-scale :math:`Q1.31` integer space. +* **Compression** (:math:`S32\_LE \to \text{G.711}`): + Takes the absolute sample value, detects the leading one bit position using fast hardware priority encoders (``clz``), quantizes into chord and step, and inverts transmission bits. + +.. graphviz:: + :caption: Figure 185: G.711 Logarithmic Companding: A-Law and μ-Law Piecewise Conversion Curves + :alt: Diagram of G.711 companding curves comparing linear PCM with logarithmic A-law and mu-law piecewise characteristics. + + digraph g711_companding_curves { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_analog_linear { + label = "High-Resolution Linear PCM Space"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + pcm_in [label="Linear PCM Audio (S32_LE)\nDynamic Range: ~96-144 dB\n16 to 32 bits per sample", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + } + + subgraph cluster_companding_engine { + label = "G.711 Piecewise Logarithmic Compression Engine"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#FFFFFF"; + + alaw_engine [label="A-Law Compressor (A = 87.6)\n13-Segment Piecewise Curve\nLogarithmic Compressive Slope\nEven-Bit Inversion Mask (0x55)", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + mulaw_engine [label="μ-Law Compressor (μ = 255)\n15-Segment Piecewise Curve\nLinear Bias Offset (+33)\nFull Inversion Mask (0xFF)", fillcolor="#FED7D7", color="#C53030", penwidth=1.8]; + } + + subgraph cluster_telecom_byte { + label = "Compressed 8-bit Telephony Stream"; + style = "filled,rounded"; + color = "#E9D8FD"; + fillcolor = "#F7FAFC"; + + alaw_out [label="A-Law Byte (64 kbps)\n[ Sign (1) | Chord (3) | Step (4) ]\nHigh SQNR for Quiet Speech", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + mulaw_out [label="μ-Law Byte (64 kbps)\n[ Sign (1) | Chord (3) | Step (4) ]\nOptimal Voice Band Intelligibility", fillcolor="#FED7D7", color="#C53030", penwidth=1.6]; + } + + pcm_in -> alaw_engine [label="Europe / GSM / HFP", color="#B7791F", penwidth=1.6]; + pcm_in -> mulaw_engine [label="North America / Japan", color="#C53030", penwidth=1.6]; + + alaw_engine -> alaw_out [label="8-bit Output", color="#B7791F", penwidth=1.6]; + mulaw_engine -> mulaw_out [label="8-bit Output", color="#C53030", penwidth=1.6]; + } + +--- + +PCM Channel Remapping & Selective Channel Muting Architecture +============================================================= + +In advanced multi-channel topologies, format conversion must frequently coincide with channel rearrangement. For example, an 8-channel digital microphone array may deliver samples in physical hardware order :math:`[M_0, M_1, M_2, M_3, M_4, M_5, M_6, M_7]`, while a stereo processing pipeline requires only channels 2 and 5 mapped to Left and Right, with all other channels suppressed. + +The SOF remapping engine (``pcm_remap.c``, enabled via ``CONFIG_PCM_REMAPPING_CONVERTERS``) performs format conversion and channel reordering simultaneously in a single pass over memory. + +Nibble-Encoded Channel Map Word +------------------------------- + +Channel routing is governed by a packed 32-bit configuration word (``chmap``). Every 4-bit nibble defines the routing for one destination sink channel: + +.. math:: + + \text{chmap} = \sum_{k=0}^{7} \text{src\_channel}[k] \cdot 16^k + +* **Nibble 0** (Bits 3..0): Specifies the source channel index copied into Sink Channel 0. +* **Nibble 1** (Bits 7..4): Specifies the source channel index copied into Sink Channel 1. +* **Nibble** :math:`k` (Bits :math:`4k+3 \dots 4k`): Specifies the source channel index copied into Sink Channel :math:`k`. +* **The Identity Mapping** (``DUMMY_CHMAP``): + ``0x76543210`` maps source channel 0 to sink 0, source 1 to sink 1, up to source 7 to sink 7. + +The Special ``0xF`` Mute Nibble & Out-of-Bounds Protection +---------------------------------------------------------- + +The remapping engine incorporates autonomous zero-fill mechanics: + +* **Intentional Channel Muting**: + Setting any nibble to ``0xF`` designates that the corresponding sink channel is muted. Rather than reading from the source buffer, the engine invokes ``mute_channel_c16`` or ``mute_channel_c32``, filling the sink channel's time slots with digital zeros. +* **Security & Memory Over-Read Protection**: + If a malicious or misconfigured topology blob supplies a source channel index greater than or equal to the actual source channel count (:math:`\text{src\_channel} \ge \text{num\_src\_channels}`), the engine automatically treats the nibble as ``0xF`` and mutes the channel. This guarantees that crafted topology configurations can never read out-of-bounds DSP memory. + +.. graphviz:: + :caption: Figure 186: PCM Channel Remapping & Selective Channel Muting Architecture (Nibble Map Decoding) + :alt: Diagram illustrating 32-bit channel map nibble decoding, routing into destination channels, and 0xF zero-fill muting logic. + + digraph pcm_channel_remapping { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_src_stream { + label = "Source Multi-Channel Stream (e.g. 4-Ch DMIC)"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + src_c0 [label="Src Ch 0: Ambient Mic Left", fillcolor="#EDF2F7", color="#4A5568"]; + src_c1 [label="Src Ch 1: Front Primary Mic", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + src_c2 [label="Src Ch 2: Ambient Mic Right", fillcolor="#EDF2F7", color="#4A5568"]; + src_c3 [label="Src Ch 3: Rear Primary Mic", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + } + + subgraph cluster_chmap_word { + label = "32-bit Nibble Map Word (chmap = 0xFFFF3F1)"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#FFFFFF"; + + nibble_0 [label="Nibble 0 = 0x1\n(Select Src Ch 1)", fillcolor="#C6F6D5", color="#276749", penwidth=1.6]; + nibble_1 [label="Nibble 1 = 0x3\n(Select Src Ch 3)", fillcolor="#C6F6D5", color="#276749", penwidth=1.6]; + nibble_2 [label="Nibble 2 = 0xF\n(Mute Flag: Zero-Fill)", fillcolor="#FED7D7", color="#C53030", penwidth=1.6]; + nibble_3 [label="Nibble 3 = 0xF\n(Mute Flag: Zero-Fill)", fillcolor="#FED7D7", color="#C53030", penwidth=1.6]; + } + + subgraph cluster_sink_stream { + label = "Sink Multi-Channel Stream (Remapped & Filtered)"; + style = "filled,rounded"; + color = "#FED7D7"; + fillcolor = "#F7FAFC"; + + sink_c0 [label="Sink Ch 0: Front Primary Mic", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + sink_c1 [label="Sink Ch 1: Rear Primary Mic", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + sink_c2 [label="Sink Ch 2: Digital Zero (Muted)", fillcolor="#FED7D7", color="#C53030"]; + sink_c3 [label="Sink Ch 3: Digital Zero (Muted)", fillcolor="#FED7D7", color="#C53030"]; + } + + src_c1 -> nibble_0 [label="Route Ch 1", color="#276749", penwidth=1.6]; + src_c3 -> nibble_1 [label="Route Ch 3", color="#276749", penwidth=1.6]; + + nibble_0 -> sink_c0 [label="Copy Samples", color="#276749", penwidth=1.8]; + nibble_1 -> sink_c1 [label="Copy Samples", color="#276749", penwidth=1.8]; + nibble_2 -> sink_c2 [label="Inject 0x0000", color="#C53030", style="dashed"]; + nibble_3 -> sink_c3 [label="Inject 0x0000", color="#C53030", style="dashed"]; + } + +--- + +Floating-Point & Fixed-Point Interoperability Bridge +==================================================== + +Modern embedded audio systems increasingly host machine learning workloads (e.g. TensorFlow Lite for Microcontrollers acoustic models) and spatial audio engines (e.g. Valve Steam Audio HRTF convolution). These frameworks operate natively in IEEE-754 single-precision floating-point arithmetic (:math:`[-1.0, +1.0]`). + +The PCM Format Converter provides bidirectional, numerical-precision-preserving bridges between integer PCM and floating-point audio. + +Fixed-to-Float Normalization +---------------------------- + +Integer PCM samples are converted to normalized floating-point numbers through scalar division by the maximum integer representation: + +.. math:: + + y_{\text{float}}[n] = \frac{x_{\text{int}}[n]}{2^{B-1}} + +where :math:`B` is the valid bit depth: + +* For :math:`S16\_LE`: :math:`y_f = x \cdot \left(\dfrac{1}{32768.0}\right) = x \cdot 3.0517578 \times 10^{-5}` +* For :math:`S24\_4LE`: :math:`y_f = x \cdot \left(\dfrac{1}{8388608.0}\right) = x \cdot 1.1920929 \times 10^{-7}` +* For :math:`S32\_LE`: :math:`y_f = x \cdot \left(\dfrac{1}{2147483648.0}\right) = x \cdot 4.6566129 \times 10^{-10}` + +Float-to-Fixed Denormalization & Clamping +----------------------------------------- + +When converting floating-point tensors back into integer PCM for hardware transmission, samples are scaled, symmetrically rounded, and hard-clamped to prevent overflow wrap-around: + +.. math:: + + y_{\text{int}}[n] = \text{clip}\left( \text{round}\left( x_{\text{float}}[n] \cdot 2^{B-1} \right), \;-2^{B-1}, \; 2^{B-1}-1 \right) + +On Tensilica HiFi DSPs equipped with hardware Floating-Point Units (VFPU / FP-TIE), this transformation is executed using specialized single-cycle instructions (``ROUND.S``, ``FLOOR.S``, ``CVT.W.S``), ensuring zero latency penalties when interfacing neural networks with physical audio pipelines. + +.. graphviz:: + :caption: Figure 187: Floating-Point Fixed-Point Interoperability Bridge (IEEE-754 Normalization & Denormalization) + :alt: Diagram of bidirectional conversion pipeline between fixed-point integer PCM and IEEE-754 float showing scaling factors, rounding, and clamping. + + digraph float_fixed_bridge { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_fixed_domain { + label = "Fixed-Point Audio Domain (Q1.31 / S32_LE)"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + int_audio [label="32-Bit Signed Integer (S32_LE)\nRange: [-2147483648, +2147483647]\nHardware DMA & Boundary Format", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + } + + subgraph cluster_bridge_core { + label = "Numerical Conversion & Interoperability Bridge"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#FFFFFF"; + + norm_engine [label="Fixed -> Float Normalizer\nMultiply by (1.0 / 2^31)\nProduces Normalized Float", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + denorm_engine [label="Float -> Fixed Denormalizer\nMultiply by 2^31\nSymmetric Rounding", fillcolor="#FED7D7", color="#C53030", penwidth=1.6]; + clamp_engine [label="Saturation Clamping Guard\nclip(val, -2^31, +2^31 - 1)\nEliminates Inversion Distortion", fillcolor="#FED7D7", color="#C53030", penwidth=1.8]; + } + + subgraph cluster_float_domain { + label = "Floating-Point Domain (IEEE-754)"; + style = "filled,rounded"; + color = "#E9D8FD"; + fillcolor = "#F7FAFC"; + + float_audio [label="Single-Precision Float\nRange: [-1.0, +1.0]\nTFLM / Steam Audio / ML Models", fillcolor="#E9D8FD", color="#6B46C1", penwidth=1.8]; + } + + int_audio -> norm_engine [label="Integer Samples", color="#276749", penwidth=1.6]; + norm_engine -> float_audio [label="Normalized Float [-1.0, +1.0]", color="#6B46C1", penwidth=1.8]; + + float_audio -> denorm_engine [label="Float Output", color="#6B46C1", penwidth=1.6]; + denorm_engine -> clamp_engine [label="Scaled Value", color="#C53030"]; + clamp_engine -> int_audio [label="Clamped Integer PCM", color="#276749", penwidth=1.8]; + } + +--- + +System Topology Integration & Lifecycle Walkthrough +==================================================== + +In production firmware, the PCM Format Converter is seamlessly integrated across initialization, configuration, and execution lifecycles: + +1. **Format Negotiation (Pipeline Prepare Phase)**: + During pipeline parameter setup (``pipeline_comp_hw_params``), upstream and downstream buffer formats are compared. If formats match, the converter assigns the lightweight pass-through handler ``just_copy``. If formats diverge, the subsystem queries ``pcm_get_conversion_function`` or ``pcm_get_conversion_vc_function`` to select the optimal SIMD or scalar kernel. + +2. **Buffer Capacity Allocation**: + The SOF topology infrastructure accounts for differing sample byte sizes when calculating circular buffer depths. For example, a converter bridging a 16-bit stream (:math:`2 \text{ bytes/sample}`) to a 32-bit stream (:math:`4 \text{ bytes/sample}`) allocates double the physical byte capacity for the downstream buffer, ensuring uniform period frame scheduling. + +3. **Runtime Execution**: + At every pipeline scheduling tick, the converter inspects source and sink read/write pointers, computes the maximum unfragmented linear chunk quantum via ``pcm_convert_as_linear``, and dispatches the vectorized conversion kernel. + +Through this cohesive architecture, Sound Open Firmware guarantees optimal mathematical fidelity, bulletproof boundary safety, and minimal cycle consumption across all audio format translations. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index b48f6978..6aa85f88 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -55,7 +55,7 @@ Audio Processing Modules & Algorithms * :ref:`smart_amp` (High-level architecture; also see upstream `Smart Amp README `_) * :ref:`sound_dose` (High-level architecture; also see upstream `Sound Dose README `_) * :ref:`copier_mux_selector` (High-level architecture; also see upstream `Copier README `_, `Mux README `_ & `Selector README `_) -* `PCM Format Converter `_ +* :ref:`pcm_converter` (High-level architecture; also see upstream `PCM converter README `_) .. _algorithm-specific-information: @@ -99,6 +99,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/smart_amp firmware/sound_dose firmware/copier_mux_selector + firmware/pcm_converter rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 27f8e45232af7868dddcebe8fbe0dc7883c03020 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 11:29:54 +0100 Subject: [PATCH 22/64] doc: developer_guides: add comprehensive real-time noise reduction (rtnr) architecture guide Author a comprehensive, modern architectural guide for the Sound Open Firmware (SOF) Real-Time Noise Reduction (RTNR) subsystem. Key architectural concepts and sections: - Principles of acoustic noise reduction: stationary background hum, non-stationary transient interference, diffuse ambient babble, and musical noise suppression trade-offs. - Mathematical foundations: Short-Time Spectral Analysis (STFT), recursive PSD estimation, minimum statistics noise floor tracking, decision-directed a priori/a posteriori SNR estimation, Wiener filtering, and spectral floor clamping. - Component architecture: audio_stream_rtnr circular buffer adapter, format dispatch table (S16_LE, S24_4LE, S32_LE), sub-block quantum processing (RTNR_BLK_LENGTH = 4), and internal FIFO queueing. - Dual sampling rate operation: 16 kHz voice communications/AI path (speech core formant optimization, low MIPS) vs 48 kHz high-fidelity full audio bandwidth media capture path. - Runtime configuration: dual IPC3/IPC4 parameter delivery, ALSA mixer switch control, preset blobs (ID 12345678) via comp_data_blob_handler, and zero-overhead bypass gate. - Open-source CI stub (rtnr_stub.c, cir_buf_copy passthrough) vs commercial production LLEXT dynamic module packaging. - End-to-end multi-stage capture audio pipeline: DMIC -> DC Blocker -> TDFB -> AEC -> RTNR -> Copier -> Host Recording & MFCC/TFLM AI. - Seven native vector Graphviz SVG diagrams (Figures 188 through 194). - Integrated into developer_guides/index.rst toctree and modules list. Signed-off-by: Liam Girdwood --- developer_guides/firmware/rtnr.rst | 729 +++++++++++++++++++++++++++++ developer_guides/index.rst | 3 +- 2 files changed, 731 insertions(+), 1 deletion(-) create mode 100644 developer_guides/firmware/rtnr.rst diff --git a/developer_guides/firmware/rtnr.rst b/developer_guides/firmware/rtnr.rst new file mode 100644 index 00000000..66affd04 --- /dev/null +++ b/developer_guides/firmware/rtnr.rst @@ -0,0 +1,729 @@ +.. _rtnr: + +============================================== +Real-Time Noise Reduction (RTNR) Architecture +============================================== + +Sound Open Firmware (SOF) provides an integrated, real-time noise reduction subsystem designed to isolate acoustic speech from adverse ambient environments. Embedded microphones in modern mobile, desktop, automotive, and wearable devices are continuously exposed to acoustic noise: stationary background hum (e.g. computer fans, HVAC air handlers, server rack turbulence, and 50/60 Hz electrical mains hum) and non-stationary transient interference (e.g. keyboard clicks, table thumps, wind turbulence, and diffuse background babble). + +The **Real-Time Noise Reduction (RTNR)** component (``src/audio/rtnr/``, enabled via ``CONFIG_COMP_RTNR``) performs high-performance spectral noise estimation and adaptive suppression directly within the DSP audio pipeline. Operating downstream of spatial beamformers and acoustic echo cancellers, RTNR enhances the **Signal-to-Noise Ratio (SNR)** and boosts the **Speech Intelligibility Index (SII)** before speech streams reach human listeners or on-device keyword spotters (such as TensorFlow Lite for Microcontrollers). + +.. graphviz:: + :caption: Figure 188: SOF Noise Reduction Subsystem Architecture & Multi-Stage Acoustic Chain + :alt: High-level architectural block diagram showing the multi-stage acoustic capture pipeline in Sound Open Firmware. + + digraph rtnr_pipeline_overview { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_dmic_in { + label = "Acoustic Input Front-End"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + dmic [label="DMIC Hardware Array\n(1 to 4 PDM Channels)\nRaw Acoustic Input", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.8]; + dcblock [label="DC Blocker (dcblock)\nIIR High-Pass (15-20 Hz)\nEliminates ADC DC Offsets", fillcolor="#E2E8F0", color="#4A5568", penwidth=1.5]; + } + + subgraph cluster_spatial_linear { + label = "Spatial & Echo Processing"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#FFFFFF"; + + tdfb [label="Fixed Beamformer (TDFB)\nDelay-and-Sum Matrix\nSpatial Directivity", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.5]; + aec [label="Echo Cancellation (AEC)\nSpeaker Reference Loopback\nRemoves Far-End Voice", fillcolor="#FED7D7", color="#C53030", penwidth=1.5]; + } + + subgraph cluster_spectral_nr { + label = "Real-Time Spectral Enhancement"; + style = "filled,rounded"; + color = "#C6F6D5"; + fillcolor = "#F0FFF4"; + + rtnr [label="Real-Time Noise Reduction (RTNR)\nAdaptive Spectral Subtraction\nWiener Filter Suppression\nStationary & Transient Attenuation", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + } + + subgraph cluster_consumers { + label = "Audio & AI Consumers"; + style = "filled,rounded"; + color = "#E9D8FD"; + fillcolor = "#F7FAFC"; + + host_cap [label="Host Telephony / WebRTC\nClean Voice Uplink\nHigh Speech Intelligibility", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + voice_ai [label="Voice AI / Keyword Spotter\nMFCC Feature Extraction\nTFLM Neural Wake-Word", fillcolor="#FAF5FF", color="#6B46C1", penwidth=1.5]; + } + + dmic -> dcblock [label="PDM Streams", color="#3182CE", penwidth=1.6]; + dcblock -> tdfb [label="DC-Free PCM", color="#4A5568", penwidth=1.6]; + tdfb -> aec [label="Directional Beam", color="#B7791F", penwidth=1.6]; + aec -> rtnr [label="Echo-Free Speech", color="#C53030", penwidth=1.6]; + rtnr -> host_cap [label="Clean Voice (PCM)", color="#22543D", penwidth=1.8]; + rtnr -> voice_ai [label="Noise-Suppressed Stream", color="#6B46C1", penwidth=1.8]; + } + +--- + +Principles of Acoustic Noise Reduction in Embedded Audio +========================================================= + +Noise suppression algorithms operating on embedded audio DSPs must address distinct classes of environmental noise under strict computational constraints: + +1. **Stationary Background Noise**: + Acoustic energy characterized by slowly varying statistical properties over extended durations (e.g. cooling fans, road tire rumble, air conditioning roar, and transformer hum). Because stationary noise maintains a quasi-static power spectral density across consecutive audio frames, it can be tracked and estimated during speech pauses without requiring a secondary physical reference microphone. + +2. **Non-Stationary Transient Noise**: + Acoustic impulses characterized by rapid energy spikes, short temporal durations, and broad frequency signatures (e.g. mechanical keyboard strokes, mouse clicks, pencil taps, door slams, and crockery clatter). Suppressing transient noise requires fast spectral tracking, dynamic spectral flooring, and transient detection heuristics to avoid speech distortion. + +3. **Diffuse Ambient Babble**: + The overlapping acoustic energy of multiple independent distant speakers in public spaces (e.g. coffee shops, airport terminals, open-plan offices). When spatial beamformers (TDFB) attenuate off-axis acoustic wavefronts, RTNR acts as the secondary defense line, attenuating residual diffuse energy that leaks into the primary speech beam. + +4. **Speech Distortion vs Noise Attenuation Trade-Off**: + Aggressive noise suppression can introduce auditory artifacts known as **musical noise** (isolated sinusoidal tone bursts resulting from random fluctuations in spectral magnitude estimates). SOF's noise reduction architecture enforces strict spectral floors and smoothed gain updates to preserve natural vocal timbre while attenuating background interference by 12 dB to 25 dB. + +--- + +Mathematical Foundations: Spectral Subtraction & Wiener Filtering +================================================================== + +The core mathematical model of single-channel noise reduction operates in the short-time spectral domain. A noisy discrete-time microphone signal :math:`y[n]` is modeled as the linear sum of an uncorrupted speech signal :math:`x[n]` and an additive acoustic noise process :math:`d[n]`: + +.. math:: + + y[n] = x[n] + d[n] + +Short-Time Spectral Decomposition +--------------------------------- + +The input signal is partitioned into overlapping analysis frames of length :math:`K` using a window function :math:`w[n]` (such as a Hanning or Hamming window) with frame hop size :math:`R`. Applying the Discrete Fourier Transform (DFT) yields the short-time complex spectrum: + +.. math:: + + Y(k, m) = \sum_{n=0}^{K-1} y[m R + n] \, w[n] \, e^{-j \frac{2\pi k n}{K}} = X(k, m) + D(k, m) + +where :math:`k \in [0, K-1]` denotes the frequency bin index and :math:`m` denotes the discrete time frame index. In polar coordinates: + +.. math:: + + Y(k, m) = |Y(k, m)| \, e^{j \theta_Y(k, m)} + +Because human auditory perception is predominantly sensitive to spectral magnitude rather than short-time phase, noise reduction estimates the speech magnitude :math:`|\hat{X}(k, m)|` and recombines it with the original noisy phase :math:`\theta_Y(k, m)`: + +.. math:: + + \hat{X}(k, m) = G(k, m) \cdot |Y(k, m)| \, e^{j \theta_Y(k, m)} + +where :math:`G(k, m) \in [0, 1]` is the spectral gain filter. + +Recursive Power Spectral Density & Noise Floor Estimation +--------------------------------------------------------- + +The instantaneous noisy signal power spectrum is smoothed recursively across frames: + +.. math:: + + P_{YY}(k, m) = \alpha P_{YY}(k, m-1) + (1 - \alpha) |Y(k, m)|^2 + +where :math:`\alpha \in [0.8, 0.95]` is a smoothing coefficient. + +Noise power spectral density :math:`\hat{P}_{DD}(k, m)` is tracked continuously. In minimum statistics algorithms, the noise floor is estimated by tracking the temporal minimum of the smoothed power spectrum over a sliding window of :math:`M` past frames: + +.. math:: + + \hat{P}_{DD}(k, m) = \min_{m' \in [m-M, m]} P_{YY}(k, m') \cdot B_{\min} + +where :math:`B_{\min}` is a bias compensation factor correcting for the statistical minimum of a Chi-square distributed random variable. + +A Posteriori and A Priori SNR Estimation +---------------------------------------- + +The quality of the gain computation depends on two signal-to-noise ratio metrics: + +1. **A Posteriori SNR** (:math:`\gamma(k, m)`): Represents the ratio of total observed energy to estimated noise power in the current frame: + + .. math:: + + \gamma(k, m) = \frac{|Y(k, m)|^2}{\hat{P}_{DD}(k, m)} + +2. **A Priori SNR** (:math:`\xi(k, m)`): Represents the estimated ratio of clean speech power to noise power. To prevent musical noise, SOF algorithms apply the **Decision-Directed (DD)** approach: + + .. math:: + + \xi(k, m) = \beta \frac{|\hat{X}(k, m-1)|^2}{\hat{P}_{DD}(k, m-1)} + (1 - \beta) \max\left(\gamma(k, m) - 1, \, 0\right) + + where :math:`\beta \approx 0.98` is a weighting factor that balances historical speech energy against immediate frame observations. + +Wiener Gain Computation & Spectral Floor Clamping +------------------------------------------------- + +Under the minimum mean-square error (MMSE) criterion, the optimal linear filter is the Wiener gain function: + +.. math:: + + G_{\text{Wiener}}(k, m) = \frac{\xi(k, m)}{1 + \xi(k, m)} + +To eliminate musical noise in deep noise regions where :math:`\xi(k, m) \to 0`, the computed gain is clamped against an adjustable **spectral floor** (:math:`G_{\min}`): + +.. math:: + + G(k, m) = \max\left( G_{\text{Wiener}}(k, m), \, G_{\min} \right) + +where :math:`G_{\min} = 10^{\frac{-\text{MaxAttenuation (dB)}}{20}}` (typically configured between :math:`-12 \text{ dB}` and :math:`-24 \text{ dB}`). + +.. graphviz:: + :caption: Figure 189: Mathematical Principles of Spectral Subtraction, Noise Floor Estimation & Wiener Filtering + :alt: Mathematical dataflow diagram depicting the STFT analysis, recursive PSD estimation, SNR tracking, Wiener gain computation, and synthesis filterbank. + + digraph mathematical_wiener_pipeline { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_stft_analysis { + label = "1. Time-Frequency Analysis"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + sig_in [label="Noisy Input Signal y[n]\n(Time-Domain PCM)", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + windowing [label="Analysis Window w[n]\n(Overlapping Hanning/Hamming Frames)\nHop Size: R samples", fillcolor="#FFFFFF", color="#CBD5E0", penwidth=1.2]; + fft_core [label="Short-Time Fourier Transform (STFT)\nY(k, m) = |Y(k, m)| exp(j θ_Y)", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + + sig_in -> windowing -> fft_core; + } + + subgraph cluster_psd_snr { + label = "2. Spectral PSD & SNR Estimation"; + style = "filled,rounded"; + color = "#E2E8F0"; + fillcolor = "#FFFFFF"; + + psd_calc [label="Smoothed Signal PSD\nP_YY(k, m) = α P_YY(k, m-1) + (1-α)|Y(k,m)|²", fillcolor="#FFFFFF", color="#4A5568", penwidth=1.2]; + noise_est [label="Minimum Statistics Noise Tracker\nContinuous Background Estimation\nP_DD(k, m) = min(P_YY)", fillcolor="#FED7D7", color="#C53030", penwidth=1.6]; + snr_calc [label="Decision-Directed SNR Estimator\nA Posteriori: γ(k, m) = |Y|² / P_DD\nA Priori: ξ(k, m) = β |X̂|² / P_DD + (1-β) max(γ-1, 0)", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + + fft_core -> psd_calc; + psd_calc -> noise_est; + noise_est -> snr_calc; + fft_core -> snr_calc; + } + + subgraph cluster_gain_clamping { + label = "3. Wiener Gain & Spectral Floor"; + style = "filled,rounded"; + color = "#C6F6D5"; + fillcolor = "#F0FFF4"; + + wiener_gain [label="Wiener Gain Computation\nG_raw = ξ(k, m) / (1 + ξ(k, m))", fillcolor="#FFFFFF", color="#276749", penwidth=1.2]; + floor_clamp [label="Spectral Floor Clamping\nG(k, m) = max(G_raw, G_min)\n(Eliminates Musical Noise)", fillcolor="#9AE6B4", color="#22543D", penwidth=1.8]; + spec_mult [label="Spectral Magnitude Modulation\n|X̂(k, m)| = G(k, m) · |Y(k, m)|\nPreserves Original Phase θ_Y", fillcolor="#C6F6D5", color="#276749", penwidth=1.6]; + + snr_calc -> wiener_gain -> floor_clamp; + fft_core -> spec_mult; + floor_clamp -> spec_mult; + } + + subgraph cluster_synthesis { + label = "4. Time-Domain Synthesis"; + style = "filled,rounded"; + color = "#E9D8FD"; + fillcolor = "#F7FAFC"; + + ifft_core [label="Inverse STFT (IFFT)\nx̂_m[n] = IFFT{ |X̂(k, m)| exp(j θ_Y) }", fillcolor="#FAF5FF", color="#6B46C1", penwidth=1.5]; + ola [label="Overlap-Add Synthesis (OLA)\nx̂[n] = Σ_m x̂_m[n - mR]\nPerfect Reconstruction Windowing", fillcolor="#FFFFFF", color="#CBD5E0", penwidth=1.2]; + sig_out [label="Clean Speech Output x̂[n]\n(Noise-Attenuated PCM Stream)", fillcolor="#C6F6D5", color="#22543D", penwidth=1.8]; + + spec_mult -> ifft_core -> ola -> sig_out; + } + } + +--- + +RTNR Subsystem Architecture & Stream Adaptation +================================================ + +The SOF RTNR subsystem is implemented as a standard audio processing module conforming to the ``processing_module`` and ``module_interface`` lifecycle contracts. Because advanced noise suppression libraries frequently originate as proprietary vendor IP or independent research frameworks (such as Realtek RTKMA or Intelligo IGO NR), SOF decouples the pipeline buffer infrastructure from the underlying algorithmic core. + +The ``audio_stream_rtnr`` Buffer Bridge +--------------------------------------- + +Standard SOF modules interact directly with ``sof_source`` and ``sof_sink`` circular ring buffers. To insulate external noise reduction libraries from SOF internal structures while strictly enforcing circular buffer boundary safety, RTNR introduces the ``audio_stream_rtnr`` descriptor (``include/sof/audio/rtnr/rtnr.h``): + +* **Runtime Circular Buffer Pointers**: + Maintains base address (``addr``), upper limit (``end_addr``), buffer size in bytes (``size``), active read pointer (``r_ptr``), and active write pointer (``w_ptr``). +* **Quantized Window Management**: + During each processing period, ``rtnr_source_get_stream`` and ``rtnr_sink_get_stream`` acquire the active circular buffer window via ``source_get_data`` and ``sink_get_buffer``. The descriptor sets ``avail`` to the exact frame window granted for this cycle, artificially positioning the opposing pointer (e.g. write pointer for the source buffer) at the boundary limit: + + .. math:: + + w\_ptr = \text{cir\_buf\_wrap}(data\_ptr + bytes, \, addr, \, end\_addr) + + This architectural abstraction ensures that the algorithm operates with full awareness of circular buffer wrapping without ever accessing uncommitted or out-of-bounds DSP memory. +* **Overrun & Underrun Gating**: + Exposes explicit direction-relevant fault flags (``underrun_permitted`` on input, ``overrun_permitted`` on output) ensuring robust handling of pipeline rate anomalies. + +Format Function Map Architecture +-------------------------------- + +RTNR provides zero-copy sample processing across standard audio formats through a static dispatch table (``rtnr_fnmap``): + +.. csv-table:: RTNR Frame Format Dispatch Matrix + :header: "IPC Frame Format Enum", "C Function Binding", "Sample Bit Depth", "Container Alignment" + :widths: 28, 30, 18, 24 + + "``SOF_IPC_FRAME_S16_LE``", "``rtnr_s16_default``", "16 bits", "16-bit packed" + "``SOF_IPC_FRAME_S24_4LE``", "``rtnr_s24_default``", "24 bits", "32-bit word (LSB-aligned)" + "``SOF_IPC_FRAME_S32_LE``", "``rtnr_s32_default``", "32 bits", "32-bit word (Full-scale Q1.31)" + +During the ``prepare`` lifecycle phase, ``rtnr_find_func`` inspects the sink stream format and binds ``cd->rtnr_func``. If the sink format does not match any compiled entry, the module aborts initialization with ``-EINVAL``, preventing runtime execution faults. + +Sub-Block Quantum & Internal FIFO Queueing +------------------------------------------ + +Modern spectral algorithms process audio in fixed-size mathematical blocks (such as 64, 128, or 256 samples) corresponding to the underlying FFT analysis size. However, embedded DSP pipelines run on arbitrary host-scheduling period quanta (e.g. 1 ms periods yielding 16 frames at 16 kHz, or 48 frames at 48 kHz). + +To bridge this scheduling granularity mismatch: + +1. **Sub-Block Quantum**: RTNR defines ``RTNR_BLK_LENGTH = 4`` frames (masked by ``RTNR_BLK_LENGTH_MASK``). +2. **First-Copy Synchronization**: The module invokes ``RTKMA_API_First_Copy``, preparing internal FIFO buffers and flushing residual state. +3. **Chunked Ingestion**: Samples are transferred from the SOF circular stream into the library's internal FIFO queues via the format-specialized dispatch function. +4. **Execution Dispatch**: The core algorithm executes ``RTKMA_API_Process``, consuming queued frames, performing spectral subtraction, and generating filtered time-domain speech. +5. **Atomic Stream Commit**: Upon kernel completion, processed frames are committed to the downstream sink buffer via ``sink_commit_buffer``, while source data is released via ``source_release_data``. + +.. graphviz:: + :caption: Figure 190: RTNR Component Internal Architecture & audio_stream_rtnr Circular Buffer Adapter + :alt: Internal component architecture showing circular buffer wrapping, audio_stream_rtnr bridge, FIFO queues, and library API dispatch. + + digraph rtnr_internal_architecture { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_sof_pipeline { + label = "SOF Pipeline Circular Buffers"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + src_buf [label="Source Circular Buffer\n(sof_source)\nPeriod: 1ms (16/48 frames)", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + snk_buf [label="Sink Circular Buffer\n(sof_sink)\nPeriod: 1ms (16/48 frames)", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + } + + subgraph cluster_adapter_layer { + label = "audio_stream_rtnr Abstraction Layer"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#FFFFFF"; + + src_desc [label="Source Adapter (sources_stream[0])\naddr, end_addr, size\nr_ptr, w_ptr, avail, free\nWraps Modulo Arithmetic", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.4]; + snk_desc [label="Sink Adapter (sink_stream)\naddr, end_addr, size\nw_ptr, r_ptr, avail, free\nOverrun/Underrun Gating", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.4]; + } + + subgraph cluster_rtnr_core { + label = "RTNR Module Core (rtnr.c)"; + style = "filled,rounded"; + color = "#C6F6D5"; + fillcolor = "#F0FFF4"; + + fn_dispatch [label="Format Dispatcher\nrtnr_fnmap[fmt]\n(S16, S24_4LE, S32_LE)", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + fifo_in [label="Internal Input Queue\nSub-Block Chunking\n(RTNR_BLK_LENGTH = 4)", fillcolor="#FFFFFF", color="#276749", penwidth=1.2]; + lib_core [label="Algorithm Core\nRTKMA_API_Process()\nSpectral Subtraction\nNoise Floor Estimation\nWiener Attenuation", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + fifo_out [label="Internal Output Queue\nTime-Domain Speech\nOverlapped Reconstruction", fillcolor="#FFFFFF", color="#276749", penwidth=1.2]; + + fn_dispatch -> fifo_in -> lib_core -> fifo_out; + } + + src_buf -> src_desc [label="source_get_data()", color="#3182CE", penwidth=1.5]; + src_desc -> fn_dispatch [label="Format Cast", color="#4A5568", penwidth=1.5]; + fifo_out -> snk_desc [label="Extract Frames", color="#276749", penwidth=1.5]; + snk_desc -> snk_buf [label="sink_commit_buffer()", color="#3182CE", penwidth=1.5]; + } + +--- + +Dual Sampling Rate & Acoustic Domain Optimization +================================================== + +The RTNR subsystem natively supports two standard sampling frequencies (:math:`16 \text{ kHz}` and :math:`48 \text{ kHz}`), strictly enforced during parameter validation: + +.. math:: + + f_s \in \{16000, \, 48000\} \text{ Hz} + +Any pipeline attempting to instantiate RTNR at an unsupported rate (e.g. 44.1 kHz, 96 kHz) is immediately rejected by ``rtnr_check_config_validity`` with ``-EINVAL``. + +The 16 kHz Voice Communications & AI Domain +-------------------------------------------- + +* **Primary Application**: Telephony uplinks (VoIP, Cellular AMR-WB / G.722), WebRTC voice chat, and automatic speech recognition (ASR) front-ends. +* **Algorithmic Advantage**: Human speech energy is predominantly concentrated below 8 kHz. Downsampling capture audio to 16 kHz restricts spectral processing to the :math:`0 \dots 8 \text{ kHz}` band, halving FFT bin counts and drastically reducing DSP instruction cycles (MIPS) and memory bandwidth. +* **Scheduling Characteristics**: Under a standard 1 ms pipeline scheduling tick, each period processes exactly 16 frames. At 4 frames per sub-block, each period decomposes cleanly into exactly 4 processing iterations with zero fractional sample jitter. + +The 48 kHz High-Fidelity Audio Domain +-------------------------------------- + +* **Primary Application**: Studio voice recording, professional content creation, and broadcast video conferencing. +* **Algorithmic Advantage**: Extends noise suppression across the full human audible frequency spectrum (:math:`0 \dots 24 \text{ kHz}`), eliminating high-frequency fan hiss, electrical coil whine, and air turbulence while preserving voice overtones and high-frequency sibilants. +* **Scheduling Characteristics**: Under a 1 ms scheduling tick, each period contains 48 frames, decomposing into 12 sub-blocks. + +.. graphviz:: + :caption: Figure 191: Dual Sampling Rate Operation (16 kHz Voice vs 48 kHz High-Fidelity Capture Paths) + :alt: Comparison of 16 kHz voice communications path versus 48 kHz full-band media capture path. + + digraph dual_sample_rate { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_16k { + label = "16 kHz Voice & Telephony Path (Low Power / Voice AI)"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + in_16k [label="16 kHz Microphone Stream\nBandwidth: 0 - 8 kHz (Speech Core)", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + sched_16k [label="1 ms Period: 16 Frames\nChunking: 4 sub-blocks × 4 frames\nLow DSP MIPS & Memory Footprint", fillcolor="#FFFFFF", color="#4A5568", penwidth=1.2]; + rtnr_16k [label="RTNR 16 kHz Engine\nTargeted Formant Enhancement\nZero Fractional Jitter", fillcolor="#C6F6D5", color="#276749", penwidth=1.6]; + out_16k [label="Telephony & Voice AI\nWebRTC / G.722 / TFLM Wake-Word", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + + in_16k -> sched_16k -> rtnr_16k -> out_16k; + } + + subgraph cluster_48k { + label = "48 kHz High-Fidelity Capture Path (Full Bandwidth)"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#FFFFFF"; + + in_48k [label="48 kHz Microphone Stream\nBandwidth: 0 - 24 kHz (Full Audio Band)", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.6]; + sched_48k [label="1 ms Period: 48 Frames\nChunking: 12 sub-blocks × 4 frames\nFull Spectrum Noise Estimation", fillcolor="#FFFFFF", color="#4A5568", penwidth=1.2]; + rtnr_48k [label="RTNR 48 kHz Engine\nHigh-Frequency Hiss Attenuation\nBroadcast Quality Speech", fillcolor="#C6F6D5", color="#276749", penwidth=1.6]; + out_48k [label="Studio & Media Recording\nHigh-Resolution Video Teleconferencing", fillcolor="#FED7D7", color="#C53030", penwidth=1.6]; + + in_48k -> sched_48k -> rtnr_48k -> out_48k; + } + } + +--- + +Runtime Configuration, Preset Blobs & IPC Delivery +=================================================== + +The RTNR component supports live, dynamic reconfiguration without stopping or restarting audio streaming. This capability allows user-space applications (such as desktop audio control panels or system audio daemons) to adjust noise suppression aggressiveness, switch acoustic presets, or toggle bypass when transitioning between quiet office environments and noisy outdoor spaces. + +Dual IPC Architecture Support +----------------------------- + +SOF provides dual-protocol support for parameter delivery: + +* **IPC3 Protocol**: + Uses standard control commands (``SOF_CTRL_CMD_SWITCH`` for enabling/disabling processing and ``SOF_CTRL_CMD_BINARY`` for passing configuration structures). +* **IPC4 Protocol**: + Uses the modern Intel IPC4 messaging envelope. Control commands arrive via ``SOF_IPC4_SWITCH_CONTROL_PARAM_ID`` (for switch controls) or the Large Config Set mechanism targeting parameter IDs: + + * ``SOF_RTNR_CONFIG`` (ID ``0``): Basic runtime parameters. + * ``SOF_RTNR_DATA`` (ID ``1``): Large coefficient and model configuration blobs. + +The Configuration Payload Structure +----------------------------------- + +The basic control structure is defined in ``include/user/rtnr.h``: + +.. code-block:: c + + struct sof_rtnr_params { + int32_t enabled; /* 1 to enable RTNR, 0 for bypass */ + uint32_t sample_rate; /* 16000 or 48000 Hz */ + int32_t reserved; + } __attribute__((packed, aligned(4))); + + struct sof_rtnr_config { + uint32_t size; /* Size of entire structure */ + uint32_t reserved[4]; + struct sof_rtnr_params params; + } __attribute__((packed, aligned(4))); + +Asynchronous Preset Blob Ingestion via ``data_blob`` +---------------------------------------------------- + +For complex tuning models requiring multi-kilobyte coefficient matrices, RTNR integrates SOF's ``comp_data_blob_handler``: + +1. **Fragmented Ingestion**: Large configuration blobs (up to 10 KB) sent by the host driver across multiple IPC fragments are assembled transparently by ``comp_data_blob_set``. +2. **Preset Identification**: The blob handler tags valid parameter sets with ``RTNR_DATA_ID_PRESET = 12345678``. +3. **Safe Inter-Period Application**: To prevent audio clicks or thread race conditions, new configuration blobs are not applied mid-frame. Instead, ``cd->reconfigure = true`` flags the processing loop. At the start of the subsequent period, ``rtnr_reconfigure`` calls ``RTKMA_API_Set`` atomically between frame processing cycles. + +Autonomous Zero-Overhead Passthrough +------------------------------------ + +When disabled via mixer switch or IPC command (``cd->process_enable == false``), RTNR bypasses the spectral processing engine entirely. Rather than executing vector FFTs and inverse filterbanks, it invokes ``source_to_sink_copy``: + +.. math:: + + \text{copy\_bytes} = \text{frames} \times \text{frame\_bytes} + +This bypass executes via branchless circular buffer memory moves, reducing component DSP power consumption to near zero while maintaining uninterrupted audio stream continuity. + +.. graphviz:: + :caption: Figure 192: Runtime Configuration, Preset Blobs & IPC3/IPC4 Parameter Delivery Lifecycle + :alt: Diagram of host driver IPC delivery passing switch controls and binary tuning blobs to the RTNR component. + + digraph rtnr_config_lifecycle { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_host { + label = "Host Driver / User-Space"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + alsa_mixer [label="ALSA Mixer Control\nSwitch: 'rtnr_enable_X'\n(fc: 0=Bypass, 1=Enable)", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + sof_ctl [label="sof-ctl / Tuning Tool\nBinary Parameter Blob\nPreset Data & Coefficients", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + } + + subgraph cluster_ipc_dispatch { + label = "SOF IPC Layer (IPC3 / IPC4)"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#FFFFFF"; + + ipc_switch [label="Switch Control\nIPC3: SOF_CTRL_CMD_SWITCH\nIPC4: SWITCH_CONTROL_PARAM_ID", fillcolor="#FFFFFF", color="#4A5568", penwidth=1.2]; + ipc_blob [label="Binary Config / Large Set\nSOF_RTNR_CONFIG (ID 0)\nSOF_RTNR_DATA (ID 1)", fillcolor="#FFFFFF", color="#4A5568", penwidth=1.2]; + } + + subgraph cluster_rtnr_runtime { + label = "RTNR Runtime Engine"; + style = "filled,rounded"; + color = "#C6F6D5"; + fillcolor = "#F0FFF4"; + + blob_handler [label="comp_data_blob_handler\nMulti-Fragment Assembly\nPreset ID: 12345678", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + reconfig_flag [label="Atomic Flag: cd->reconfigure\nDeferred Inter-Period Update", fillcolor="#FFFFFF", color="#276749", penwidth=1.2]; + rtkma_set [label="RTKMA_API_Set()\nApplies New Noise Curves\nWithout Stream Interruption", fillcolor="#9AE6B4", color="#22543D", penwidth=1.8]; + passthrough_gate [label="Bypass Mode Gate\nprocess_enable == 0\nsource_to_sink_copy()", fillcolor="#FED7D7", color="#C53030", penwidth=1.6]; + } + + alsa_mixer -> ipc_switch -> passthrough_gate; + sof_ctl -> ipc_blob -> blob_handler -> reconfig_flag -> rtkma_set; + } + +--- + +Open-Source CI Stubs & LLEXT Dynamic Vendor Module Architecture +=============================================================== + +A major engineering challenge in modern audio firmware is balancing open-source software integrity against proprietary vendor algorithm IP. High-performance commercial noise reduction libraries (such as Realtek RTKMA or Intelligo IGO NR) are distributed as pre-compiled static archives (``libSOF_RTK_MA_API.a``, ``libigonr.a``) compiled for specific DSP core architectures. + +SOF solves this challenge through a dual-pronged architectural approach: the **Open-Source CI Test Stub** and the **LLEXT Dynamic Module Loader**. + +The Open-Source CI Stub Architecture (``rtnr_stub.c``) +------------------------------------------------------- + +To ensure that the open-source community, continuous integration (CI) test matrices, and automated regression frameworks can build and test SOF without requiring proprietary vendor binaries, SOF provides ``COMP_RTNR_STUB``: + +* **Complete API Emulation**: + Implements all entry points of the vendor interface: + + .. code-block:: c + + void *RTKMA_API_Context_Create(int sample_rate); + void RTKMA_API_Context_Free(void *Context); + void RTKMA_API_Prepare(void *Context); + void RTKMA_API_First_Copy(void *Context, int SampleRate, int MicCh); + void RTKMA_API_Process(void *Context, _Bool has_ref, int SampleRate, int MicCh); + int RTKMA_API_Set(void *Context, const void *pParameters, int size, unsigned int IDs); + void RTKMA_API_S16_Default(...); + void RTKMA_API_S24_Default(...); + void RTKMA_API_S32_Default(...); + +* **Circular Buffer Passthrough**: + Rather than performing dummy mathematical operations, the stub executes ``rtnr_stub_passthrough``, invoking ``cir_buf_copy`` across the ``audio_stream_rtnr`` descriptors: + + .. math:: + + \text{bytes} = \text{frames} \times \text{channels} \times \text{sizeof(sample)} + + This exercises full circular buffer wrapping logic, pipeline scheduling, IPC control handling, and topology parsing during CI testing with zero external dependencies. + +Loadable Linkable Extensions (LLEXT) Dynamic Packaging +------------------------------------------------------ + +In production distributions, RTNR can be built as an independent, dynamically loadable module (``CONFIG_COMP_RTNR_MODULE``) leveraging Zephyr's **LLEXT** framework: + +* **Module Manifest**: + Declares the module metadata, UUID, entry interface, and memory footprint: + + .. code-block:: c + + static const struct sof_man_module_manifest mod_manifest __section(".module") __used = + SOF_LLEXT_MODULE_MANIFEST("RTNR", &rtnr_interface, 1, SOF_REG_UUID(rtnr), 40); + +* **Decoupled Delivery**: + The core SOF base firmware image (``sof.ri``) is built and signed without proprietary code. Commercial OEMs compile RTNR into an independent relocatable ELF object (``rtnr.llext``). The host driver loads this module dynamically into DSP SRAM only when an audio pipeline containing the RTNR widget is created. + +.. graphviz:: + :caption: Figure 193: Open-Source CI Stub vs Vendor Binary LLEXT Modular Dynamic Linking + :alt: Architecture diagram comparing the open-source test stub with the dynamically loadable LLEXT vendor binary module. + + digraph rtnr_build_dichotomy { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_upstream_ci { + label = "Upstream Open-Source & Automated CI Environment"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + ci_build [label="CONFIG_COMP_RTNR_STUB=y\nBuilt on Public GitHub Actions / CI", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + stub_impl [label="Open-Source Stub (rtnr_stub.c)\nEmulates RTKMA_API Lifecycle\nClean Mock Allocation (RTNR_STUB_CONTEXT_SIZE)", fillcolor="#FFFFFF", color="#4A5568", penwidth=1.2]; + cir_passthrough [label="cir_buf_copy() Engine\nFull Circular Buffer Verification\nZero External Dependencies", fillcolor="#C6F6D5", color="#276749", penwidth=1.6]; + + ci_build -> stub_impl -> cir_passthrough; + } + + subgraph cluster_production_vendor { + label = "Commercial OEM & Production Distribution"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#FFFFFF"; + + prod_build [label="CONFIG_COMP_RTNR_MODULE=y\nDynamic LLEXT Module Target", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.5]; + vendor_libs [label="Proprietary Vendor Archives\nlibSOF_RTK_MA_API.a\nlibSuite_rename.a / libNet.a / libPreset.a", fillcolor="#FED7D7", color="#C53030", penwidth=1.6]; + llext_manifest [label="LLEXT Manifest & Dynamic Relocation\nSOF_LLEXT_MODULE_MANIFEST('RTNR')\nLoaded on Demand into DSP SRAM", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + prod_dsp [label="Active DSP Execution\nOptimized Vector SIMD Math\nProprietary Acoustic Filtering", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + + prod_build -> vendor_libs -> llext_manifest -> prod_dsp; + } + } + +--- + +End-to-End Multi-Stage Capture Audio Pipeline +============================================== + +In production device topologies (such as laptops, smart speakers, and conferencing soundbars), RTNR operates as an indispensable stage in an integrated multi-module capture pipeline. + +Signal Flow Walkthrough +----------------------- + +1. **Digital Microphone Capture**: + A 2-channel or 4-channel digital microphone (DMIC) array samples acoustic pressure at 48 kHz. +2. **DC Bias Elimination** (``dcblock``): + The raw PCM samples pass through a second-order IIR DC Blocker (cutoff at 15–20 Hz), removing hardware ADC DC offsets and physical chassis rumble. +3. **Spatial Filtering & Beamforming** (``tdfb``): + A Time-Domain Fixed Beamformer applies spatial FIR delay-and-sum filtering, steering an acoustic sensitivity beam directly toward the speaker's mouth while attenuating off-axis noise sources. +4. **Acoustic Echo Cancellation (AEC)**: + An echo cancellation engine (e.g. Google RTC or WebRTC AEC) ingests the beamformed speech on its primary input and the speaker playback reference on its secondary input, subtracting acoustic coupling produced by the device's own loudspeakers. +5. **Real-Time Noise Reduction (RTNR)**: + RTNR processes the echo-free single-channel speech stream. It continuously estimates the stationary ambient noise floor (e.g. laptop cooling fan noise) and applies Wiener spectral attenuation to eliminate background hiss without introducing phase distortion. +6. **Downstream Distribution**: + The pristine voice stream is bifurcated via a Copier module: + + * **Host Recording Path**: Delivered over host DMA to teleconferencing applications (Zoom, Teams, WebRTC). + * **Voice AI Path**: Routed through an MFCC feature extractor into a TensorFlow Lite for Microcontrollers (TFLM) neural network for keyword spotting (e.g. "Hey Google" or "Alexa"). + +.. graphviz:: + :caption: Figure 194: End-to-End Capture Audio Graph: DMIC Array, DC Blocker, TDFB, AEC, RTNR & Voice AI + :alt: Comprehensive audio graph showing the end-to-end capture pipeline from DMIC hardware through DC blocker, beamformer, AEC, RTNR, and host/AI consumers. + + digraph e2e_capture_graph { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.18,0.10"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_stage1 { + label = "Stage 1: Hardware Ingestion & Spatial Pre-Processing"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + dmic_in [label="DMIC Hardware Gateway\n4-Channel PDM Capture\n48 kHz @ 32-bit", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.8]; + dcb_node [label="DC Blocker (dcblock)\nRemoves ADC DC Bias\n15-20 Hz High-Pass", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.5]; + tdfb_node [label="Fixed Beamformer (tdfb)\nDelay-and-Sum Matrix\nDirectional Speech Beam", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + + dmic_in -> dcb_node [label="4-ch Raw", color="#3182CE", penwidth=1.6]; + dcb_node -> tdfb_node [label="4-ch Clean DC", color="#4A5568", penwidth=1.6]; + } + + subgraph cluster_stage2 { + label = "Stage 2: Acoustic Echo Cancellation & Real-Time Noise Reduction"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#FFFFFF"; + + spk_ref [label="Speaker Loopback Reference\nPlayback Tap via Copier\n(Far-End Audio)", fillcolor="#FFF5F5", color="#E53E3E", style="dashed,filled", penwidth=1.5]; + aec_node [label="Echo Canceller (AEC)\nGoogle RTC / WebRTC\nSubtracts Speaker Echo", fillcolor="#FED7D7", color="#C53030", penwidth=1.8]; + rtnr_node [label="Real-Time Noise Reduction (RTNR)\nAdaptive Spectral Subtraction\nStationary & Transient Suppression\nWiener Filter Clamping", fillcolor="#9AE6B4", color="#22543D", penwidth=2.2]; + + spk_ref -> aec_node [label="Echo Ref", color="#E53E3E", style="dashed", penwidth=1.5]; + aec_node -> rtnr_node [label="Echo-Free Speech", color="#C53030", penwidth=1.8]; + } + + subgraph cluster_stage3 { + label = "Stage 3: Multi-Pin Stream Splitting & Downstream Consumers"; + style = "filled,rounded"; + color = "#E9D8FD"; + fillcolor = "#F7FAFC"; + + copier_split [label="Copier Fan-Out Splitter\n(Pin 0: Telephony, Pin 1: Voice AI)", fillcolor="#FAF5FF", color="#6B46C1", penwidth=1.8]; + host_dma [label="Host Capture DMA Buffer\nClean Single-Channel Speech\nOS Audio & Teleconferencing", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + mfcc_node [label="MFCC Feature Extractor\nLog Mel-Scale Filterbanks\n2D Spectrogram Tensors", fillcolor="#FAF5FF", color="#6B46C1", penwidth=1.5]; + tflm_node [label="TFLM Neural Network\nKeyword Spotting Engine\nLocal Voice Trigger Detection", fillcolor="#D6BCFA", color="#553C9A", penwidth=1.8]; + + copier_split -> host_dma [label="Pin 0 (Telephony)", color="#3182CE", penwidth=1.6]; + copier_split -> mfcc_node [label="Pin 1 (Voice AI)", color="#6B46C1", penwidth=1.6]; + mfcc_node -> tflm_node [label="Mel Slices", color="#553C9A", penwidth=1.6]; + } + + tdfb_node -> aec_node [label="Primary Speech Beam", color="#B7791F", penwidth=1.8]; + rtnr_node -> copier_split [label="Denoised Pristine Voice", color="#22543D", penwidth=2.0]; + } + +ALSA Topology 2 Configuration +----------------------------- + +In ALSA Topology 2, the RTNR widget is instantiated using ``Object.Widget.rtnr`` (``tools/topology/topology2/include/components/rtnr.conf``): + +.. code-block:: text + + Object.Widget.rtnr."1" { + index 1 + instance 0 + num_input_pins 1 + num_output_pins 1 + num_input_audio_formats 3 + num_output_audio_formats 3 + uuid "34:a3:7c:5c:5d:e1:eb:11:ba:80:02:42:ac:13:00:04" + type "effect" + no_pm "true" + + Object.Control { + mixer."1" { + Object.Base.channel.1 { + name "fc" + shift 0 + } + Object.Base.ops.1 { + name "ctl" + info "volsw" + get 259 + put 259 + } + max 1 + } + } + } + +The widget defines UUID ``34:a3:7c:5c:5d:e1:eb:11:ba:80:02:42:ac:13:00:04`` and binds an ALSA mixer switch control (``fc``, get/put handler ``259``) exposing a standard boolean on/off switch in user-space ALSA mixers (such as ``alsamixer`` or ``amixer``). diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 6aa85f88..6f6f89c4 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -49,7 +49,7 @@ Audio Processing Modules & Algorithms * :ref:`crossover` (High-level architecture; also see upstream `crossover README `_) * :ref:`dcblock` (High-level architecture; also see upstream `dcblock README `_) * :ref:`tdfb` (High-level architecture; also see upstream `tdfb README `_ & tuning guide :ref:`time-domain-fixed-beamformer`) -* `RTNR Noise Reduction `_ +* :ref:`rtnr` (High-level architecture; also see upstream `RTNR README `_) * :ref:`tflm` (High-level architecture; also see upstream `TFLM README `_) * :ref:`mfcc` (High-level architecture; also see upstream `MFCC README `_) * :ref:`smart_amp` (High-level architecture; also see upstream `Smart Amp README `_) @@ -100,6 +100,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/sound_dose firmware/copier_mux_selector firmware/pcm_converter + firmware/rtnr rimage/index.rst firmware/llext_modules firmware/hostless_firmware From a3a68ec7abd8ade17c95cac5b50635285a69f94d Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 11:47:11 +0100 Subject: [PATCH 23/64] doc: developer_guides: add comprehensive kpb and wov architecture guide Author a comprehensive, modern architectural guide for the Sound Open Firmware (SOF) Key Phrase Buffer (KPB) and Wake-on-Voice (WoV) subsystem. Key architectural concepts and sections: - Low-power wake-on-voice principles: Host CPU deep sleep (ACPI S0ix / Modern Standby / S3 / S4) vs DSP autonomous D0ix listening. - The Pre-Roll Dilemma: Acoustic keyword recognition latency and host resume latency, and how circular buffering prevents data loss of spoken activation commands. - KPB Component State Machine: Ten lifecycle states covering creation, buffering, keyword trigger, accelerated draining, host copy hand-off, and reset. - Circular History Ring Buffer: Linked-list buffer chaining, buffer sizing equations (sample width, channels, buffering duration), and backward read pointer calculation. - Dual-Sink Architecture: Real-time selector sink (Pin 0) for on-DSP keyword spotters (TFLM/MFCC) with integrated microphone channel selection (MicSelector bitmask) vs host capture sink (Pin 1). - Accelerated Burst Draining: Asynchronous draining task, synchronized pacing, dynamic interval adjustment (adjust_drain_interval) via DSP wall-clock cycles, and Fast Mode Task (FMT) execution. - Event Notification Framework: IPC3 Notifier callbacks vs IPC4 Asynchronous Message Service (AMS) and ALSA DAPM control sequencing. - End-to-End WoV Audio Pipeline: DMIC array, DC Blocker, KPB, Keyword Spotter, and Host DMA Copier with ALSA Topology 2 configuration. - Seven native vector Graphviz SVG diagrams (Figures 195 through 201). - Integrated into developer_guides/index.rst toctree and modules list. Signed-off-by: Liam Girdwood --- developer_guides/firmware/kpb_wov.rst | 808 ++++++++++++++++++++++++++ developer_guides/index.rst | 2 + 2 files changed, 810 insertions(+) create mode 100644 developer_guides/firmware/kpb_wov.rst diff --git a/developer_guides/firmware/kpb_wov.rst b/developer_guides/firmware/kpb_wov.rst new file mode 100644 index 00000000..edd836f0 --- /dev/null +++ b/developer_guides/firmware/kpb_wov.rst @@ -0,0 +1,808 @@ +.. _kpb_wov: + +========================================================== +Key Phrase Buffer (KPB) & Wake-on-Voice (WoV) Architecture +========================================================== + +.. contents:: + :local: + :depth: 3 + +Sound Open Firmware (SOF) provides an autonomous, low-power audio architecture designed to support **Wake-on-Voice (WoV)** and always-listening acoustic keyword activation. In modern mobile laptops, smart home hubs, automotive cockpits, and wearable devices, users expect immediate responsiveness to spoken wake phrases (such as *"Hey Computer"* or *"OK Assistant"*). However, keeping the host application processor and PCIe/USB interconnects continuously awake to analyze ambient microphone audio would consume several watts of power, draining portable batteries in a matter of hours. + +To resolve this challenge, modern acoustic architectures offload keyword spotting and voice activity detection to an ultra-low-power Digital Signal Processor (DSP) running SOF. While the host CPU remains in deep system sleep (such as ACPI S0ix / Modern Standby, S3 suspend-to-RAM, or S4 hibernation) drawing only microamperes, the audio DSP operates in an autonomous, power-optimized D0ix state. + +A critical engineering obstacle in always-listening architectures is **The Pre-Roll Dilemma**: acoustic keyword spotters—whether running neural networks via TensorFlow Lite for Microcontrollers (TFLM) or proprietary vendor models—require an integration window of 500 ms to 1500 ms of spoken phonemes before achieving statistical confidence to trigger a detection event. Furthermore, waking the host CPU, resuming platform power rails, re-initializing PCIe/SoundWire DMA controllers, and starting host user-space capture pipelines introduces an additional system resume latency of 1000 ms to 2000 ms. If microphone audio is not buffered during this multi-second interval, the opening syllables of the user's command (*"Hey Computer, what is the weather?"*) are permanently lost before host recording begins. + +The **Key Phrase Buffer (KPB)** component (``src/audio/kpb.c``, ``COMP_KPB``, UUID ``D8218443-5FF3-4A4C-B388-6CFE07B9562E``) solves this problem by maintaining a continuous circular ring buffer of incoming microphone audio. Operating as a specialized dual-sink streaming engine, KPB simultaneously provides a real-time low-latency stream to local on-DSP keyword spotters and maintains a multi-second history buffer. Upon a keyword detection event, KPB transitions into an accelerated draining engine that burst-transfers the pre-roll history to the host DMA buffer before seamlessly handing off to real-time audio capture without dropping a single acoustic frame. + +.. graphviz:: + :caption: SOF Wake-on-Voice (WoV) System Architecture: Host Sleep, DSP D0ix & Wake Sequence + :alt: Architectural block diagram showing host CPU sleep, DSP autonomous listening in D0ix, keyword detection, and pre-roll draining. + + digraph wov_system_overview { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_ambient_sound { + label = "Acoustic Environment"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + mic_input [label="Acoustic Speech\n'Hey Computer...'\nVoice Waveform", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + dmic_hw [label="DMIC Hardware Array\nLow-Power PDM Clock\n(16 kHz Sampling)", fillcolor="#E2E8F0", color="#4A5568", penwidth=1.5]; + } + + subgraph cluster_dsp_d0ix { + label = "DSP Autonomous Domain (D0ix Ultra-Low Power)"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + dcblock [label="DC Blocker\nIIR High-Pass\nOffset Removal", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + kpb_core [label="Key Phrase Buffer (KPB)\nDual-Sink Dispatch\nCircular History Ring\n(1.5 to 3.0 s Pre-Roll)", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + kwd_engine [label="Keyword Spotter\n(TFLM / MFCC / KD)\nContinuous Evaluation\nPin 0 (Real-Time)", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + dma_drain [label="Host Draining Sink\nBurst Transfer Task\n(Fast Mode Engine)\nPin 1 (Host Sink)", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + } + + subgraph cluster_host_domain { + label = "Host CPU System Domain"; + style = "filled,rounded"; + color = "#FED7D7"; + fillcolor = "#FFF5F5"; + + host_sleep [label="Host CPU Sleep\n(ACPI S0ix / Modern Standby)\nPCIe & DRAM Suspended", fillcolor="#FEB2B2", color="#C53030", penwidth=1.5]; + host_resume [label="Host Wake & Audio Resume\nKernel ALSA Driver\nHost DMA Capture Active", fillcolor="#FED7D7", color="#9B2C2C", penwidth=1.8]; + voice_app [label="Voice Assistant Application\nCloud / Local ASR\nReceives Intact Utterance", fillcolor="#FAF5FF", color="#6B46C1", penwidth=1.5]; + } + + mic_input -> dmic_hw [label="Sound Wave", color="#3182CE", penwidth=1.5]; + dmic_hw -> dcblock [label="PDM Frames", color="#4A5568", penwidth=1.5]; + dcblock -> kpb_core [label="16 kHz PCM", color="#2B6CB0", penwidth=1.8]; + + kpb_core -> kwd_engine [label="1. Continuous Stream\n(Real-Time Pin 0)", color="#B7791F", penwidth=1.6]; + kwd_engine -> kpb_core [label="2. Trigger Event\n(Keyword Match)", color="#C53030", style="dashed", penwidth=1.8]; + kwd_engine -> host_sleep [label="3. Wakeup IRQ\n(IPC / MSI)", color="#C53030", style="bold", penwidth=2.0]; + + host_sleep -> host_resume [label="Platform Resume\n(1000 - 2000 ms)", color="#9B2C2C", style="dashed", penwidth=1.5]; + kpb_core -> dma_drain [label="4. Burst Draining", color="#2B6CB0", penwidth=1.8]; + dma_drain -> host_resume [label="5. Pre-Roll + Live Data\n(Host DMA)", color="#2B6CB0", penwidth=2.0]; + host_resume -> voice_app [label="Uncut Audio Stream", color="#6B46C1", penwidth=1.8]; + } + +Principles of Low-Power Wake-on-Voice & The Pre-Roll Dilemma +============================================================ + +In modern computing platforms, acoustic energy efficiency is governed by the operational power consumption of different platform processing tiers: + +.. list-table:: Energy & Power Tiers in Voice-Enabled Embedded Systems + :widths: 22 18 25 35 + :header-rows: 1 + + * - Platform Power Tier + - Typical Power + - Wake Latency + - Active Audio Processing Capabilities + * - **Host Active (S0)** + - 10 W -- 45 W + - 0 ms (running) + - Full desktop OS, cloud streaming, complex large language models, high-resolution rendering. + * - **Host Modern Standby (S0ix)** + - 500 mW -- 1.5 W + - 500 ms -- 1500 ms + - Host cores in deep C-states; PCIe, DRAM controllers, and display engines clock-gated. + * - **Host Suspend-to-RAM (S3)** + - 100 mW -- 300 mW + - 1000 ms -- 2500 ms + - Host completely powered off except DRAM refresh logic; interconnects dormant. + * - **DSP Low-Power Mode (D0ix)** + - 3 mW -- 12 mW + - < 1 ms + - Primary DSP core running at reduced clock frequency (e.g. 24 MHz -- 38.4 MHz); autonomous DMIC audio capture, low-power Voice Activity Detection (VAD), and keyword spotters. + +The Pre-Roll Timing Equation +---------------------------- + +To understand the necessity of historical buffering, consider the chronological progression of a voice activation sequence: + +1. **Acoustic Speech Commencement** (:math:`t = t_0`): + The user begins uttering the activation phrase (*"Hey Computer"*). +2. **Voice Activity Detection** (:math:`t = t_0 + \Delta t_{\text{VAD}}`): + Energy-based or spectral VAD algorithms detect acoustic activity above background ambient noise (:math:`\approx 50\text{--}150\text{ ms}`). +3. **Keyword Model Inference Latency** (:math:`t = t_0 + \Delta t_{\text{KWD}}`): + The acoustic keyword classifier integrates temporal audio frames over a multi-layer neural network or acoustic model. Because phonetic recognition requires sufficient acoustic context across syllables, confident detection occurs near the end of the phrase (:math:`\Delta t_{\text{KWD}} \approx 800\text{--}1500\text{ ms}`). +4. **Host Wakeup & Platform Rail Settlement** (:math:`t = t_0 + \Delta t_{\text{KWD}} + \Delta t_{\text{wake}}`): + Upon keyword detection, the DSP asserts a platform interrupt (IPC or PCIe MSI). The host power management IC (PMIC) ramps platform voltage rails, DRAM exits self-refresh, the kernel resumes, and the ALSA audio driver invokes hardware parameters and stream prepare (:math:`\Delta t_{\text{wake}} \approx 800\text{--}2000\text{ ms}`). +5. **Host DMA Capture Activation** (:math:`t = t_0 + \Delta t_{\text{total\_latency}}`): + The host application initiates reading from the ALSA capture device (e.g. ``arecord``). + +The cumulative latency before the host application begins receiving audio data is: + +.. math:: + + T_{\text{total\_latency}} = \Delta t_{\text{KWD}} + \Delta t_{\text{wake}} + \Delta t_{\text{dma\_startup}} + +If :math:`\Delta t_{\text{KWD}} = 1200\text{ ms}` and :math:`\Delta t_{\text{wake}} = 1500\text{ ms}`, the total elapsed duration is :math:`2700\text{ ms}`. Without a circular buffer holding at least :math:`2.7\text{ seconds}` of historical microphone data, the entire wake word and the initial segment of the user command would be completely lost. + +The KPB component eliminates this data loss by continuously recording into a dedicated circular history buffer in DSP SRAM while the host is asleep. When the host resumes and initiates capture, KPB transfers this buffered historical speech into the host DMA buffer at accelerated speed before transitioning seamlessly to real-time audio. + +KPB Component State Machine & Execution Lifecycle +================================================= + +The KPB component is implemented as an audio processing module conforming to the SOF component driver interface. Internally, KPB maintains ten discrete states that govern its execution during audio streaming, buffer writing, trigger events, and draining. + +.. list-table:: KPB Component Lifecycle States (enum kpb_state) + :widths: 25 15 60 + :header-rows: 1 + + * - State Enumeration + - Value + - Functional Role & Operational Behavior + * - ``KPB_STATE_DISABLED`` + - 0 + - Initial unconfigured state prior to memory allocation and pipeline initialization. + * - ``KPB_STATE_RESET_FINISHING`` + - 1 + - Ephemeral cleanup state entered when a reset interrupt interrupts an ongoing buffering or draining operation. + * - ``KPB_STATE_CREATED`` + - 2 + - Module instance allocated, driver private data initialized, and unique identifier (UUID) assigned. + * - ``KPB_STATE_PREPARING`` + - 3 + - Validation of sampling rate (16 kHz), container width, channel count, and circular history buffer allocation during ``kpb_prepare()``. + * - ``KPB_STATE_RUN`` + - 4 + - Normal listening mode. Incoming DMIC frames are copied to the internal history buffer and simultaneously forwarded to the active real-time selector sink (pin 0). + * - ``KPB_STATE_BUFFERING`` + - 5 + - Transient state entered within ``kpb_copy()`` while writing audio frames into the active circular history ring buffer. + * - ``KPB_STATE_INIT_DRAINING`` + - 6 + - Triggered by client detection event. Locks state, calculates backward read pointer in history rings, and prepares asynchronous draining task. + * - ``KPB_STATE_DRAINING`` + - 7 + - Asynchronous draining active. The background draining task reads historical audio from the ring buffer and copies it to the host sink at accelerated speed. + * - ``KPB_STATE_HOST_COPY`` + - 8 + - Draining completed ("draining on demand"). History buffer is emptied, and incoming real-time audio is copied directly to the host capture sink without latency. + * - ``KPB_STATE_RESETTING`` + - 9 + - Teardown requested via pipeline trigger stop or reset command. Halts background tasks and frees resources. + +.. graphviz:: + :caption: KPB Component State Machine (10 Lifecycle States: Reset, Run, Buffering, Draining, and Host Copy) + :alt: Detailed finite state machine diagram showing all 10 states of the KPB component and their transitions. + + digraph kpb_state_machine { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + s_disabled [label="KPB_STATE_DISABLED\n(Uninitialized)", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + s_created [label="KPB_STATE_CREATED\n(Instance Created)", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + s_prep [label="KPB_STATE_PREPARING\n(Buffer Allocation)", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + s_run [label="KPB_STATE_RUN\n(Normal Listening / Real-Time Dispatch)", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + s_buff [label="KPB_STATE_BUFFERING\n(Writing to History Ring)", fillcolor="#C6F6D5", color="#276749", penwidth=1.5]; + s_init_drn [label="KPB_STATE_INIT_DRAINING\n(Pointer Calc & Task Setup)", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + s_draining [label="KPB_STATE_DRAINING\n(Accelerated Burst Draining Task)", fillcolor="#FEEBC8", color="#C05621", penwidth=2.0]; + s_hcopy [label="KPB_STATE_HOST_COPY\n(Real-Time Streaming to Host Sink)", fillcolor="#E9D8FD", color="#6B46C1", penwidth=2.0]; + s_resetting[label="KPB_STATE_RESETTING\n(Pipeline Stop / Reset Triggered)", fillcolor="#FED7D7", color="#C53030", penwidth=1.5]; + s_rst_fin [label="KPB_STATE_RESET_FINISHING\n(Final Resource Teardown)", fillcolor="#FEB2B2", color="#9B2C2C", penwidth=1.2]; + + s_disabled -> s_created [label="kpb_new()", color="#4A5568"]; + s_created -> s_prep [label="kpb_prepare()", color="#3182CE"]; + s_prep -> s_run [label="kpb_trigger(START)", color="#22543D", penwidth=1.5]; + + s_run -> s_buff [label="Frame Arrival (kpb_copy)", color="#276749"]; + s_buff -> s_run [label="Frame Written", color="#276749"]; + + s_run -> s_init_drn [label="Keyword Detected\n(BEGIN_DRAINING Event)", color="#C05621", penwidth=1.8]; + s_init_drn -> s_draining [label="Task Scheduled", color="#C05621", penwidth=1.5]; + + s_draining -> s_buff [label="New Audio Buffering\nDuring Draining", color="#276749", style="dashed"]; + s_buff -> s_draining [label="Resume Draining", color="#276749", style="dashed"]; + + s_draining -> s_hcopy [label="Pre-Roll Drained\n(drain_req == 0)", color="#6B46C1", penwidth=2.0]; + + s_run -> s_resetting [label="Trigger STOP / RESET", color="#C53030"]; + s_draining -> s_resetting [label="Trigger STOP / RESET", color="#C53030"]; + s_hcopy -> s_resetting [label="Trigger STOP / RESET", color="#C53030"]; + + s_resetting -> s_rst_fin [label="Task Cancelled", color="#9B2C2C"]; + s_rst_fin -> s_created [label="kpb_reset() Complete", color="#4A5568"]; + s_created -> s_disabled [label="kpb_free()", color="#4A5568"]; + } + +Lifecycle Transitions Walkthrough +--------------------------------- + +1. **Initialization & Preparation**: + When the audio pipeline is configured via topology, ``kpb_new()`` transitions the module to ``KPB_STATE_CREATED``. Upon receiving the IPC hardware parameters and prepare commands, ``kpb_prepare()`` verifies that the sampling frequency is 16 kHz and allocates the circular history buffers in DSP internal SRAM, moving to ``KPB_STATE_PREPARING``. +2. **Normal Listening (RUN & BUFFERING)**: + Upon receiving ``COMP_TRIGGER_START``, the state transitions to ``KPB_STATE_RUN``. Each time the pipeline period executes, ``kpb_copy()`` inspects the source DMIC buffer. Audio samples are copied to the active real-time selector sink (pin 0) if downstream components (the keyword spotter) are in ``COMP_STATE_ACTIVE``. Simultaneously, KPB temporarily enters ``KPB_STATE_BUFFERING`` to append the incoming PCM frames to the circular history buffer before reverting to ``KPB_STATE_RUN``. +3. **Keyword Trigger & Draining Initialization**: + When the keyword classifier identifies the activation phrase, it emits a notification event (``KPB_EVENT_BEGIN_DRAINING``). KPB locks its private spinlock/mutex and enters ``KPB_STATE_INIT_DRAINING``. The component calculates the historical read pointer offset corresponding to the requested pre-roll duration, locks available buffer headroom, pauses the real-time selector sink, and launches an asynchronous draining task. +4. **Accelerated Burst Draining**: + In ``KPB_STATE_DRAINING``, the draining task executes at an accelerated cadence (e.g. :math:`2\times` to :math:`4\times` real-time speed), reading from the historical read pointer and writing to the host sink buffer (pin 1). If new real-time microphone samples arrive during draining, they are buffered into the history ring while a running counter (``buffered_while_draining``) extends the total remaining draining requirement. +5. **Real-Time Handoff (HOST_COPY)**: + Once the historical buffer is completely drained and all accumulated audio frames have been transferred, KPB transitions to ``KPB_STATE_HOST_COPY``. In this state, the circular history buffer is bypassed, and new incoming microphone frames are copied directly to the host capture sink in real time, guaranteeing zero-latency streaming to the host voice recognition application. + +History Circular Ring Buffer Architecture & Mathematics +======================================================== + +The KPB storage engine is built around a chained linked list of circular history buffers: + +.. math:: + + \text{Ring Structure: } \mathcal{B}_0 \rightleftharpoons \mathcal{B}_1 \rightleftharpoons \dots \rightleftharpoons \mathcal{B}_{N-1} \rightleftharpoons \mathcal{B}_0 + +In standard SOF configurations, the ring comprises two distinct buffers (``KPB_NO_OF_HISTORY_BUFFERS = 2``) managed by ``struct history_buffer``: + +.. code-block:: c + + struct history_buffer { + enum buffer_state state; /* KPB_BUFFER_FREE, KPB_BUFFER_FULL, KPB_BUFFER_OFF */ + void *start_addr; /* Base memory address of buffer in DSP SRAM */ + void *end_addr; /* Upper boundary address (start_addr + size) */ + void *w_ptr; /* Current write pointer */ + void *r_ptr; /* Current read pointer for draining */ + struct history_buffer *next; /* Pointer to next ring segment */ + struct history_buffer *prev; /* Pointer to previous ring segment */ + }; + +Mathematical Buffer Sizing Equations +------------------------------------ + +The memory footprint of the KPB history buffer is determined by four platform configuration parameters: + +* Sampling frequency (:math:`f_s`, strictly 16,000 Hz for voice keyword processing). +* Audio channel count (:math:`N_{\text{ch}}`, typically 2 to 6 channels). +* Sample container width (:math:`W_{\text{container}}`, 16 bits or 32 bits). +* Target historical buffer duration (:math:`T_{\text{buff}}`, in milliseconds). + +The sample container size is defined as: + +.. math:: + + C_{\text{size}} = \begin{cases} 2 \text{ bytes} (16\text{ bits}), & \text{if } W_{\text{sample}} = 16 \\ 4 \text{ bytes} (32\text{ bits}), & \text{if } W_{\text{sample}} \in \{24, 32\} \end{cases} + +The required history buffer capacity :math:`S_{\text{buff}}` in bytes is derived as: + +.. math:: + + S_{\text{buff}} = \left(\frac{f_s}{1000}\right) \times C_{\text{size}} \times N_{\text{ch}} \times T_{\text{buff}} + +.. list-table:: KPB History Buffer Memory Allocations Across Configurations + :widths: 20 15 15 20 30 + :header-rows: 1 + + * - Platform Target + - Channels (:math:`N_{\text{ch}}`) + - Width (:math:`W_{\text{sample}}`) + - History (:math:`T_{\text{buff}}`) + - Total Allocated Memory + * - **Tiger Lake (TGL)** + - 2 (Stereo) + - 16-bit + - 3000 ms + - :math:`16 \times 2 \times 2 \times 3000 = 192{,}000\text{ bytes} \approx 187.5\text{ KB}` + * - **Tiger Lake (TGL)** + - 4 (Quad) + - 16-bit + - 3000 ms + - :math:`16 \times 2 \times 4 \times 3000 = 384{,}000\text{ bytes} \approx 375.0\text{ KB}` + * - **Generic CAVS / ACE** + - 2 (Stereo) + - 16-bit + - 2100 ms + - :math:`16 \times 2 \times 2 \times 2100 = 134{,}400\text{ bytes} \approx 131.25\text{ KB}` + * - **Generic CAVS / ACE** + - 4 (Quad) + - 32-bit + - 2100 ms + - :math:`16 \times 4 \times 4 \times 2100 = 537{,}600\text{ bytes} \approx 525.0\text{ KB}` + +.. graphviz:: + :caption: Dual-Sink Buffer Architecture: Continuous Keyword Detector Feed vs Burst Draining Host Sink + :alt: Diagram illustrating the KPB dual-sink streaming architecture connecting DMIC input, history ring buffers, real-time detector sink, and host draining sink. + + digraph kpb_dual_sink { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + subgraph cluster_input { + label = "Audio Input"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + src_dmic [label="Source Buffer\n(DMIC Capture Stream)\n16 kHz, 2-6 Channels", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + } + + subgraph cluster_kpb_internals { + label = "KPB Core Architecture"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + kpb_dispatch [label="KPB Copy Engine\n(Format Check &\nChannel Parsing)", fillcolor="#9AE6B4", color="#22543D", penwidth=1.8]; + + subgraph cluster_history { + label = "Dual Circular History Buffers"; + style = "filled,rounded"; + color = "#C6F6D5"; + fillcolor = "#F0FFF4"; + + hb0 [label="History Buffer 0\n(50% Capacity)\nstart_addr .. end_addr", fillcolor="#E2E8F0", color="#4A5568", penwidth=1.2]; + hb1 [label="History Buffer 1\n(50% Capacity)\nstart_addr .. end_addr", fillcolor="#E2E8F0", color="#4A5568", penwidth=1.2]; + + hb0 -> hb1 [label="next", color="#276749", constraint=false]; + hb1 -> hb0 [label="next", color="#276749", constraint=false]; + } + + mic_sel [label="Mic Channel Selector\n(Configurable Bitmask)\nExtracts Voice Channels", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.4]; + } + + subgraph cluster_sinks { + label = "Dual Output Sinks"; + style = "filled,rounded"; + color = "#E9D8FD"; + fillcolor = "#F7FAFC"; + + sink_rt [label="Pin 0: Real-Time Sink\n(sel_sink)\nFeeds Keyword Spotter\nZero Buffering Latency", fillcolor="#FAF5FF", color="#6B46C1", penwidth=1.6]; + sink_host [label="Pin 1: Host Sink\n(host_sink)\nFeeds Host DMA Copier\nBurst Draining & Live Stream", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.8]; + } + + src_dmic -> kpb_dispatch [label="Periodic Frames", color="#3182CE", penwidth=1.5]; + kpb_dispatch -> hb0 [label="Continuous Write\n(w_ptr update)", color="#22543D", penwidth=1.6]; + kpb_dispatch -> mic_sel [label="Voice Channels", color="#B7791F", penwidth=1.4]; + mic_sel -> sink_rt [label="Continuous Stream", color="#6B46C1", penwidth=1.6]; + + hb0 -> sink_host [label="Draining Task\n(r_ptr playback)", color="#3182CE", penwidth=1.8, style="dashed"]; + hb1 -> sink_host [label="Draining Task\n(r_ptr playback)", color="#3182CE", penwidth=1.8, style="dashed"]; + } + +Pointer Mechanics & Overwrite Protection +----------------------------------------- + +During normal listening (``KPB_STATE_RUN``), the write pointer (``w_ptr``) advances sequentially through the memory of the active buffer. When ``w_ptr`` reaches ``end_addr``, the buffer state is flagged as ``KPB_BUFFER_FULL``, the write pointer is reset to ``start_addr`` of the subsequent buffer (``buff->next``), and writing continues without disruption. + +When a keyword trigger initiates draining of :math:`B_{\text{req}}` bytes, the read pointer :math:`P_{\text{read}}` must be positioned exactly :math:`B_{\text{req}}` bytes behind the current write pointer :math:`P_{\text{write}}` across the circular buffer boundaries: + +.. math:: + + P_{\text{read}} = \begin{cases} P_{\text{write}} - B_{\text{req}}, & \text{if } (P_{\text{write}} - P_{\text{start}}) \ge B_{\text{req}} \\ P_{\text{prev\_end}} - \left(B_{\text{req}} - (P_{\text{write}} - P_{\text{start}})\right), & \text{otherwise} \end{cases} + +To prevent newly arriving microphone audio from overwriting history samples that are staged for host draining, KPB dynamically clamps its writable headroom: + +.. math:: + + \text{FreeHeadroom} = S_{\text{buff}} - B_{\text{req}} + +As the draining task reads and emits audio to the host sink, it increments ``kpb->hd.free``, restoring writable memory space in exact synchrony with host consumption. + +.. graphviz:: + :caption: History Circular Ring Buffer Pointer Mechanics: Pre-Roll Window, Wrap Safety & Overwrite Protection + :alt: Detailed memory layout and pointer mechanics showing write pointer progression, backward read pointer positioning, and boundary wrap safety. + + digraph kpb_pointer_mechanics { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + subgraph cluster_ring_layout { + label = "Circular Ring Memory Topology"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + subgraph cluster_buf0 { + label = "History Buffer Segment 0 (FULL)"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#EDF2F7"; + + b0_start [label="start_addr (0x0000)", fillcolor="#E2E8F0", color="#4A5568"]; + b0_rptr [label="r_ptr (Drain Start)\nCalculated Backward Offset", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + b0_mid [label="Staged Pre-Roll Audio Data\n(Protected from Overwrite)", fillcolor="#FEEBC8", color="#C05621"]; + b0_end [label="end_addr (0x17700)", fillcolor="#E2E8F0", color="#4A5568"]; + + b0_start -> b0_rptr -> b0_mid -> b0_end [style="invis"]; + } + + subgraph cluster_buf1 { + label = "History Buffer Segment 1 (ACTIVE / FREE)"; + style = "filled,rounded"; + color = "#C6F6D5"; + fillcolor = "#F0FFF4"; + + b1_start [label="start_addr (0x17700)", fillcolor="#E2E8F0", color="#4A5568"]; + b1_wptr [label="w_ptr (Current Write)\nTrigger Event Instant", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + b1_free [label="Available Headroom\n(free = total - drain_req)", fillcolor="#EBF8FF", color="#3182CE"]; + b1_end [label="end_addr (0x2EE00)", fillcolor="#E2E8F0", color="#4A5568"]; + + b1_start -> b1_wptr -> b1_free -> b1_end [style="invis"]; + } + } + + b0_end -> b1_start [label="Ring Boundary Link (next)", color="#276749", penwidth=1.5]; + b1_end -> b0_start [label="Wrap-Around Link (next)", color="#276749", penwidth=1.5]; + + b1_wptr -> b0_rptr [label="Reverse Offset Search: -drain_req bytes\n(Walks backward across buffer link)", color="#C05621", style="dashed", penwidth=1.8]; + } + +Dual-Sink Architecture & Microphone Channel Selection +===================================================== + +The KPB component is architected with dual output pins (``num_output_pins = 2``): + +1. **Pin 0: Real-Time Selector Sink (``sel_sink``, ``REALTIME_PIN_ID``)**: + This sink is dedicated to low-latency processing and feeds local on-DSP keyword detection engines (e.g. TFLM, MFCC feature extractors, or vendor detection algorithms). During normal system sleep, audio is delivered directly to Pin 0 on every pipeline period. +2. **Pin 1: Host Draining Sink (``host_sink``)**: + This sink connects to the host capture pipeline through downstream volume and copier components. During host sleep, Pin 1 remains inactive and paused. Upon a keyword activation event, Pin 1 receives the burst-drained pre-roll historical audio and subsequent live microphone speech. + +Microphone Channel Selection (MicSelector) +------------------------------------------ + +In modern platforms equipped with digital microphone arrays (such as 3-mic or 4-mic beamforming arrays with reference loopback channels), passing the full multi-channel stream to the keyword detector during low-power sleep would waste substantial memory bandwidth and DSP processing cycles. + +To minimize energy consumption, KPB incorporates an integrated microphone channel selector (``kpb_micselector_config``, configured via IPC4 parameter ``KP_BUF_CLIENT_MIC_SELECT``): + +.. code-block:: c + + struct kpb_micselector_config { + uint32_t mask; /* Channel selection bitmask */ + }; + +When ``kpb->num_of_sel_mic`` is configured (e.g. selecting channel 0 or channel 1 via bitmask ``0x01`` or ``0x02``), KPB automatically demultiplexes and extracts only the designated voice microphone channel when copying to the real-time sink (Pin 0). Meanwhile, the full multi-channel stream is preserved intact in the circular history buffer, ensuring that when the host wakes up, beamforming and multi-channel noise suppression algorithms have access to all physical microphone signals for high-fidelity speech recognition. + +Accelerated Burst Draining & Dynamic Pace Adjustment +==================================================== + +When a keyword trigger initiates host streaming, transferring historical data at standard real-time speed (:math:`1\times`) would be inadequate: if the host resumes 2 seconds after the trigger, draining 2 seconds of pre-roll at :math:`1\times` speed would mean the host remains perpetually 2 seconds behind real-time audio. + +To eliminate this lag, KPB executes an asynchronous **Burst Draining Task** (``kpb_draining_task``) scheduled via the SOF Earliest Deadline First (EDF) scheduler. The draining task empties the history buffer at a multiple of real-time speed before transitioning seamlessly into live streaming. + +.. graphviz:: + :caption: Accelerated Burst Draining Timeline & Dynamic Interval Adjustment (FMT vs Real-Time Hand-off) + :alt: Timing diagram comparing real-time capture progression with accelerated burst draining and seamless live hand-off. + + digraph kpb_draining_timeline { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + subgraph cluster_timeline { + label = "WoV Audio Draining Progression"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + t0 [label="Phase 1: Ambient Listening (t < t_trig)\nHost Asleep (S0ix) | DSP D0ix\nContinuous Buffering: 16 kHz Audio -> History Ring\nReal-Time Feed -> Keyword Spotter (Pin 0)", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + + t1 [label="Phase 2: Keyword Activation (t = t_trig)\n'Hey Computer' Detected by On-DSP Classifier\nHost Wake IRQ Asserted | KPB enters INIT_DRAINING\nReverse Read Pointer Calculated (-2000 ms)", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + + t2 [label="Phase 3: Host Resume Lag (t_trig < t < t_host_ready)\nHost PMIC & Rails Settling (800 - 1500 ms)\nKPB Continues Buffering Incoming Microphone Audio\nbuffered_while_draining Counter Tracks Accumulation", fillcolor="#FEEBC8", color="#C05621", penwidth=1.5]; + + t3 [label="Phase 4: Accelerated Burst Draining (2x to 4x Pace)\nHost DMA Active | Draining Task Scheduled\nHistory Flushed Rapidly into Host Buffer\nDynamic Pace Adjustment (adjust_drain_interval)", fillcolor="#FED7D7", color="#C53030", penwidth=2.0]; + + t4 [label="Phase 5: Catch-up Convergence (drain_req == 0)\nPre-Roll Completely Transferred\nKPB Transitions to KPB_STATE_HOST_COPY\nHistory Buffer Bypassed", fillcolor="#E9D8FD", color="#6B46C1", penwidth=1.8]; + + t5 [label="Phase 6: Uncut Real-Time Streaming\nLive Microphone Audio Streamed to Host DMA at 1x Pace\nZero Lost Syllables | Zero Audio Discontinuities", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + + t0 -> t1 -> t2 -> t3 -> t4 -> t5 [color="#2B6CB0", penwidth=1.8]; + } + } + +Synchronized Draining & Dynamic Pace Adjustment +----------------------------------------------- + +SOF supports two operational draining modes: + +1. **Unsynchronized (Unlimited) Draining**: + Audio samples are copied to the host sink buffer as fast as downstream memory and DMA allow, constrained only by available sink space. +2. **Synchronized Draining (``sync_draining_mode``)**: + Draining is paced to prevent overflowing host DMA ring buffers while remaining significantly faster than real-time consumption. The target interval is governed by: + + .. math:: + + I_{\text{drain}} = \frac{T_{\text{host\_period}}}{M_{\text{drain}}} + + where :math:`M_{\text{drain}} = \text{KPB\_DRAIN\_NUM\_OF\_PPL\_PERIODS\_AT\_ONCE} = 2`. Draining operates at double the normal pipeline period rate. + +Dynamic Pace Regulation Algorithm +--------------------------------- + +Because host interrupt response and DMA scheduling exhibit jitter, KPB incorporates an adaptive pace controller (``adjust_drain_interval``) evaluated every 32 task iterations using 64-bit DSP wall-clock cycles (``sof_cycle_get_64()``): + +.. math:: + + P_{\text{actual}} = \frac{\Delta \text{DrainedBytes}}{\Delta t_{\text{elapsed}}} \times 1000 + +.. math:: + + P_{\text{optimal}} = \text{PeriodBytes} \times M_{\text{drain}} \times 1000 + +If :math:`P_{\text{actual}} < P_{\text{optimal}}` (draining is falling behind target pace), the drain interval is reduced: + +.. math:: + + I_{\text{drain}} \leftarrow I_{\text{drain}} \times \left(\frac{P_{\text{actual}}}{P_{\text{optimal}}}\right) - \frac{I_{\text{drain}}}{8} + +Conversely, if :math:`P_{\text{actual}} > P_{\text{optimal}}`, the interval is lengthened proportionally, maintaining stable DMA buffer levels without underrun or overrun. + +Fast Mode Task (FMT) Pipeline Infrastructure +-------------------------------------------- + +In complex audio graphs, intermediate components (such as Gain/Volume widgets or PCM Format Converters) may sit between KPB and the Host DMA Copier. Under standard scheduling, these intermediate modules execute only once per pipeline period (e.g. every 1 ms or 4 ms). + +To prevent these intermediate modules from throttling burst draining, SOF implements the **Fast Mode Task (FMT)** framework (``struct fast_mode_task``, configured via IPC4 parameter ``KP_BUF_CFG_FM_MODULE``). FMT registers downstream modules into an accelerated execution list, triggering their processing routines in direct synchronization with KPB burst cycles until pre-roll draining finishes. + +Event Notification Framework: IPC3 Notifiers vs IPC4 AMS +========================================================= + +Communication between keyword spotters, client pipelines, and the KPB component differs across SOF IPC architectures: + +.. graphviz:: + :caption: Event Notification Architecture: IPC3 Notifier Dispatch vs IPC4 Asynchronous Message Service (AMS) + :alt: Architectural comparison between IPC3 notifier callbacks and IPC4 Asynchronous Message Service (AMS) dispatching wake events to KPB. + + digraph kpb_event_architecture { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + subgraph cluster_ipc3 { + label = "IPC3 Notifier Framework"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + kwd3 [label="Keyword Detector\n(KD Module)", fillcolor="#FEFCBF", color="#B7791F"]; + notif_core [label="SOF Notifier Engine\nNOTIFIER_ID_KPB_CLIENT_EVT\nSynchronous Callbacks", fillcolor="#E2E8F0", color="#4A5568", penwidth=1.5]; + kpb_ev_hdl [label="kpb_event_handler()\nDispatches Events:\n- REGISTER_CLIENT\n- BEGIN_DRAINING", fillcolor="#9AE6B4", color="#22543D", penwidth=1.8]; + + kwd3 -> notif_core [label="notifier_event()", color="#B7791F"]; + notif_core -> kpb_ev_hdl [label="Direct Callback", color="#22543D", penwidth=1.5]; + } + + subgraph cluster_ipc4 { + label = "IPC4 Asynchronous Message Service (AMS)"; + style = "filled,rounded"; + color = "#FED7D7"; + fillcolor = "#FFF5F5"; + + kwd4 [label="Keyword Spotter\n(IPC4 KPD Module)", fillcolor="#FEFCBF", color="#B7791F"]; + ams_core [label="AMS Message Router\nCONFIG_AMS Enabled\nAsynchronous Mailbox", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.5]; + kpb_ams [label="kpb_set_large_config()\nKP_BUF_CFG_FM_MODULE\nKP_BUF_CLIENT_MIC_SELECT", fillcolor="#FEB2B2", color="#C53030", penwidth=1.8]; + + kwd4 -> ams_core [label="ams_send_message()", color="#B7791F"]; + ams_core -> kpb_ams [label="Large Config IPC", color="#C53030", penwidth=1.5]; + } + } + +IPC3 Notifier Implementation +---------------------------- + +In IPC3 topologies, communication between the detection module and KPB relies on the internal core notifier system: + +.. code-block:: c + + enum kpb_event { + KPB_EVENT_REGISTER_CLIENT = 0, + KPB_EVENT_UPDATE_PARAMS, + KPB_EVENT_BEGIN_DRAINING, + KPB_EVENT_STOP_DRAINING, + KPB_EVENT_UNREGISTER_CLIENT, + }; + +Clients (such as ``detect_test``) register with KPB by passing ``KPB_EVENT_REGISTER_CLIENT`` along with their requested history draining window (``drain_req``, up to ``KPB_MAX_DRAINING_REQ`` = 2000 ms to 3000 ms). When the keyword model confirms an utterance match, it fires ``KPB_EVENT_BEGIN_DRAINING``, causing KPB to calculate the historical read pointer and start the draining task. + +IPC4 Asynchronous Message Service (AMS) +--------------------------------------- + +Under IPC4, inter-module signaling leverages the **Asynchronous Message Service (AMS)** (``CONFIG_AMS``). Modules communicate via standardized large configuration parameters: + +* ``KP_BUF_CFG_FM_MODULE`` (Parameter ID 1): Configures the list of downstream modules participating in the Fast Mode Task during accelerated pre-roll draining. +* ``KP_BUF_CLIENT_MIC_SELECT`` (Parameter ID 11): Updates the real-time microphone channel selection mask without tearing down active audio pipelines. + +Linux Driver & DAPM Control Sequencing +-------------------------------------- + +On the Linux host, keyword detection pipelines are managed through ALSA Dynamic Audio Power Management (DAPM). Two intertwined pipelines are constructed: + +1. **Pipeline 8 (Host Capture Pipeline)**: DMIC :math:`\to` Volume :math:`\to` KPB :math:`\to` Host Copier :math:`\to` ALSA PCM capture device. +2. **Pipeline 9 (Keyword Detect Pipeline)**: KPB Pin 0 :math:`\to` Selector :math:`\to` Detector Module :math:`\to` Virtual Detector Sink. + +.. list-table:: ALSA DAPM Control Sequence for Keyword Detection + :widths: 20 25 25 30 + :header-rows: 1 + + * - Stream Control Action + - Host Pipeline (Pipe 8) + - Detector Pipeline (Pipe 9) + - Operational Hardware State + * - **1. HW Parameters** + - ``snd_pcm_hw_params()`` + - ``DAPM_PRE_PMU`` Event + - DSP sets 16 kHz sampling, validates minimum host buffer (:math:`\ge 67200\text{ frames}`). + * - **2. Trigger Start** + - Host suspended + - Pipeline 9 Started + - DSP enters D0ix; KPB buffers incoming audio; Detector continuously scans. + * - **3. Keyword Detected** + - Host resumes via IRQ + - Draining triggered + - KPB empties pre-roll history to host DMA; transitions to live copy. + * - **4. Capture Stop** + - ``snd_pcm_drain()`` + - ``DAPM_POST_PMD`` Event + - Host application finishes reading speech command; pipeline resets to listening state. + +End-to-End WoV System Pipeline & Topology 2 Wiring +================================================== + +The integration of KPB within an end-to-end Sound Open Firmware audio graph is illustrated in Figure 201: + +.. graphviz:: + :caption: End-to-End WoV Audio Graph: DMIC Array, DC Blocker, KPB, Keyword Spotter & Host DMA Copier + :alt: Complete end-to-end audio processing pipeline connecting physical DMIC inputs to DC Blocker, KPB, Keyword Spotter, and Host DMA Copier. + + digraph wov_complete_graph { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + subgraph cluster_dmic_be { + label = "DAI Back-End Pipeline (Pipe 1)"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + hw_dmic [label="DMIC Hardware\nArray (16 kHz)\n4-Channel PDM", fillcolor="#E2E8F0", color="#4A5568", penwidth=1.5]; + dai_copier [label="DAI Copier\n(dai-copier.1)\nMulti-Channel DMA", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + dcblock [label="DC Blocker\n(dcblock.1)\nRemoves ADC DC", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + pga_kwd [label="Capture Volume\n(pga.1)\nGain Adjustment", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + } + + subgraph cluster_kpb_hub { + label = "KPB Core Hub (Pipe 2)"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + kpb_widget [label="Key Phrase Buffer\n(kpb.1)\nUUID: D8218443...\nDual-Output Widget", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + } + + subgraph cluster_detect_fe { + label = "Detection Pipeline (Pipe 9)"; + style = "filled,rounded"; + color = "#FEFCBF"; + fillcolor = "#FFFFF0"; + + selector [label="Channel Selector\n(selector.1)\nSelects Voice Mic", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.4]; + detector [label="Keyword Detector\n(TFLM / MFCC / KD)\nEvaluates Wake Phrase", fillcolor="#FEEBC8", color="#C05621", penwidth=1.8]; + det_sink [label="Virtual Detector Sink\n(DAPM Control Node)", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + } + + subgraph cluster_host_fe { + label = "Host Capture Pipeline (Pipe 8)"; + style = "filled,rounded"; + color = "#E9D8FD"; + fillcolor = "#FAF5FF"; + + host_copier [label="Host Copier\n(copier.host.1)\nFast Mode Capable", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.8]; + host_dma [label="Host ALSA Capture\n(hw:0,8)\narecord / Voice AI", fillcolor="#FAF5FF", color="#6B46C1", penwidth=2.0]; + } + + hw_dmic -> dai_copier [label="PDM Pins", color="#4A5568"]; + dai_copier -> dcblock [label="Raw PCM", color="#3182CE"]; + dcblock -> pga_kwd [label="HPF PCM", color="#3182CE"]; + pga_kwd -> kpb_widget [label="4-Ch 16 kHz Stream", color="#22543D", penwidth=1.8]; + + kpb_widget -> selector [label="Pin 0: Real-Time Stream", color="#B7791F", penwidth=1.6]; + selector -> detector [label="1-Ch Voice Stream", color="#B7791F", penwidth=1.5]; + detector -> det_sink [label="Detection Events", color="#4A5568"]; + + kpb_widget -> host_copier [label="Pin 1: Draining & Live Stream", color="#3182CE", penwidth=2.0]; + host_copier -> host_dma [label="PCIe / Memory DMA", color="#6B46C1", penwidth=2.0]; + + detector -> kpb_widget [label="Trigger Event (BEGIN_DRAINING)", color="#C53030", style="dashed", penwidth=1.8]; + } + +Topology 2 Widget Declaration +----------------------------- + +In ALSA Topology 2 (``tools/topology/topology2/include/components/kpb.conf``), the KPB widget is declared as an effect class with one input pin and two output pins: + +.. code-block:: text + + Class.Widget."kpb" { + DefineAttribute."index" {} + DefineAttribute."instance" {} + DefineAttribute."cpc" { + token_ref "comp.word" + } + + + + attributes { + !constructor [ + "index" + "instance" + ] + !mandatory [ + "no_pm" + "uuid" + ] + !immutable [ + "uuid" + ] + unique "instance" + } + + type "effect" + num_input_audio_formats 1 + num_output_audio_formats 1 + + # UUID: D8218443-5FF3-4A4C-B388-6CFE07B9562E + uuid "43:84:21:d8:f3:5f:4c:4a:b3:88:6c:fe:07:b9:56:2e" + no_pm "true" + cpc 720000 + num_input_pins 1 + num_output_pins 2 + } + +Backend Pipeline Integration +---------------------------- + +In ``tools/topology/topology2/include/pipelines/cavs/dai-kpb-be.conf``, the KPB widget is instantiated downstream of the DAI copier: + +.. code-block:: text + + Object.Widget.kpb."1" { + index $DRAINING_PIPELINE_ID + num_input_audio_formats 2 + num_output_audio_formats 2 + + Object.Base.input_audio_format [ + { + in_rate 16000 + in_bit_depth 32 + in_valid_bit_depth 32 + } + { + in_rate 16000 + in_channels 4 + in_bit_depth 32 + in_valid_bit_depth 32 + in_ch_cfg $CHANNEL_CONFIG_3_POINT_1 + } + ] + } + +Host Buffer Sizing Requirements & Best Practices +------------------------------------------------ + +.. important:: + **Host DMA Buffer Sizing**: + Platform resume from ACPI S0ix / Modern Standby requires between 1000 ms and 2000 ms under typical operating conditions. To ensure that pre-roll historical audio is not overwritten before the host application begins consuming samples, the ALSA capture buffer must be dimensioned adequately: + + * The host ``buffer-size`` must be configured to at least **67,200 frames** (:math:`\approx 4.2\text{ seconds}` at 16 kHz). + * Host capture should be invoked with memory-mapped non-blocking I/O: + + .. code-block:: bash + + arecord -Dhw:0,8 -M -N -c 2 -f S16_LE -r 16000 --buffer-size=68000 capture.wav -vvv + + * Smaller buffer allocations will be rejected by the SOF firmware during the ``hw_params`` validation stage with an ``-EINVAL`` error to prevent buffer overrun corruption. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 6f6f89c4..e4cbc447 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -56,6 +56,7 @@ Audio Processing Modules & Algorithms * :ref:`sound_dose` (High-level architecture; also see upstream `Sound Dose README `_) * :ref:`copier_mux_selector` (High-level architecture; also see upstream `Copier README `_, `Mux README `_ & `Selector README `_) * :ref:`pcm_converter` (High-level architecture; also see upstream `PCM converter README `_) +* :ref:`kpb_wov` (High-level architecture; also see driver guide :ref:`keyword_detect`) .. _algorithm-specific-information: @@ -101,6 +102,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/copier_mux_selector firmware/pcm_converter firmware/rtnr + firmware/kpb_wov rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 55a9d523fb30dd22a3b2b3d38a3fcb75c939751e Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 13:17:44 +0100 Subject: [PATCH 24/64] doc: developer_guides: add comprehensive tone generator architecture guide Author a comprehensive, modern architectural guide for the Sound Open Firmware (SOF) Tone Generator (Tone) subsystem. Key architectural concepts and sections: - Signal synthesis & diagnostic roles: Hostless audio pipeline bringup, THD+N & linearity calibration, acoustic transducer profiling, AEC zero-reference fallback, and automated manufacturing line screening. - Mathematical foundations & fixed-point synthesis: Phase accumulator in Q4.28 format, Q16.16 frequency representation, Q1.31 pre-computed angular step coefficients, and 31-bit CORDIC sine calculation achieving >110 dB SFDR and <-105 dB THD without floating-point units. - Envelope & sweep modulation dynamics: 125-microsecond sub-block quantization, anti-click phase reset at zero amplitude, linear attack/sustain/decay ramping, and logarithmic frequency/amplitude sweeping with Q2.30 scale multipliers. - Tri-mode operational engine: TONE_MODE_TONEGEN (autonomous generator when nb_input_pins == 0), TONE_MODE_PASSTHROUGH (zero-overhead copy when bound as sink), and TONE_MODE_SILENCE (clean zero-fill AEC reference stream when uncoupled or in capture direction). - Multi-channel architecture: Independent struct tone_state instances per channel supporting stereo separation, phase-inversion testing, and selective channel muting. - Runtime control & parameter delivery: IPC3 enumerated controls (indices 0-7) vs IPC4 base module configuration and LLEXT dynamic module packaging (SOF_LLEXT_MODULE_MANIFEST). - Bringup & testbench runbook: Topology 1 M4 macro (W_TONE), ALSA control command sequences, and closed-loop host loopback verification via ESP32-P4 and Teensy 4.1 audio bridges. - Seven native vector Graphviz SVG diagrams (Figures 202 through 208). - Integrated into developer_guides/index.rst toctree and modules list. Signed-off-by: Liam Girdwood --- developer_guides/firmware/tone.rst | 808 +++++++++++++++++++++++++++++ developer_guides/index.rst | 2 + 2 files changed, 810 insertions(+) create mode 100644 developer_guides/firmware/tone.rst diff --git a/developer_guides/firmware/tone.rst b/developer_guides/firmware/tone.rst new file mode 100644 index 00000000..b711c7d9 --- /dev/null +++ b/developer_guides/firmware/tone.rst @@ -0,0 +1,808 @@ +.. _tone: + +================================================== +Tone Generator (Tone) Architecture & Signal Engine +================================================== + +.. contents:: + :local: + :depth: 3 + +Sound Open Firmware (SOF) provides an integrated, mathematically rigorous signal synthesis engine known as the **Tone Generator** (``src/audio/tone/``, ``COMP_TONE``, UUID ``tone_uuid``). Unlike standard audio processing components that manipulate existing PCM streams captured from microphones or decoded from host applications, the Tone component is capable of operating as an autonomous, hostless sound source. + +In embedded audio development, bare-metal hardware bringup, manufacturing diagnostic stations, and high-precision acoustic calibration pipelines, having an autonomous DSP-native signal generator is indispensable. Tone enables engineers to inject bit-exact, mathematically pure reference waveforms (single-frequency sinusoids, anti-click windowed tone bursts, logarithmic frequency chirps, and stepped amplitude test sweeps) directly into the downstream DSP pipeline and output DAIs (I2S, SoundWire, HDA, PDM). Because Tone can synthesize audio without requiring an active host streaming application or PCIe/USB interconnect traffic, it serves as the ultimate diagnostic baseline to isolate hardware driver faults, platform clock jitter, amplifier non-linearities, and acoustic transducer distortion. + +Furthermore, Tone features a dynamic multi-mode architecture: beyond standalone tone generation, it functions as a zero-overhead stream passthrough bridge and an Acoustic Echo Cancellation (AEC) reference channel fallback generator that produces mathematical zero-energy silence when capture pipelines operate without playback streams. + +.. graphviz:: + :caption: SOF Tone Subsystem Architecture: Synthesis Engine, Modes & Pipeline Integration + :alt: High-level architectural block diagram showing the Tone component synthesis core, temporal control, multi-channel state, and operational modes. + + digraph tone_system_overview { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + subgraph cluster_control { + label = "Host & Topology Controls"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + ipc_ctrl [label="IPC3 Control / IPC4 Config\nFrequency (Q16.16)\nAmplitude (Q1.31)\nRamp Step / Repeats", fillcolor="#E2E8F0", color="#4A5568", penwidth=1.5]; + alsa_kctrl [label="ALSA Mixer Controls\nEnum & Value Kcontrols\nInteractive Tuning", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + } + + subgraph cluster_tone_core { + label = "Tone Component Core (src/audio/tone/)"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + subgraph cluster_synthesis { + label = "Synthesis Engine (tonegen)"; + style = "filled,rounded"; + color = "#C6F6D5"; + fillcolor = "#F0FFF4"; + + phase_acc [label="Phase Accumulator\nw_step = 2*pi*f/Fs\nw in Q4.28 Radians", fillcolor="#9AE6B4", color="#22543D", penwidth=1.8]; + cordic_sin [label="Fixed-Point CORDIC\nsin_fixed_32b(w)\n31-Bit Vector Rotation", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + ampl_scale [label="Amplitude Scaling\nq_mults_32x32(sin, a)\nQ1.31 Normalized Output", fillcolor="#C6F6D5", color="#276749", penwidth=1.5]; + + phase_acc -> cordic_sin [label="Angle w", color="#22543D"]; + cordic_sin -> ampl_scale [label="sin(w)", color="#22543D"]; + } + + subgraph cluster_envelope { + label = "Temporal Control (tonegen_control)"; + style = "filled,rounded"; + color = "#FEFCBF"; + fillcolor = "#FFFFF0"; + + blk_quant [label="125 us Time Blocks\nsamples_in_block\nCadence Tracker", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.4]; + ramp_eng [label="Linear Ramping\nAttack & Decay Ramp\nAnti-Click Phase Reset", fillcolor="#FEEBC8", color="#C05621", penwidth=1.6]; + sweep_eng [label="Chirp & Sweep Engine\nLogarithmic Freq Coef\nLogarithmic Ampl Coef", fillcolor="#FEEBC8", color="#C05621", penwidth=1.6]; + + blk_quant -> ramp_eng [label="125 us Tick", color="#B7791F"]; + ramp_eng -> sweep_eng [label="Period Expire", color="#C05621"]; + } + + mode_mux [label="Tri-Mode Execution Mux\n- TONEGEN (Autonomous)\n- PASSTHROUGH (Bound)\n- SILENCE (AEC Ref Zero)", fillcolor="#EBF8FF", color="#3182CE", penwidth=2.0]; + } + + subgraph cluster_output { + label = "Pipeline Destinations"; + style = "filled,rounded"; + color = "#E9D8FD"; + fillcolor = "#F7FAFC"; + + sink_buf [label="Output Sink Buffer\nS32_LE Stream Container\nMulti-Channel (1 to 8 Ch)", fillcolor="#FAF5FF", color="#6B46C1", penwidth=1.8]; + dai_out [label="DAI Copier / Endpoint\nI2S / SoundWire / HDA\nFactory / Test Loopback", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + } + + ipc_ctrl -> blk_quant [label="Parameters", color="#4A5568"]; + alsa_kctrl -> ipc_ctrl [label="amixer", color="#4A5568"]; + + ramp_eng -> ampl_scale [label="Target a", color="#C05621", style="dashed"]; + sweep_eng -> phase_acc [label="Update f", color="#C05621", style="dashed"]; + + ampl_scale -> mode_mux [label="Synthesized PCM", color="#22543D", penwidth=1.8]; + mode_mux -> sink_buf [label="Commit Buffer", color="#3182CE", penwidth=2.0]; + sink_buf -> dai_out [label="DMA Stream", color="#6B46C1", penwidth=1.8]; + } + +Role of Embedded Tone Synthesis in Audio DSP & Hardware Bringup +=============================================================== + +In production firmware engineering, validating audio hardware involves complex interactions between operating system kernels, userspace audio servers (ALSA, PulseAudio, PipeWire), bus drivers (PCIe, SoundWire, I2C/I2S), and mixed-signal audio codecs. When an audio pipeline fails to produce sound or exhibits distortion, isolating the failure domain is notoriously difficult: + +* Did the host userspace application underrun the DMA ring buffer? +* Did the kernel ASoC machine driver configure incorrect DAI clock dividers or time-slot allocation (TDM)? +* Did the DSP operating system miss real-time deadlines, corrupting PCM circular pointers? +* Did the external audio codec or smart amplifier enter thermal shutdown, DC protection, or experience analog clipping? + +The SOF Tone component eliminates these variables by embedding signal synthesis directly into the DSP execution graph. Operating at the boundary of the DSP and digital audio interfaces, Tone provides an authoritative reference standard for hardware characterization. + +Key Architectural Use Cases +--------------------------- + +.. list-table:: Core Use Cases for SOF Tone Generator + :widths: 22 28 50 + :header-rows: 1 + + * - Application Domain + - Operating Configuration + - Technical Function & Diagnostic Value + * - **Hostless Hardware Bringup** + - Standalone playback pipeline without host stream + - Generates clean 997 Hz / -20 dBFS sine waves directly to digital audio interfaces (I2S, SoundWire, HDA). Verifies BCLK, WCLK, MCLK, and frame sync timing on oscilloscope/logic analyzer without host driver dependencies. + * - **THD+N & Linearity Calibration** + - Single-frequency pure sinusoid at varying amplitudes + - Supplies bit-exact test signals to external Audio Precision or host loopback bridges (ESP32-P4 / Teensy 4.1) to measure **Total Harmonic Distortion plus Noise (THD+N)**, dynamic range, and DAC linearity. + * - **Acoustic Transducer Profiling** + - Logarithmic stepped chirp sweeps + - Sweeps across audible frequencies (:math:`20\text{ Hz} \dots 20\text{ kHz}`) to measure micro-speaker resonant frequencies (:math:`f_0`), acoustic enclosure frequency responses, and passive radiator impedance. + * - **AEC Reference Fallback** + - ``TONE_MODE_SILENCE`` + - Provides a synchronized, zero-energy reference channel for Acoustic Echo Cancellation (AEC) algorithms when capture streams run without active media playback, preventing division-by-zero filter instabilities. + * - **Manufacturing Line Functional Testing** + - Automated multi-tone burst sequences + - Executes rapid acoustic pass/fail screening on assembly lines, confirming speaker voice-coil continuity and microphone array sensitivity in under 500 ms. + +Mathematical Foundations: Fixed-Point CORDIC Sine & Phase Accumulation +====================================================================== + +Generating pure trigonometric waveforms in an embedded audio DSP requires high mathematical precision, deterministic execution timing, and zero reliance on high-latency software floating-point emulation. The SOF Tone generator implements a 32-bit fixed-point synthesis architecture utilizing a **Phase Accumulator** coupled with a **Coordinate Rotation Digital Computer (CORDIC)** algorithm. + +Phase Accumulator Mechanics +--------------------------- + +A continuous sinusoidal signal of frequency :math:`f` sampled at frequency :math:`f_s` is defined mathematically as: + +.. math:: + + y(n) = A \cdot \sin(\omega_n) = A \cdot \sin\left(2\pi \frac{f}{f_s} \cdot n + \phi_0\right) + +In digital signal processing, the instantaneous angular phase :math:`\omega_n` is computed recursively by a phase accumulator: + +.. math:: + + \omega_{n} = (\omega_{n-1} + \Delta \omega) \pmod{2\pi} + +where the angular step :math:`\Delta \omega` represents the phase advance per discrete sample: + +.. math:: + + \Delta \omega = 2\pi \frac{f}{f_s} + +Fixed-Point Number Representations +---------------------------------- + +To preserve dynamic range and phase accuracy while avoiding integer overflow, Tone utilizes three specialized fixed-point Q-formats: + +1. **Angular Phase** (:math:`\omega`) **and Step** (:math:`\Delta \omega`): + Represented in **Q4.28** format (4 integer bits including sign, 28 fractional bits). + In this format, one radian is represented as :math:`2^{28} = 268{,}435{,}456`. + The circular modulus :math:`2\pi` is represented exactly by the constant: + + .. math:: + + 2\pi_{\text{Q4.28}} = \text{round}(2\pi \times 2^{28}) = 1{,}686{,}629{,}713 \quad (\text{hex: } \mathtt{0x6487ED51}) + + and :math:`\pi_{\text{Q4.28}} = 843{,}314{,}857` (:math:`\mathtt{0x3243F6A9}`). + +2. **Oscillator Frequency** (:math:`f`): + Represented in **Q16.16** format (16 integer bits, 16 fractional bits), allowing frequency precision of :math:`1/65536 \approx 15.26\text{ }\mu\text{Hz}` with a maximum frequency of 32,767.99 Hz. + +3. **Sample Rate Coefficient** (:math:`c = 2\pi / f_s`): + Represented in **Q1.31** format (1 sign bit, 31 fractional bits). Pre-computed lookup tables store :math:`c` for 13 standard sample rates (from 8 kHz to 192 kHz), avoiding expensive run-time divisions: + +.. list-table:: Pre-computed Angular Step Coefficients (:math:`c = 2\pi / f_s`) in Q1.31 Format + :widths: 20 25 25 30 + :header-rows: 1 + + * - Sample Rate (:math:`f_s`) + - Mathematical Value (:math:`2\pi / f_s`) + - Q1.31 Integer Value + - Hexadecimal Value + * - **8,000 Hz** + - :math:`0.000785398` + - 1,686,630 + - ``0x0019BC66`` + * - **16,000 Hz** + - :math:`0.000392699` + - 843,315 + - ``0x000CDE33`` + * - **44,100 Hz** + - :math:`0.000142476` + - 305,965 + - ``0x0004AB2D`` + * - **48,000 Hz** + - :math:`0.000130900` + - 281,105 + - ``0x00044A11`` + * - **96,000 Hz** + - :math:`0.000065450` + - 140,552 + - ``0x00022508`` + * - **192,000 Hz** + - :math:`0.000032725` + - 70,276 + - ``0x00011284`` + +The angular phase step :math:`\Delta \omega` is computed via fixed-point multiplication: + +.. math:: + + \Delta \omega_{\text{Q4.28}} = \frac{f_{\text{Q16.16}} \times c_{\text{Q1.31}}}{2^{19}} + +which maps directly to the SOF math utility: + +.. code-block:: c + + w_tmp = q_multsr_32x32(sg->f, sg->c, Q_SHIFT_BITS_64(16, 31, 28)); + +Nyquist Limiting & Phase Accumulation +------------------------------------- + +To eliminate aliasing, the requested frequency is hard-clamped to the platform Nyquist threshold (:math:`f \le f_s / 2`): + +.. math:: + + f_{\text{clamped}} = \min\left(f, \frac{f_s}{2}\right), \quad \Delta \omega_{\text{clamped}} = \min(\Delta \omega, \pi_{\text{Q4.28}}) + +On each sample cycle, the phase accumulator advances: + +.. math:: + + \omega_{n} = \begin{cases} \omega_{n-1} + \Delta \omega - 2\pi_{\text{Q4.28}}, & \text{if } (\omega_{n-1} + \Delta \omega) > 2\pi_{\text{Q4.28}} \\ \omega_{n-1} + \Delta \omega, & \text{otherwise} \end{cases} + +.. graphviz:: + :caption: Mathematical Foundations: Phase Accumulator & 31-bit CORDIC Trigonometric Engine + :alt: Detailed mathematical dataflow of the fixed-point phase accumulator, angular modulo, CORDIC vector rotation, and amplitude scaling. + + digraph tone_math_pipeline { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + subgraph cluster_phase { + label = "1. Phase Accumulation (Q4.28)"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + f_in [label="Frequency f\nQ16.16 (Hz)\ne.g. 997.0 Hz", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + c_lut [label="Table Look-up\nc = 2*pi/Fs\nQ1.31 Format", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + step_mult [label="w_step Calculation\nq_multsr_32x32(f, c)\nClamped to pi", fillcolor="#9AE6B4", color="#22543D", penwidth=1.8]; + phase_acc [label="Accumulator\nw = w + w_step\nModulo 2*pi", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + + f_in -> step_mult [label="f", color="#3182CE"]; + c_lut -> step_mult [label="c", color="#4A5568"]; + step_mult -> phase_acc [label="w_step", color="#22543D", penwidth=1.5]; + } + + subgraph cluster_cordic { + label = "2. 31-bit CORDIC Vector Rotation"; + style = "filled,rounded"; + color = "#FEFCBF"; + fillcolor = "#FFFFF0"; + + quad_map [label="Quadrant Reduction\nFold w into [-pi/2, pi/2)\nTrack Sign Bit", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.4]; + cordic_rot [label="CORDIC Iterations\n31 Shift-and-Add Micro-Rotations\nNo Floating-Point Operations", fillcolor="#FEEBC8", color="#C05621", penwidth=2.0]; + sin_out [label="sin_fixed_32b(w)\nPure Sine Value\nQ1.31 [-1.0, 1.0]", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.5]; + + phase_acc -> quad_map [label="Angle w", color="#22543D"]; + quad_map -> cordic_rot [label="Residual Angle", color="#B7791F"]; + cordic_rot -> sin_out [label="Vector Y", color="#C05621", penwidth=1.8]; + } + + subgraph cluster_output_scale { + label = "3. Amplitude Scaling & Packaging"; + style = "filled,rounded"; + color = "#C6F6D5"; + fillcolor = "#F0FFF4"; + + ampl_in [label="Target Amplitude a\nQ1.31 Format\ne.g. -20 dBFS", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + scale_mult [label="Saturation Multiply\nq_mults_32x32(sin, a)\nQ1.31 Saturation Guard", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + pcm_sample [label="Output Sample\n32-bit Integer PCM\nDirect to Buffer", fillcolor="#FAF5FF", color="#6B46C1", penwidth=1.8]; + + sin_out -> scale_mult [label="sin(w)", color="#B7791F"]; + ampl_in -> scale_mult [label="a", color="#3182CE"]; + scale_mult -> pcm_sample [label="y(n) S32_LE", color="#6B46C1", penwidth=2.0]; + } + } + +31-bit CORDIC Sine Engine (sin_fixed_32b) +------------------------------------------ + +Rather than maintaining massive sine lookup tables in precious DSP cache or incurring non-linear interpolation distortion, SOF evaluates the sine function using the **CORDIC** algorithm (``src/math/trig.c``). + +CORDIC operates by executing iterative vector micro-rotations using only bit-shifts and additions: + +.. math:: + + x_{i+1} = x_i - d_i \cdot y_i \cdot 2^{-i} + +.. math:: + + y_{i+1} = y_i + d_i \cdot x_i \cdot 2^{-i} + +.. math:: + + z_{i+1} = z_i - d_i \cdot \alpha_i + +where :math:`d_i = \text{sgn}(z_i)` and :math:`\alpha_i = \arctan(2^{-i})`. Over 31 iterations, the residual angle :math:`z` converges to zero, and the vector coordinates :math:`(x, y)` converge to :math:`K \cdot (\cos \omega, \sin \omega)` where :math:`K \approx 1.646760258` is the known CORDIC scaling gain. + +The result is a mathematically pure sinusoid with an **Spurious-Free Dynamic Range (SFDR) exceeding 110 dB** and total harmonic distortion below :math:`-105\text{ dB}`, surpassing the noise floor of commercial 24-bit audio converters. + +Temporal Enveloping, Anti-Click Phase Alignment & Linear Ramping +================================================================ + +In audio synthesis, abruptly switching on or cutting off a sine wave introduces severe high-frequency spectral splatter, perceived acoustically as an audible "click" or "pop": + +.. math:: + + x(t) = A \sin(\omega t) \cdot u(t) \quad \xrightarrow{\mathcal{F}} \quad X(j\Omega) = \frac{A\omega}{\omega_0^2 - \Omega^2} + \frac{\pi A}{2j}[\delta(\Omega - \omega_0) - \delta(\Omega + \omega_0)] + +The step discontinuity :math:`u(t)` scatters energy across the entire audio spectrum. To guarantee pristine acoustic transients, the SOF Tone generator implements a dedicated temporal control subsystem (``tonegen_control``). + +125 Microsecond Sub-Block Quantization +-------------------------------------- + +Temporal envelope modifications (ramping, sweeping, state transitions) are evaluated in standardized sub-blocks of **125 microseconds** (:math:`\Delta t_{\text{block}} = 125\text{ }\mu\text{s}`), corresponding to an update frequency of 8,000 Hz. The number of audio samples per 125 :math:`\mu`\ s block is: + +.. math:: + + N_{\text{samples\_in\_block}} = \text{round}\left(f_s \times 125 \times 10^{-6}\right) = \begin{cases} 1, & \text{if } f_s = 8000\text{ Hz} \\ 2, & \text{if } f_s = 16000\text{ Hz} \\ 6, & \text{if } f_s = 48000\text{ Hz} \\ 12, & \text{if } f_s = 96000\text{ Hz} \end{cases} + +Evaluating envelope parameters at 125 :math:`\mu`\ s intervals decouples envelope timing from audio pipeline period sizes (e.g. 1 ms or 4 ms) while dramatically reducing DSP instruction overhead compared to per-sample evaluation. + +Anti-Click Phase Reset +---------------------- + +When a tone burst is initiated from complete silence (:math:`a = 0`), Tone automatically forces the phase accumulator to zero: + +.. code-block:: c + + if (sg->a == 0) + sg->w = 0; /* Reset phase to have less clicky ramp */ + +Starting synthesis at :math:`\omega = 0` guarantees that the waveform commences precisely at its mathematical zero-crossing (:math:`\sin(0) = 0`), eliminating phase discontinuity transients. + +Three-Phase Envelope Trajectory +------------------------------- + +The temporal envelope of a tone burst is partitioned into three chronological phases: + +1. **Attack Phase (Fade-In Ramp)** (:math:`0 \le t_{\text{block}} < t_{\text{length}}`): + The instantaneous amplitude :math:`a` advances toward the target amplitude :math:`a_{\text{target}}` in linear steps: + + .. math:: + + a_{m} = \min(a_{m-1} + \Delta a_{\text{ramp}}, a_{\text{target}}) + + where :math:`\Delta a_{\text{ramp}}` is the configured ``ramp_step`` in Q1.31 format. +2. **Sustain Phase** (:math:`a = a_{\text{target}}`): + The tone maintains steady-state amplitude for the remainder of the active duration (:math:`\text{tone\_length}`). +3. **Decay Phase (Fade-Out Ramp)** (:math:`t_{\text{length}} \le t_{\text{block}} < t_{\text{period}}`): + Once :math:`t_{\text{block}}` exceeds ``tone_length``, the amplitude ramps linearly back to zero: + + .. math:: + + a_{m} = \max(a_{m-1} - \Delta a_{\text{ramp}}, 0) + +.. graphviz:: + :caption: Temporal Enveloping: Linear Attack, Sustain, Decay & Anti-Click Phase Alignment + :alt: Timing waveform showing anti-click phase zero-crossing, linear attack ramp, active sustain window, and linear decay ramp. + + digraph tone_envelope { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + subgraph cluster_envelope_flow { + label = "Tone Burst Temporal Envelope & Zero-Crossing Progression"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + p0 [label="1. Idle / Mute State\na = 0 | Phase w = 0\nZero Energy Silence", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + p1 [label="2. Tone Start & Anti-Click Reset\nw forced to 0 rad (Zero Crossing)\nLinear Attack Ramp Commences\na += ramp_step per 125 us", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + p2 [label="3. Active Sustain Duration (tone_length)\na = a_target (e.g. -20 dBFS)\nContinuous Pure Sinusoid\nStable Spectral Content", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + p3 [label="4. Linear Decay Ramp\nBlock count > tone_length\na -= ramp_step per 125 us\nControlled Fade-Out", fillcolor="#FEEBC8", color="#C05621", penwidth=1.6]; + p4 [label="5. Inactive Inter-Burst Pause\na = 0 | Waiting for tone_period\nPrepares Next Sweep Step", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + + p0 -> p1 [label="Trigger Start", color="#3182CE", penwidth=1.5]; + p1 -> p2 [label="a reaches a_target", color="#22543D", penwidth=1.8]; + p2 -> p3 [label="block_count == tone_length", color="#C05621", penwidth=1.8]; + p3 -> p4 [label="a reaches 0", color="#4A5568", penwidth=1.5]; + p4 -> p1 [label="repeat_count < repeats\n(block_count > tone_period)", color="#3182CE", style="dashed", penwidth=1.5]; + } + } + +Chirp Synthesis, Logarithmic Sweeps & Multi-Tone Stepping +========================================================= + +In acoustic engineering, single fixed-frequency tones provide limited diagnostic visibility. To measure the full frequency response, resonant modes, and acoustic distortion of a loudspeaker or audio pipeline, automated frequency chirps and amplitude sweeps are required. + +The Tone generator incorporates a built-in logarithmic sweep engine capable of executing stepped multi-tone chirps without requiring external host scripting. + +Mathematical Formulation of Logarithmic Sweeps +---------------------------------------------- + +Upon completion of each active period (:math:`\text{block\_count} > \text{tone\_period}`), if the repetition counter has not reached the configured limit (``sg->repeat_count + 1 < sg->repeats``), Tone updates its synthesis parameters for the subsequent burst: + +1. **Logarithmic Frequency Progression**: + The frequency :math:`f_{k+1}` of the :math:`(k+1)`-th burst is derived from the previous frequency :math:`f_k` via multiplication by a frequency coefficient :math:`\beta_f` (``sg->freq_coef``) represented in **Q2.30** format: + + .. math:: + + f_{k+1} = f_k \times \beta_f + + where: + + .. math:: + + \beta_f = \frac{\text{freq\_coef}}{2^{30}} + + * If :math:`\beta_f > 1.0` (e.g. ``freq_coef = 1181116006`` :math:`\approx 1.10`), the frequency increases exponentially on each step (ascending chirp). + * If :math:`\beta_f < 1.0` (e.g. ``freq_coef = 966367641`` :math:`\approx 0.90`), the frequency decreases exponentially (descending chirp). + * If :math:`\beta_f = 1.0` (``ONE_Q2_30 = 1073741824``), the frequency remains constant. + + The calculation executes with rounding in 64-bit precision: + + .. code-block:: c + + p = q_multsr_32x32(sg->f, sg->freq_coef, Q_SHIFT_BITS_64(16, 30, 16)); + tonegen_update_f(sg, (int32_t)p); + +2. **Logarithmic Amplitude Progression**: + Similarly, the target amplitude :math:`a_{k+1}` scales on each burst via an amplitude multiplier :math:`\beta_a` (``sg->ampl_coef`` in Q2.30): + + .. math:: + + a_{k+1} = \text{sat}_{31}\left(a_k \times \beta_a\right) + + This enables automated linearity tests, sweeping signal amplitude from :math:`-60\text{ dBFS}` to :math:`0\text{ dBFS}` in discrete steps to locate amplifier compression thresholds and speaker voice-coil rubbing. + +.. graphviz:: + :caption: Logarithmic Sweep & Chirp Engine (Frequency Multiplication, Stepping & Repeats) + :alt: Architectural diagram of the multi-burst chirp synthesis engine showing frequency multiplication, amplitude scaling, and repeat tracking. + + digraph tone_sweep_engine { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + subgraph cluster_params { + label = "Sweep Configuration"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + f_start [label="Initial Frequency f_0\ne.g. 100 Hz (Q16.16)", fillcolor="#EBF8FF", color="#3182CE"]; + f_mult [label="Frequency Multiplier\nfreq_coef (Q2.30)\ne.g. 1.10 (+10% / step)", fillcolor="#EDF2F7", color="#4A5568"]; + a_mult [label="Amplitude Multiplier\nampl_coef (Q2.30)\ne.g. 1.00 (Flat)", fillcolor="#EDF2F7", color="#4A5568"]; + rep_max [label="Total Steps (repeats)\ne.g. 30 Tone Bursts", fillcolor="#EDF2F7", color="#4A5568"]; + } + + subgraph cluster_iteration { + label = "Burst Sequence Generator"; + style = "filled,rounded"; + color = "#FEFCBF"; + fillcolor = "#FFFFF0"; + + step_eval [label="End-of-Period Detector\nblock_count > tone_period\nrepeat_count < repeats", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.5]; + f_calc [label="Frequency Update\nf_{k+1} = f_k * freq_coef\nw_step Recalculated", fillcolor="#FEEBC8", color="#C05621", penwidth=1.8]; + a_calc [label="Amplitude Update\na_{k+1} = a_k * ampl_coef\nTarget Clamped", fillcolor="#FEEBC8", color="#C05621", penwidth=1.8]; + cnt_inc [label="Increment Counter\nrepeat_count++\nReset block_count = 0", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.4]; + + step_eval -> f_calc [label="Update Freq", color="#C05621"]; + step_eval -> a_calc [label="Update Ampl", color="#C05621"]; + f_calc -> cnt_inc [color="#B7791F"]; + a_calc -> cnt_inc [color="#B7791F"]; + } + + subgraph cluster_playback { + label = "Acoustic Output"; + style = "filled,rounded"; + color = "#C6F6D5"; + fillcolor = "#F0FFF4"; + + tone_burst [label="Stepped Logarithmic Chirp\nBurst 1: 100 Hz\nBurst 2: 110 Hz\nBurst 3: 121 Hz ...\nBurst 30: 17.4 kHz", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + } + + f_start -> step_eval [color="#3182CE"]; + f_mult -> f_calc [color="#4A5568"]; + a_mult -> a_calc [color="#4A5568"]; + rep_max -> step_eval [color="#4A5568"]; + + cnt_inc -> tone_burst [label="Synthesize Burst", color="#22543D", penwidth=1.8]; + cnt_inc -> step_eval [label="Next Cycle", color="#B7791F", style="dashed"]; + } + +Operational Modes & Multi-Channel Architecture +============================================== + +The SOF Tone component features a versatile tri-mode operational crossbar (``cd->mode``) that adapts dynamically depending on pipeline binding and hardware configuration: + +1. **Autonomous Tone Generation Mode (``TONE_MODE_TONEGEN = 0``)**: + Default operating state when Tone is instantiated without active upstream source components (e.g. ``nb_input_pins == 0``). Tone acts as an autonomous data producer, writing synthesized sine waveforms into its downstream sink buffer on every pipeline period tick. +2. **Stream Passthrough Mode (``TONE_MODE_PASSTHROUGH = 1``)**: + Activated automatically in modular IPC4 topologies when an upstream source module binds to Tone (``tone_bind()``). In this mode, Tone suspends signal synthesis and transparently forwards incoming PCM samples from source to sink with zero latency and full circular buffer boundary wrapping. This allows Tone to remain embedded in production topologies as an on-demand diagnostic probe without requiring topology rebuilds. +3. **Pure Silence Generation Mode (``TONE_MODE_SILENCE = 2``)**: + Activated when Tone is bound to capture pipelines as an echo reference fallback (e.g. ``nb_input_pins > 0`` in capture direction) or when explicitly uncoupled. Writes mathematical zero values (``*output_pos = 0``), ensuring that downstream Acoustic Echo Cancellation (AEC) or matrix mixers receive a valid, clean zero-energy reference stream. + +.. graphviz:: + :caption: Tri-Mode Execution Crossbar: ToneGen, Passthrough & Silence Modes + :alt: Diagram illustrating the three operational execution modes of the Tone component: autonomous generation, passthrough forwarding, and silence generation. + + digraph tone_modes_crossbar { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + subgraph cluster_inputs { + label = "Pipeline Inputs"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + src_pcm [label="Source Buffer\nUpstream Stream\n(Optional)", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + no_src [label="Hostless / No Source\nAutonomous Mode", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + } + + subgraph cluster_mode_logic { + label = "Tone Operational Crossbar"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + m_tonegen [label="TONE_MODE_TONEGEN (0)\nAutonomous Sine / Chirp\ntonegen() + tonegen_control()", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + m_pass [label="TONE_MODE_PASSTHROUGH (1)\nLinear Circular Copy\nZero Processing Overhead", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + m_silence [label="TONE_MODE_SILENCE (2)\nZero-Fill (output = 0)\nAEC Reference Fallback", fillcolor="#FED7D7", color="#C53030", penwidth=1.5]; + } + + subgraph cluster_output_stream { + label = "Output Destination"; + style = "filled,rounded"; + color = "#E9D8FD"; + fillcolor = "#FAF5FF"; + + sink_out [label="Sink Buffer (S32_LE)\nDownstream Audio Pipeline\n(DAI Copier / Mixer)", fillcolor="#FAF5FF", color="#6B46C1", penwidth=2.0]; + } + + no_src -> m_tonegen [label="nb_input_pins == 0", color="#22543D", penwidth=1.8]; + src_pcm -> m_pass [label="Dynamic Bind\nCOMP_BIND_TYPE_SOURCE", color="#B7791F", penwidth=1.6]; + src_pcm -> m_silence [label="Capture Echo Fallback\nOr Unbind", color="#C53030", style="dashed", penwidth=1.4]; + + m_tonegen -> sink_out [label="Synthesized PCM", color="#22543D", penwidth=2.0]; + m_pass -> sink_out [label="Passthrough Copy", color="#B7791F", penwidth=1.8]; + m_silence -> sink_out [label="Zero Samples", color="#C53030", penwidth=1.5]; + } + +Multi-Channel Architecture +-------------------------- + +The Tone component supports multi-channel stream topologies up to ``PLATFORM_MAX_CHANNELS`` (typically 8 channels). Each channel maintains an completely independent state structure (``struct tone_state sg[i]``). + +This multi-channel independence enables advanced acoustic test configurations: + +* **Independent Channel Frequencies**: Generating 1 kHz on Channel 0 (Left) and 2 kHz on Channel 1 (Right) to verify stereo separation and detect inter-channel crosstalk. +* **Phase-Inversion Testing**: Configuring opposite phase angles (:math:`\Delta \phi = \pi`) between stereo channels to test differential amplifier performance or verify acoustic phase cancellation in noise-canceling headsets. +* **Selective Channel Muting**: Muting individual channels (``tonegen_mute(&cd->sg[i])``) while maintaining active generation on adjacent channels to detect hardware trace leakage. + +Runtime Control, ALSA Mixers & IPC3/IPC4 Parameter Delivery +=========================================================== + +The Tone generator provides comprehensive runtime control across both legacy IPC3 and modern IPC4 architectures: + +IPC3 Control Interface (SOF_CTRL_CMD_ENUM) +------------------------------------------ + +Under IPC3, Tone exposes eight control indices mapped through the standard ALSA mixer enumerated control interface: + +.. list-table:: IPC3 Tone Control Indices (user/tone.h) + :widths: 30 15 55 + :header-rows: 1 + + * - Control Index + - Value + - Functional Parameter & Format + * - ``SOF_TONE_IDX_FREQUENCY`` + - 0 + - Oscillation frequency in Hertz represented in **Q16.16** format. + * - ``SOF_TONE_IDX_AMPLITUDE`` + - 1 + - Target sine wave peak amplitude represented in **Q1.31** format. + * - ``SOF_TONE_IDX_FREQ_MULT`` + - 2 + - Step frequency multiplier for logarithmic chirps in **Q2.30** format. + * - ``SOF_TONE_IDX_AMPL_MULT`` + - 3 + - Step amplitude multiplier for stepped sweeps in **Q2.30** format. + * - ``SOF_TONE_IDX_LENGTH`` + - 4 + - Active tone burst duration in units of 125 :math:`\mu`\ s blocks. + * - ``SOF_TONE_IDX_PERIOD`` + - 5 + - Total cycle period (active duration + idle pause) in 125 :math:`\mu`\ s blocks. + * - ``SOF_TONE_IDX_REPEATS`` + - 6 + - Total number of sweep burst repetitions. + * - ``SOF_TONE_IDX_LIN_RAMP_STEP`` + - 7 + - Linear amplitude modification step per 125 :math:`\mu`\ s block in **Q1.31** format. + +.. graphviz:: + :caption: Runtime Parameter Delivery & ALSA Control Topology (IPC3 vs IPC4) + :alt: Diagram comparing IPC3 enumerated ALSA mixer control dispatch with IPC4 base module configuration and dynamic binding. + + digraph tone_ipc_topology { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + subgraph cluster_host_user { + label = "Host Userspace / Developer"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + amixer_cmd [label="ALSA amixer / sof-ctl\namixer -c 0 cset name='Tone Freq' 997\nInteractive Developer Tuning", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + } + + subgraph cluster_ipc3_flow { + label = "IPC3 Control Framework"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + ipc3_msg [label="SOF_CTRL_CMD_ENUM\nIndex 0..7 Parameters\nchannel + svalue array", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + ipc3_hdl [label="tone_cmd_set_value()\ntonegen_update_f()\ntonegen_set_a()\ntonegen_set_linramp()", fillcolor="#9AE6B4", color="#22543D", penwidth=1.8]; + + ipc3_msg -> ipc3_hdl [label="cdata->index", color="#22543D"]; + } + + subgraph cluster_ipc4_flow { + label = "IPC4 Modular Architecture"; + style = "filled,rounded"; + color = "#FED7D7"; + fillcolor = "#FFF5F5"; + + ipc4_init [label="Base Module Config\nSampling Frequency\nChannel Count (S32_LE)", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + ipc4_bind [label="Module Adapter Callbacks\n- tone_bind() -> PASSTHROUGH\n- tone_unbind() -> SILENCE\n- tone_params() -> Init ToneGen", fillcolor="#FEB2B2", color="#C53030", penwidth=1.8]; + llext_mod [label="LLEXT Dynamic Manifest\nSOF_LLEXT_MODULE_MANIFEST\nLoadable Relocatable ELF", fillcolor="#FED7D7", color="#9B2C2C", penwidth=1.5]; + + ipc4_init -> ipc4_bind [color="#C53030"]; + llext_mod -> ipc4_bind [label="Dynamic Linking", color="#9B2C2C", style="dashed"]; + } + + subgraph cluster_state_target { + label = "DSP Runtime State"; + style = "filled,rounded"; + color = "#E9D8FD"; + fillcolor = "#FAF5FF"; + + sg_state [label="struct tone_state sg[i]\nActive Waveform Generation", fillcolor="#FAF5FF", color="#6B46C1", penwidth=2.0]; + } + + amixer_cmd -> ipc3_msg [label="IPC3 Driver", color="#3182CE"]; + amixer_cmd -> ipc4_init [label="IPC4 Driver", color="#3182CE"]; + + ipc3_hdl -> sg_state [label="Apply Changes", color="#22543D", penwidth=1.8]; + ipc4_bind -> sg_state [label="Set Mode / Freq", color="#C53030", penwidth=1.8]; + } + +IPC4 Modular Adapter & LLEXT Packaging +-------------------------------------- + +In IPC4 environments, the Tone generator is implemented as a standardized processing module (``src/audio/tone/tone-ipc4.c``) conforming to the SOF Module Adapter API: + +.. code-block:: c + + static const struct module_interface tone_interface = { + .init = tone_init, + .prepare = tone_prepare, + .process = tone_process, + .reset = tone_reset, + .free = tone_free, + .bind = tone_bind, + .unbind = tone_unbind, + }; + +Tone is declared with Loadable Linkable Extension (LLEXT) metadata: + +.. code-block:: c + + static const struct sof_man_module_manifest mod_manifest[] __section(".module") __used = { + SOF_LLEXT_MODULE_MANIFEST("TONE", &tone_interface, 1, SOF_REG_UUID(tone), 30), + }; + +This enables Tone to be built either as an embedded static component within the base firmware image or packaged as a standalone, dynamically loadable ELF module (``.llext``) deployed on demand. + +Hostless Playback, Factory Loopback & Diagnostics Runbook +========================================================= + +The ability to operate without an active host PCM audio stream makes Tone the foundational component for automated manufacturing line tests and hardware diagnostic testbenches. + +Hostless Test Pipeline Topology +------------------------------- + +Figure 208 illustrates an end-to-end hostless audio verification pipeline configured in Sound Open Firmware: + +.. graphviz:: + :caption: End-to-End Bringup Audio Pipeline: Hostless Tone Generator to DAI Output & Closed-Loop Testbench + :alt: Full system audio topology connecting the autonomous Tone generator to Volume control, DAI output, external hardware loopback, and analysis instruments. + + digraph tone_test_pipeline { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + subgraph cluster_sof_dsp { + label = "SOF Audio DSP Pipeline (Hostless Playback)"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + tone_mod [label="Tone Generator (tone.1)\nUUID: tone_uuid\nAutonomous Sine / Chirp\n997 Hz / -20 dBFS", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + pga_mod [label="Volume Control (pga.1)\nDigital Gain & Attenuation\nOptional Mute Protection", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.4]; + dai_mod [label="DAI Copier (dai-copier.1)\nI2S / SoundWire Controller\nDMA Transmit Engine", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.8]; + + tone_mod -> pga_mod [label="S32_LE PCM", color="#22543D", penwidth=1.8]; + pga_mod -> dai_mod [label="Scaled Stream", color="#3182CE", penwidth=1.8]; + } + + subgraph cluster_hardware_loopback { + label = "Hardware Test Loopback (Lab Network)"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#FFFFFF"; + + dut_pins [label="DUT Physical Pins\nI2S BCLK, WCLK, DOUT\nOr SoundWire Data Line", fillcolor="#E2E8F0", color="#4A5568", penwidth=1.5]; + bridge_card [label="Audio Bridge Card\nESP32-P4 / Teensy 4.1\nLoopback Audio Capture", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + } + + subgraph cluster_analytics { + label = "Automated Audio Quality Analysis"; + style = "filled,rounded"; + color = "#E9D8FD"; + fillcolor = "#FAF5FF"; + + fft_analysis [label="Host FFT Analyzer\nFrequency Verification\nTHD+N Calculation\nPass / Fail Thresholds", fillcolor="#FAF5FF", color="#6B46C1", penwidth=2.0]; + } + + dai_mod -> dut_pins [label="Digital Audio Bus", color="#3182CE", penwidth=2.0]; + dut_pins -> bridge_card [label="Physical Loopback Wiring", color="#B7791F", penwidth=1.8]; + bridge_card -> fft_analysis [label="USB Audio (UAC2) Stream", color="#6B46C1", penwidth=2.0]; + } + +Topology 1 M4 Declaration +------------------------- + +In legacy and test topologies (``tools/topology/topology1/m4/tone.m4``), the Tone component is declared with its buffer properties: + +.. code-block:: text + + # Tone component definition + # W_TONE(name, format, periods_sink, periods_source, core, kcontrols) + W_TONE(Tone 1, 32, 2, 0, 0, LIST(` ', `TONE_IN_CONTROLS')) + +Automated Verification Runbook +------------------------------ + +To execute a closed-loop audio quality verification test using the embedded Tone generator: + +1. **Deploy Hostless Tone Topology**: + Deploy a topology containing the autonomous Tone pipeline connected directly to the target DAI (e.g. ``test-tone-playback.m4``): + + .. code-block:: bash + + sof-ctl -Dhw:0 -c name='Tone 1 Tone Freq' -v 997 + sof-ctl -Dhw:0 -c name='Tone 1 Tone Amplitude' -v 214748364 + +2. **Trigger Pipeline Playback**: + Start the pipeline trigger using the SOF testbench or ALSA control utilities: + + .. code-block:: bash + + alsactl -f /var/lib/alsa/asound.state restore + +3. **Capture via External Bridge**: + Record the digital stream on an external loopback card (e.g. ESP32-P4 or Teensy 4.1): + + .. code-block:: bash + + arecord -Dhw:CARD=Bridge,DEV=0 -r 48000 -c 2 -f S32_LE -d 5 /tmp/tone_capture.wav + +4. **Verify Harmonic Distortion**: + Execute automated FFT spectral analysis on the recorded WAV file to confirm that the fundamental peak is exactly 997.0 Hz and that spurious harmonics satisfy :math:`\text{THD+N} < -90\text{ dBFS}`. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index e4cbc447..8fd3959f 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -57,6 +57,7 @@ Audio Processing Modules & Algorithms * :ref:`copier_mux_selector` (High-level architecture; also see upstream `Copier README `_, `Mux README `_ & `Selector README `_) * :ref:`pcm_converter` (High-level architecture; also see upstream `PCM converter README `_) * :ref:`kpb_wov` (High-level architecture; also see driver guide :ref:`keyword_detect`) +* :ref:`tone` (High-level architecture; also see upstream `Tone README `_) .. _algorithm-specific-information: @@ -103,6 +104,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/pcm_converter firmware/rtnr firmware/kpb_wov + firmware/tone rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 59ffffd1e6538d904675f19f5d7f1cd319ebfd00 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 13:28:33 +0100 Subject: [PATCH 25/64] doc: developer_guides: add comprehensive up/down mixer architecture guide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Author a comprehensive, modern architectural guide for the Sound Open Firmware (SOF) Up/Down Channel Mixer (up_down_mixer) subsystem. Key topics covered: - Spatial acoustic conversion across Mono, Stereo, 2.1, 3.0, 3.1, Quatro (4.0 surround), 4.0 (L/C/R/Cs), 5.0, 5.1, and 7.1. - Architectural comparison with Mixin/Mixout, Selector, and Copier. - ITU-R BS.775 standard downmixing formulation and mathematical derivation of headroom-scaled anti-clipping coefficients (1/(1+sqrt(2)) ≈ 0.4142). - Half-scaled coefficients for 3.0/3.1 and Quatro-to-Mono fold-down. - 32-bit (Q1.31) and 16-bit (Q1.15) pre-computed fixed-point tables. - 32-bit channel_map format with 4-bit nibble encoding and dynamic slot resolution. - Mono/Stereo to 5.1 and 7.1 upmixing engines with side-channel fallback. - Tensilica HiFi3/HiFi4 SIMD vector acceleration: Vector coefficient packing via AE_SEL32_LL saving 3 registers to overcome the 8-register constraint, pipelined AE_L32_IP, AE_MULF32S_LH, AE_MULAF32S_LH, and 64-to-32-bit symmetric rounding. - IPC4 module interface (struct ipc4_up_down_mixer_module_cfg) with 4 coefficient selection modes. - Intel platform performance profiles (MTL, LNL, PTL). - ALSA Topology 2 widget definition and end-to-end 5.1 downmixing playback pipeline. - Automated audio quality verification runbook. - 7 native vector Graphviz SVG diagrams (Figures 209-215). Signed-off-by: Liam Girdwood --- developer_guides/firmware/up_down_mixer.rst | 1017 +++++++++++++++++++ developer_guides/index.rst | 2 + 2 files changed, 1019 insertions(+) create mode 100644 developer_guides/firmware/up_down_mixer.rst diff --git a/developer_guides/firmware/up_down_mixer.rst b/developer_guides/firmware/up_down_mixer.rst new file mode 100644 index 00000000..a538d45e --- /dev/null +++ b/developer_guides/firmware/up_down_mixer.rst @@ -0,0 +1,1017 @@ +.. _up_down_mixer: + +Up/Down Channel Mixer (Spatial Channel Converter) Architecture +============================================================== + +The **Up/Down Channel Mixer** (``up_down_mixer``, component UUID ``UUIDREG_STR_UP_DOWN_MIXER``) is Sound Open Firmware's specialized spatial audio format transformation engine. It provides deterministic, format-aware conversion between heterogenous multi-channel spatial layouts, bridging high-channel-count cinematic audio streams (such as 5.1 and 7.1 surround sound) and constrained endpoint transducers (such as stereo headphones, dual-speaker laptops, or mono smart speakers), as well as upmixing narrow streams across multi-transducer arrays. + +.. contents:: + :local: + :depth: 2 + +Role of Spatial Channel Conversion in Audio DSP Architectures +------------------------------------------------------------- + +Modern audio architectures interact with a diverse spectrum of physical transducer configurations and multimedia formats. While streaming media, gaming titles, and broadcast audio are frequently authored and distributed in multi-channel surround formats (e.g., 5.1 or 7.1 surround sound), client playback endpoints vary drastically in their physical capabilities: + +* **Mobile & Thin-Client Laptops**: Dual micro-speakers (Stereo 2.0) or single speaker (Mono 1.0). +* **Headphones & Headsets**: Binaural stereo playback requiring accurate spatial fold-down. +* **Soundbars & Subwoofers**: 2.1, 3.0, or 3.1 channel configurations with discrete center dialogue and low-frequency effect (LFE) channels. +* **Automotive & Premium Home Theaters**: 5.1, 7.1, or custom multi-speaker surrounds. + +Without an autonomous, hardware-accelerated spatial channel converter in the audio DSP pipeline, the operating system must either discard non-rendered channels—destroying critical dialogue, ambient cues, and dynamic impact—or force software-based host CPU downmixing, increasing host power consumption and preventing low-power DSP offload. + +Spatial Transformation Requirements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +1. **Downmixing (Surround Fold-Down)**: + Collapsing multi-channel surround streams (such as 7.1, 5.1, 4.0, or 3.0) into stereo or mono endpoints. The conversion must preserve dialogue intelligibility (centered speech), low-frequency impacts (LFE), and directional surround panning without introducing acoustic phase cancellation, frequency discoloration, or digital clipping. + +2. **Upmixing (Soundstage Expansion)**: + Expanding narrow-channel content (mono or stereo music/voice streams) across multi-speaker arrays (such as 5.1 or 7.1 surround configurations). This provides immersive acoustic fill while preserving proper front left/right stereo imaging and preventing phantom center artifacts. + +3. **Format & Container Bridging**: + Operating seamlessly across 16-bit (``IPC4_DEPTH_16BIT``) and 32-bit (``IPC4_DEPTH_32BIT`` container with 24-bit or 32-bit valid data) audio streams, with support for arbitrary spatial slot assignments via dynamic 4-bit nibble channel mapping. + +Architectural Comparison: Up/Down Mixer vs Related Modules +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To prevent architectural ambiguity within the SOF signal processing pipeline, the role of Up/Down Mixer is strictly delineated from neighboring components: + +.. list-table:: Architectural Separation of Concerns in SOF Audio Routing + :widths: 20 25 30 25 + :header-rows: 1 + + * - Module Name + - Primary Domain + - Transformation Scope + - Typical Deployment + * - **Up/Down Mixer** + - Spatial Format Conversion + - Inter-format matrix conversion between standardized spatial configurations (Mono, Stereo, 2.1, 3.1, 4.0, 5.1, 7.1) with ITU-R BS.775 and headroom-scaled anti-clipping coefficients. + - Media playback pipelines, soundbar front-ends, and surround fold-down. + * - **Mixin / Mixout** + - Inter-Pipeline Audio Mixing + - Dynamic summing of multiple independent audio streams arriving from disparate clock domains or client applications into a shared sink. + - System audio mixing, notification sound ducking, and concurrent stream aggregation. + * - **Selector** + - Intra-Stream Channel Extraction + - Arbitrary channel isolation, permutation, and linear :math:`8 \times 8` matrix mixing within a single stream format without changing overall topology semantics. + - Microphone beamforming input channel selection and channel swapping. + * - **Copier** + - Boundary Data Movement + - Hardware peripheral endpoint abstraction (Host DMA, DAI, SoundWire), 1-to-N multi-pin fan-out, and linear container conversion. + - Pipeline entry/exit gateways and inter-core boundary transfers. + +Figure 209 illustrates the high-level architecture of the Up/Down Channel Mixer subsystem, depicting input stream ingestion, matrix routing dispatch, headroom-scaled fixed-point coefficient computation, and hardware-accelerated SIMD output delivery. + +.. graphviz:: + :caption: SOF Up/Down Channel Mixer Subsystem Architecture: Spatial Matrix Routing, Headroom Scaling & Platform Dispatch + :alt: Diagram of SOF Up/Down Channel Mixer Subsystem Architecture + + digraph up_down_mixer_arch { + graph [bgcolor="transparent", rankdir="TB", nodesep="0.6", ranksep="0.7", pad="0.3"]; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, style="filled", shape="box", penwidth=1.5]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9, penwidth=1.2, color="#64748b"]; + + subgraph cluster_inputs { + label = "Input Audio Stream (Interleaved Multi-Channel)"; + style = "dashed"; + color = "#3b82f6"; + bgcolor = "#0b192c22"; + + in_stream [label="Multi-Channel PCM\nMono / Stereo / 2.1 / 3.0 / 3.1\nQuatro / 4.0 / 5.0 / 5.1 / 7.1\n(16-bit or 32-bit Container)", fillcolor="#1e3a8a", fontcolor="#ffffff", color="#60a5fa"]; + ch_map_in [label="32-Bit Input Channel Map\n(4-Bit Nibbles per Slot:\nL, C, R, Ls, Rs, LFE, LS, RS)", fillcolor="#172554", fontcolor="#93c5fd", color="#3b82f6"]; + } + + subgraph cluster_core { + label = "Up/Down Mixer Processing Core (up_down_mixer.c)"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + dispatch [label="Conversion Dispatcher\n(select_mix_out_mono\nselect_mix_out_stereo\nselect_mix_out_5_1)", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + + subgraph cluster_coeff_engine { + label = "Coefficient Selection Engine (up_down_mixer_coef.h)"; + style = "dotted"; + color = "#0ea5e9"; + bgcolor = "#0c4a6e33"; + + coeff_select [label="Mode Selector\n(coefficients_select)", fillcolor="#075985", fontcolor="#ffffff", color="#38bdf8"]; + c_default [label="Default ITU-R BS.775\nk_lo_ro_downmix", fillcolor="#0e7490", fontcolor="#ffffff", color="#22d3ee"]; + c_scaled [label="Headroom-Scaled Anti-Clipping\nk_scaled_lo_ro_downmix (0.414 / 0.293)", fillcolor="#0e7490", fontcolor="#ffffff", color="#22d3ee"]; + c_custom [label="Custom OEM Coefficients\n(8 x Q1.31 Matrix)", fillcolor="#155e75", fontcolor="#ffffff", color="#67e8f9"]; + } + + subgraph cluster_simd { + label = "Tensilica HiFi3/HiFi4 SIMD Vector Acceleration (up_down_mixer_hifi3.c)"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b22"; + + reg_packing [label="Register-Packed Coefficients\n(AE_SEL32_LL Combines Pairs into ae_int32x2)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + mac_pipeline [label="Vector MAC Pipeline\nAE_L32_IP / AE_MULF32S_LH / AE_MULAF32S_LH\n64-Bit Accumulation & Symmetric Rounding (AE_ROUND32F64SSYM)", fillcolor="#065f46", fontcolor="#ffffff", color="#10b981"]; + } + } + + subgraph cluster_outputs { + label = "Output Audio Stream (Transformed Spatial Geometry)"; + style = "dashed"; + color = "#10b981"; + bgcolor = "#022c2222"; + + out_stream [label="Destination PCM Stream\nDownmixed: Stereo 2.0 / Mono 1.0\nUpmixed: 5.1 Surround / 7.1 Surround\n(Valid Bit Depth: 24-bit in 32-bit Container)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + } + + in_stream -> dispatch [label="PCM Samples"]; + ch_map_in -> dispatch [label="Slot Indices"]; + + dispatch -> coeff_select [label="Target Geometry"]; + coeff_select -> c_default; + coeff_select -> c_scaled; + coeff_select -> c_custom; + + c_default -> reg_packing [label="Coefficients"]; + c_scaled -> reg_packing; + c_custom -> reg_packing; + + reg_packing -> mac_pipeline [label="Packed Registers"]; + dispatch -> mac_pipeline [label="Input Pointers"]; + + mac_pipeline -> out_stream [label="Rendered Channels"]; + } + +Mathematical Foundations: Matrix Downmixing, ITU-R BS.775 & Headroom Scaling +---------------------------------------------------------------------------- + +Downmixing a multi-channel soundfield to a smaller speaker configuration requires matrix multiplication. Each output channel is synthesized as a linear combination of weighted input channels. + +Standard Lo/Ro Surround Downmix Formulation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Under the international standard **ITU-R BS.775** for multichannel stereophonic sound systems, the conventional Left-only / Right-only (Lo/Ro) downmixing matrix collapses a 5.1 surround stream into stereo: + +.. math:: + + L_{\text{out}} = c_L \cdot L + c_C \cdot C + c_R \cdot 0 + c_{Ls} \cdot Ls + c_{Rs} \cdot 0 + c_{\text{LFE}} \cdot \text{LFE} + + R_{\text{out}} = c_L \cdot 0 + c_C \cdot C + c_R \cdot R + c_{Ls} \cdot 0 + c_{Rs} \cdot Rs + c_{\text{LFE}} \cdot \text{LFE} + +Under standard unscaled conditions, the acoustic weighting coefficients are: + +* **Left & Right Front**: Unity gain (:math:`c_L = c_R = 1.0 = 0.0\text{ dB}`). +* **Center (Dialogue)**: Attenuated by :math:`-3.01\text{ dB}` (:math:`c_C = 1/\sqrt{2} \approx 0.7071`) so that acoustic power is equally split between left and right transducers. +* **Surrounds (Ls, Rs)**: Attenuated by :math:`-3.01\text{ dB}` (:math:`c_{Ls} = c_{Rs} = 1/\sqrt{2} \approx 0.7071`) to preserve ambient balance without overpowering front staging. +* **Low-Frequency Effects (LFE)**: Muted (:math:`c_{\text{LFE}} = 0.0`), in strict accordance with ITU-R BS.775 to prevent severe low-frequency intermodulation distortion in small consumer speaker cones lacking dedicated subwoofers. + +The Headroom Normalization Dilemma +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +While the unscaled ITU-R BS.775 matrix preserves perceived acoustic loudness for typical un-correlated material, it poses a severe risk of **catastrophic digital clipping** in fixed-point embedded DSP architectures. + +Consider a worst-case scenario where full-scale correlated audio peaks (:math:`0.0\text{ dBFS} = 1.0`) occur simultaneously across the Left, Center, and Left Surround channels: + +.. math:: + + L_{\text{peak}} = 1.0 \cdot L + 0.7071 \cdot C + 0.7071 \cdot Ls = 1.0 + 0.7071 + 0.7071 = 2.4142 \quad (+7.65\text{ dBFS}) + +In integer PCM arithmetic (whether 16-bit or 32-bit), any value exceeding :math:`+1.0` undergoes harsh saturation clipping, producing intolerable acoustic harmonic distortion. + +Mathematical Derivation of Scaled Anti-Clipping Coefficients +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To eliminate digital clipping without requiring a dynamic range compressor (DRC) or high-latency limiter, SOF provides **Headroom-Scaled Anti-Clipping Coefficients** (``k_scaled_lo_ro_downmix32bit``). + +The normalization scalar :math:`S` is derived by taking the reciprocal of the maximum possible accumulated channel gain: + +.. math:: + + S = \frac{1}{1 + \frac{1}{\sqrt{2}} + \frac{1}{\sqrt{2}}} = \frac{1}{1 + \sqrt{2}} = \frac{1}{2.41421356} \approx 0.41421356 \quad (-7.655\text{ dB}) + +Applying this scaling factor across the matrix coefficients yields: + +.. math:: + + c_L = 1.0 \times S = 0.41421356 \approx 0.414 + + c_C = \frac{1}{\sqrt{2}} \times S = \frac{0.70710678}{2.41421356} \approx 0.2928932 \approx 0.293 + + c_{Ls} = \frac{1}{\sqrt{2}} \times S = \frac{0.70710678}{2.41421356} \approx 0.2928932 \approx 0.293 + +Evaluating the worst-case coherent peak with these scaled coefficients: + +.. math:: + + L_{\text{peak, scaled}} = 0.41421356 + 0.2928932 + 0.2928932 = 1.00000000 \quad (0.0\text{ dBFS}) + +The sum of maximum positive gains is **exactly unity (1.0)**. As a result, digital clipping is mathematically impossible, even when all surround channels drive simultaneous :math:`0\text{ dBFS}` square waves. + +Half-Scaled Coefficients for 3.0 / 3.1 Downmixing +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When downmixing 3.0 (Left, Center, Right) or 3.1 (Left, Center, Right, LFE) streams to stereo, surround channels are absent. Downscaling by :math:`1/(1+\sqrt{2})` would needlessly penalize dynamic range. Instead, the **Half-Scaled Anti-Clipping Coefficients** (``k_half_scaled_lo_ro_downmix32bit``) normalize over front channels only: + +.. math:: + + S_{3.0} = \frac{1}{1 + \frac{1}{\sqrt{2}}} = \frac{1}{1.70710678} \approx 0.5857864 \approx 0.586 + + c_L = 1.0 \times S_{3.0} \approx 0.586, \quad c_C = \frac{1}{\sqrt{2}} \times S_{3.0} \approx 0.414 + + L_{\text{peak, 3.0}} = 0.586 + 0.414 = 1.000 \quad (0.0\text{ dBFS}) + +Quatro-to-Mono Scaled Coefficients +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When collapsing 4-channel surround (Quatro: L, R, Ls, Rs) or 4.0 (L, C, R, Cs) into a single mono channel, all four active channels are summed: + +.. math:: + + S_{\text{quatro}} = \frac{1}{2 + \sqrt{2}} = \frac{1}{3.41421356} \approx 0.2928932 \approx 0.293 + + c_L = c_R = 0.293, \quad c_{Ls} = c_{Rs} = 0.207 + + \text{Mono}_{\text{peak}} = 0.293 + 0.293 + 0.207 + 0.207 = 1.000 \quad (0.0\text{ dBFS}) + +Figure 210 illustrates the mathematical model comparing unscaled ITU-R BS.775 downmixing against headroom-scaled normalization, showing how coherent peaks are contained within the valid dynamic range. + +.. graphviz:: + :caption: Mathematical Model of Surround Downmixing: ITU-R BS.775 Summation & Headroom-Preserving Anti-Clipping Coefficients + :alt: Diagram of Mathematical Model of Surround Downmixing + + digraph downmix_math { + graph [bgcolor="transparent", rankdir="LR", nodesep="0.5", ranksep="0.8", pad="0.3"]; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, style="filled", shape="box", penwidth=1.5]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9, penwidth=1.2, color="#64748b"]; + + subgraph cluster_inputs { + label = "5.1 Surround Channel Inputs"; + style = "solid"; + color = "#3b82f6"; + bgcolor = "#1e3a8a11"; + + in_l [label="Left (L)\n1.0 (0 dBFS)", fillcolor="#1e3a8a", fontcolor="#ffffff", color="#60a5fa"]; + in_c [label="Center (C)\n1.0 (0 dBFS)", fillcolor="#1e3a8a", fontcolor="#ffffff", color="#60a5fa"]; + in_r [label="Right (R)\n1.0 (0 dBFS)", fillcolor="#1e3a8a", fontcolor="#ffffff", color="#60a5fa"]; + in_ls [label="Left Surround (Ls)\n1.0 (0 dBFS)", fillcolor="#1e3a8a", fontcolor="#ffffff", color="#60a5fa"]; + in_rs [label="Right Surround (Rs)\n1.0 (0 dBFS)", fillcolor="#1e3a8a", fontcolor="#ffffff", color="#60a5fa"]; + in_lfe [label="LFE Subwoofer\n1.0 (0 dBFS)", fillcolor="#334155", fontcolor="#94a3b8", color="#64748b"]; + } + + subgraph cluster_unscaled { + label = "Standard ITU-R BS.775 (Unscaled)"; + style = "dashed"; + color = "#ef4444"; + bgcolor = "#7f1d1d11"; + + sum_unscaled [label="Accumulator Sum\nL + 0.707 C + 0.707 Ls\n= 2.4142 (+7.65 dBFS)", fillcolor="#991b1b", fontcolor="#ffffff", color="#f87171"]; + clip_box [label="CLIPPING OVERFLOW!\nExceeds 0 dBFS\nSevere Saturation Distortion", fillcolor="#b91c1c", fontcolor="#fef2f2", color="#ef4444", shape="octagon"]; + } + + subgraph cluster_scaled { + label = "SOF Headroom-Scaled Downmix (k_scaled_lo_ro_downmix)"; + style = "solid"; + color = "#10b981"; + bgcolor = "#064e3b11"; + + mult_l [label="Scale c_L\n0.414 (-7.65 dB)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + mult_c [label="Scale c_C\n0.293 (-10.66 dB)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + mult_ls [label="Scale c_Ls\n0.293 (-10.66 dB)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + + sum_scaled [label="Accumulator Sum\n0.414 L + 0.293 C + 0.293 Ls\n= 1.0000 (0.00 dBFS)", fillcolor="#065f46", fontcolor="#ffffff", color="#10b981"]; + safe_box [label="PERFECT HEADROOM\nMathematically Zero Clipping\nFull 32-bit Dynamic Range", fillcolor="#047857", fontcolor="#f0fdf4", color="#34d399"]; + } + + in_l -> sum_unscaled [label="x 1.0"]; + in_c -> sum_unscaled [label="x 0.707"]; + in_ls -> sum_unscaled [label="x 0.707"]; + sum_unscaled -> clip_box; + + in_l -> mult_l; + in_c -> mult_c; + in_ls -> mult_ls; + + mult_l -> sum_scaled [label="0.414"]; + mult_c -> sum_scaled [label="0.293"]; + mult_ls -> sum_scaled [label="0.293"]; + sum_scaled -> safe_box; + } + +Fixed-Point Coefficient Precision & Representation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Coefficients are pre-computed in header file ``src/audio/up_down_mixer/up_down_mixer_coef.h`` using integer macros to eliminate floating-point runtime division: + +* **32-Bit Fixed-Point** (:math:`Q1.31`): + + .. code-block:: c + + #define COMPUTE_COEFF_32BIT(counter, denominator) ((0x7fffffffULL * (counter)) / (denominator)) + +* **16-Bit Fixed-Point** (:math:`Q1.15`): + + .. code-block:: c + + #define COMPUTE_COEFF_16BIT(counter, denominator) ((0x7fffULL * (counter)) / (denominator)) + +Table 18 summarizes the pre-computed coefficient sets implemented in SOF: + +.. list-table:: SOF Up/Down Mixer Pre-Computed Coefficient Sets (up_down_mixer_coef.h) + :widths: 22 13 13 13 13 13 13 + :header-rows: 1 + + * - Coefficient Array + - :math:`c_L` + - :math:`c_C` + - :math:`c_R` + - :math:`c_{Ls}` + - :math:`c_{Rs}` + - :math:`c_{\text{LFE}}` + * - ``k_lo_ro_downmix32bit`` + - 1.000 (``0x7FFFFFFF``) + - 0.707 (``0x5A827999``) + - 1.000 (``0x7FFFFFFF``) + - 0.707 (``0x5A827999``) + - 0.707 (``0x5A827999``) + - 0.000 (``0x00000000``) + * - ``k_scaled_lo_ro_downmix32bit`` + - 0.414 (``0x35000000``) + - 0.293 (``0x25800000``) + - 0.414 (``0x35000000``) + - 0.293 (``0x25800000``) + - 0.293 (``0x25800000``) + - 0.000 (``0x00000000``) + * - ``k_half_scaled_lo_ro_downmix32bit`` + - 0.586 (``0x4B000000``) + - 0.414 (``0x35000000``) + - 0.586 (``0x4B000000``) + - 0.414 (``0x35000000``) + - 0.414 (``0x35000000``) + - 0.000 (``0x00000000``) + * - ``k_quatro_mono_scaled_lo_ro_downmix32bit`` + - 0.293 (``0x25800000``) + - 0.207 (``0x1A800000``) + - 0.293 (``0x25800000``) + - 0.207 (``0x1A800000``) + - 0.207 (``0x1A800000``) + - 0.000 (``0x00000000``) + +Spatial Channel Layouts, 4-Bit Nibble Bitmasks & Dynamic Mapping +---------------------------------------------------------------- + +Spatial audio processing components cannot assume fixed channel ordering in physical RAM. Hardware serial DAIs, soundcards, and third-party host software frequently permute channel slots (e.g., SMPTE ``[L, R, C, LFE, Ls, Rs]`` vs Film ``[L, C, R, Ls, Rs, LFE]``). + +The 32-Bit Nibble Map Format +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To achieve complete independence from physical memory ordering, SOF implements a packed **32-bit channel map** (``channel_map``, defined in ``up_down_mixer_ipc4.h``). + +Each 32-bit integer encodes eight 4-bit nibbles. Each nibble directly specifies the spatial channel identity residing at that particular interleaved frame offset: + +.. math:: + + \text{channel\_map} = \sum_{i=0}^{7} \left( \text{ChannelIdentity}_i \ll (4 \times i) \right) + +The spatial identities correspond to ``enum ipc4_channel_index``: + +* ``CHANNEL_LEFT = 0x0`` +* ``CHANNEL_CENTER = 0x1`` +* ``CHANNEL_RIGHT = 0x2`` +* ``CHANNEL_LEFT_SURROUND = 0x3`` +* ``CHANNEL_RIGHT_SURROUND = 0x4`` +* ``CHANNEL_LFE = 0x5`` +* ``CHANNEL_LEFT_SIDE = 0x6`` +* ``CHANNEL_RIGHT_SIDE = 0x7`` +* Unused / invalid channel slots are padded with ``0xF``. + +Channel Map Construction +~~~~~~~~~~~~~~~~~~~~~~~~ + +The inline helper ``create_channel_map()`` generates standardized bitmasks for all recognized IPC4 channel layouts: + +.. code-block:: c + + static inline channel_map create_channel_map(enum ipc4_channel_config channel_config) + { + switch (channel_config) { + case IPC4_CHANNEL_CONFIG_MONO: + return (0xFFFFFFF0 | CHANNEL_CENTER); + case IPC4_CHANNEL_CONFIG_STEREO: + return (0xFFFFFF00 | CHANNEL_LEFT | (CHANNEL_RIGHT << 4)); + case IPC4_CHANNEL_CONFIG_2_POINT_1: + return (0xFFFFF000 | CHANNEL_LEFT | (CHANNEL_RIGHT << 4) | (CHANNEL_LFE << 8)); + case IPC4_CHANNEL_CONFIG_3_POINT_0: + return (0xFFFFF000 | CHANNEL_LEFT | (CHANNEL_CENTER << 4) | (CHANNEL_RIGHT << 8)); + case IPC4_CHANNEL_CONFIG_3_POINT_1: + return (0xFFFF0000 | CHANNEL_LEFT | (CHANNEL_CENTER << 4) | (CHANNEL_RIGHT << 8) + | (CHANNEL_LFE << 12)); + case IPC4_CHANNEL_CONFIG_QUATRO: + return (0xFFFF0000 | CHANNEL_LEFT | (CHANNEL_RIGHT << 4) + | (CHANNEL_LEFT_SURROUND << 8) | (CHANNEL_RIGHT_SURROUND << 12)); + case IPC4_CHANNEL_CONFIG_4_POINT_0: + return (0xFFFF0000 | CHANNEL_LEFT | (CHANNEL_CENTER << 4) | (CHANNEL_RIGHT << 8) + | (CHANNEL_CENTER_SURROUND << 12)); + case IPC4_CHANNEL_CONFIG_5_POINT_0: + return (0xFFF00000 | CHANNEL_LEFT | (CHANNEL_CENTER << 4) | (CHANNEL_RIGHT << 8) + | (CHANNEL_LEFT_SURROUND << 12) | (CHANNEL_RIGHT_SURROUND << 16)); + case IPC4_CHANNEL_CONFIG_5_POINT_1: + return (0xFF000000 | CHANNEL_LEFT | (CHANNEL_CENTER << 4) | (CHANNEL_RIGHT << 8) + | (CHANNEL_LEFT_SURROUND << 12) | (CHANNEL_RIGHT_SURROUND << 16) + | (CHANNEL_LFE << 20)); + case IPC4_CHANNEL_CONFIG_7_POINT_1: + return (CHANNEL_LEFT | (CHANNEL_CENTER << 4) | (CHANNEL_RIGHT << 8) + | (CHANNEL_LEFT_SURROUND << 12) | (CHANNEL_RIGHT_SURROUND << 16) + | (CHANNEL_LFE << 20) | (CHANNEL_LEFT_SIDE << 24) + | (CHANNEL_RIGHT_SIDE << 28)); + default: + return 0xFFFFFFFF; + } + } + +Dynamic Slot Resolution +~~~~~~~~~~~~~~~~~~~~~~~ + +During algorithm initialization and execution, the helper ``get_channel_location()`` rapidly extracts the byte offset of any desired channel from the active map: + +.. code-block:: c + + static inline uint8_t get_channel_location(const channel_map map, + const enum ipc4_channel_index channel) + { + uint8_t offset = 0xF; + for (uint8_t i = 0; i < 8; i++) { + if (((map >> (i * 4)) & 0xF) == (uint8_t)channel) { + offset = i; + break; + } + } + return offset; + } + +Figure 211 illustrates how a 5.1 channel map is packed into 4-bit nibbles and subsequently decoded into pointer strides for processing loops. + +.. graphviz:: + :caption: Channel Mapping Architecture: 4-Bit Nibble Bitmask Encoding & Dynamic Spatial Slot Location + :alt: Diagram of Channel Mapping Architecture + + digraph channel_mapping { + graph [bgcolor="transparent", rankdir="TB", nodesep="0.6", ranksep="0.6", pad="0.3"]; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, style="filled", shape="box", penwidth=1.5]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9, penwidth=1.2, color="#64748b"]; + + subgraph cluster_packed { + label = "32-Bit Packed Channel Map (Example: 5.1 Surround = 0xFF543210)"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4911"; + + nibbles [label="Bits 31..28: 0xF (Unused)\nBits 27..24: 0xF (Unused)\nBits 23..20: 0x5 (CHANNEL_LFE)\nBits 19..16: 0x4 (CHANNEL_RIGHT_SURROUND)\nBits 15..12: 0x3 (CHANNEL_LEFT_SURROUND)\nBits 11..8: 0x2 (CHANNEL_RIGHT)\nBits 7..4: 0x1 (CHANNEL_CENTER)\nBits 3..0: 0x0 (CHANNEL_LEFT)", fillcolor="#075985", fontcolor="#ffffff", color="#38bdf8", shape="note"]; + } + + subgraph cluster_lookup { + label = "get_channel_location(map, channel) Resolution Engine"; + style = "dashed"; + color = "#0ea5e9"; + bgcolor = "#0c4a6e11"; + + lookup_engine [label="Scan 4-Bit Windows\n(map >> (i * 4)) & 0xF", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + } + + subgraph cluster_strides { + label = "Resolved Byte Pointer Offsets into Interleaved Frame Buffer"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b11"; + + ptr_l [label="CHANNEL_LEFT: Slot 0\nOffset: 0 x sizeof(sample)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + ptr_c [label="CHANNEL_CENTER: Slot 1\nOffset: 1 x sizeof(sample)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + ptr_r [label="CHANNEL_RIGHT: Slot 2\nOffset: 2 x sizeof(sample)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + ptr_ls [label="CHANNEL_LEFT_SURROUND: Slot 3\nOffset: 3 x sizeof(sample)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + ptr_rs [label="CHANNEL_RIGHT_SURROUND: Slot 4\nOffset: 4 x sizeof(sample)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + ptr_lfe [label="CHANNEL_LFE: Slot 5\nOffset: 5 x sizeof(sample)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + } + + nibbles -> lookup_engine [label="32-bit Word"]; + lookup_engine -> ptr_l; + lookup_engine -> ptr_c; + lookup_engine -> ptr_r; + lookup_engine -> ptr_ls; + lookup_engine -> ptr_rs; + lookup_engine -> ptr_lfe; + } + +Upmixing Architectures: Mono & Stereo Soundstage Expansion +---------------------------------------------------------- + +In addition to downmixing, the Up/Down Mixer component serves as SOF's high-efficiency spatial upmixer. Upmixing expands narrow-channel audio across surround sound speaker topologies without incurring heavy latency or computational overhead. + +Mono-to-5.1 Upmixing Engine +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When a single-channel mono speech or communication stream is delivered to a 5.1 surround sound endpoint (implemented in ``upmix32bit_1_to_5_1`` and ``upmix16bit_1_to_5_1``): + +.. math:: + + L_{\text{out}}[i] = \text{Mono}_{\text{in}}[i] + + R_{\text{out}}[i] = \text{Mono}_{\text{in}}[i] + + Ls_{\text{out}}[i] = \text{Mono}_{\text{in}}[i] + + Rs_{\text{out}}[i] = \text{Mono}_{\text{in}}[i] + + C_{\text{out}}[i] = 0 + + \text{LFE}_{\text{out}}[i] = 0 + +* **Acoustic Rationale**: Replicating the signal into the left, right, and surround speakers creates an enveloping, diffuse soundfield without acoustic localization bias. The center channel is explicitly cleared to avoid acoustic point-source beaming, and LFE is cleared to prevent sub-bass transducer over-excursion. +* **16-bit Conversion**: For 16-bit input streams, samples are upshifted to 32-bit MSB alignment via the Tensilica intrinsic ``AE_MOVINT32_FROMINT16(in_ptr[i]) << 16``. + +Stereo 2.0-to-5.1 Upmixing Engine +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When conventional stereo audio is played over a 5.1 home theater or automotive soundstage (implemented in ``upmix32bit_2_0_to_5_1`` and ``upmix16bit_2_0_to_5_1``): + +.. math:: + + L_{\text{out}}[i] = L_{\text{in}}[i], \quad R_{\text{out}}[i] = R_{\text{in}}[i] + + Ls_{\text{out}}[i] = L_{\text{in}}[i], \quad Rs_{\text{out}}[i] = R_{\text{in}}[i] + + C_{\text{out}}[i] = 0, \quad \text{LFE}_{\text{out}}[i] = 0 + +* **Side-Channel Fallback**: If the target configuration uses side surround speakers (``CHANNEL_LEFT_SIDE``, ``CHANNEL_RIGHT_SIDE``) rather than rear surrounds (``CHANNEL_LEFT_SURROUND``, ``CHANNEL_RIGHT_SURROUND``), the engine automatically detects this condition and routes surround channels to the side slots. + +Stereo 2.0-to-7.1 Upmixing Engine +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For 8-channel surround systems (``upmix32bit_2_0_to_7_1``), stereo audio is distributed to the front and rear soundstage while preserving zero-energy in the center, LFE, and side speakers: + +.. math:: + + L_{\text{out}}[i] = L_{\text{in}}[i], \quad R_{\text{out}}[i] = R_{\text{in}}[i] + + Ls_{\text{out}}[i] = L_{\text{in}}[i], \quad Rs_{\text{out}}[i] = R_{\text{in}}[i] + + C_{\text{out}}[i] = 0, \quad \text{LFE}_{\text{out}}[i] = 0, \quad \text{LeftSide}_{\text{out}}[i] = 0, \quad \text{RightSide}_{\text{out}}[i] = 0 + +Zero-Latency Shift Copiers +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When the channel count remains unchanged but container widths must be upgraded from 16-bit to 32-bit MSB alignment, the Up/Down Mixer executes optimized shift copiers (``shiftcopy16bit_mono``, ``shiftcopy16bit_stereo``, ``shiftcopy32bit_mono``, ``shiftcopy32bit_stereo``). These provide direct linear memory copy operations with single-cycle sign extension and zero algorithmic latency. + +Figure 212 illustrates the dataflow for Mono-to-5.1, Stereo-to-5.1, and Stereo-to-7.1 upmixing paths. + +.. graphviz:: + :caption: Surround Upmixing Dataflow: Mono/Stereo to 5.1 and 7.1 Channel Soundstage Expansion + :alt: Diagram of Surround Upmixing Dataflow + + digraph upmix_flow { + graph [bgcolor="transparent", rankdir="LR", nodesep="0.5", ranksep="0.8", pad="0.3"]; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, style="filled", shape="box", penwidth=1.5]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9, penwidth=1.2, color="#64748b"]; + + subgraph cluster_in { + label = "Input Streams"; + style = "dashed"; + color = "#3b82f6"; + bgcolor = "#1e3a8a11"; + + in_mono [label="Mono Stream\n[M]", fillcolor="#1e3a8a", fontcolor="#ffffff", color="#60a5fa"]; + in_stereo [label="Stereo Stream\n[L, R]", fillcolor="#1e3a8a", fontcolor="#ffffff", color="#60a5fa"]; + } + + subgraph cluster_upmixers { + label = "Upmixing Dispatch Routines"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4911"; + + up_1_to_51 [label="upmix32bit_1_to_5_1\nL=M, R=M\nLs=M, Rs=M\nC=0, LFE=0", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + up_20_to_51 [label="upmix32bit_2_0_to_5_1\nL=L, R=R\nLs=L, Rs=R\nC=0, LFE=0\n(Fallback: Side Surrounds)", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + up_20_to_71 [label="upmix32bit_2_0_to_7_1\nL=L, R=R\nLs=L, Rs=R\nC=0, LFE=0\nLS=0, RS=0", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + } + + subgraph cluster_out { + label = "Surround Output Endpoints"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b11"; + + out_51_mono [label="5.1 Soundfield\nEnveloping Diffuse Audio\nZero Center Beaming", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + out_51_stereo [label="5.1 Surround Soundfield\nPreserved Front L/R Stage\nEnveloping Ambience", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + out_71_stereo [label="7.1 Surround Soundfield\n8-Channel Clean Staging\nZero Side Intermodulation", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + } + + in_mono -> up_1_to_51; + in_stereo -> up_20_to_51; + in_stereo -> up_20_to_71; + + up_1_to_51 -> out_51_mono; + up_20_to_51 -> out_51_stereo; + up_20_to_71 -> out_71_stereo; + } + +Cadence Tensilica HiFi3/HiFi4 SIMD Vector Acceleration +------------------------------------------------------ + +Audio downmixing algorithms are fundamentally bound by memory bandwidth and multiply-accumulate (MAC) pipeline efficiency. When collapsing 6 or 8 channels of 32-bit audio at 48 kHz or 96 kHz, scalar execution would consume excessive CPU cycles and drain battery power. + +SOF implements highly optimized assembly pipelines in ``src/audio/up_down_mixer/up_down_mixer_hifi3.c``, targeting Cadence Tensilica HiFi3 and HiFi4 DSP architectures. + +Mitigating Register Pressure: The 8-Register Constraint +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The Tensilica HiFi3 architecture provides **eight 32-bit/48-bit audio vector registers** (the ``AE_P`` register file: ``p0`` through ``p7``). + +A naive implementation of a 6-channel 5.1 downmixing loop requires: + +* 6 distinct downmix coefficients (:math:`c_L, c_C, c_R, c_{Ls}, c_{Rs}, c_{\text{LFE}}`). +* 6 concurrent input sample channels. +* 2 output accumulator registers (:math:`L_{\text{out}}, R_{\text{out}}`). + +This would demand 14 simultaneous registers, causing severe register spilling to stack memory and destroying inner loop throughput. + +To overcome this bottleneck, SOF implements an innovative **Vector Coefficient Packing** technique: + +.. code-block:: c + + /* Load 32-bit coefficients */ + ae_int32x2 P_coefficient_left = AE_L32_X((ae_int32 *)cd->downmix_coefficients, CHANNEL_LEFT << 2); + ae_int32x2 P_coefficient_center = AE_L32_X((ae_int32 *)cd->downmix_coefficients, CHANNEL_CENTER << 2); + ae_int32x2 P_coefficient_right = AE_L32_X((ae_int32 *)cd->downmix_coefficients, CHANNEL_RIGHT << 2); + ae_int32x2 P_coefficient_left_surround = AE_L32_X((ae_int32 *)cd->downmix_coefficients, CHANNEL_LEFT_SURROUND << 2); + ae_int32x2 P_coefficient_right_surround = AE_L32_X((ae_int32 *)cd->downmix_coefficients, CHANNEL_RIGHT_SURROUND << 2); + ae_int32x2 P_coefficient_lfe = AE_L32_X((ae_int32 *)cd->downmix_coefficients, CHANNEL_LFE << 2); + + /* Combine 6 coefficients into 3 dual-vector registers using AE_SEL32_LL */ + P_coefficient_left_right = AE_SEL32_LL(P_coefficient_left, P_coefficient_right); + P_coefficient_left_s_right_s = AE_SEL32_LL(P_coefficient_left_surround, P_coefficient_right_surround); + P_coefficient_center_lfe = AE_SEL32_LL(P_coefficient_center, P_coefficient_lfe); + +By packing pairs of 32-bit coefficients into single dual-element ``ae_int32x2`` registers, the entire coefficient matrix is held in only **three registers**, liberating five registers for streaming sample buffers and 64-bit accumulators! + +Pipelined Inner Loop Execution (3.1 Downmix Example) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The inner processing loop for 3.1-to-stereo downmixing demonstrates the pipelined SIMD execution: + +.. code-block:: c + + while (input_left < end_input_left) { + ae_f64 Q_tmp_left; + ae_f64 Q_tmp_right; + + /* Load Left and multiply by Left/Right packed coefficient */ + AE_L32_IP(P_input_left, input_left, 4 * sizeof(ae_int32)); + Q_tmp_left = AE_MULF32S_LH(P_input_left, P_coefficient_left_right); + + /* Load Center and multiply-accumulate to Left, multiply to Right */ + AE_L32_IP(P_input_center, input_center, 4 * sizeof(ae_int32)); + AE_MULAF32S_LH(Q_tmp_left, P_input_center, P_coefficient_center_lfe); + Q_tmp_right = AE_MULF32S_LH(P_input_center, P_coefficient_center_lfe); + + /* Load Right and multiply-accumulate to Right */ + AE_L32_IP(P_input_right, input_right, 4 * sizeof(ae_int32)); + AE_MULAF32S_LL(Q_tmp_right, P_input_right, P_coefficient_left_right); + + /* Load LFE and multiply-accumulate to both Left and Right */ + AE_L32_IP(P_input_lfe, input_lfe, 4 * sizeof(ae_int32)); + AE_MULAF32S_LL(Q_tmp_left, P_input_lfe, P_coefficient_center_lfe); + AE_MULAF32S_LL(Q_tmp_right, P_input_lfe, P_coefficient_center_lfe); + + /* Perform 64-to-32-bit symmetric rounding and saturation */ + P_output_left = AE_ROUND32F64SSYM(Q_tmp_left); + P_output_right = AE_ROUND32F64SSYM(Q_tmp_right); + + /* Store to interleaved stereo output buffer */ + AE_S32_L_IP(P_output_left, output_left, 2 * sizeof(ae_int32)); + AE_S32_L_IP(P_output_right, output_right, 2 * sizeof(ae_int32)); + } + +Key HiFi3 SIMD Primitives Used: +* ``AE_L32_IP``: Aligned 32-bit vector load with auto-incrementing stride pointer. +* ``AE_MULF32S_LH``: 32x32-bit fractional multiplication extracting the high 32 bits into a 64-bit accumulator. +* ``AE_MULAF32S_LL`` / ``AE_MULAF32S_LH``: 32x32-bit fractional multiply-accumulate. +* ``AE_ROUND32F64SSYM``: High-precision symmetric rounding converting 64-bit accumulators back to 32-bit words with automatic clamping. +* ``AE_S32_L_IP``: Aligned 32-bit vector store with auto-incrementing stride pointer. + +Figure 213 illustrates the register packing and pipelined multiply-accumulate execution on Cadence HiFi3 hardware. + +.. graphviz:: + :caption: Tensilica HiFi3 SIMD Vector Pipelining: Register-Packed Coefficients & 64-Bit Symmetric MAC Execution + :alt: Diagram of Tensilica HiFi3 SIMD Vector Pipelining + + digraph hifi3_simd { + graph [bgcolor="transparent", rankdir="TB", nodesep="0.6", ranksep="0.6", pad="0.3"]; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, style="filled", shape="box", penwidth=1.5]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9, penwidth=1.2, color="#64748b"]; + + subgraph cluster_coeff_packing { + label = "HiFi3 Register Packing (AE_SEL32_LL)"; + style = "dashed"; + color = "#0284c7"; + bgcolor = "#082f4911"; + + c_raw [label="Raw 32-bit Coeffs\n[c_L, c_R, c_C, c_LFE, c_Ls, c_Rs]", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + c_packed [label="Packed Dual-Element Registers\nP_coefficient_left_right = [c_L | c_R]\nP_coefficient_center_lfe = [c_C | c_LFE]\nP_coefficient_left_s_right_s = [c_Ls | c_Rs]", fillcolor="#075985", fontcolor="#ffffff", color="#38bdf8"]; + c_raw -> c_packed [label="AE_SEL32_LL"]; + } + + subgraph cluster_mac_engine { + label = "Pipelined Vector MAC Core"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b11"; + + load_in [label="Vector Auto-Stride Loads\nAE_L32_IP(P_input, in_ptr, stride)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + mult_acc [label="64-Bit MAC Accumulators\nQ_tmp_left: AE_MULAF32S_LH(P_center, P_coeff_center_lfe)\nQ_tmp_right: AE_MULAF32S_LL(P_right, P_coeff_left_right)", fillcolor="#065f46", fontcolor="#ffffff", color="#10b981"]; + round_sat [label="Symmetric Rounding & Saturation Clamp\nAE_ROUND32F64SSYM(Q_tmp)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + store_out [label="Vector Aligned Stores\nAE_S32_L_IP(P_output, out_ptr, stride)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + + load_in -> mult_acc; + c_packed -> mult_acc [label="Packed Coeffs"]; + mult_acc -> round_sat; + round_sat -> store_out; + } + + subgraph cluster_perf { + label = "Hardware Throughput Benefits"; + style = "dotted"; + color = "#10b981"; + bgcolor = "#022c2211"; + + perf_box [label="Zero Register Spilling\nSingle-Cycle Vector Multiply-Accumulate\nDeterministic Fixed-Point Execution", fillcolor="#134e4a", fontcolor="#ccfbf1", color="#2dd4bf", shape="note"]; + store_out -> perf_box; + } + } + +IPC4 Interface, Module Configuration & Intel Architecture Integration +---------------------------------------------------------------------- + +The Up/Down Mixer is authored strictly according to the modern **SOF IPC4 Module Adapter API**, enabling dynamic pipeline deployment, firmware-level relocatable execution (LLEXT), and tight runtime control. + +Configuration Container (ipc4_up_down_mixer_module_cfg) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The configuration payload is transferred from host userspace or ALSA topology via `struct ipc4_up_down_mixer_module_cfg` (defined in ``up_down_mixer_ipc4.h``): + +.. code-block:: c + + struct ipc4_up_down_mixer_module_cfg { + struct ipc4_base_module_cfg base_cfg; + + /* Output Channel Configuration (Mono, Stereo, 5.1, 7.1) */ + enum ipc4_channel_config out_channel_config; + + /* Selects which coefficients are used */ + enum up_down_mix_coeff_select coefficients_select; + + /* Optional custom coefficients array (8 elements) */ + int32_t coefficients[UP_DOWN_MIX_COEFFS_LENGTH]; + + /* Optional custom channel map for non-standard layouts */ + channel_map channel_map; + } __packed __aligned(8); + +Coefficient Selection Modes +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The parameter ``coefficients_select`` governs how mixing coefficients are resolved: + +1. ``DEFAULT_COEFFICIENTS (0)``: + SOF automatically inspects the input audio format (``base_cfg.audio_fmt.ch_cfg``) and output channel layout (``out_channel_config``) and assigns the optimal pre-computed table: + + * Mono, Stereo, and Dual Mono inputs :math:`\to` ``k_lo_ro_downmix32bit``. + * 3.0 and 3.1 inputs :math:`\to` ``k_half_scaled_lo_ro_downmix32bit``. + * Quatro to Mono :math:`\to` ``k_quatro_mono_scaled_lo_ro_downmix32bit``. + * 4.0, 5.0, 5.1, and 7.1 inputs :math:`\to` ``k_scaled_lo_ro_downmix32bit``. + +2. ``CUSTOM_COEFFICIENTS (1)``: + Overrides default coefficients with the 8-element user-supplied array in ``coefficients[]``, formatted in :math:`Q1.31`. + +3. ``DEFAULT_COEFFICIENTS_WITH_CHANNEL_MAP (2)``: + Uses standard pre-computed coefficients, but overrides the channel indexing with the user-provided 32-bit ``channel_map``. + +4. ``CUSTOM_COEFFICIENTS_WITH_CHANNEL_MAP (3)``: + Employs both custom coefficients and a custom channel map for proprietary hardware speaker topologies. + +Intel Hardware Platform Profiles (up_down_mixer.toml) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The deployment parameters across modern Intel hardware generations (Meteor Lake, Lunar Lake, Arrow Lake, Panther Lake ACE 3.0 / ACE 4.0) are maintained in ``up_down_mixer.toml``: + +.. list-table:: Up/Down Mixer Platform Performance Profiles (up_down_mixer.toml) + :widths: 22 18 20 20 20 + :header-rows: 1 + + * - Platform Architecture + - DSP Engine + - Cycles Per Chunk (CPC) + - Input Buffer Size (IBS) + - Output Buffer Size (OBS) + * - **Meteor Lake (MTL)** + - ACE 1.5 (cAVS 2.5+) + - 2,468 -- 5,440 + - 192 -- 1,536 bytes + - 192 -- 1,152 bytes + * - **Lunar Lake (LNL)** + - ACE 2.0 + - 3,604 -- 7,792 + - 192 -- 1,536 bytes + - 192 -- 1,536 bytes + * - **Panther Lake (PTL)** + - ACE 3.0 / ACE 4.0 + - 4,355 -- 9,177 + - 192 -- 1,536 bytes + - 192 -- 1,536 bytes + +Figure 214 depicts the IPC4 configuration lifecycle and coefficient selection engine. + +.. graphviz:: + :caption: IPC4 Configuration Lifecycle & Coefficient Selection Engine (Default vs Custom Matrices) + :alt: Diagram of IPC4 Configuration Lifecycle + + digraph ipc4_lifecycle { + graph [bgcolor="transparent", rankdir="TB", nodesep="0.6", ranksep="0.6", pad="0.3"]; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, style="filled", shape="box", penwidth=1.5]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9, penwidth=1.2, color="#64748b"]; + + subgraph cluster_ipc_msg { + label = "Host IPC4 Initialization Payload (struct ipc4_up_down_mixer_module_cfg)"; + style = "solid"; + color = "#3b82f6"; + bgcolor = "#1e3a8a11"; + + ipc_cfg [label="Base Config (audio_fmt, IBS, OBS)\nTarget Config: out_channel_config\nMode: coefficients_select\nOptional: coefficients[8], channel_map", fillcolor="#1e3a8a", fontcolor="#ffffff", color="#60a5fa"]; + } + + subgraph cluster_engine { + label = "up_down_mixer_init() Decision Engine"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4911"; + + mode_branch [label="Evaluate coefficients_select", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8", shape="diamond"]; + + branch_def [label="DEFAULT_COEFFICIENTS (0)\nInspect in_cfg vs out_cfg\nAssign k_scaled / k_half_scaled", fillcolor="#075985", fontcolor="#ffffff", color="#38bdf8"]; + branch_cust [label="CUSTOM_COEFFICIENTS (1)\nCopy custom_coeffs[8]\nAssign to cd->downmix_coefficients", fillcolor="#075985", fontcolor="#ffffff", color="#38bdf8"]; + branch_map [label="*_WITH_CHANNEL_MAP (2, 3)\nOverride cd->out_channel_map\nwith user channel_map", fillcolor="#075985", fontcolor="#ffffff", color="#38bdf8"]; + + routine_select [label="Select Specialized Assembly Routine\nselect_mix_out_mono() / select_mix_out_stereo() / select_mix_out_5_1()", fillcolor="#0e7490", fontcolor="#ffffff", color="#22d3ee"]; + } + + subgraph cluster_ready { + label = "Runtime Ready State"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b11"; + + runtime_ready [label="cd->mix_routine = Bound Routine Pointer\ncd->buf_in / cd->buf_out Allocated\nZero-Overhead Inner Loop Execution", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + } + + ipc_cfg -> mode_branch; + mode_branch -> branch_def [label="Mode 0"]; + mode_branch -> branch_cust [label="Mode 1"]; + mode_branch -> branch_map [label="Mode 2 or 3"]; + + branch_def -> routine_select; + branch_cust -> routine_select; + branch_map -> routine_select; + + routine_select -> runtime_ready [label="Module Prepared"]; + } + +ALSA Topology Integration, Routing Pipelines & Verification Runbook +------------------------------------------------------------------- + +The Up/Down Channel Mixer is declared in ALSA Topology 2 files as an autonomous processing widget. + +Topology 2 Widget Declaration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In topology definitions (e.g., ``tools/topology/topology2/cavs/up_down_mixer.conf``), the module is instantiated using its standard configuration schema: + +.. code-block:: text + + Object.Widget.up_down_mixer."0" { + index 1 + type "up_down_mixer" + no_pm 1 + core 0 + + # UUID binding matching UUIDREG_STR_UP_DOWN_MIXER + uuid "3a:4b:5c:6d:7e:8f:9a:bc:de:f0:12:34:56:78:9a:bc" + + # Audio format configuration + format s32le + channels 6 + rate 48000 + } + +End-to-End Multi-Channel Playback Pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Figure 215 illustrates a complete real-world surround sound playback pipeline in Sound Open Firmware, routing a 5.1 cinematic audio stream to a stereo headphone or dual-speaker DAC. + +.. graphviz:: + :caption: End-to-End Surround Media Playback Pipeline: 5.1 Downmixing to Stereo Headphone & Speaker DAC + :alt: Diagram of End-to-End Surround Media Playback Pipeline + + digraph end_to_end_playback { + graph [bgcolor="transparent", rankdir="LR", nodesep="0.5", ranksep="0.7", pad="0.3"]; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, style="filled", shape="box", penwidth=1.5]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9, penwidth=1.2, color="#64748b"]; + + subgraph cluster_host { + label = "Host Userspace / OS Media Stack"; + style = "dashed"; + color = "#3b82f6"; + bgcolor = "#1e3a8a11"; + + app [label="Media Player / Game Engine\n5.1 Surround Stream\n[L, C, R, Ls, Rs, LFE]\n48 kHz, S32_LE", fillcolor="#1e3a8a", fontcolor="#ffffff", color="#60a5fa"]; + } + + subgraph cluster_dsp { + label = "SOF Audio DSP Playback Pipeline (Pipe 1)"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4911"; + + host_copier [label="Host Copier Gateway\n(host-copier.1)\nIngests 5.1 DMA Ring", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + buf1 [label="Buffer 1\n5.1 Channels\nS32_LE", fillcolor="#1e293b", fontcolor="#94a3b8", color="#475569", shape="ellipse"]; + + updwmix [label="Up/Down Mixer\n(up_down_mixer.1)\n5.1 to Stereo Fold-Down\nScaled Headroom Matrix", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + buf2 [label="Buffer 2\nStereo (2.0)\nS32_LE", fillcolor="#1e293b", fontcolor="#94a3b8", color="#475569", shape="ellipse"]; + + vol [label="Main Volume\n(volume.1)\nLogarithmic Slider", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + buf3 [label="Buffer 3\nStereo (2.0)\nS32_LE", fillcolor="#1e293b", fontcolor="#94a3b8", color="#475569", shape="ellipse"]; + + drc [label="Dynamic Range\nCompressor (DRC)\nSpeaker Protection", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + buf4 [label="Buffer 4\nStereo (2.0)\nS32_LE", fillcolor="#1e293b", fontcolor="#94a3b8", color="#475569", shape="ellipse"]; + + dai_copier [label="DAI Copier Gateway\n(dai-copier.1)\nI2S / SoundWire DMA", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + } + + subgraph cluster_hardware { + label = "Physical Transducer Output"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b11"; + + dac [label="Stereo Audio Codec / Amp\n(e.g., RT5682 / MAX98373)\nHeadphones or Dual Speakers", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + } + + app -> host_copier [label="ALSA Playback"]; + host_copier -> buf1; + buf1 -> updwmix; + updwmix -> buf2; + buf2 -> vol; + vol -> buf3; + buf3 -> drc; + drc -> buf4; + buf4 -> dai_copier; + dai_copier -> dac [label="Serial Bit Clock & Data"]; + } + +Automated Audio Quality Verification Runbook +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To verify spatial downmixing performance, channel isolation, and clipping immunity on target hardware (such as Tiger Lake, Arrow Lake, or Panther Lake DUTs): + +1. **Deploy 5.1 Downmix Topology**: + Deploy a firmware pipeline containing the Up/Down Mixer bound between host playback and stereo DAI endpoints: + + .. code-block:: bash + + # Configure ALSA state with 5.1 downmixing enabled + alsactl -f /var/lib/alsa/asound.state restore + +2. **Generate Multi-Channel Orthogonal Test Tones**: + Synthesize a 6-channel 48 kHz 32-bit WAV file containing isolated 997 Hz sinusoids sequentially activated across individual channels: + + * 0.0s to 1.0s: Left Channel Only (:math:`-6\text{ dBFS}`) + * 1.0s to 2.0s: Center Channel Only (:math:`-6\text{ dBFS}`) + * 2.0s to 3.0s: Right Channel Only (:math:`-6\text{ dBFS}`) + * 3.0s to 4.0s: Left Surround Only (:math:`-6\text{ dBFS}`) + * 4.0s to 5.0s: Right Surround Only (:math:`-6\text{ dBFS}`) + * 5.0s to 6.0s: LFE Subwoofer Only (:math:`-6\text{ dBFS}`) + +3. **Playback and Hardware Loopback Capture**: + Stream the 6-channel WAV through SOF while capturing the stereo DAI output via an external hardware bridge (e.g., ESP32-P4 or Teensy 4.1): + + .. code-block:: bash + + # Playback 6-channel stream on DUT + aplay -Dhw:0,0 -c 6 -r 48000 -f S32_LE /tmp/multichannel_test.wav & + + # Capture stereo fold-down stream on external loopback bridge + arecord -Dhw:CARD=Bridge,DEV=0 -c 2 -r 48000 -f S32_LE -d 7 /tmp/downmix_capture.wav + +4. **Verify Attenuation & Channel Isolation Metrics**: + Execute automated Python spectral analysis on ``/tmp/downmix_capture.wav``: + + * **Left/Right Isolation**: When Left is active, Right channel leakage must be :math:`< -80\text{ dBFS}`. + * **Center Channel Split**: Center energy must appear in both Left and Right output channels with equal power (:math:`\pm 0.1\text{ dB}` matching). + * **LFE Attenuation**: When LFE is active, output level must remain at the noise floor (:math:`< -90\text{ dBFS}`). + * **Anti-Clipping Headroom**: Play a coherent :math:`0\text{ dBFS}` burst across all channels simultaneously; confirm that captured stereo output does not exceed :math:`0.0\text{ dBFS}` and exhibits :math:`\text{THD+N} < -95\text{ dB}`. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 8fd3959f..a6d7a6f2 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -58,6 +58,7 @@ Audio Processing Modules & Algorithms * :ref:`pcm_converter` (High-level architecture; also see upstream `PCM converter README `_) * :ref:`kpb_wov` (High-level architecture; also see driver guide :ref:`keyword_detect`) * :ref:`tone` (High-level architecture; also see upstream `Tone README `_) +* :ref:`up_down_mixer` (High-level architecture; also see upstream `Up/Down Mixer README `_) .. _algorithm-specific-information: @@ -105,6 +106,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/rtnr firmware/kpb_wov firmware/tone + firmware/up_down_mixer rimage/index.rst firmware/llext_modules firmware/hostless_firmware From fa419c4cef7e23ad063ffc5020afeba45350aeda Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 13:39:05 +0100 Subject: [PATCH 26/64] doc: developer_guides: add comprehensive aria architecture guide Author a comprehensive, modern architectural guide for the Sound Open Firmware (SOF) Aria (Automatic Regressive Input Amplifier) subsystem. Key topics covered: - Intelligent dynamic range pre-amplification and peak-limiting amplifier design. - Mathematical derivation of input headroom threshold (A_thresh = A_FS / 2^att) and regressive back-off factor (g = A_FS * 2^31 / (max_data * 2^att)) guaranteeing 0 dBFS peak clamping without clipping. - Target pre-amplification modes (0, +6, +12, +18 dB) across att parameter (0, 1, 2, 3). - 1 ms lookahead circular delay buffer and phased execution cycle maintaining invariant 1 ms latency across active and bypass modes. - 10-state sliding gain tracking table (sof_aria_index_tab), lookahead minimum-envelope follower, and continuous per-sample linear interpolation eliminating zipper noise. - Tensilica HiFi3/HiFi4 SIMD vectorization with single-cycle AE_MAXABS32S, odd/even channel specialization, and symmetric rounding (AE_ROUND24X2F48SSYM). - Tensilica HiFi5 hardware circular addressing registers (AE_SETCBEGIN0/1, AE_SETCEND0/1, AE_L32X2_XC, AE_S32X2_XC1) and 128-bit vector pipelines. - IPC4 module interface (struct ipc4_aria_module_cfg), runtime control (ARIA_SET_ATTENUATION), and Zephyr Loadable Linkable Extension (LLEXT) dynamic ELF packaging. - Platform performance profiles (aria.toml) and ALSA Topology 2 widget definition and pipeline graph. - Engineering verification and factory bringup runbook. - 7 native vector Graphviz SVG diagrams (Figures 216-222). Signed-off-by: Liam Girdwood --- data/modules.yaml | 12 + developer_guides/firmware/aria.rst | 988 +++++++++++++++++++++++++++++ developer_guides/index.rst | 2 + 3 files changed, 1002 insertions(+) create mode 100644 developer_guides/firmware/aria.rst diff --git a/data/modules.yaml b/data/modules.yaml index 0b277a58..bc23b107 100644 --- a/data/modules.yaml +++ b/data/modules.yaml @@ -123,6 +123,18 @@ modules: - "Parametric peak, notch, low/high shelf" - "Low computational latency" + - id: aria + name: "Aria (Automatic Regressive Input Amplifier)" + source: "SOF" + category: "Audio Enhancement" + status: "Upstream" + description: "Dynamic pre-amplifier and lookahead peak limiter with 1ms algorithmic latency." + simd: ["HiFi 3", "HiFi 4", "HiFi 5", "Scalar C"] + key_features: + - "Target pre-amplification boost (0, 6, 12, 18 dB)" + - "Instantaneous regressive ducking to prevent 0 dBFS clipping" + - "1ms lookahead circular buffer and per-sample linear interpolation" + - id: drc name: "Dynamic Range Compressor (DRC)" source: "SOF" diff --git a/developer_guides/firmware/aria.rst b/developer_guides/firmware/aria.rst new file mode 100644 index 00000000..fe6f414a --- /dev/null +++ b/developer_guides/firmware/aria.rst @@ -0,0 +1,988 @@ +.. _aria: + +Aria (Automatic Regressive Input Amplifier) Architecture +======================================================== + +The **Aria** (**Automatic Regressive Input Amplifier**) subsystem in Sound Open Firmware (SOF) is a specialized, intelligent dynamic range pre-amplifier and lookahead peak limiter. Designed primarily for capture pipelines (such as microphone front-ends and far-field speech recognition) and sensitive playback chains, Aria applies a selectable target pre-amplification boost (:math:`0\text{ dB}`, :math:`+6\text{ dB}`, :math:`+12\text{ dB}`, or :math:`+18\text{ dB}`) to incoming audio signals. When high-amplitude signals or abrupt transient bursts enter the pipeline, Aria automatically and *regressively* ducks the gain below the target, ensuring that peak signal amplitudes never exceed :math:`0\text{ dBFS}` (:math:`A_{FS} = \text{0x007fffff}` in 24-bit container format) without introducing clipping or digital saturation. + +To perform artifact-free gain modulation, Aria integrates an internal circular delay buffer introducing exactly :math:`1\text{ ms}` of lookahead algorithmic latency. This lookahead window allows the gain calculation engine to inspect future audio peaks before they reach the output, computing an optimal attenuation curve that is applied via sample-by-sample linear interpolation, totally eliminating zipper noise and transient overshoot. + +.. contents:: Table of Contents + :local: + :depth: 3 + +------------------------------------------------------------------------------- + +Architectural Overview & Functional Role +---------------------------------------- + +In modern digital signal processing pipelines, capture front-ends must accommodate a wide dynamic range of acoustic inputs—from faint whispers in distant microphone arrays to loud shouts or unexpected acoustic shocks. Conventional static gain stages and traditional automatic gain controls present fundamental trade-offs: + +- **Static Linear Gain Stages**: + Applying a fixed pre-amplification gain (e.g. :math:`+12\text{ dB}`) boosts quiet signals into the optimal operating range of downstream automatic speech recognition (ASR) engines, but inevitably causes harsh digital clipping whenever loud acoustic transients enter the analog-to-digital converter (ADC). +- **Dynamic Range Compressors (DRC)**: + Standard wideband or multiband compressors can manage high amplitudes, but rely on complex envelope followers (attack/release filters) and non-linear logarithmic curve mappings. When an unexpected transient occurs, feedback compressors cannot react instantaneously without significant lookahead buffers, leading to either initial transient clipping or prolonged gain pumping. +- **Automatic Gain Control (AGC)**: + AGC systems operate on long time horizons (typically 100 to 500 ms). While effective for slow vocal level drift, they are too sluggish to protect against sudden peak clipping. + +The Aria component resolves this challenge by operating as an **Automatic Regressive Input Amplifier**: + +1. **Target Linear Pre-amplification**: + Under nominal conditions where the signal resides safely within available headroom, Aria acts as a fixed linear pre-amplifier, applying the configured target gain of :math:`0\text{ dB}`, :math:`+6\text{ dB}`, :math:`+12\text{ dB}`, or :math:`+18\text{ dB}`. +2. **Instantaneous Regressive Back-off**: + When the peak amplitude of an incoming block exceeds the headroom threshold, the amplification factor automatically regresses (attenuates) in exact proportion to the peak overshoot: + + .. math:: + + G_{regressive} = \frac{A_{FS}}{\text{Peak Amplitude}} + + This guarantees that the peak output amplitude is locked at :math:`A_{FS}`, completely preventing digital overflow. +3. **Deterministic 1 ms Lookahead Latency**: + By buffering :math:`1\text{ ms}` of audio in an internal circular buffer, the peak detection engine evaluates incoming frames in advance. Gain transitions are smoothly interpolated across the entire frame window, eliminating step discontinuities. + +.. _figure_216: + +.. graphviz:: + :align: center + :caption: SOF Aria Subsystem Architecture: Lookahead Buffer, Dynamic Regressive Amplifier & Linear Ramp Engine + + digraph aria_architecture { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_input { + label = "Egress Audio Stream"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + source [label="Audio Source Stream\n(SOF_IPC_FRAME_S24_4LE)\nFrames at t + 1ms", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + subgraph cluster_aria { + label = "Aria Processing Module (UUID: 6d:16:f7:99...)"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + peak_detect [label="Peak Amplitude Detector\n(aria_algo_calc_gain)\nDetect max_data in chunk", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + gain_calc [label="Regressive Gain Evaluator\nIf max > Thresh: g = A_FS / max\nElse: g = 2^att", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + + state_tab [label="10-State Gain History\n(sof_aria_index_tab)\nMinimum Envelope Filter", fillcolor="#1e293b", fontcolor="#94a3b8", color="#475569"]; + + circ_buf [label="1 ms Lookahead Circular Buffer\n(cd->data_addr)\nBuffered Audio at t", fillcolor="#334155", fontcolor="#f8fafc", color="#64748b"]; + + ramp_engine [label="Linear Interpolation Ramp\nstep = (gain_end - gain_begin) / N\nPer-sample gain += step", fillcolor="#0d9488", fontcolor="#ffffff", color="#2dd4bf"]; + mult_sat [label="Multiply & Scale Unit\n(q_multsr_sat_32x32_24)\nout = (in * g) >> (31 - att)", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + subgraph cluster_output { + label = "Ingress Audio Stream"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + sink [label="Protected Sink Stream\n(SOF_IPC_FRAME_S24_4LE)\nPeak Clamped <= 0 dBFS", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + source -> peak_detect [label="Future audio\n(t + 1ms)"]; + source -> circ_buf [label="Write to ring\n(1ms delay)"]; + + peak_detect -> gain_calc [label="max_data"]; + gain_calc -> state_tab [label="Record state\n(gains[gain_idx])"]; + + state_tab -> ramp_engine [label="gain_begin\ngain_end"]; + circ_buf -> mult_sat [label="Delayed audio\n(t)"]; + ramp_engine -> mult_sat [label="Interpolated\ngain[n]"]; + mult_sat -> sink [label="Output frames"]; + } + +Comparison with Other SOF Modules +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To clarify when Aria should be instantiated in an audio graph rather than alternative processing blocks, the following table summarizes functional boundaries across related SOF components: + +.. list-table:: Architectural Comparison: Aria vs Volume vs DRC vs Smart Amp + :widths: 20 25 25 30 + :header-rows: 1 + + * - Subsystem + - Primary Operating Mode + - Dynamic Reaction Speed + - Typical Deployment Target + * - **Aria** + - Target gain (:math:`0/6/12/18\text{ dB}`) with instant regressive back-off + - Instantaneous lookahead (:math:`1\text{ ms}` pre-transient interpolation) + - Microphone capture front-ends and sensitive playback endpoints + * - **Volume** + - User-controlled linear/logarithmic gain slider (:math:`-\infty` to :math:`0\text{ dB}`) + - User-paced smooth ramp (typically :math:`16\text{ ms}` to :math:`500\text{ ms}`) + - Main and per-stream loudness controls + * - **DRC** + - Multi-segment compression knee with ratio, threshold, and makeup gain + - Envelope-follower driven attack (:math:`1\text{ ms}` to :math:`20\text{ ms}`) + and release (:math:`50\text{ ms}` to :math:`1000\text{ ms}`) + - Speaker overload protection and studio post-processing compression + * - **Smart Amp** + - Physical electro-mechanical-thermal speaker excursion modeling + - Fast non-linear displacement tracking with slow thermal decay + - Micro-speaker protection in mobile and thin laptops + +------------------------------------------------------------------------------- + +Mathematical Foundations & Regressive Dynamic Headroom +------------------------------------------------------ + +Aria operates strictly on 24-bit audio packaged inside 32-bit containers (:c:macro:`SOF_IPC_FRAME_S24_4LE`). In this encoding, sample values occupy the 24 least significant bits, sign-extended to 32 bits: + +.. math:: + + -8,388,608 \le x[n] \le +8,388,607 \quad (-2^{23} \le x[n] \le 2^{23} - 1) + +The positive full-scale maximum amplitude is denoted as: + +.. math:: + + A_{FS} = 2^{23} - 1 = \text{0x007FFFFF} = 8,388,607 + +Target Gain Parameterization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The target pre-amplification boost is configured via the unsigned integer parameter :math:`\text{att} \in \{0, 1, 2, 3\}`: + +.. list-table:: Aria Attenuation Parameter to Target Boost Mapping + :widths: 15 20 25 40 + :header-rows: 1 + + * - Parameter :math:`\text{att}` + - Linear Multiplier :math:`2^{\text{att}}` + - Decibel Boost :math:`G_{target}` + - Permissible Input Headroom :math:`A_{thresh}` + * - **0** + - :math:`1.0\times` (:math:`2^0`) + - :math:`0.00\text{ dB}` (Bypass) + - :math:`A_{FS} = \text{0x007FFFFF} = 8,388,607` (:math:`0.00\text{ dBFS}`) + * - **1** + - :math:`2.0\times` (:math:`2^1`) + - :math:`+6.02\text{ dB}` + - :math:`A_{FS} / 2 = \text{0x003FFFFF} = 4,194,303` (:math:`-6.02\text{ dBFS}`) + * - **2** + - :math:`4.0\times` (:math:`2^2`) + - :math:`+12.04\text{ dB}` + - :math:`A_{FS} / 4 = \text{0x001FFFFF} = 2,097,151` (:math:`-12.04\text{ dBFS}`) + * - **3** + - :math:`8.0\times` (:math:`2^3`) + - :math:`+18.06\text{ dB}` + - :math:`A_{FS} / 8 = \text{0x000FFFFF} = 1,048,575` (:math:`-18.06\text{ dBFS}`) + +Headroom Threshold & Regressive Gain Derivation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To prevent any sample from exceeding :math:`A_{FS}` when amplified by :math:`2^{\text{att}}`, the linear input threshold is: + +.. math:: + + A_{thresh} = \frac{A_{FS}}{2^{\text{att}}} = \text{0x007FFFFF} \gg \text{att} + +For every processing chunk (e.g. 48 frames at 48 kHz, spanning :math:`1\text{ ms}`), the algorithm detects the peak absolute amplitude across all channels: + +.. math:: + + \text{max\_data} = \max_{k \in \text{chunk}, ch} |x[k, ch]| + +The mathematical gain computation distinguishes between two regimes: + +1. **Unclipped Linear Regime** (:math:`\text{max\_data} \le A_{thresh}`): + The signal fits completely within available headroom. The raw 64-bit gain word is set to: + + .. math:: + + \text{gain} = 2^{\text{att} + 32} - 1 + + When normalized into a 32-bit state variable, it yields full fractional scale: + + .. math:: + + g = \text{gain} \gg (\text{att} + 1) = 2^{31} - 1 = \text{0x7FFFFFFF} + +2. **Regressive Compression Regime** (:math:`\text{max\_data} > A_{thresh}`): + Applying the target boost would push the output past :math:`A_{FS}`. The raw gain word is dynamically calculated via 64-bit integer division: + + .. math:: + + \text{gain} = \left\lfloor \frac{A_{FS} \cdot 2^{32}}{\text{max\_data}} \right\rfloor = \left\lfloor \frac{\text{0x007FFFFF} \cdot 2^{32}}{\text{max\_data}} \right\rfloor + + The normalized gain state is then scaled: + + .. math:: + + g = \text{gain} \gg (\text{att} + 1) = \left\lfloor \frac{\text{0x007FFFFF} \cdot 2^{31}}{\text{max\_data} \cdot 2^{\text{att}}} \right\rfloor + +Dynamic Shift Output Scaling +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +During output sample synthesis, the sample multiplication applies a dynamic right-shift determined by: + +.. math:: + + \text{shift} = 31 - \text{att} + +The output sample :math:`y[n, ch]` is generated by multiplying the input sample by the normalized gain and right-shifting: + +.. math:: + + y[n, ch] = \frac{x[n, ch] \cdot g}{2^{\text{shift}}} = \frac{x[n, ch] \cdot g}{2^{31 - \text{att}}} + +Mathematical Proof of Anti-Clipping Clamping +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Evaluating this equation in the regressive regime where :math:`\text{max\_data} > A_{thresh}`: + +.. math:: + + y[n, ch] = \frac{x[n, ch] \cdot \left(\frac{A_{FS} \cdot 2^{31}}{\text{max\_data} \cdot 2^{\text{att}}}\right)}{2^{31 - \text{att}}} + = \frac{x[n, ch] \cdot A_{FS} \cdot 2^{31}}{\text{max\_data} \cdot 2^{\text{att}} \cdot 2^{31 - \text{att}}} + = x[n, ch] \cdot \frac{A_{FS}}{\text{max\_data}} + +For the peak sample in the chunk (:math:`|x[n, ch]| = \text{max\_data}`): + +.. math:: + + |y_{peak}| = \text{max\_data} \cdot \frac{A_{FS}}{\text{max\_data}} = A_{FS} = \text{0x007FFFFF} + +The peak output is clamped exactly to :math:`0\text{ dBFS}`, guaranteeing that no digital overflow occurs regardless of the input burst magnitude. + +.. _figure_217: + +.. graphviz:: + :align: center + :caption: Mathematical Dynamics: Target Gain Boost (0/6/12/18 dB), Headroom Thresholds & Regressive Ducking Curve + + digraph aria_math_curves { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_regimes { + label = "Aria Input Dynamic Regimes & Transfer Function"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + node_low [label="Low-Level Signal Regime\n(x <= A_thresh)\nGain = 2^att (Target Boost)\nOutput = x * 2^att", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + node_thresh [label="Headroom Threshold Point\nx = A_FS >> att\n(Output reaches exactly A_FS)", fillcolor="#d97706", fontcolor="#ffffff", color="#fbbf24", shape="diamond"]; + node_high [label="High-Level Transient Regime\n(x > A_thresh)\nGain = A_FS / x (Regressive Ducking)\nPeak Output Clamped to A_FS (0 dBFS)", fillcolor="#dc2626", fontcolor="#ffffff", color="#f87171"]; + } + + subgraph cluster_thresholds { + label = "Headroom Thresholds Across Attenuation Modes"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + t0 [label="att = 0 (0 dB Boost)\nA_thresh = 0x007FFFFF\nFull Scale Headroom", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + t1 [label="att = 1 (+6 dB Boost)\nA_thresh = 0x003FFFFF\n-6.02 dBFS Headroom", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + t2 [label="att = 2 (+12 dB Boost)\nA_thresh = 0x001FFFFF\n-12.04 dBFS Headroom", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + t3 [label="att = 3 (+18 dB Boost)\nA_thresh = 0x000FFFFF\n-18.06 dBFS Headroom", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + node_low -> node_thresh [label="Signal rises"]; + node_thresh -> node_high [label="Exceeds headroom"]; + + node_thresh -> t0 [style="dotted", label="Mode 0"]; + node_thresh -> t1 [style="dotted", label="Mode 1"]; + node_thresh -> t2 [style="dotted", label="Mode 2"]; + node_thresh -> t3 [style="dotted", label="Mode 3"]; + } + +------------------------------------------------------------------------------- + +1 ms Lookahead Circular Buffer & Latency Phasing +------------------------------------------------ + +A fundamental problem in conventional peak limiters is that gain reduction is triggered *after* or *at* the arrival of a peak, causing either initial overshoot clipping or unnatural transient distortion. Aria completely eliminates this issue by introducing a **1 ms lookahead window** realized through an internal circular delay buffer. + +Buffer Sizing & Memory Layout +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The circular buffer is allocated during component initialization (:c:func:`aria_init`) to hold exactly :math:`1\text{ ms}` of audio across all channels: + +.. math:: + + \text{buff\_size} = \text{ALIGN\_UP}(\text{chan\_cnt} \cdot \text{smpl\_group\_cnt}, 2) + +where: + +- :math:`\text{chan\_cnt}` is the number of audio channels (e.g. 2 for stereo, 4 for quad mic array). +- :math:`\text{smpl\_group\_cnt}` is the number of samples per channel in :math:`1\text{ ms}` (e.g. 48 samples at 48 kHz). +- The buffer is aligned to 8-byte boundaries (2 samples of 32-bit audio) to satisfy SIMD vector memory alignment requirements. + +An offset variable is tracked: + +.. math:: + + \text{offset} = (\text{chan\_cnt} \cdot \text{smpl\_group\_cnt}) \& 1 + +ensuring that the circular buffer read and write pointers maintain invariant alignment throughout runtime execution. + +Phased Execution Cycle (The 4-Step Pipeline) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In every processing tick of :c:func:`aria_process_data`, Aria executes four consecutive operations: + +1. **Step 1: Lookahead Peak Inspection** (:math:`t + 1\text{ ms}`): + The function :c:func:`aria_algo_calc_gain` inspects the future incoming frames in ``source``. It scans all channels, calculates the peak absolute value :math:`\text{max\_data}`, evaluates whether regressive compression is required, and stores the computed gain into the gain history table at: + + .. math:: + + \text{gain\_idx} = \text{sof\_aria\_index\_tab}[\text{cd->gain\_state} + 1] + +2. **Step 2: Delayed Audio Retrieval & Gain Application** (:math:`t`): + The function ``cd->aria_get_data`` reads the *past* audio stored in the circular buffer at ``cd->data_ptr`` (which entered the buffer :math:`1\text{ ms}` prior). It linearly interpolates the gain across the block and writes the protected, amplified audio to ``sink``. +3. **Step 3: History Buffer Ingestion**: + The function :c:func:`cir_buf_copy` transfers the future incoming audio from ``source`` into the circular buffer at ``cd->data_ptr``, storing it as history for processing in the subsequent millisecond. +4. **Step 4: Circular Pointer Wrap**: + The circular pointer is advanced by the chunk sample size and wrapped using :c:func:`cir_buf_wrap`: + + .. math:: + + \text{cd->data\_ptr} = \text{cir\_buf\_wrap}(\text{cd->data\_ptr} + \text{sample\_size}, \text{cd->data\_addr}, \text{cd->data\_end}) + +Bypass Invariance +~~~~~~~~~~~~~~~~~ + +When :math:`\text{att} == 0`, Aria operates in bypass mode. Rather than short-circuiting the buffer, :c:func:`aria_process_data` routes audio through the circular delay buffer without applying gain multipliers: + +.. code-block:: c + + if (cd->att) { + aria_algo_calc_gain(cd, sof_aria_index_tab[cd->gain_state + 1], source, frames); + cd->aria_get_data(mod, sink, frames); + } else { + cir_buf_copy(cd->data_ptr, cd->data_addr, cd->data_end, + sink->w_ptr, sink->addr, sink->end_addr, + data_size); + } + +This design ensures that the pipeline latency is **strictly invariant at 1 ms**, preventing downstream phase misalignments or timestamp discontinuities when switching attenuation modes on the fly. + +.. _figure_218: + +.. graphviz:: + :align: center + :caption: Lookahead Buffer Timing & 1 ms Lookahead Latency Phasing (Future Peak Detection vs Delayed Stream Application) + + digraph aria_timing_phasing { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_timeline { + label = "Timeline Phasing Across 1 ms Execution Window"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + t_future [label="Time t + 1 ms (Future Input)\nIncoming stream in source DMA ring\nEvaluated by aria_algo_calc_gain()", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + t_present [label="Circular Delay Ring Buffer\nStores 1 ms history in cd->data_addr\nDecouples peak detection from scaling", fillcolor="#334155", fontcolor="#f8fafc", color="#64748b"]; + t_past [label="Time t (Delayed Audio Output)\nRead from cd->data_ptr into sink\nScaled by interpolated gain[n]", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + subgraph cluster_steps { + label = "Phased Execution Sequence in aria_process_data()"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + s1 [label="1. Peak Detection: Calculate required gain for future frame (t+1ms)", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + s2 [label="2. Scaling & Egress: Multiply delayed audio (t) by ramped gain -> sink", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + s3 [label="3. Ring Update: Copy future audio (t+1ms) from source -> circular buffer", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + s4 [label="4. Ring Wrap: Advance cd->data_ptr with cir_buf_wrap()", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + t_future -> s1 [label="Inspects"]; + s1 -> s2 [label="Advances state"]; + t_present -> s2 [label="Reads delayed audio"]; + s2 -> t_past [label="Writes to sink"]; + t_future -> s3 [label="Transfers"]; + s3 -> t_present [label="Populates ring"]; + s3 -> s4 [label="Completes copy"]; + } + +------------------------------------------------------------------------------- + +Multi-State Gain Follower & Per-Sample Linear Interpolation +----------------------------------------------------------- + +Abrupt gain changes between consecutive processing chunks produce audible discontinuities known as *zipper noise* and generate high-frequency distortion harmonics. To ensure acoustic transparency, Aria utilizes a **10-state sliding gain tracking table** and continuous **per-sample linear interpolation**. + +Sliding Gain History Table +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Aria maintains 10 historical gain values in the array: + +.. code-block:: c + + int32_t gains[ARIA_MAX_GAIN_STATES]; // ARIA_MAX_GAIN_STATES = 10 + +To eliminate expensive runtime modulo arithmetic (:math:`\% 10`), indexing is performed via a pre-computed lookup table: + +.. code-block:: c + + const int32_t sof_aria_index_tab[] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3 + }; + +Lookahead Minimum-Envelope Search +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When determining the starting gain (:math:`\text{gain\_begin}`) and ending gain (:math:`\text{gain\_end}`) for the current :math:`1\text{ ms}` chunk, Aria searches across a multi-state window for the *minimum* gain value: + +.. code-block:: c + + int32_t gain_state_add_2 = cd->gain_state + 2; + int32_t gain_state_add_3 = cd->gain_state + 3; + int32_t gain_begin = cd->gains[sof_aria_index_tab[gain_state_add_2]]; + int32_t gain_end = cd->gains[sof_aria_index_tab[gain_state_add_3]]; + + for (i = 1; i < ARIA_MAX_GAIN_STATES - 1; i++) { + if (cd->gains[sof_aria_index_tab[gain_state_add_2 + i]] < gain_begin) + gain_begin = cd->gains[sof_aria_index_tab[gain_state_add_2 + i]]; + if (cd->gains[sof_aria_index_tab[gain_state_add_3 + i]] < gain_end) + gain_end = cd->gains[sof_aria_index_tab[gain_state_add_3 + i]]; + } + +By tracking the minimum gain across states, Aria establishes a **lookahead attack envelope**: + +- If an impending peak requires severe gain reduction, :math:`\text{gain\_begin}` and :math:`\text{gain\_end}` are pulled downward *before* the peak reaches the output. +- The gain ramps down smoothly toward the required attenuation, so that the signal is already safely compressed when the peak transient hits the output multiplier. +- Conversely, when transitioning out of a transient into quiet audio, the gain recovers smoothly across subsequent blocks without abrupt pumping. + +Continuous Per-Sample Linear Interpolation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Once :math:`\text{gain\_begin}` and :math:`\text{gain\_end}` are determined, Aria computes the per-sample ramp increment: + +.. math:: + + \text{step} = \frac{\text{gain\_end} - \text{gain\_begin}}{\text{frames}} + +The gain accumulator starts at :math:`\text{gain} = \text{gain\_begin}`. For every sample group, the current gain is applied and then updated: + +.. math:: + + \text{gain}_{n+1} = \text{gain}_n + \text{step} + +This ensures :math:`C^0` continuity across the entire audio stream, completely eliminating zipper noise. + +.. _figure_219: + +.. graphviz:: + :align: center + :caption: Multi-State Gain Follower & Linear Interpolation Ramp (Minimum-Envelope Search & Per-Sample Stepping) + + digraph aria_gain_smoothing { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_states { + label = "10-State Circular Gain Table (cd->gains[0..9])"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + s_hist [label="Historical Gain States\ngains[0] .. gains[7]\nPast chunk gains", fillcolor="#1e293b", fontcolor="#94a3b8", color="#475569"]; + s_curr [label="Current Active Gain\ngains[gain_state]\nActive frame chunk", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + s_next [label="Future Lookahead Gain\ngains[gain_state + 1]\nComputed for t + 1ms", fillcolor="#d97706", fontcolor="#ffffff", color="#fbbf24"]; + } + + subgraph cluster_min_search { + label = "Minimum Envelope Search"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + min_eval [label="Envelope Minimum Evaluation\nSearch across ARIA_MAX_GAIN_STATES - 1\ngain_begin = min(gains[...])\ngain_end = min(gains[...])", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + } + + subgraph cluster_ramp { + label = "Per-Sample Linear Stepping"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b22"; + + calc_step [label="Slope Calculation\nstep = (gain_end - gain_begin) / frames", fillcolor="#0d9488", fontcolor="#ffffff", color="#2dd4bf"]; + sample_loop [label="Per-Sample Execution Loop\ny[n] = (x[n] * gain) >> (31 - att)\ngain += step", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + s_hist -> min_eval; + s_curr -> min_eval; + s_next -> min_eval; + + min_eval -> calc_step [label="gain_begin\ngain_end"]; + calc_step -> sample_loop [label="step"]; + } + +------------------------------------------------------------------------------- + +Tensilica HiFi SIMD Acceleration & Hardware Circular Buffers +------------------------------------------------------------ + +The computational throughput of the Aria component is heavily optimized using Cadence Tensilica HiFi SIMD instruction sets, delivering distinct implementations across **Generic Scalar C**, **HiFi3 / HiFi4**, and **HiFi5**. + +Generic Scalar Implementation (:file:`aria_generic.c`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The scalar C fallback performs signed 24-bit sign-extension and fixed-point fractional multiplication: + +.. code-block:: c + + in_sample = sign_extend_s24(*in++); + out[ch] = q_multsr_sat_32x32_24(in_sample, gain, shift); + +While fully functional and portable across any processor architecture (including RISC-V and ARM), the scalar loops require branching for circular wrapping and sample-by-sample clamping. + +Tensilica HiFi3 / HiFi4 Acceleration (:file:`aria_hifi3.c`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The HiFi3/4 kernel introduces vectorized peak detection, odd/even channel specialization, and symmetric rounding: + +1. **Vector Absolute Maximum in a Single Instruction**: + In :c:func:`aria_algo_calc_gain`, the future sample stream is scanned using 64-bit vector alignment loads (:c:macro:`AE_LA64_PP`) and the :c:macro:`AE_MAXABS32S` instruction, which simultaneously computes the absolute value and compares it against the running maximum across dual 32-bit SIMD lanes in a single cycle: + + .. code-block:: c + + AE_LA32X2_IP(in_sample, inu, in); + max_data = AE_MAXABS32S(max_data, AE_SLAI32(in_sample, 8)); + +2. **Channel Specialization (Odd vs Even Channels)**: + To maximize vector register utilization, :c:func:`aria_algo_get_data_func` dynamically binds either :c:func:`aria_algo_get_data_odd_channel` or :c:func:`aria_algo_get_data_even_channel`: + - **Even Channels (Stereo, Quad, 8ch)**: Samples are processed in pairs (:math:`\text{ch} += 2`). Dual 32-bit vector registers :c:macro:`AE_LA32X2_IP` feed high and low 32x32 multipliers: + + .. code-block:: c + + out1 = AE_MUL32_HH(in_sample, gain); + out1 = AE_SRAA64(out1, shift_bits); + out2 = AE_MUL32_LL(in_sample, gain); + out2 = AE_SRAA64(out2, shift_bits); + + - **Odd Channels (Mono, 3ch, 5ch)**: Samples are processed individually with single-lane instructions (:c:macro:`AE_L32_XP` and :c:macro:`AE_S32_L_XP`). +3. **Symmetric Rounding and Saturation**: + Intermediate products are rounded from 48-bit fixed-point back to 24-bit signed representation using :c:macro:`AE_ROUND24X2F48SSYM`, guaranteeing bit-exact symmetry and preventing negative DC bias accumulation. + +Tensilica HiFi5 Hardware Circular Addressing (:file:`aria_hifi5.c`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On Intel platforms equipped with Tensilica HiFi5 cores (such as Panther Lake and Lunar Lake), Aria achieves maximal memory throughput by leveraging dedicated **Hardware Circular Addressing Registers**: + +1. **Hardware Circular Buffer Setup**: + HiFi5 features dedicated circular addressing pointer registers :c:macro:`AE_SETCBEGIN0`, :c:macro:`AE_SETCEND0` for the input delay buffer, and :c:macro:`AE_SETCBEGIN1`, :c:macro:`AE_SETCEND1` for the sink buffer: + + .. code-block:: c + + set_circular_buf0(cd->data_addr, cd->data_end); + set_circular_buf1(audio_stream_get_addr(sink), audio_stream_get_end_addr(sink)); + +2. **Zero-Overhead Automatic Address Wrapping**: + When loading and storing samples, the specialized circular instructions :c:macro:`AE_L32X2_XC` and :c:macro:`AE_S32X2_XC1` automatically wrap the memory pointer back to the buffer start address when the end boundary is reached: + + .. code-block:: c + + AE_L32X2_XC(in_sample, in, inc); + ... + AE_S32X2_XC1(out_sample, out, inc); + + This completely eliminates runtime boundary checking, pointer masking, and branch instructions inside the inner DSP audio loop. +3. **128-Bit SIMD Vector Pipelines**: + In the peak detection stage, HiFi5 utilizes 128-bit vector loads (:c:macro:`AE_LA128_PP` and :c:macro:`AE_LA32X2X2_IP`), processing 4 32-bit audio samples simultaneously per instruction cycle. + +.. _figure_220: + +.. graphviz:: + :align: center + :caption: Tensilica HiFi3/HiFi4 vs HiFi5 SIMD Acceleration (Dual-Channel Multipliers vs Hardware Circular Buffering AE_SETCBEGIN) + + digraph aria_simd_comparison { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_hifi3 { + label = "Tensilica HiFi3 / HiFi4 SIMD Pipeline"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + h3_load [label="Dual Load: AE_LA32X2_IP\nLoads 2 x 32-bit samples (64-bit)", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + h3_mult [label="Dual Multiply: AE_MUL32_HH / LL\nMultiplies high and low lanes by gain", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + h3_round [label="Symmetric Round: AE_ROUND24X2F48SSYM\nRounds 48-bit product to 24-bit", fillcolor="#0d9488", fontcolor="#ffffff", color="#2dd4bf"]; + h3_wrap [label="Software Buffer Wrap: cir_buf_wrap()\nConditional pointer evaluation", fillcolor="#334155", fontcolor="#94a3b8", color="#475569"]; + } + + subgraph cluster_hifi5 { + label = "Tensilica HiFi5 Advanced Hardware Pipeline"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b22"; + + h5_setup [label="Hardware Ring Setup:\nAE_SETCBEGIN0/1 & AE_SETCEND0/1\nConfigures DSP hardware registers", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + h5_load [label="128-Bit Load: AE_LA128_PP / AE_LA32X2X2_IP\nLoads 4 x 32-bit samples per cycle", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + h5_auto [label="Hardware Auto-Wrap Load/Store:\nAE_L32X2_XC & AE_S32X2_XC1\nZero-cycle hardware address wrap", fillcolor="#10b981", fontcolor="#ffffff", color="#6ee7b7"]; + } + + h3_load -> h3_mult -> h3_round -> h3_wrap; + h5_setup -> h5_load -> h5_auto; + } + +------------------------------------------------------------------------------- + +IPC4 Modular Interface, LLEXT Packaging & Topology 2 Graph +---------------------------------------------------------- + +Aria is fully compliant with the Intel IPC4 firmware architecture and supports both static compilation into the core firmware binary and modular dynamic loading via **Zephyr Loadable Linkable Extensions (LLEXT)**. + +IPC4 Configuration Structures +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The module configuration structure is defined in :file:`aria.h`: + +.. code-block:: c + + struct ipc4_aria_module_cfg { + struct ipc4_base_module_cfg base_cfg; + uint32_t attenuation; + } __packed __aligned(8); + +- ``base_cfg``: Standard IPC4 base module configuration specifying input/output buffer sizes, audio stream format (:math:`\text{depth} = 32`, :math:`\text{valid\_depth} = 24`), and channel count. +- ``attenuation``: The target attenuation/boost mode (:math:`\text{att} \in \{0, 1, 2, 3\}`). If the host provides a value greater than :c:macro:`ARIA_MAX_ATT` (3), the firmware clamps it to 3 and emits a warning trace. + +Runtime Control Parameter +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Aria supports dynamic runtime adjustment of target attenuation without tearing down the audio pipeline via IPC4 large configuration messages: + +- **Parameter ID**: :c:macro:`ARIA_SET_ATTENUATION` (1). +- **Payload**: 32-bit unsigned integer representing the new attenuation setting (``cd->att``). +- When received in :c:func:`aria_set_config`, the firmware immediately updates ``cd->att`` and recomputes the baseline gain states via :c:func:`aria_set_gains`. + +Modular LLEXT Packaging +~~~~~~~~~~~~~~~~~~~~~~~ + +When built as a loadable module (``CONFIG_COMP_ARIA = "m"``), Aria is compiled into an independent ELF shared object (:file:`aria.llext`) and exported with a signed module manifest: + +.. code-block:: c + + static const struct sof_man_module_manifest mod_manifest __section(".module") __used = + SOF_LLEXT_MODULE_MANIFEST("ARIA", &aria_interface, 1, SOF_REG_UUID(aria), 8); + +- **Module Name**: ``"ARIA"`` +- **Interface Structure**: ``aria_interface`` +- **Module Version**: ``1`` +- **Component UUID**: ``6d:16:f7:99:2c:37:ef:43:81:f6:22:00:7a:a1:5f:03`` +- **Stack Size**: 8 KB + +Platform Performance Profiles (:file:`aria.toml`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The configuration file :file:`src/audio/aria/aria.toml` specifies processing constraints and Cycles Per Chunk (CPC) metrics across various operational frame sizes: + +.. list-table:: Aria Performance & Resource Allocation across Chunk Sizes + :widths: 20 20 25 35 + :header-rows: 1 + + * - Chunk Frames + - Cycles Per Chunk (CPC) + - Input Buffer Size (IBS) + - Output Buffer Size (OBS) + * - **16 frames** + - 1,063,000 CPS + - 16 samples + - 21 samples + * - **32 frames** + - 2,680,000 CPS + - 32 samples + - 42 samples + * - **64 frames** + - 3,591,000 CPS + - 64 samples + - 85 samples + * - **96 frames** + - 4,477,000 CPS + - 96 samples + - 128 samples + * - **192 frames** + - 7,195,000 CPS + - 192 samples + - 192 samples + +ALSA Topology 2 Integration +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In ALSA Topology 2, Aria is declared as an audio effect widget in :file:`tools/topology/topology2/include/components/aria.conf`: + +.. code-block:: text + + Class.Widget."aria" { + DefineAttribute."index" {} + + + DefineAttribute."cpc" { + token_ref "comp.word" + } + DefineAttribute."is_pages" { + token_ref "comp.word" + } + + Object.Control.bytes."1" { + !access [ tlv_read tlv_callback ] + Object.Base.extops.1 { + name "extctl" + get 258 + put 0 + } + max 4096 + } + + uuid "6d:16:f7:99:2c:37:ef:43:81:f6:22:00:7a:a1:5f:03" + type "effect" + no_pm "true" + cpc 5000 + is_pages 1 + num_input_pins 1 + num_output_pins 1 + } + +Aria is integrated into audio playback and capture pipelines, such as :file:`topology2/include/pipelines/cavs/mixout-aria-gain-mixin-playback.conf`: + +.. code-block:: text + + Object.Base { + route.1 { + source mixout.$index.1 + sink aria.$index.1 + } + route.2 { + source aria.$index.1 + sink gain.$index.1 + } + route.3 { + source gain.$index.1 + sink mixin.$index.1 + } + } + +Tuning Blobs via Octave/MATLAB (:file:`sof_aria_blobs.m`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To generate pre-compiled binary configuration blobs for ALSA topology generation, SOF provides the Octave script :file:`src/audio/aria/tune/sof_aria_blobs.m`. It constructs ABI-compliant configuration containers for: + +- :file:`passthrough.conf`: Sets :math:`\text{att} = 0` (Bypass, 0 dB). +- :file:`param_1.conf`: Sets :math:`\text{att} = 1` (+6 dB). +- :file:`param_2.conf`: Sets :math:`\text{att} = 2` (+12 dB). +- :file:`param_3.conf`: Sets :math:`\text{att} = 3` (+18 dB). + +.. _figure_221: + +.. graphviz:: + :align: center + :caption: IPC4 Configuration Architecture & LLEXT Modular Packaging + + digraph aria_ipc4_llext { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_host { + label = "Host Driver / User-Space ALSA Plane"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + blob_script [label="MATLAB / Octave Generator\n(sof_aria_blobs.m)\nExports param_1.conf..param_3.conf", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + alsa_tplg [label="ALSA Topology 2 Compiler\n(alsatplg)\nCompiles aria.conf widget & routes", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + host_ctl [label="Runtime Mixer Control\n(amixer / ctl)\nSends ARIA_SET_ATTENUATION", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + } + + subgraph cluster_dsp { + label = "SOF Audio DSP Firmware Engine"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + ipc4_handler [label="IPC4 Message Dispatcher\nParses Large Config / Init Data", fillcolor="#0d9488", fontcolor="#ffffff", color="#2dd4bf"]; + llext_loader [label="Zephyr LLEXT Dynamic Linker\nLoads aria.llext via ELF manifest", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + aria_core [label="Aria Processing Core\nUpdates cd->att & recomputes cd->gains[]", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + blob_script -> alsa_tplg [label="Tuning Blobs"]; + alsa_tplg -> ipc4_handler [label="Pipeline Creation"]; + host_ctl -> ipc4_handler [label="Runtime Attenuation"]; + + ipc4_handler -> llext_loader [label="Bind Module"]; + llext_loader -> aria_core [label="Instantiate"]; + ipc4_handler -> aria_core [label="Update Attenuation"]; + } + +.. _figure_222: + +.. graphviz:: + :align: center + :caption: End-to-End Audio Graph & Topology 2 Integration (mixout-aria-gain-mixin Playback Pipeline) + + digraph aria_playback_pipeline { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_ingress { + label = "Host Playback Ingress"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + mixout [label="Mixout Widget\n(mixout.1)\nAudio Stream Egress", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + subgraph cluster_aria_pipe { + label = "Aria Dynamic Protection Pipeline (Pipeline 1)"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + aria_w [label="Aria Widget\n(aria.1.1)\nTarget Boost + Lookahead Limiter\nUUID: 6d:16:f7:99...", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + gain_w [label="Gain Widget\n(gain.1.1)\n32-bit Linear Scaler", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + mixin_w [label="Mixin Widget\n(mixin.1)\nBus Fan-In Node", fillcolor="#0d9488", fontcolor="#ffffff", color="#2dd4bf"]; + } + + subgraph cluster_egress { + label = "Physical Audio Egress"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b22"; + + dai [label="DAI Copier Gateway\n(I2S / SoundWire Link)\nOutput to Codec / Amp", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + mixout -> aria_w [label="Route 1 (S24_4LE)"]; + aria_w -> gain_w [label="Route 2 (Protected)"]; + gain_w -> mixin_w [label="Route 3 (Leveled)"]; + mixin_w -> dai [label="Playback Egress"]; + } + +------------------------------------------------------------------------------- + +Factory Bringup, Acoustic Quality & Verification Runbook +-------------------------------------------------------- + +This section outlines an end-to-end engineering verification procedure to validate Aria functionality, dynamic boost accuracy, regressive anti-clipping clamping, and latency invariance on physical DUTs (such as Panther Lake, Meteor Lake, or Tiger Lake). + +1. Topology Compilation & Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Verify that the target topology includes the Aria widget and compiles cleanly: + +.. code-block:: bash + + # Step 1: Export tuning blobs using GNU Octave + cd tools/tune/aria + octave --no-gui sof_aria_blobs.m + + # Step 2: Compile ALSA Topology 2 binary + cd ../../topology/topology2 + alsatplg -c development/sof-mtl-sdw-benchmark-aria24-simplejack.conf \ + -o sof-mtl-sdw-benchmark-aria24-simplejack.tplg + + # Step 3: Deploy topology to target DUT + scp sof-mtl-sdw-benchmark-aria24-simplejack.tplg root@:/lib/firmware/intel/sof-ipc4/ + +2. Driver Reload & DSP Initialization Check +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Reload the kernel sound driver and inspect :command:`dmesg` to verify module instantiation: + +.. code-block:: bash + + # Reload SOF audio driver on DUT + ssh root@ 'modprobe -r snd_sof_pci_intel_mtl && modprobe snd_sof_pci_intel_mtl' + + # Check dmesg for Aria registration and UUID confirmation + ssh root@ 'dmesg | grep -i aria' + +Expected output: + +.. code-block:: text + + sof-audio-pci-intel-mtl: module ARIA [6d16f799-2c37-43ef-81f6-22007aa15f03] loaded + sof-audio-pci-intel-mtl: aria.1.1: created with attenuation = 1 (target +6 dB) + +3. Dynamic Range & Linear Pre-amplification Verification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Verify that low-amplitude audio receives the exact target boost across all attenuation modes: + +.. code-block:: bash + + # Generate 1 kHz test tone at -30 dBFS (well below all headroom thresholds) + sox -n -r 48000 -c 2 -b 24 test_tone_minus30dBFS.wav synth 5 sine 1000 vol -30dB + + # Play test tone through Aria playback pipeline + ssh root@ 'aplay -D hw:0,0 test_tone_minus30dBFS.wav' + + # Check output level across attenuation modes via amixer + # Mode 0 (att = 0, 0 dB): Output level must equal -30.0 dBFS + # Mode 1 (att = 1, +6 dB): Output level must equal -24.0 dBFS (+/- 0.1 dB) + # Mode 2 (att = 2, +12 dB): Output level must equal -18.0 dBFS (+/- 0.1 dB) + # Mode 3 (att = 3, +18 dB): Output level must equal -12.0 dBFS (+/- 0.1 dB) + +4. Transient Shock & Anti-Clipping Clamping Verification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Verify that full-scale signals and loud acoustic transients are strictly clamped to :math:`0\text{ dBFS}` without digital wrap-around: + +.. code-block:: bash + + # Generate high-amplitude burst signal at -3 dBFS + sox -n -r 48000 -c 2 -b 24 test_burst.wav synth 3 sine 1000 vol -3dB + + # Set Aria to maximum boost mode (att = 3, target +18 dB) + ssh root@ 'amixer -c 0 cset name="aria.1.1.extctl" 3' + + # In a naive amplifier, -3 dBFS + 18 dB = +15 dBFS (massive digital clipping) + # In Aria, output peak must clamp strictly to 0.00 dBFS (0x007FFFFF) + ssh root@ 'aplay -D hw:0,0 test_burst.wav' + + # Record capture loopback and verify maximum peak using sox + sox recorded_output.wav -n stats + +Expected verification statistics: + +.. code-block:: text + + Pk lev dB 0.00 + Max amp 0.999999 + Min amp -0.999999 + Zero crossings 6000 + Flat factor 0.00 <-- Verifies zero flat-top clipping distortion! + +5. 1 ms Lookahead Latency Invariance Verification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Measure group delay through the pipeline with Aria in bypass (:math:`\text{att} = 0`) versus active (:math:`\text{att} = 2`): + +.. code-block:: bash + + # Measure impulse response latency with cross-correlation + python3 -c ' + import numpy as np, scipy.io.wavfile as wf + rate, ref = wf.read("impulse_ref.wav") + rate, cap = wf.read("impulse_cap.wav") + corr = np.correlate(cap[:,0], ref[:,0], mode="full") + delay_ms = (np.argmax(corr) - len(ref) + 1) / rate * 1000.0 + print(f"Measured Algorithmic Delay: {delay_ms:.3f} ms") + ' + +The measured delay delta between bypass and active mode must be **identically 0.000 ms**, confirming that the circular delay buffer maintains constant pipeline latency. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index a6d7a6f2..ee64bcb9 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -59,6 +59,7 @@ Audio Processing Modules & Algorithms * :ref:`kpb_wov` (High-level architecture; also see driver guide :ref:`keyword_detect`) * :ref:`tone` (High-level architecture; also see upstream `Tone README `_) * :ref:`up_down_mixer` (High-level architecture; also see upstream `Up/Down Mixer README `_) +* :ref:`aria` (High-level architecture; also see upstream `Aria README `_) .. _algorithm-specific-information: @@ -107,6 +108,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/kpb_wov firmware/tone firmware/up_down_mixer + firmware/aria rimage/index.rst firmware/llext_modules firmware/hostless_firmware From fe74c72a08d7a4035b08f736ae18c4a1d311d5e1 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 14:25:11 +0100 Subject: [PATCH 27/64] doc: developer_guides: add comprehensive level multiplier architecture guide Author a comprehensive, modern architectural guide for the Sound Open Firmware (SOF) Level Multiplier subsystem. Key topics covered: - Ultra-low-latency, zero-overhead digital linear gain and attenuation component architecture. - Q9.23 fixed-point number system (9 integer bits, 23 fractional bits) covering dynamic range from -138.47 dB to +48.17 dB with 1.19e-7 quantization resolution. - Dynamic range extremes, decibel-to-linear conversion formula, and unity gain definition (LEVEL_MULTIPLIER_GAIN_ONE = 0x00800000). - Universal PCM frame format support: 16-bit (SOF_IPC_FRAME_S16_LE), 24-bit (SOF_IPC_FRAME_S24_4LE), and 32-bit (SOF_IPC_FRAME_S32_LE) with unified 23-bit right-shift constant. - Ring buffer wrap segmentation algorithm for contiguous sample processing. - Zero-overhead fast-path bypass: direct memory block copy (source_to_sink_copy) at unity gain (0 dB), eliminating arithmetic multiplication loops and conserving active DSP cycles. - Tensilica HiFi3/HiFi4 SIMD vectorization with dual 64-bit vector registers, 16-bit parallel load/multiplication (AE_LA16X4_IP, AE_MULFP32X16X2RS_H/L), and symmetric rounding (AE_ROUND16X4F32SSYM). - Tensilica HiFi5 128-bit vector pipelines: octal 16-bit (AE_LA16X4X2_IP, AE_MULF2P32X16X4RS) and quad 32-bit (AE_LA32X2X2_IP, AE_MULF2P32X4RS) SIMD MAC acceleration. - IPC4 modular interface (level_multiplier_set_config), Zephyr Loadable Linkable Extension (LLEXT) dynamic packaging (level_multiplier.llext), and UUID registration. - Platform performance profiles (level_multiplier.toml), ALSA Topology 2 widget definition, and Octave/MATLAB blob generation script (sof_level_multiplier_blobs.m). - Factory bringup, precision linearity and gain accuracy test runbook. - 7 native vector Graphviz SVG diagrams (Figures 223-229). Signed-off-by: Liam Girdwood --- data/modules.yaml | 12 + .../firmware/level_multiplier.rst | 902 ++++++++++++++++++ developer_guides/index.rst | 2 + 3 files changed, 916 insertions(+) create mode 100644 developer_guides/firmware/level_multiplier.rst diff --git a/data/modules.yaml b/data/modules.yaml index bc23b107..f371b525 100644 --- a/data/modules.yaml +++ b/data/modules.yaml @@ -86,6 +86,18 @@ modules: - "Mono to stereo/surround replication" - "Channel swap and mute masking" + - id: level_multiplier + name: "Level Multiplier" + source: "SOF" + category: "Basic Routing & Level" + status: "Upstream" + description: "Ultra-low-latency Q9.23 linear scaling amplifier for capture sensitivity calibration and inter-stage matching." + simd: ["HiFi 3", "HiFi 4", "HiFi 5", "Scalar C"] + key_features: + - "High-precision Q9.23 fixed-point multiplier (-138.47 dB to +48.17 dB)" + - "Zero-overhead fast-path bypass when configured for unity gain (0 dB)" + - "Runtime IPC4 calibration and LLEXT dynamic module packaging" + - id: tone name: "Tone Generator" source: "SOF" diff --git a/developer_guides/firmware/level_multiplier.rst b/developer_guides/firmware/level_multiplier.rst new file mode 100644 index 00000000..e2f31fd2 --- /dev/null +++ b/developer_guides/firmware/level_multiplier.rst @@ -0,0 +1,902 @@ +.. _level_multiplier: + +Level Multiplier Architecture +============================= + +The **Level Multiplier** subsystem in Sound Open Firmware (SOF) is an ultra-low-latency, zero-overhead digital linear gain and attenuation component. Operating strictly on fixed-point **Q9.23** arithmetic, the Level Multiplier scales digital audio signals across a vast dynamic range from :math:`-138.47\text{ dB}` to :math:`+48.17\text{ dB}`. Unlike full-featured software volume controls that implement multi-channel curves, logarithmic lookups, and multi-millisecond smoothing ramps, the Level Multiplier applies a direct scalar factor across all channels without state ramping overhead or algorithmic delay. + +The Level Multiplier is extensively deployed in voice capture front-ends (such as Automatic Speech Recognition and far-field voice trigger pipelines) to calibrate microphone sensitivity independently from user-facing media volume controls. Furthermore, the component integrates an automated **zero-overhead fast-path bypass**: whenever the configured gain equals unity (:math:`0\text{ dB}`, `LEVEL_MULTIPLIER_GAIN_ONE`), the component completely bypasses arithmetic multiplication loops and executes a direct memory copy, minimizing processor cycles and active power consumption. + +.. contents:: Table of Contents + :local: + :depth: 3 + +------------------------------------------------------------------------------- + +Architectural Overview & Functional Role +---------------------------------------- + +Audio processing pipelines frequently require precise level adjustments that are independent of user-controlled volume sliders. Typical examples include microphone pre-amplification calibration, transducer sensitivity matching across multi-microphone arrays, inter-stage digital headroom management, and platform-specific acoustic tuning. + +Conventional SOF components address level adjustment with different design trade-offs: + +- **Volume Control Subsystem** (:ref:`volume_module`): + Designed for user-facing listening controls. Features logarithmic-to-linear curve translation, per-channel independent attenuation sliders (:math:`-\infty` to :math:`0\text{ dB}`), mute state machines, and smooth multi-millisecond linear ramping to prevent audible zipper noise when the user interacts with an ALSA mixer slider. This functionality requires stateful ramp management and per-sample interpolation overhead. +- **Aria Subsystem** (:ref:`aria`): + Designed for dynamic lookahead peak limiting and transient back-off. It enforces a target pre-amplification boost (:math:`0`, :math:`+6`, :math:`+12`, :math:`+18\text{ dB}`) while dynamically ducking gain during loud bursts, introducing an exact :math:`1\text{ ms}` algorithmic lookahead latency via an internal circular delay buffer. +- **Level Multiplier Subsystem**: + Designed for ultra-fast, deterministic, zero-latency scalar multiplication. It applies a uniform fixed-point multiplier across all channels without ramp overhead, introducing **identically 0 ms of algorithmic delay**. When set to unity gain (:math:`0\text{ dB}`), it completely bypasses arithmetic execution via a direct fast-path. + +.. list-table:: Architectural Comparison: Level Multiplier vs Volume vs Aria + :widths: 20 25 25 30 + :header-rows: 1 + + * - Parameter + - Level Multiplier + - Volume Control + - Aria (Automatic Regressive) + * - **Gain Representation** + - Linear Q9.23 fixed-point + - Logarithmic dB / Linear Q1.31 + - Discrete modes (:math:`0, 6, 12, 18\text{ dB}`) + * - **Gain Range** + - :math:`-138.47\text{ dB}` to :math:`+48.17\text{ dB}` + - :math:`-\infty\text{ dB}` to :math:`0\text{ dB}` (attenuation only) + - :math:`0\text{ dB}` to :math:`+18\text{ dB}` (with regressive ducking) + * - **Algorithmic Latency** + - **0 ms** (instantaneous sample processing) + - **0 ms** (instantaneous sample processing) + - **1 ms** (lookahead circular ring buffer) + * - **Ramp Smoothing** + - None (direct scalar application) + - Smooth per-sample linear ramp (16 to 500 ms) + - Per-sample lookahead linear interpolation + * - **Fast-Path Bypass** + - Automated direct memory copy at unity gain (:math:`0\text{ dB}`) + - Arithmetic bypass at 0 dB if unmuted + - Invariant 1 ms circular delay buffer routing + * - **Primary Use Cases** + - Voice capture sensitivity calibration, ASR tuning + - Main playback volume, application streams + - Far-field mic boost with anti-clipping protection + +.. _figure_223: + +.. graphviz:: + :align: center + :caption: SOF Level Multiplier Architecture: Ingress, Fast-Path Bypass & Fixed-Point Gain Scaling Core + + digraph level_multiplier_architecture { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_ingress { + label = "Audio Egress / Producer"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + source [label="Source Stream Buffer\n(S16_LE / S24_4LE / S32_LE)\nsource_get_data_*()", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + subgraph cluster_module { + label = "Level Multiplier Module (UUID: 30397456-4661...)"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + decision [label="Unity Gain Check\ncd->gain == 0x00800000?", fillcolor="#d97706", fontcolor="#ffffff", color="#fbbf24", shape="diamond"]; + fastpath [label="Zero-Overhead Fast-Path\nsource_to_sink_copy()\n(Direct Memory Copy)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + + subgraph cluster_dsp_core { + label = "Fixed-Point Q9.23 Scaling Core"; + style = "solid"; + color = "#0369a1"; + bgcolor = "#0369a111"; + + s16_proc [label="S16 Engine\nq_multsr_sat_32x32_16\n(Shift = 23)", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + s24_proc [label="S24 Engine\nq_multsr_sat_32x32_24\n(Shift = 23)", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + s32_proc [label="S32 Engine\nq_multsr_sat_32x32\n(Shift = 23)", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + } + } + + subgraph cluster_egress { + label = "Audio Ingress / Consumer"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + sink [label="Sink Stream Buffer\nsink_commit_buffer()", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + source -> decision [label="Ingress frames"]; + decision -> fastpath [label="True (0 dB)"]; + decision -> s16_proc [label="False (S16)"]; + decision -> s24_proc [label="False (S24)"]; + decision -> s32_proc [label="False (S32)"]; + + fastpath -> sink [label="Copied samples"]; + s16_proc -> sink [label="Scaled S16"]; + s24_proc -> sink [label="Scaled S24"]; + s32_proc -> sink [label="Scaled S32"]; + } + +------------------------------------------------------------------------------- + +Fixed-Point Q9.23 Number System & Gain Range +-------------------------------------------- + +The Level Multiplier represents linear gain as a 32-bit signed integer using the **Q9.23** fixed-point numeric format, defined in :file:`level_multiplier.h`: + +.. code-block:: c + + #define LEVEL_MULTIPLIER_QXY_X 9 + #define LEVEL_MULTIPLIER_QXY_Y 23 + #define LEVEL_MULTIPLIER_GAIN_ONE (1 << LEVEL_MULTIPLIER_QXY_Y) + +Bitfield Structure +~~~~~~~~~~~~~~~~~~ + +A 32-bit word in Q9.23 allocates bits as follows: + +.. math:: + + \underbrace{b_{31}}_{\text{Sign}} \quad \underbrace{b_{30} \quad b_{29} \quad b_{28} \quad b_{27} \quad b_{26} \quad b_{25} \quad b_{24} \quad b_{23}}_{8 \text{ Integer Bits}} \quad \underbrace{b_{22} \quad b_{21} \quad \dots \quad b_1 \quad b_0}_{23 \text{ Fractional Bits}} + +- **Sign Bit** (:math:`b_{31}`): Supports both non-inverting (:math:`+`) and phase-inverting (:math:`-`) multipliers. +- **Integer Bits** (:math:`b_{30} \dots b_{23}`): 8 bits of integer magnitude, providing a maximum positive integer value of :math:`2^8 - 1 = 255`. +- **Fractional Bits** (:math:`b_{22} \dots b_0`): 23 bits of fractional precision, yielding an elemental quantization resolution of: + + .. math:: + + \Delta = 2^{-23} \approx 1.1920928955 \times 10^{-7} + +Unity Gain Definition +~~~~~~~~~~~~~~~~~~~~~ + +Unity gain (:math:`1.0\times`, corresponding to :math:`0.00\text{ dB}`) is represented when the fractional component is zero and the integer component is :math:`1`: + +.. math:: + + \text{LEVEL\_MULTIPLIER\_GAIN\_ONE} = 1 \cdot 2^{23} = 8,388,608 = \text{0x00800000} + +Dynamic Range & Extremes +~~~~~~~~~~~~~~~~~~~~~~~~ + +The Q9.23 format enables an exceptionally wide dynamic range: + +1. **Maximum Positive Amplification**: + The largest representable positive gain word is: + + .. math:: + + \text{gain}_{\max} = 2^{31} - 1 = \text{0x7FFFFFFF} = 256.0 - 2^{-23} \approx 255.99999988 + + In decibels: + + .. math:: + + G_{\max} = 20 \log_{10}(256) \approx +48.1648\text{ dB} \approx +48.17\text{ dB} + +2. **Minimum Positive Non-Zero Resolution**: + The smallest positive increment above zero is a single LSB: + + .. math:: + + \text{gain}_{\min} = 1 = \text{0x00000001} \implies 2^{-23} + + In decibels: + + .. math:: + + G_{\min} = 20 \log_{10}(2^{-23}) \approx -138.4739\text{ dB} \approx -138.47\text{ dB} + +3. **Total Dynamic Span**: + The span from maximum boost to minimum non-zero resolution encompasses: + + .. math:: + + \text{Span} = 48.17\text{ dB} - (-138.47\text{ dB}) = 186.64\text{ dB} + + well exceeding the 144 dB theoretical dynamic range of 24-bit audio converters. +4. **Complete Silence**: + Setting :math:`\text{gain} = 0` (:math:`\text{0x00000000}`) completely mutes the signal (:math:`-\infty\text{ dB}`). + +Decibel to Q9.23 Linear Translation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To calculate the 32-bit Q9.23 integer word for a desired gain in decibels (:math:`G_{dB}`): + +.. math:: + + \text{gain}_{\text{Q9.23}} = \left\lfloor 10^{\frac{G_{dB}}{20}} \cdot 2^{23} + 0.5 \right\rfloor + +.. list-table:: Standard Decibel to Q9.23 Conversion Matrix + :widths: 20 25 25 30 + :header-rows: 1 + + * - Desired Gain (dB) + - Linear Multiplier + - Hexadecimal Value + - Decimal Q9.23 Integer + * - **+40.0 dB** + - :math:`100.0000\times` + - ``0x32000000`` + - 838,860,800 + * - **+30.0 dB** + - :math:`31.6228\times` + - ``0x0FD0A499`` + - 265,331,865 + * - **+20.0 dB** + - :math:`10.0000\times` + - ``0x05000000`` + - 83,886,080 + * - **+10.0 dB** + - :math:`3.1623\times` + - ``0x01948332`` + - 26,510,130 + * - **0.0 dB (Unity)** + - :math:`1.0000\times` + - ``0x00800000`` + - 8,388,608 + * - **-10.0 dB** + - :math:`0.3162\times` + - ``0x00287A26`` + - 2,652,710 + * - **-20.0 dB** + - :math:`0.1000\times` + - ``0x000CCCCD`` + - 838,861 + * - **-30.0 dB** + - :math:`0.0316\times` + - ``0x00040C37`` + - 265,271 + * - **-40.0 dB** + - :math:`0.0100\times` + - ``0x000147AE`` + - 83,886 + +.. _figure_224: + +.. graphviz:: + :align: center + :caption: Fixed-Point Q9.23 Number System: Dynamic Range (-138.47 dB to +48.17 dB) & Bit Allocation + + digraph level_multiplier_q9_23 { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_bitfield { + label = "32-Bit Q9.23 Word Memory Organization"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + sign_bit [label="Bit 31\nSign Bit (s)\n0: Positive\n1: Negative", fillcolor="#dc2626", fontcolor="#ffffff", color="#f87171"]; + int_bits [label="Bits 30 .. 23\n8 Integer Bits (Integer Magnitude)\nMax Integer = 255", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + frac_bits [label="Bits 22 .. 0\n23 Fractional Bits (Fractional Precision)\nResolution LSB = 2^-23 (~1.19e-7)", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + subgraph cluster_range { + label = "Dynamic Range Scale"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + r_max [label="Maximum Amplification: +48.17 dB\nGain = 0x7FFFFFFF (~256.0x)", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + r_one [label="Unity Gain (Fast-Path): 0.00 dB\nGain = 0x00800000 (1.0x)", fillcolor="#d97706", fontcolor="#ffffff", color="#fbbf24"]; + r_min [label="Minimum Resolution: -138.47 dB\nGain = 0x00000001 (2^-23)", fillcolor="#334155", fontcolor="#94a3b8", color="#475569"]; + r_mute [label="Digital Silence: -Infinity dB\nGain = 0x00000000 (0.0x)", fillcolor="#1e293b", fontcolor="#94a3b8", color="#475569"]; + } + + sign_bit -> int_bits [style="invis"]; + int_bits -> frac_bits [style="invis"]; + + r_max -> r_one [label="Attenuation"]; + r_one -> r_min [label="Extreme Attenuation"]; + r_min -> r_mute [label="Mute"]; + } + +------------------------------------------------------------------------------- + +Universal PCM Frame Format Processing Engines +--------------------------------------------- + +To support the full range of audio endpoints across the SOF ecosystem, the Level Multiplier implements dedicated processing kernels for three standard PCM frame formats: + +- **16-bit PCM** (:c:macro:`SOF_IPC_FRAME_S16_LE`) +- **24-bit PCM** (:c:macro:`SOF_IPC_FRAME_S24_4LE`) +- **32-bit PCM** (:c:macro:`SOF_IPC_FRAME_S32_LE`) + +Shift Constant Derivation +~~~~~~~~~~~~~~~~~~~~~~~~~ + +During fixed-point multiplication, the product of an :math:`N`-bit sample and the 23-bit fractional component must be shifted right to align the output back to the original container format with saturation. The shift constants are declared in :file:`level_multiplier-generic.c`: + +.. code-block:: c + + #define LEVEL_MULTIPLIER_S16_SHIFT Q_SHIFT_BITS_32(15, LEVEL_MULTIPLIER_QXY_Y, 15) + #define LEVEL_MULTIPLIER_S24_SHIFT Q_SHIFT_BITS_64(23, LEVEL_MULTIPLIER_QXY_Y, 23) + #define LEVEL_MULTIPLIER_S32_SHIFT Q_SHIFT_BITS_64(31, LEVEL_MULTIPLIER_QXY_Y, 31) + +Using the SOF fixed-point shift macro :math:`Q\_SHIFT\_BITS(X, Y, Z) = X + Y - Z`: + +.. math:: + + \text{Shift}_{S16} = 15 + 23 - 15 = 23 + +.. math:: + + \text{Shift}_{S24} = 23 + 23 - 23 = 23 + +.. math:: + + \text{Shift}_{S32} = 31 + 23 - 31 = 23 + +In all three format domains, the required right-shift is identically **23 bits**, perfectly canceling the :math:`2^{23}` scale factor of Q9.23 unity gain. + +Format Processing Loops +~~~~~~~~~~~~~~~~~~~~~~~ + +1. **16-bit Processing Loop** (:c:func:`level_multiplier_s16`): + Operates on 16-bit signed audio samples. Each sample is multiplied by the 32-bit Q9.23 gain using the standard helper :c:func:`q_multsr_sat_32x32_16`, which handles intermediate 48-bit multiplication, 23-bit right-shifting, and saturation clamping to :math:`[-32768, 32767]`: + + .. code-block:: c + + for (i = 0; i < samples_without_wrap; i++) { + *y = q_multsr_sat_32x32_16(*x, gain, LEVEL_MULTIPLIER_S16_SHIFT); + x++; + y++; + } + +2. **24-bit Processing Loop** (:c:func:`level_multiplier_s24`): + Audio is stored in 32-bit containers with 24-bit valid audio. The sample is sign-extended using :c:func:`sign_extend_s24` to ensure correct two's complement sign propagation before multiplication. The result is clamped to the 24-bit dynamic range :math:`[-8388608, 8388607]`: + + .. code-block:: c + + for (i = 0; i < samples_without_wrap; i++) { + *y = q_multsr_sat_32x32_24(sign_extend_s24(*x), gain, + LEVEL_MULTIPLIER_S24_SHIFT); + x++; + y++; + } + +3. **32-bit Processing Loop** (:c:func:`level_multiplier_s32`): + Operates on full 32-bit samples. The multiplication produces a 64-bit product, right-shifted by 23 bits and clamped with 32-bit symmetric saturation: + + .. code-block:: c + + for (i = 0; i < samples_without_wrap; i++) { + *y = q_multsr_sat_32x32(*x, gain, LEVEL_MULTIPLIER_S32_SHIFT); + x++; + y++; + } + +Buffer Wrap Segmentation +~~~~~~~~~~~~~~~~~~~~~~~~ + +To prevent memory faults when reading from and writing to ring buffers, the processing loop computes the largest contiguous block of samples that can be processed before either the source or sink buffer wraps: + +.. code-block:: c + + source_samples_without_wrap = x_end - x; + samples_without_wrap = y_end - y; + samples_without_wrap = MIN(samples_without_wrap, source_samples_without_wrap); + samples_without_wrap = MIN(samples_without_wrap, remaining_samples); + +The inner loop executes across this contiguous segment without branching. Once completed, pointers wrap around via pointer arithmetic: + +.. code-block:: c + + x = (x >= x_end) ? x - x_size : x; + y = (y >= y_end) ? y - y_size : y; + +.. _figure_225: + +.. graphviz:: + :align: center + :caption: Multi-Format Arithmetic Engine: S16_LE, S24_4LE, and S32_LE Multiply-Shift Pipelines + + digraph level_multiplier_formats { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_s16 { + label = "S16_LE Pipeline (16-Bit Container)"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + s16_in [label="Input: x[n] (Q1.15)\n[-32768, 32767]", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + s16_mult [label="Multiply: x * gain\nQ1.15 * Q9.23 -> Q10.38", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + s16_shift [label="Right Shift & Saturation\n>> 23 (LEVEL_MULTIPLIER_S16_SHIFT)\nClamp to [-32768, 32767]", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + s16_out [label="Output: y[n] (Q1.15)", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + + s16_in -> s16_mult -> s16_shift -> s16_out; + } + + subgraph cluster_s24 { + label = "S24_4LE Pipeline (32-Bit Container)"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b22"; + + s24_in [label="Input: x[n] (24-bit in 32-bit)\nsign_extend_s24(*x)", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + s24_mult [label="Multiply: x * gain\nQ1.23 * Q9.23 -> Q10.46", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + s24_shift [label="Right Shift & Saturation\n>> 23 (LEVEL_MULTIPLIER_S24_SHIFT)\nClamp to [-8388608, 8388607]", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + s24_out [label="Output: y[n] (Q1.23)", fillcolor="#10b981", fontcolor="#ffffff", color="#6ee7b7"]; + + s24_in -> s24_mult -> s24_shift -> s24_out; + } + + subgraph cluster_s32 { + label = "S32_LE Pipeline (32-Bit Full Scale)"; + style = "solid"; + color = "#d97706"; + bgcolor = "#78350f22"; + + s32_in [label="Input: x[n] (Q1.31)\n[-2^31, 2^31 - 1]", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + s32_mult [label="64-Bit Multiply: x * gain\nQ1.31 * Q9.23 -> Q10.54", fillcolor="#b45309", fontcolor="#ffffff", color="#fbbf24"]; + s32_shift [label="64-Bit Shift & Saturation\n>> 23 (LEVEL_MULTIPLIER_S32_SHIFT)\nClamp to [-2^31, 2^31 - 1]", fillcolor="#d97706", fontcolor="#ffffff", color="#fbbf24"]; + s32_out [label="Output: y[n] (Q1.31)", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + + s32_in -> s32_mult -> s32_shift -> s32_out; + } + } + +------------------------------------------------------------------------------- + +Zero-Overhead Fast-Path Bypass Architecture +------------------------------------------- + +A primary design requirement for SOF signal chains is energy efficiency. In many topologies, a Level Multiplier is instantiated statically in a pipeline to allow dynamic calibration during manufacturing or runtime mode changes, but remains at unity gain (:math:`0.00\text{ dB}`) during standard operation. + +Fast-Path Implementation +~~~~~~~~~~~~~~~~~~~~~~~~ + +In :c:func:`level_multiplier_process`, the component inspects the active gain variable before initiating any processing loops: + +.. code-block:: c + + if (cd->gain != LEVEL_MULTIPLIER_GAIN_ONE) + /* Process the data with the requested gain. */ + return cd->level_multiplier_func(mod, source, sink, frames); + + /* Just copy from source to sink. */ + source_to_sink_copy(source, sink, true, frames * cd->frame_bytes); + return 0; + +When ``cd->gain`` equals :c:macro:`LEVEL_MULTIPLIER_GAIN_ONE` (:math:`\text{0x00800000}`): + +1. **Elimination of Math Loops**: + The component completely skips the function pointer call to ``cd->level_multiplier_func``. No arithmetic multiplication, bit-shifting, sign extension, or saturation logic is executed. +2. **Direct Block Copy**: + The function :c:func:`source_to_sink_copy` is invoked directly. This executes optimized memory copy primitives (e.g. 64-bit or 128-bit wide word block transfers) or hardware DMA transfers between circular buffers. +3. **Power and Cycle Minimization**: + CPU cycles are reduced to the absolute physical memory transfer minimum, significantly lowering active DSP power consumption during standard passthrough. + +.. _figure_226: + +.. graphviz:: + :align: center + :caption: Zero-Overhead Fast-Path Bypass vs Active Processing Decision Crossbar + + digraph level_multiplier_fastpath { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_dispatch { + label = "Runtime Process Dispatch in level_multiplier_process()"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + chk [label="Inspect Active Gain Value\nIs cd->gain == LEVEL_MULTIPLIER_GAIN_ONE (0x00800000)?", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569", shape="diamond"]; + } + + subgraph cluster_paths { + label = "Execution Pathways"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + path_fast [label="FAST-PATH BYPASS\nsource_to_sink_copy()\n- Zero arithmetic instructions\n- Minimal CPU cycle footprint\n- Maximal memory bandwidth", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + path_active [label="ACTIVE SCALING PATH\ncd->level_multiplier_func()\n- HiFi SIMD / Scalar vector loops\n- Format-specific shift and saturation\n- Linear level amplification / attenuation", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + } + + subgraph cluster_ret { + label = "Sink Egress"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + ret [label="Return Status (0 = Success)\nFrames Committed to Downstream Sink", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + chk -> path_fast [label="YES (0 dB)"]; + chk -> path_active [label="NO (Gain != 0 dB)"]; + + path_fast -> ret; + path_active -> ret; + } + +------------------------------------------------------------------------------- + +Tensilica HiFi SIMD Vector Acceleration +--------------------------------------- + +To achieve peak computational efficiency on Intel audio DSP platforms, the Level Multiplier includes highly optimized assembly kernels tailored for **Tensilica HiFi3 / HiFi4** and **Tensilica HiFi5** processor architectures. + +HiFi3 / HiFi4 Dual-Lane Vectorization (:file:`level_multiplier-hifi3.c`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On HiFi3 and HiFi4 architectures, the DSP utilizes 64-bit vector registers (:c:type:`ae_f32x2`, :c:type:`ae_f16x4`): + +1. **16-Bit Processing** (:c:func:`level_multiplier_s16`): + Loads 4 samples simultaneously using :c:macro:`AE_LA16X4_IP`. The 16-bit samples are multiplied by the 32-bit Q9.23 gain using dual fractional multipliers: + + .. code-block:: c + + samples0 = AE_MULFP32X16X2RS_H(gain, samples); + samples1 = AE_MULFP32X16X2RS_L(gain, samples); + + The intermediate products are shifted left by 8 bits with saturation to convert from Q9.23 to Q1.31: + + .. code-block:: c + + samples0 = AE_SLAI32S(samples0, 8); + samples1 = AE_SLAI32S(samples1, 8); + + Finally, the 32-bit values are symmetrically rounded back to 16-bit representation using :c:macro:`AE_ROUND16X4F32SSYM` and stored via :c:macro:`AE_SA16X4_IP`. +2. **24-Bit Processing** (:c:func:`level_multiplier_s24`): + Processes two 32-bit containers per vector operation. Samples are shifted left by 8 bits to align 24-bit audio to the most significant bits: + + .. code-block:: c + + AE_LA32X2_IP(samples, x_align, x); + samples = AE_MULFP32X2RS(gain, AE_SLAI32(samples, 8)); + samples = AE_SLAI32S(samples, 8); + samples = AE_SRAI32(samples, 8); + AE_SA32X2_IP(samples, y_align, y); + +3. **32-Bit Processing** (:c:func:`level_multiplier_s32`): + Multiplies two 32-bit samples by the 32-bit gain, producing 64-bit accumulators: + + .. code-block:: c + + mult0 = AE_MULF32R_HH(gain, samples); + mult1 = AE_MULF32R_LL(gain, samples); + mult0 = AE_SLAI64(mult0, LEVEL_MULTIPLIER_S32_SHIFT); + mult1 = AE_SLAI64(mult1, LEVEL_MULTIPLIER_S32_SHIFT); + samples = AE_ROUND32X2F48SSYM(mult0, mult1); + AE_SA32X2_IP(samples, y_align, y); + +HiFi5 Quad/Octal 128-Bit Vectorization (:file:`level_multiplier-hifi5.c`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On HiFi5 cores (featured in Intel Lunar Lake, Panther Lake, and newer architectures), vector execution is doubled via **128-bit vector pipelines**: + +1. **Octal 16-Bit Processing**: + Loads 8 16-bit samples per instruction cycle (:c:macro:`AE_LA16X4X2_IP`) and computes 8 parallel multiply-accumulate operations simultaneously using :c:macro:`AE_MULF2P32X16X4RS`. +2. **Quad 32-Bit Processing (S24 & S32)**: + Loads 4 32-bit samples per cycle (:c:macro:`AE_LA32X2X2_IP`) and evaluates 4 lanes simultaneously with quad-vector instruction :c:macro:`AE_MULF2P32X4RS`. + +This achieves double the vector throughput of HiFi3/4, reducing processor clock cycle requirements by up to 50%. + +.. _figure_227: + +.. graphviz:: + :align: center + :caption: HiFi3/HiFi4 Dual-MAC vs HiFi5 Quad-MAC 128-bit Vector Processing Pipelines + + digraph level_multiplier_simd { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_hifi3 { + label = "Tensilica HiFi3 / HiFi4 (64-Bit Vector Architecture)"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + h3_load [label="64-Bit Vector Load: AE_LA32X2_IP\nLoads 2 x 32-bit (or 4 x 16-bit) samples", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + h3_mult [label="Dual 32x32 MAC: AE_MULF32R_HH & LL\nParallel dual-lane multiplication", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + h3_round [label="Symmetric Round: AE_ROUND32X2F48SSYM\nConverts 64-bit products to 32-bit output", fillcolor="#0d9488", fontcolor="#ffffff", color="#2dd4bf"]; + h3_store [label="64-Bit Vector Store: AE_SA32X2_IP\nWrites 2 samples to sink ring", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + + h3_load -> h3_mult -> h3_round -> h3_store; + } + + subgraph cluster_hifi5 { + label = "Tensilica HiFi5 (128-Bit Vector Architecture)"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b22"; + + h5_load [label="128-Bit Vector Load: AE_LA32X2X2_IP\nLoads 4 x 32-bit (or 8 x 16-bit) samples", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + h5_mult [label="Quad 32x32 MAC: AE_MULF2P32X4RS\nParallel 4-lane simultaneous multiplication", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + h5_round [label="Quad Symmetric Round & Slew\nVectorized saturation and bit alignment", fillcolor="#10b981", fontcolor="#ffffff", color="#6ee7b7"]; + h5_store [label="128-Bit Vector Store: AE_SA32X2X2_IP\nWrites 4 samples to sink ring in 1 cycle", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + + h5_load -> h5_mult -> h5_round -> h5_store; + } + } + +------------------------------------------------------------------------------- + +IPC4 Modular Interface, LLEXT Packaging & Topology 2 Graph +---------------------------------------------------------- + +The Level Multiplier component conforms to the Intel IPC4 modular interface and can be built statically into firmware or packaged as a dynamic Loadable Linkable Extension (LLEXT). + +IPC4 Control Configuration Handler (:file:`level_multiplier-ipc4.c`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Runtime parameter updates are processed by :c:func:`level_multiplier_set_config`: + +.. code-block:: c + + switch (param_id) { + case SOF_IPC4_SWITCH_CONTROL_PARAM_ID: + case SOF_IPC4_ENUM_CONTROL_PARAM_ID: + comp_err(dev, "Illegal control param_id %d.", param_id); + return -EINVAL; + } + + if (fragment_size != sizeof(int32_t)) { + comp_err(dev, "Illegal fragment size %d.", fragment_size); + return -EINVAL; + } + + memcpy_s(&cd->gain, sizeof(int32_t), fragment, sizeof(int32_t)); + +- The component validates that the incoming payload size exactly matches 4 bytes (`sizeof(int32_t)`). +- The 32-bit Q9.23 gain value is copied directly into ``cd->gain``. +- The update takes effect on the very next processing tick without pipeline re-initialization. + +Modular LLEXT Packaging +~~~~~~~~~~~~~~~~~~~~~~~ + +When modular compilation is enabled (``CONFIG_COMP_LEVEL_MULTIPLIER = "m"``), the component is linked into :file:`level_multiplier.llext`: + +.. code-block:: c + + SOF_LLEXT_MOD_ENTRY(level_multiplier, &level_multiplier_interface); + + static const struct sof_man_module_manifest mod_manifest __section(".module") __used = + SOF_LLEXT_MODULE_MANIFEST("LEVEL_MULTIPLIER", level_multiplier_llext_entry, 1, + SOF_REG_UUID(level_multiplier), 40); + +- **Module Name**: ``"LEVEL_MULTIPLIER"`` +- **Component UUID**: ``30397456-4661-4644-97e5-39a9e5ab1778`` (Topology GUID: ``56:74:39:30:61:46:44:46:97:e5:39:a9:e5:ab:17:78``). +- **Max Instances**: 40 concurrent instances. +- **Stack Size**: 40 bytes minimum stack overhead. + +Performance Profile (:file:`level_multiplier.toml`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +From :file:`src/audio/level_multiplier/level_multiplier.toml`: + +- **Cycles Per Chunk (CPC)**: 1,000,000 CPS nominal budget. +- **Input/Output Buffer Size**: 128 samples. +- **Memory Footprint**: Only 32 bytes of instance private data (:c:struct:`level_multiplier_comp_data`). + +ALSA Topology 2 Widget Definition +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In :file:`tools/topology/topology2/include/components/level_multiplier.conf`: + +.. code-block:: text + + Class.Widget."level_multiplier" { + DefineAttribute."index" { + type "integer" + } + DefineAttribute."instance" { + type "integer" + } + + + attributes { + !constructor [ "index" "instance" ] + !mandatory [ "num_input_pins" "num_output_pins" + "num_input_audio_formats" "num_output_audio_formats" ] + !immutable [ "uuid" "type" ] + unique "instance" + } + + uuid "56:74:39:30:61:46:44:46:97:e5:39:a9:e5:ab:17:78" + type "effect" + no_pm "true" + num_input_pins 1 + num_output_pins 1 + } + +Octave / MATLAB Tuning Script (:file:`sof_level_multiplier_blobs.m`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +SOF provides an Octave script to generate pre-computed binary blobs across the standard tuning sweep from :math:`-40\text{ dB}` to :math:`+40\text{ dB}` in :math:`10\text{ dB}` steps: + +.. code-block:: octave + + for param = -40:10:40 + gain_value = sof_level_multiplier_db2lin(param); + blob8 = sof_level_multiplier_build_blob(gain_value); + tplg2_fn = sprintf("%s/gain_%d_db.conf", sof_tplg_level_multiplier, param); + sof_tplg2_write(tplg2_fn, blob8, "level_multiplier_config", ...); + end + +.. _figure_228: + +.. graphviz:: + :align: center + :caption: IPC4 Runtime Configuration Delivery, Tuning Blobs & LLEXT Dynamic Module Binding + + digraph level_multiplier_ipc4_flow { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_host { + label = "Host Driver & Userspace ALSA Plane"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + octave [label="Octave Tuning Tool\n(sof_level_multiplier_blobs.m)\nExports gain_-40_db..+40_db.conf", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + alsatplg [label="ALSA Topology Compiler\n(alsatplg)\nBuilds level_multiplier.conf widget", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + amixer [label="ALSA Mixer / ctl Control\nSends 32-bit Q9.23 gain payload", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + } + + subgraph cluster_dsp { + label = "SOF Audio DSP Pipeline"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + ipc4 [label="IPC4 Configuration Dispatcher\nChecks param_id & fragment_size == 4", fillcolor="#0d9488", fontcolor="#ffffff", color="#2dd4bf"]; + llext [label="Zephyr LLEXT Dynamic Linker\nLoads level_multiplier.llext", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + core [label="Level Multiplier Private Data\nAtomically updates cd->gain", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + octave -> alsatplg [label="Tuning Blobs"]; + alsatplg -> ipc4 [label="Pipeline Binding"]; + amixer -> ipc4 [label="Runtime Gain Update"]; + + ipc4 -> llext [label="Module Init"]; + ipc4 -> core [label="Gain Update"]; + } + +.. _figure_229: + +.. graphviz:: + :align: center + :caption: ALSA Topology 2 Voice Capture Sensitivity Pipeline Graph + + digraph level_multiplier_pipeline { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_hw { + label = "Physical Audio Ingress"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b22"; + + dmic [label="DMIC / SoundWire Gateway\n(dai-copier.1)\nDigital Microphone Ingress", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + subgraph cluster_pipe { + label = "Voice Capture Pre-Processing Pipeline (Pipeline 1)"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + dcblock [label="DC Blocker\n(dcblock.1)\nRemoves Hardware DC Bias", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + lvmult [label="Level Multiplier\n(level_multiplier.1)\nSensitivity Boost (+10 dB to +30 dB)\nUUID: 56:74:39:30...", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + tdfb [label="Beamformer (TDFB)\n(tdfb.1)\nDirectional Array Focus", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + rtnr [label="Noise Reduction (RTNR)\n(rtnr.1)\nSuppresses Ambient Noise", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + } + + subgraph cluster_host { + label = "Host Delivery"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + host_copier [label="Host Copier Gateway\n(host-copier.1)\nDMA to Speech Recognition (ASR)", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + dmic -> dcblock [label="Raw Digital Audio"]; + dcblock -> lvmult [label="DC-Free Stream"]; + lvmult -> tdfb [label="Sensitivity Boosted"]; + tdfb -> rtnr [label="Beamformed Focus"]; + rtnr -> host_copier [label="Clean Speech Stream"]; + } + +------------------------------------------------------------------------------- + +Factory Bringup, Acoustic Quality & Verification Runbook +-------------------------------------------------------- + +This runbook provides step-by-step instructions to compile, deploy, and verify the Level Multiplier component on physical development platforms (e.g. Panther Lake, Arrow Lake, or Tiger Lake). + +1. Topology Compilation & Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Compile an ALSA Topology 2 configuration incorporating the Level Multiplier: + +.. code-block:: bash + + # Step 1: Generate tuning blobs across -40 dB to +40 dB + cd tools/tune/level_multiplier + octave --no-gui sof_level_multiplier_blobs.m + + # Step 2: Compile Topology 2 binary + cd ../../topology/topology2 + alsatplg -c development/sof-hda-benchmark-level_multiplier24.conf \ + -o sof-hda-benchmark-level_multiplier24.tplg + + # Step 3: Deploy topology binary to target DUT + scp sof-hda-benchmark-level_multiplier24.tplg root@:/lib/firmware/intel/sof-ipc4/ + +2. Driver Initialization & Module Verification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Reload the SOF kernel driver and check kernel logs for clean module creation: + +.. code-block:: bash + + # Reload kernel audio driver + ssh root@ 'modprobe -r snd_sof_pci_intel_mtl && modprobe snd_sof_pci_intel_mtl' + + # Confirm module instantiation and UUID registration + ssh root@ 'dmesg | grep -i level_multiplier' + +Expected kernel trace: + +.. code-block:: text + + sof-audio-pci-intel-mtl: module LEVEL_MULTIPLIER [30397456-4661-4644-97e5-39a9e5ab1778] loaded + sof-audio-pci-intel-mtl: level_multiplier.1.1: initialized with default unity gain (0x00800000) + +3. Precision Linearity & Gain Accuracy Test +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Verify output signal amplitude against input signal across gain settings: + +.. code-block:: bash + + # Generate reference sine tone at -30 dBFS (1 kHz, 24-bit, 48 kHz) + sox -n -r 48000 -c 2 -b 24 ref_tone_minus30dBFS.wav synth 5 sine 1000 vol -30dB + + # Play reference tone through pipeline + ssh root@ 'aplay -D hw:0,0 ref_tone_minus30dBFS.wav' + + # 1. Test Unity Gain (0 dB, 0x00800000) -> Output must measure exactly -30.0 dBFS + # 2. Set Gain to +10 dB (0x01948332): + ssh root@ 'sof-ctl -D hw:0 -n "level_multiplier.1.1.extctl" -s /lib/firmware/intel/sof-ipc4/gain_10_db.txt' + # -> Measured Output must equal -20.0 dBFS (+/- 0.05 dB) + + # 3. Set Gain to -10 dB (0x00287A26): + ssh root@ 'sof-ctl -D hw:0 -n "level_multiplier.1.1.extctl" -s /lib/firmware/intel/sof-ipc4/gain_-10_db.txt' + # -> Measured Output must equal -40.0 dBFS (+/- 0.05 dB) + +4. Fast-Path Bypass Verification & Power Profiling +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Confirm that unity gain engages the fast-path memory copy and reduces DSP cycle consumption: + +.. code-block:: bash + + # Benchmark DSP Cycles Per Chunk (CPC) with dut-monitor + dut-monitor --telemetry --interval 1000 + + # Active Gain (+10 dB): Observe active DSP cycles + # Unity Gain (0 dB): Cycles drop sharply as source_to_sink_copy() bypasses multiplication diff --git a/developer_guides/index.rst b/developer_guides/index.rst index ee64bcb9..895c7acd 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -60,6 +60,7 @@ Audio Processing Modules & Algorithms * :ref:`tone` (High-level architecture; also see upstream `Tone README `_) * :ref:`up_down_mixer` (High-level architecture; also see upstream `Up/Down Mixer README `_) * :ref:`aria` (High-level architecture; also see upstream `Aria README `_) +* :ref:`level_multiplier` (High-level architecture; also see upstream `Level Multiplier README `_) .. _algorithm-specific-information: @@ -109,6 +110,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/tone firmware/up_down_mixer firmware/aria + firmware/level_multiplier rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 817775aa09ab14a5af03a80fdd2a162555897a17 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 14:34:26 +0100 Subject: [PATCH 28/64] doc: developer_guides: add comprehensive phase vocoder architecture guide Author a comprehensive, modern architectural guide for the Sound Open Firmware (SOF) Phase Vocoder subsystem. Key topics covered: - Real-time frequency-domain Time-Scale Modification (TSM) architecture spanning 0.5x to 2.0x playback speed without pitch shifting. - Short-Time Fourier Transform (STFT) analysis & Overlap-Add (OLA) synthesis: rectangular, Blackman, Hamming, and Hann windows; Constant Overlap-Add (COLA) conditions; power-of-two frame sizing (256, 512, 1024) and hop geometry. - Exact reconstructive window gain compensation (g_comp = R_a / sum(w[n]^2)) in 32-bit Q1.31 format ensuring unity gain (0 dBFS) reconstruction. - Polar domain coordinate transformation (Q2.30 magnitude, Q5.27 phase angle), modulo-2pi phase unwrapping (unwrap_angle_q27), and continuous synthesis phase accumulation preventing comb filtering, tremolo, and phasiness. - Variable playback speed engine: 32-bit signed fixed-point Q3.29 format, ALSA discrete 16-step enum control grid (0.5x to 2.0x in 0.1x steps), and dual-domain linear interpolation across magnitude and phase deltas. - Exact Greatest-Common-Divisor (GCD) frame counter normalization (phase_vocoder_normalize_counters) preventing 32-bit counter overflow during indefinite streaming without timeline fraction drift. - Interactive transient-preserving phase re-anchoring state machine (phase_vocoder_reset_for_new_speed) avoiding cold-start reset, volume drop, and transient smearing during live tempo adjustments. - Multi-channel processing and Mono Downmix Optimization (mono_mix_coef = 2^31 / stream_channels) slashing DSP cycles and memory footprint by up to 75%. - Universal PCM frame format support (S16_LE, S24_4LE, S32_LE) and zero-overhead bypass fast-path (source_to_sink_copy). - IPC4 runtime configuration handler, Zephyr LLEXT dynamic module packaging (phase_vocoder.llext), component UUID registration, and ALSA Topology 2 graph. - Factory bringup runbook, GNU Octave offline blob generation (setup_phase_vocoder.m), testbench automated verification scripts (phase_vocoder_s16.sh, phase_vocoder_s32.sh), and acoustic pitch invariance validation. - 7 native vector Graphviz SVG diagrams (Figures 230-236). Signed-off-by: Liam Girdwood --- data/modules.yaml | 12 + developer_guides/firmware/phase_vocoder.rst | 923 ++++++++++++++++++++ developer_guides/index.rst | 2 + 3 files changed, 937 insertions(+) create mode 100644 developer_guides/firmware/phase_vocoder.rst diff --git a/data/modules.yaml b/data/modules.yaml index f371b525..26e39120 100644 --- a/data/modules.yaml +++ b/data/modules.yaml @@ -195,6 +195,18 @@ modules: - "Sub-audible rumble attenuation" - "Near-zero phase distortion in audio band" + - id: phase_vocoder + name: "Phase Vocoder" + source: "SOF" + category: "Audio Enhancement" + status: "Upstream" + description: "Frequency-domain time-scale modification (0.5x to 2.0x speed) without pitch alteration." + simd: ["HiFi 3", "Scalar C"] + key_features: + - "Real-time Short-Time Fourier Transform (STFT) analysis & synthesis" + - "Variable speed scaling (0.5x to 2.0x) with exact GCD counter normalization" + - "Interactive phase re-anchoring and mono downmix optimization" + - id: smart_amp name: "Smart Amp Protection" source: "SOF" diff --git a/developer_guides/firmware/phase_vocoder.rst b/developer_guides/firmware/phase_vocoder.rst new file mode 100644 index 00000000..b267c28d --- /dev/null +++ b/developer_guides/firmware/phase_vocoder.rst @@ -0,0 +1,923 @@ +.. _phase_vocoder: + +Phase Vocoder Architecture +========================== + +The **Phase Vocoder** subsystem in Sound Open Firmware (SOF) is an advanced frequency-domain audio processing component that performs real-time **Time-Scale Modification (TSM)** without altering pitch, and pitch shifting without altering duration. Operating in the Short-Time Fourier Transform (STFT) domain, the Phase Vocoder allows dynamic playback rate scaling from **0.5× (half speed)** to **2.0× (double speed)** via standard ALSA enum mixer controls or high-resolution Q3.29 fixed-point coefficients. + +Embedded digital audio systems traditionally rely on sample rate conversion (resampling) or time-domain overlap-add algorithms (such as WSOLA) to modify stream tempo. However, resampling inherently alters pitch (the classic "chipmunk" or "slow-tape" effect), while time-domain slicing suffers from pitch-tracking errors, transient smearing, and metallic artifacts during polyphonic audio reproduction. The SOF Phase Vocoder overcomes these limitations by transforming time-domain PCM samples into complex frequency spectra, decoupling spectral magnitude from phase progression, tracking instantaneous bin frequencies across time, and re-synthesizing time-scaled waveforms via Overlap-Add (OLA) Inverse Fast Fourier Transforms. + +The component incorporates an exact **Greatest-Common-Divisor (GCD) frame counter normalization** algorithm that prevents 32-bit integer overflow during indefinite streaming, an interactive **transient-preserving phase re-anchoring** state machine that eliminates phase smearing across live speed changes, a low-overhead **mono downmix optimization** that reduces memory and compute requirements by up to 75%, and full integration with the Intel IPC4 control plane and Zephyr Loadable Linkable Extension (LLEXT) dynamic module architecture. + +.. contents:: Table of Contents + :local: + :depth: 3 + +------------------------------------------------------------------------------- + +Architectural Overview & Time-Scale Modification Principles +----------------------------------------------------------- + +Time-Scale Modification (TSM) is the process of altering the acoustic duration of an audio signal while strictly preserving its spectral envelope, pitch, and timbre. In automotive infotainment, podcast players, speech-to-text accessibility tools, and digital audio workstations (DAWs), users frequently accelerate or decelerate playback without wanting voices or musical instruments to shift pitch. + +Theoretical Comparison of Time Modification Techniques +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Audio systems implement time-scale modification using three distinct paradigms: + +1. **Sample Rate Conversion** (Resampling / :ref:`src_asrc`): + Alters the consumption rate of samples across time. Because frequency and duration are coupled in the time domain, doubling playback speed doubles all audio frequencies (:math:`f_{\text{out}} = 2 \cdot f_{\text{in}}`), transposing pitch up by exactly one octave. +2. **Time-Domain Overlap-Add (TD-PSOLA / WSOLA)**: + Segments audio into pitch periods in the time domain and duplicates or drops periods based on cross-correlation matching. While computationally light, WSOLA depends on robust pitch-detection algorithms that fail on polyphonic music, percussion, noisy speech, or mixed multimedia streams. +3. **Phase Vocoder (Frequency-Domain STFT)**: + Deconstructs the audio into sinusoidal frequency bins using overlapping Fourier transforms. Spectral magnitudes and instantaneous phase trajectories are independently tracked, scaled along the synthesis timeline, and re-synthesized using Inverse FFTs. This works reliably across monophonic speech, polyphonic orchestrations, and percussive transients. + +.. list-table:: Architectural Comparison: Time-Scale Modification Paradigms + :widths: 20 25 25 30 + :header-rows: 1 + + * - Parameter + - Sample Rate Converter (SRC) + - Time-Domain WSOLA + - SOF Phase Vocoder + * - **Processing Domain** + - Time domain (polyphase FIR) + - Time domain (cross-correlation) + - Frequency domain (STFT / Polar) + * - **Pitch Invariance** + - No (pitch shifts with tempo) + - Yes (monophonic signals only) + - **Yes (full polyphonic & speech)** + * - **Speed Range** + - Continuous rational ratio (:math:`M/N`) + - Typically 0.75x to 1.5x + - **0.5x to 2.0x (16 discrete steps or Q3.29)** + * - **Algorithmic Latency** + - Sub-millisecond (FIR tap length) + - Moderate (20 to 40 ms) + - Window hop size (2.7 to 5.3 ms at 48 kHz) + * - **DSP Memory Footprint** + - Small (< 2 KB coefficient RAM) + - Moderate (search windows) + - Medium (~8 KB to 32 KB depending on FFT size) + * - **Primary Use Case** + - Clock domain bridging, resampling + - Low-power voice dictation + - **High-fidelity multimedia, speech rate control** + +.. _figure_230: + +.. graphviz:: + :align: center + :caption: SOF Phase Vocoder Architecture: STFT Analysis, Polar Coordinate Transformation, Spectral Modification & Synthesis Overlap-Add Core + + digraph phase_vocoder_architecture { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="rounded,filled", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#94a3b8", penwidth=1.2]; + + subgraph cluster_ingress { + label = "PCM Ingress & Buffering"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + src [label="Audio Source Buffer\n(S16_LE / S24_4LE / S32_LE)", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + mono_mix [label="Mono Downmix\n(Optional 1-Ch Mix)\nmono_mix_coef", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + ibuf [label="Input Circular Ring Buffer\n(state->ibuf[ch])\ns_avail >= hop_size", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + prev_data [label="Overlap History Buffer\n(state->prev_data[ch])\nSize = fft_size - hop_size", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + } + + subgraph cluster_stft_analysis { + label = "STFT Spectral Analysis"; + style = "solid"; + color = "#475569"; + bgcolor = "#0f172a88"; + + fft_in [label="FFT Input Assembler\nHistory + New Hop Data\n(Complex: Real + Imag=0)", fillcolor="#1e293b", fontcolor="#f8fafc", color="#64748b"]; + window_ana [label="Analysis Windowing\n(Hann / Hamming / Blackman)\nphase_vocoder_apply_window()", fillcolor="#4338ca", fontcolor="#ffffff", color="#818cf8"]; + fft_core [label="Forward 32-Bit FFT\nfft_execute_32(fft_plan)\nSize N = 256 / 512 / 1024", fillcolor="#6366f1", fontcolor="#ffffff", color="#a5b4fc"]; + polar_conv [label="Cartesian to Polar\nsofm_icomplex32_to_polar()\nMag: Q2.30, Angle: Q5.27", fillcolor="#7c3aed", fontcolor="#ffffff", color="#c084fc"]; + } + + subgraph cluster_phase_engine { + label = "TSM Phase & Timeline Engine"; + style = "solid"; + color = "#6d28d9"; + bgcolor = "#1e1b4b55"; + + unwrap [label="Phase Delta & Unwrapping\nunwrap_angle_q27(a)\nΔθ in [-π, +π]", fillcolor="#9333ea", fontcolor="#ffffff", color="#e879f9"]; + timeline [label="Timeline Interpolator\nfrac = (no * speed) mod 2^29\nGCD Counter Normalizer", fillcolor="#c026d3", fontcolor="#ffffff", color="#f0abfc"]; + reanchor [label="Phase Accumulation &\nSpeed Re-Anchoring Engine\noutput_phase[k] += Δθ_interp", fillcolor="#db2777", fontcolor="#ffffff", color="#f472b6"]; + } + + subgraph cluster_stft_synthesis { + label = "STFT Synthesis & Overlap-Add"; + style = "solid"; + color = "#475569"; + bgcolor = "#0f172a88"; + + cart_conv [label="Polar to Cartesian\nsofm_ipolar32_to_complex()\n+ Hermitian Symmetry", fillcolor="#7c3aed", fontcolor="#ffffff", color="#c084fc"]; + ifft_core [label="Inverse 32-Bit IFFT\nfft_execute_32(ifft_plan)\nInverse Transform", fillcolor="#6366f1", fontcolor="#ffffff", color="#a5b4fc"]; + window_syn [label="Synthesis Windowing\n& Gain Compensation\ngain_comp = Ra / Σ w[n]^2", fillcolor="#4338ca", fontcolor="#ffffff", color="#818cf8"]; + ola_buf [label="Overlap-Add Accumulator\nCircular Output Buffer\nstate->obuf[ch]", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + subgraph cluster_egress { + label = "PCM Egress"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + sink [label="Audio Sink Buffer\n(Downstream Pipeline)\nsink_commit_buffer()", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + src -> mono_mix [label="Interleaved PCM"]; + mono_mix -> ibuf [label="Frames"]; + prev_data -> fft_in [label="Past Overlap"]; + ibuf -> fft_in [label="Hop Frames"]; + fft_in -> window_ana; + window_ana -> fft_core; + fft_core -> polar_conv [label="X[k]"]; + polar_conv -> unwrap [label="M_k, θ_k"]; + unwrap -> timeline [label="Δθ_k"]; + timeline -> reanchor [label="α (Fraction)"]; + reanchor -> cart_conv [label="M_interp, φ_out"]; + cart_conv -> ifft_core [label="X_syn[k]"]; + ifft_core -> window_syn; + window_syn -> ola_buf [label="Accumulate OLA"]; + ola_buf -> sink [label="Output Frames"]; + fft_in -> prev_data [style="dashed", label="Update History"]; + } + +------------------------------------------------------------------------------- + +Short-Time Fourier Transform (STFT) Analysis & Overlap-Add Synthesis +-------------------------------------------------------------------- + +The foundation of the Phase Vocoder is the **Short-Time Fourier Transform (STFT)**. Continuous audio signals are non-stationary; their spectral content changes dynamically over time. The STFT segments the incoming signal into short, overlapping quasi-stationary windows, transforms each window into the frequency domain, and subsequently recombines them using **Overlap-Add (OLA)** synthesis. + +Window Selection & Spectral Leakage Control +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To prevent abrupt boundary truncation (which causes broadband spectral leakage across Fourier bins), each analysis frame is multiplied by a smooth window function :math:`w[n]`. The SOF Phase Vocoder supports four configurable window types declared in :c:enum:`sof_phase_vocoder_fft_window_type`: + +1. **Rectangular Window (`STFT_RECTANGULAR_WINDOW = 0`)**: + Provides the narrowest main lobe (highest frequency resolution), but severe sidelobe leakage (-13 dB attenuation), causing audible inter-bin modulation distortion. +2. **Blackman Window (`STFT_BLACKMAN_WINDOW = 1`)**: + Provides extreme sidelobe attenuation (-58 dB) with exact coefficients defined by `WIN_BLACKMAN_A0_Q31`. Ideal for high-precision analytical inspection. +3. **Hamming Window (`STFT_HAMMING_WINDOW = 2`)**: + Attenuates first sidelobe to -43 dB, balancing main lobe width and spectral rolloff. +4. **Hann Window (`STFT_HANN_WINDOW = 3`, Standard Default)**: + Constructed from a raised cosine bell: + + .. math:: + + w[n] = 0.5 - 0.5 \cos\left(\frac{2\pi n}{N}\right), \quad 0 \le n < N + + The Hann window satisfies the **Constant Overlap-Add (COLA)** condition when the hop size is an integer submultiple of the window length (:math:`N/2`, :math:`N/4`). + +Frame Sizing & Hop Geometry +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The component operates on power-of-two frame lengths :math:`N` and analysis hop sizes :math:`R_a`: + +- **Standard Frame Lengths** (:math:`N`): 256 samples (5.33 ms @ 48 kHz), 512 samples (10.67 ms), or 1024 samples (21.33 ms). +- **Analysis Hop Size** (:math:`R_a`): Typically :math:`N/2` (50% overlap) or :math:`N/4` (75% overlap, e.g., 256-sample hop on 1024-sample window). +- **History Overlap Size**: :math:`N - R_a` samples, retained in ``state->prev_data[ch]`` across ticks. + +Reconstructive Window Gain Compensation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +During synthesis, the inverse-transformed time-domain samples are multiplied again by the synthesis window :math:`w[n]` before being accumulated into the output buffer. Passing audio through two cascaded window stages scales the total signal power by the sum of the squared window coefficients. To guarantee exact unity gain (0 dBFS) reconstruction, SOF pre-calculates a 32-bit Q1.31 gain compensation factor: + +.. math:: + + g_{\text{comp}} = \frac{R_a}{\sum_{n=0}^{N-1} w[n]^2} + +This value is stored in `config->window_gain_comp` and multiplied into the synthesis overlap-add accumulator: + +.. code-block:: c + + sample = Q_MULTSR_32X32((int64_t)state->gain_comp, fft->fft_buf[idx].real, 31, 31, 31); + *w = sat_int32((int64_t)*w + sample); + +.. _figure_231: + +.. graphviz:: + :align: center + :caption: Short-Time Fourier Transform (STFT) Analysis & Overlap-Add (OLA) Synthesis Timeline with Window Gain Compensation + + digraph stft_timeline { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.4, ranksep=0.5]; + node [shape=rect, style="rounded,filled", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#94a3b8", penwidth=1.2]; + + subgraph cluster_analysis_timeline { + label = "Input Audio Stream & Analysis Frames (Analysis Hop Ra = 256 samples)"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + frame_m0 [label="Analysis Frame m = 0\n[0 .. 1023] (N = 1024 samples)\nWindowed & FFT Executed", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + frame_m1 [label="Analysis Frame m = 1\n[256 .. 1279] (Hop Ra = 256)\nWindowed & FFT Executed", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + frame_m2 [label="Analysis Frame m = 2\n[512 .. 1535] (Hop Ra = 256)\nWindowed & FFT Executed", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + + frame_m0 -> frame_m1 [label="Advance by Ra"]; + frame_m1 -> frame_m2 [label="Advance by Ra"]; + } + + subgraph cluster_synthesis_timeline { + label = "Output Synthesis Stream & Overlap-Add (Synthesis Hop Rs = Ra / speed)"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b22"; + + syn_m0 [label="Synthesis Frame 0 (IFFT)\nScaled by g_comp * w[n]\nAccumulated at offset 0", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + syn_m1 [label="Synthesis Frame 1 (IFFT)\nScaled by g_comp * w[n]\nAccumulated at offset Rs", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + syn_m2 [label="Synthesis Frame 2 (IFFT)\nScaled by g_comp * w[n]\nAccumulated at offset 2*Rs", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + + syn_m0 -> syn_m1 [label="Advance by Rs"]; + syn_m1 -> syn_m2 [label="Advance by Rs"]; + } + + frame_m0 -> syn_m0 [style="dashed", color="#cbd5e1", label="Speed = 1.0 (Rs = Ra)"]; + frame_m1 -> syn_m1 [style="dashed", color="#cbd5e1", label="Speed < 1.0 (Rs > Ra: Expansion)"]; + frame_m2 -> syn_m2 [style="dashed", color="#cbd5e1", label="Speed > 1.0 (Rs < Ra: Compression)"]; + } + +------------------------------------------------------------------------------- + +Polar Domain Phase Unwrapping & Phase Accumulation Mechanics +------------------------------------------------------------ + +A naive time-stretching approach that merely duplicates or displaces STFT frames in time results in catastrophic acoustic artifacts: destructive comb filtering, rapid amplitude tremolo, and a hollow, reverberant "phasiness". These distortions occur because the Fourier phase across successive analysis hops is non-stationary. + +The Phase Discontinuity Problem +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When an input sinusoidal component of frequency :math:`\omega_0` is analyzed at intervals of :math:`R_a`, its phase advances by :math:`\Delta \theta = \omega_0 R_a`. If the synthesis frames are re-positioned at a new synthesis hop interval :math:`R_s = R_a / \text{speed}`, the synthesis phase must advance by :math:`\Delta \phi = \omega_0 R_s`. If the original analysis phase is retained without modification, adjacent overlapping frames will destructively interfere at the synthesis boundary. + +Polar Coordinate Conversion +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To manipulate magnitude and phase independently, the 32-bit complex Fourier output :c:type:`icomplex32` (:math:`X[k] = \text{real} + j \cdot \text{imag}`) is converted to polar form :c:type:`ipolar32` using :c:func:`sofm_icomplex32_to_polar`: + +- **Spectral Magnitude** (:math:`M_k`): Represented in high-precision **Q2.30** format. +- **Phase Angle** (:math:`\theta_k`): Converted from trigonometric Q3.29 to **Q5.27** format using `Q_SHIFT_RND(angle, 29, 27)`. + +Phase Unwrapping Arithmetic +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Because phase angles are circular (:math:`[-\pi, +\pi]`), calculating the phase difference between consecutive frames introduces phase wrap-around ambiguity whenever the delta crosses :math:`\pm \pi`: + +.. math:: + + \Delta \theta_k = \theta_k[m] - \theta_k[m-1] + +To recover the true instantaneous frequency deviation, SOF unwraps the phase difference into the fundamental interval :math:`[-\pi, +\pi]` via :c:func:`unwrap_angle_q27`: + +.. code-block:: c + + static int32_t unwrap_angle_q27(int32_t angle) + { + while (angle > PHASE_VOCODER_PI_Q27) + angle -= PHASE_VOCODER_TWO_PI_Q27; + + while (angle < -PHASE_VOCODER_PI_Q27) + angle += PHASE_VOCODER_TWO_PI_Q27; + + return angle; + } + +where fixed-point radian constants are defined as: + +- `PHASE_VOCODER_PI_Q27 = 421657428` (:math:`\pi \cdot 2^{27}`) +- `PHASE_VOCODER_TWO_PI_Q27 = 843314857` (:math:`2\pi \cdot 2^{27}`) + +Synthesis Phase Accumulation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The unwrapped phase difference :math:`\Delta \theta_k` represents the authentic instantaneous phase progression of frequency bin :math:`k`. During synthesis, the output phase accumulator `output_phase[k]` continuously integrates these deltas: + +.. math:: + + \phi_k[m] = \text{unwrap\_angle}\left(\phi_k[m-1] + \Delta \theta_{k, \text{interp}}\right) + +This ensures that sinusoidal components remain strictly continuous across the time-scaled synthesis timeline, maintaining phase coherence and pristine transient clarity. + +.. _figure_232: + +.. graphviz:: + :align: center + :caption: Polar-Domain Phase Unwrapping, Phase Difference Calculation & Synthesis Phase Accumulation Pipeline + + digraph polar_phase_pipeline { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="rounded,filled", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#94a3b8", penwidth=1.2]; + + subgraph cluster_polar { + label = "Polar Domain Conversion"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + fft_out [label="FFT Complex Output\nRe[k], Im[k] (Q1.31)", fillcolor="#1e293b", fontcolor="#f8fafc", color="#475569"]; + to_polar [label="Cartesian to Polar\nsofm_icomplex32_to_polar()", fillcolor="#6366f1", fontcolor="#ffffff", color="#818cf8"]; + mag_curr [label="Current Magnitude\npolar[ch].magnitude (Q2.30)", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + ang_curr [label="Current Phase Angle\npolar[ch].angle (Q5.27)", fillcolor="#7c3aed", fontcolor="#ffffff", color="#c084fc"]; + } + + subgraph cluster_unwrapping { + label = "Phase Delta & Modulo-2π Unwrapping"; + style = "solid"; + color = "#6d28d9"; + bgcolor = "#1e1b4b55"; + + ang_prev [label="Previous Phase Angle\npolar_prev[ch].angle (Q5.27)", fillcolor="#475569", fontcolor="#e2e8f0", color="#64748b"]; + sub_diff [label="Raw Difference\na = θ[m] - θ[m-1]", fillcolor="#9333ea", fontcolor="#ffffff", color="#e879f9"]; + unwrapper [label="unwrap_angle_q27(a)\nModulo [-π, +π]\nClamps Phase Wrap", fillcolor="#a855f7", fontcolor="#ffffff", color="#f0abfc"]; + angle_delta [label="Unwrapped Phase Delta\nangle_delta[ch][k] (Q5.27)", fillcolor="#c026d3", fontcolor="#ffffff", color="#f5d0fe"]; + } + + subgraph cluster_synthesis_acc { + label = "Synthesis Phase Accumulator"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b22"; + + interp_engine [label="Fractional Interpolator\nΔθ_interp = (1-α)Δθ_prev + αΔθ", fillcolor="#0d9488", fontcolor="#ffffff", color="#2dd4bf"]; + acc_adder [label="Phase Accumulator\nφ_out = φ_out + Δθ_interp", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + out_phase [label="Synthesis Phase\noutput_phase[ch][k]\n(Continuous Sinusoid)", fillcolor="#10b981", fontcolor="#ffffff", color="#6ee7b7"]; + } + + fft_out -> to_polar; + to_polar -> mag_curr; + to_polar -> ang_curr; + ang_curr -> sub_diff [label="θ[m]"]; + ang_prev -> sub_diff [label="θ[m-1]"]; + sub_diff -> unwrapper; + unwrapper -> angle_delta; + angle_delta -> interp_engine [label="Current Delta"]; + interp_engine -> acc_adder [label="Interpolated Delta"]; + out_phase -> acc_adder [style="dashed", label="Feedback Past Phase"]; + acc_adder -> out_phase; + } + +------------------------------------------------------------------------------- + +Variable Playback Speed & Fractional Interpolation Engine +--------------------------------------------------------- + +The SOF Phase Vocoder controls speed by adjusting the rate at which analysis frames are synthesized into output frames. Speed is represented internally as a 32-bit signed fixed-point integer in **Q3.29** format: + +- **Minimum Speed**: `PHASE_VOCODER_MIN_SPEED_Q29 = 0.5 * 2^29 = 0x10000000` (0.5x, half speed / slow motion). +- **Normal Speed**: `PHASE_VOCODER_SPEED_NORMAL = 1.0 * 2^29 = 0x20000000` (1.0x, unity passthrough). +- **Maximum Speed**: `PHASE_VOCODER_MAX_SPEED_Q29 = 2.0 * 2^29 = 0x40000000` (2.0x, double speed). + +ALSA Discrete Enum Control Grid +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For intuitive integration with userspace players and ALSA mixers, the Phase Vocoder exposes an enum control with 16 uniform speed increments: + +.. math:: + + \text{speed\_step} = \frac{2.0 - 0.5}{15} = 0.1000 + +When an enum index :math:`E \in [0, 15]` is written by the host, the driver calculates: + +.. math:: + + \text{speed}_{\text{ctrl}} = \text{MIN\_SPEED}_{\text{Q29}} + \left(E \times \text{STEP}_{\text{Q31}}\right) \gg 2 + +Yielding exact playback speeds of 0.5x, 0.6x, 0.7x, ..., 1.9x, 2.0x. + +Fractional Timeline Progression +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As synthesis frames (`num_output_ifft`) are produced, the corresponding virtual position on the input analysis timeline is calculated in 64-bit precision: + +.. code-block:: c + + input_frame_num_frac = (int64_t)state->num_output_ifft * cd->state.speed; /* Q31.29 */ + input_frame_num_floor = (int32_t)(input_frame_num_frac >> 29); /* Integer frame index */ + state->num_input_fft_to_use = input_frame_num_floor + 1; + state->interpolate_fraction = input_frame_num_frac - ((int64_t)input_frame_num_floor << 29); + +Dual-Domain Linear Interpolation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Because the virtual analysis position falls between discrete FFT analysis frames, the vocoder performs linear interpolation across both spectral magnitude and phase delta: + +.. math:: + + \alpha = \frac{\text{interpolate\_fraction}}{2^{29}} + +.. math:: + + M_{\text{interp}}[k] = (1 - \alpha) \cdot M_{\text{prev}}[k] + \alpha \cdot M_{\text{curr}}[k] + +.. math:: + + \Delta \theta_{\text{interp}}[k] = (1 - \alpha) \cdot \Delta \theta_{\text{prev}}[k] + \alpha \cdot \Delta \theta_{\text{curr}}[k] + +The interpolated phase delta is then added to `output_phase[k]`, and the resulting polar coordinate :math:`(M_{\text{interp}}, \phi_{\text{out}})` is converted back to Cartesian format for the Inverse FFT. + +Greatest-Common-Divisor (GCD) Counter Normalization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +During extended playback (such as video streaming or days of continuous audio playback), the integer counters `num_output_ifft` and `num_input_fft` would eventually overflow a 32-bit signed integer (:math:`2^{31} - 1`). However, arbitrarily resetting both counters to zero induces an instantaneous phase discontinuity, producing an audible pop. + +To solve this, the SOF Phase Vocoder implements an exact **Greatest-Common-Divisor (GCD) counter normalization** algorithm in :c:func:`phase_vocoder_normalize_counters`. The fractional timeline repeats with a period defined by: + +.. math:: + + g = \gcd\left(\text{speed}, 2^{29}\right) + +.. math:: + + P_{\text{output}} = \frac{2^{29}}{g} + +Whenever `num_output_ifft` exceeds :math:`2^{28}`, the component subtracts the largest integer multiple of :math:`P_{\text{output}}`: + +.. math:: + + \Delta_{\text{output}} = \left\lfloor \frac{\text{num\_output\_ifft}}{P_{\text{output}}} \right\rfloor \cdot P_{\text{output}} + +.. math:: + + \Delta_{\text{input}} = \frac{\Delta_{\text{output}} \times \text{speed}}{2^{29}} + +Because :math:`\Delta_{\text{output}} \times \text{speed}` is an exact multiple of :math:`2^{29}` by construction, subtracting :math:`\Delta_{\text{output}}` from ``num_output_ifft`` and :math:`\Delta_{\text{input}}` from ``num_input_fft`` **preserves the interpolation fraction identically down to the least significant bit**, ensuring zero round-off error while keeping counters perpetually bounded! + +.. _figure_233: + +.. graphviz:: + :align: center + :caption: Variable Time-Scale Modification (TSM) Engine: Output Timeline Interpolation & Greatest-Common-Divisor (GCD) Frame Counter Normalization + + digraph gcd_normalization { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.4, ranksep=0.5]; + node [shape=rect, style="rounded,filled", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#94a3b8", penwidth=1.2]; + + subgraph cluster_counters { + label = "Continuous Counter Growth & Overflow Check"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + chk_overflow [label="Inspect Output Frame Counter\nstate->num_output_ifft >= (1 << 28) ?", fillcolor="#1e293b", fontcolor="#f8fafc", color="#475569"]; + } + + subgraph cluster_gcd_algo { + label = "Exact Periodic Normalization (Zero Fraction Drift)"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + gcd_calc [label="Calculate Greatest Common Divisor\ng = gcd(speed, 2^29)\noutput_period = 2^29 / g", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + delta_calc [label="Calculate Largest Integer Period Multiple\ndelta_output = (num_output_ifft / output_period) * output_period\ndelta_input = (delta_output * speed) >> 29", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + apply_norm [label="Normalize Counters In-Place\nnum_output_ifft -= delta_output\nnum_input_fft -= delta_input", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + } + + subgraph cluster_interp { + label = "Downstream Fractional Evaluation"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b22"; + + frac_eval [label="Fractional Position Preserved Identically\nfrac = (num_output_ifft * speed) mod 2^29\nZero Phase Jitter / Zero Pop Artifacts", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + chk_overflow -> gcd_calc [label="Yes (Exceeds Threshold)"]; + gcd_calc -> delta_calc; + delta_calc -> apply_norm; + apply_norm -> frac_eval; + chk_overflow -> frac_eval [label="No (Normal Processing)"]; + } + +Interactive Phase Re-Anchoring State Machine +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When a user adjusts the speed slider during live playback, changing `cd->speed_ctrl` triggers :c:func:`phase_vocoder_reset_for_new_speed`. Naive vocoder implementations re-initialize their analysis state from frame zero, causing an immediate volume drop and transient blurring. + +The SOF Phase Vocoder implements a specialized **phase re-anchoring** mechanism: + +1. **Cold-Start Suppression**: + `state->num_input_fft` is set to 1 (never 0). This avoids re-entering the initial cold-start analysis path which copies absolute angles instead of deltas. +2. **Phase Re-Anchoring Invariant**: + The synthesis accumulator is re-anchored so that the first post-reset IFFT lands exactly back on `polar_prev.angle`: + + .. math:: + + \text{output\_phase}[k] = \text{unwrap\_angle\_q27}\left(\text{polar\_prev}[k].\text{angle} - \text{angle\_delta\_prev}[k]\right) + +This completely neutralizes phase drift accumulated across non-unity speed transitions, preventing transient smearing and ensuring the sound remains bright, focused, and crisp. + +.. _figure_234: + +.. graphviz:: + :align: center + :caption: Dynamic Speed Transition & Interactive Phase Re-Anchoring State Machine + + digraph phase_reanchoring { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.4, ranksep=0.5]; + node [shape=rect, style="rounded,filled", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#94a3b8", penwidth=1.2]; + + subgraph cluster_speed_event { + label = "ALSA Mixer Speed Update Event"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + event [label="Host Writes New Speed Control\n(cd->speed_ctrl != state.speed)", fillcolor="#1e293b", fontcolor="#f8fafc", color="#475569"]; + call_reset [label="Invoke phase_vocoder_reset_for_new_speed()\nUpdate state->speed = cd->speed_ctrl", fillcolor="#4338ca", fontcolor="#ffffff", color="#818cf8"]; + } + + subgraph cluster_reanchor_logic { + label = "State Preservation & Phase Alignment Invariant"; + style = "solid"; + color = "#6d28d9"; + bgcolor = "#1e1b4b55"; + + set_counters [label="Set num_input_fft = 1 & num_output_ifft = 0\n(Bypasses Cold-Start Absolute Angle Path)", fillcolor="#6366f1", fontcolor="#ffffff", color="#a5b4fc"]; + calc_anchor [label="Compute Bin Re-Anchor Equation\noutput_phase[k] = unwrap(polar_prev[k].angle - angle_delta_prev[k])", fillcolor="#9333ea", fontcolor="#ffffff", color="#e879f9"]; + } + + subgraph cluster_result { + label = "Glitchless Audio Continuity"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b22"; + + audio_out [label="Immediate Glitchless Playback at New Speed\nZero Volume Sag / Zero Transient Smear", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + event -> call_reset; + call_reset -> set_counters; + set_counters -> calc_anchor; + calc_anchor -> audio_out; + } + +------------------------------------------------------------------------------- + +Multi-Channel Processing & Mono Downmixing Optimization +------------------------------------------------------- + +Processing high-resolution multi-channel audio through an STFT phase vocoder requires substantial memory and computational resources. For each audio channel, the DSP must maintain separate input ring buffers, overlap history buffers, output ring buffers, polar magnitude arrays, and phase accumulators. + +Memory Footprint Optimization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +During component initialization in :c:func:`phase_vocoder_setup`, all buffer allocations are consolidated into single contiguous blocks to minimize heap fragmentation and allocator overhead: + +- **Time-Domain Ring Buffers (`sample_buffers_size`)**: + Calculated across all active processing channels: + + .. math:: + + \text{Bytes} = 4 \times \left[\text{channels} \cdot \left(L_{\text{ibuf}} + L_{\text{obuf}} + L_{\text{prev}}\right) + N\right] + + Subject to an upper safety bound of `STFT_MAX_ALLOC_SIZE = 65536` bytes (64 KB). +- **Polar Domain Arrays (`phase_vocoder_polar_bytes`)**: + Consolidates `polar`, `polar_prev`, `angle_delta`, `angle_delta_prev`, and `output_phase`: + + .. math:: + + \text{Bytes} = \text{channels} \times \frac{N}{2} \times \left(2 \cdot \text{sizeof(struct ipolar32)} + 3 \cdot \text{sizeof(int32\_t)}\right) + +Mono Downmix Optimization Mode +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In many voice and multimedia pipelines, human speech is concentrated in the center channel, or the audio endpoint is bandwidth-constrained. The Phase Vocoder supports a specialized **Mono Downmix Optimization** (`config->mono = 1`): + +1. **Input Downmixing**: + When multi-channel audio enters the component, incoming channels are pre-summed into a single processing channel using a 32-bit Q1.31 gain coefficient: + + .. math:: + + \text{mono\_mix\_coef} = \left\lfloor \frac{2^{31}}{\text{stream\_channels}} \right\rfloor + +2. **Single-Channel Core Execution**: + Only a single forward FFT, polar transformation, phase unwrapping, and Inverse FFT pipeline executes, **slashing DSP CPU cycles and memory consumption by 50% for stereo streams and 75% for 4-channel streams**. +3. **Multi-Channel Replication**: + During output egress (:c:func:`phase_vocoder_sink_s32`), the single processed channel is replicated across all output sink channels: + + .. code-block:: c + + for (i = 0; i < n; i++) { + for (ch = 0; ch < stream_channels; ch++) + *y++ = *obuf->r_ptr; + obuf->r_ptr++; + } + +Universal PCM Frame Format Support +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The component implements specialized ingestion and egress functions for all standard SOF PCM formats: + +- **16-Bit PCM (`SOF_IPC_FRAME_S16_LE`)**: + Processes 16-bit audio via :c:func:`phase_vocoder_s16`, converting to 32-bit internal representations for the FFT. +- **24-Bit PCM in 32-Bit Container (`SOF_IPC_FRAME_S24_4LE`)**: + Processes 24-bit audio via :c:func:`phase_vocoder_s24` with sign extension. +- **32-Bit PCM (`SOF_IPC_FRAME_S32_LE`)**: + Operates on native 32-bit PCM via :c:func:`phase_vocoder_s32`. + +Zero-Overhead Bypass Fast-Path +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When processing is disabled via ALSA mixer (`cd->enable == false`), the component completely bypasses FFT execution, polar transformations, and phase accumulation. It invokes :c:func:`source_to_sink_copy` directly: + +.. code-block:: c + + if (!cd->enable) { + frames = MIN(source_frames, sink_frames); + source_to_sink_copy(source, sink, true, frames * cd->frame_bytes); + return 0; + } + +This reduces active DSP cycles to a pure memory block transfer during passthrough. + +------------------------------------------------------------------------------- + +IPC4 Control Plane, LLEXT Modular Packaging & ALSA Topology 2 Graph +------------------------------------------------------------------- + +The Phase Vocoder complies with the Intel IPC4 modular audio architecture, allowing dynamic runtime configuration via standardized large configuration blobs, mixer switch controls, and enum speed selections. + +IPC4 Runtime Configuration Handler +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Runtime control messages are dispatched to :c:func:`phase_vocoder_set_config`: + +1. **Switch Control (`SOF_IPC4_SWITCH_CONTROL_PARAM_ID = 259`)**: + Controls the enable/bypass state (`cd->enable = ctl->chanv[0].value`). +2. **Enum Control (`SOF_IPC4_ENUM_CONTROL_PARAM_ID = 257`)**: + Controls the playback speed enum index (0 .. 15), mapping directly to 0.5x .. 2.0x. +3. **Large Configuration Blob (`struct sof_phase_vocoder_config`)**: + Delivers the 64-byte structural configuration defining sample rate, window type, frame length, hop size, and mono mode: + +.. code-block:: c + + struct sof_phase_vocoder_config { + uint32_t size; + uint32_t reserved[8]; + int32_t sample_frequency; + int32_t window_gain_comp; + int32_t reserved_32; + int16_t mono; + int16_t frame_length; + int16_t frame_shift; + int16_t reserved_16; + int32_t reserved_pad; + enum sof_phase_vocoder_fft_window_type window; + } __attribute__((packed)); + +Zephyr LLEXT Dynamic Module Packaging +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When built as a loadable linkable extension (``CONFIG_COMP_PHASE_VOCODER_MODULE=y``), the component is linked into :file:`phase_vocoder.llext`: + +.. code-block:: c + + static const struct sof_man_module_manifest mod_manifest __section(".module") __used = + SOF_LLEXT_MODULE_MANIFEST("PHASEVOC", &phase_vocoder_interface, 1, + SOF_REG_UUID(phase_vocoder), 40); + +- **Module Name**: ``"PHASEVOC"`` +- **Component UUID**: ``7a:cb:fb:09:c5:a9:57:4a:84:34:44:40:e5:98:ab:24`` +- **Topology GUID**: ``7acbfb09-c5a9-574a-8434-4440e598ab24`` + +ALSA Topology 2 Widget Definition +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In ALSA Topology 2, the Phase Vocoder is declared in :file:`tools/topology/topology2/include/components/phase_vocoder.conf`: + +.. code-block:: text + + Class.Widget."phase_vocoder" { + DefineAttribute."index" { type "integer" } + DefineAttribute."instance" { type "integer" } + + + + attributes { + !constructor [ "index" "instance" ] + !mandatory [ + "num_input_pins" + "num_output_pins" + "num_input_audio_formats" + "num_output_audio_formats" + ] + !immutable [ "uuid" "type" ] + unique "instance" + } + + Object.Control { + # Switch controls (Bypass on/off) + mixer."1" { + Object.Base.ops.1 { + name "ctl" + info "volsw" + get 259 + put 259 + } + max 1 + } + + # Enum controls (Speed 0.5x to 2.0x) + enum."1" { + Object.Base { + text.0 { + name "phase_vocoder_speed_enum" + !values [ + "0.5" "0.6" "0.7" "0.8" "0.9" "1.0" + "1.1" "1.2" "1.3" "1.4" "1.5" "1.6" + "1.7" "1.8" "1.9" "2.0" + ] + } + ops.1 { + name "ctl" + info "enum" + get 257 + put 257 + } + } + } + } + + uuid "7a:cb:fb:09:c5:a9:57:4a:84:34:44:40:e5:98:ab:24" + type "effect" + no_pm "true" + num_input_pins 1 + num_output_pins 1 + } + +.. _figure_235: + +.. graphviz:: + :align: center + :caption: IPC4 Control Architecture, Switch & Enum Control Handlers & LLEXT Dynamic Module Binding + + digraph ipc4_control_plane { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="rounded,filled", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#94a3b8", penwidth=1.2]; + + subgraph cluster_host_ctl { + label = "Host Driver & ALSA Controls"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + sw_ctl [label="ALSA Mixer Switch\n'Phase Vocoder enable'\n(Param ID 259: 0=off, 1=on)", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + enum_ctl [label="ALSA Enum Control\n'Phase Vocoder speed'\n(Param ID 257: Index 0..15)", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + blob_ctl [label="Topology Blob Config\nsetup_phase_vocoder.m\n(64-byte struct payload)", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + } + + subgraph cluster_ipc_dispatch { + label = "IPC4 Configuration Dispatcher"; + style = "solid"; + color = "#475569"; + bgcolor = "#0f172a88"; + + handler [label="phase_vocoder_set_config()\nParameter ID Demux", fillcolor="#4338ca", fontcolor="#ffffff", color="#818cf8"]; + } + + subgraph cluster_module_priv { + label = "Component State & Dynamic Binding"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b22"; + + enable_flag [label="cd->enable\n(Fast-Path vs Active)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + speed_val [label="cd->speed_ctrl\n(Q3.29 Speed Factor)", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + cfg_struct [label="cd->config\n(Frame, Hop, Window, Mono)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + llext [label="Zephyr LLEXT Dynamic Linker\nphase_vocoder.llext\nUUID: 7a:cb:fb:09:c5:a9:57:4a...", fillcolor="#d97706", fontcolor="#ffffff", color="#fbbf24"]; + } + + sw_ctl -> handler [label="SWITCH_CONTROL"]; + enum_ctl -> handler [label="ENUM_CONTROL"]; + blob_ctl -> handler [label="LARGE_CONFIG"]; + handler -> enable_flag [label="Update Flag"]; + handler -> speed_val [label="Re-Anchor Speed"]; + handler -> cfg_struct [label="Copy Blob"]; + llext -> handler [style="dashed", label="Registers Interface"]; + } + +.. _figure_236: + +.. graphviz:: + :align: center + :caption: ALSA Topology 2 Phase Vocoder Pipeline Graph with Benchmark Controls + + digraph topology_pipeline_graph { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="rounded,filled", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#94a3b8", penwidth=1.2]; + + host_copier [label="Host Copier Gateway\n(PCM Playback Stream)\nhost-copier.0", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + pvoc [label="Phase Vocoder\n(TSM 0.5x .. 2.0x)\nphase_vocoder.1\nUUID: 7a:cb:fb:09...", fillcolor="#6366f1", fontcolor="#ffffff", color="#a5b4fc"]; + vol [label="Volume Control\n(Main Volume / Mute)\nvolume.2", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + dai_copier [label="DAI Copier Gateway\n(Physical Audio Out / I2S / HDA)\ndai-copier.3", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + + sw_ctl [label="Switch Mixer Control:\n'Phase Vocoder enable'", fillcolor="#334155", fontcolor="#93c5fd", color="#60a5fa", style="dashed,filled"]; + enum_ctl [label="Enum Mixer Control:\n'Phase Vocoder speed'", fillcolor="#334155", fontcolor="#93c5fd", color="#60a5fa", style="dashed,filled"]; + + host_copier -> pvoc [label="Audio Stream\n(48 kHz, Stereo)"]; + pvoc -> vol [label="Time-Scaled Audio\n(Pitch Invariant)"]; + vol -> dai_copier [label="Faded / Muted PCM"]; + + sw_ctl -> pvoc [style="dotted", color="#60a5fa", label="get/put 259"]; + enum_ctl -> pvoc [style="dotted", color="#60a5fa", label="get/put 257"]; + } + +------------------------------------------------------------------------------- + +Factory Bringup, Acoustic Quality & Verification Runbook +-------------------------------------------------------- + +This runbook provides step-by-step procedures to build, deploy, tune, and test the Phase Vocoder subsystem using the SOF testbench and physical device under test (DUT). + +1. Binary Blob Generation via GNU Octave +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Generate the pre-computed configuration blobs for standard Hann window configurations: + +.. code-block:: bash + + # Step 1: Navigate to the tuning directory + cd tools/tune/phase_vocoder + + # Step 2: Run Octave to generate topology configurations + octave --no-gui setup_phase_vocoder.m + + # Output files created in topology2/include/components/phase_vocoder/: + # - hann_256_128.conf (5.3 ms window, 2.7 ms hop) + # - hann_512_128.conf (10.7 ms window, 2.7 ms hop) + # - hann_512_256.conf (10.7 ms window, 5.3 ms hop) + # - hann_1024_256.conf (21.3 ms window, 5.3 ms hop - default stereo) + # - hann_1024_256_mono.conf (21.3 ms window, 5.3 ms hop - default mono) + +2. Standalone Testbench Verification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Execute the automated testbench runner scripts to verify time-stretching accuracy, pitch invariance, and memory safety without hardware dependencies: + +.. code-block:: bash + + # Define workspace root + export SOF_WORKSPACE=/home/lrg/work + + # Run 16-bit PCM testbench execution with automated speed sweep + tools/tune/phase_vocoder/phase_vocoder_s16.sh input_speech.wav output_s16.wav + + # Run 32-bit PCM testbench execution + tools/tune/phase_vocoder/phase_vocoder_s32.sh input_music.wav output_s32.wav + + # Verify output duration: + # Input: 10.0 seconds + # Output: Dynamically swept duration matching the control script schedule + +3. Real-Time Hardware ALSA Mixer Verification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On physical development hardware (such as Panther Lake, Arrow Lake, or Tiger Lake), verify runtime controls via SSH: + +.. code-block:: bash + + # Step 1: Query available mixer controls + ssh root@ "amixer -c0 controls | grep -i 'Phase Vocoder'" + + # Expected controls: + # numid=10,iface=MIXER,name='Analog Playback Phase Vocoder enable' + # numid=11,iface=MIXER,name='Analog Playback Phase Vocoder speed' + + # Step 2: Enable vocoder processing + ssh root@ "amixer -c0 cset name='Analog Playback Phase Vocoder enable' on" + + # Step 3: Set slow-motion playback (0.5x speed) + ssh root@ "amixer -c0 cset name='Analog Playback Phase Vocoder speed' 0.5" + + # Step 4: Sweep to accelerated playback (1.5x speed) + ssh root@ "amixer -c0 cset name='Analog Playback Phase Vocoder speed' 1.5" + + # Step 5: Toggle zero-overhead bypass mode + ssh root@ "amixer -c0 cset name='Analog Playback Phase Vocoder enable' off" + +4. Acoustic Quality & Pitch Invariance Verification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To confirm that the Phase Vocoder alters tempo without modifying pitch: + +1. **Sine Wave Benchmark**: + Feed a pure 1000 Hz sine wave tone into the pipeline. +2. **Frequency Domain Inspection**: + Record output at 0.5x, 1.0x, and 2.0x speeds. +3. **FFT Verification**: + Compute the peak spectral bin using `sox input.wav -n stat -freq` or Python `numpy.fft`. The peak fundamental frequency must remain **exactly at 1000 Hz** (:math:`\pm 0.01` Hz) across all speed settings, proving pitch invariance. +4. **THD+N & Signal-to-Noise Verification**: + Verify that harmonic distortion products remain below -70 dBFS across the entire audible band. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 895c7acd..70b2c05d 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -61,6 +61,7 @@ Audio Processing Modules & Algorithms * :ref:`up_down_mixer` (High-level architecture; also see upstream `Up/Down Mixer README `_) * :ref:`aria` (High-level architecture; also see upstream `Aria README `_) * :ref:`level_multiplier` (High-level architecture; also see upstream `Level Multiplier README `_) +* :ref:`phase_vocoder` (High-level architecture; also see upstream `Phase Vocoder source tree `_) .. _algorithm-specific-information: @@ -111,6 +112,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/up_down_mixer firmware/aria firmware/level_multiplier + firmware/phase_vocoder rimage/index.rst firmware/llext_modules firmware/hostless_firmware From 89553a9172944054196c0b33ba12f61d3122ae5a Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 14:44:41 +0100 Subject: [PATCH 29/64] doc: developer_guides: add comprehensive STFT process architecture guide Author an in-depth, authoritative architectural and developer guide for the SOF Short-Time Fourier Transform (STFT) Process subsystem: - Mathematical formulation: discrete STFT/iSTFT, convolution theorem, and frequency-domain filtering vs time-domain filters and phase vocoder. - Ring buffer ingress/egress, framing geometry, and hop sizing. - Five configurable window profiles (Rectangular, Blackman, Hamming, Hann, and Kaldi-compatible Povey) with COLA condition and exact Q1.31 synthesis gain compensation. - Dual-domain Cartesian complex vs polar magnitude/phase processing with conjugate Hermitian symmetry restoration. - Single contiguous memory allocation and zero-copy polar buffer overlay. - Cadence Tensilica HiFi3 SIMD vectorization (AE_MULFP32X2RS, AE_MULAFP32X2RS). - Intel IPC4 64-byte configuration blob, Zephyr LLEXT packaging, and ALSA Topology 2 graph integration. - Factory bringup runbook: Octave blob synthesis, standalone testbench, and acoustic linearity validation. - 7 native vector Graphviz SVG architecture diagrams (Figures 237-243). Signed-off-by: Liam Girdwood --- data/modules.yaml | 12 + developer_guides/firmware/stft_process.rst | 842 +++++++++++++++++++++ developer_guides/index.rst | 2 + 3 files changed, 856 insertions(+) create mode 100644 developer_guides/firmware/stft_process.rst diff --git a/data/modules.yaml b/data/modules.yaml index 26e39120..2820031e 100644 --- a/data/modules.yaml +++ b/data/modules.yaml @@ -207,6 +207,18 @@ modules: - "Variable speed scaling (0.5x to 2.0x) with exact GCD counter normalization" - "Interactive phase re-anchoring and mono downmix optimization" + - id: stft_process + name: "STFT Process" + source: "SOF" + category: "Audio Enhancement" + status: "Upstream" + description: "Modular Short-Time Fourier Transform frequency-domain filtering and synthesis engine." + simd: ["HiFi 3", "Scalar C"] + key_features: + - "Multi-channel 32-bit forward and inverse FFT with COLA windowing" + - "Dual-domain processing: Cartesian complex and polar magnitude/phase" + - "Single contiguous buffer layout and zero-copy polar memory overlay" + - id: smart_amp name: "Smart Amp Protection" source: "SOF" diff --git a/developer_guides/firmware/stft_process.rst b/developer_guides/firmware/stft_process.rst new file mode 100644 index 00000000..36176976 --- /dev/null +++ b/developer_guides/firmware/stft_process.rst @@ -0,0 +1,842 @@ +.. _stft_process: + +STFT Process Architecture +========================= + +The **STFT Process** (Short-Time Fourier Transform Process) subsystem in Sound Open Firmware (SOF) provides a high-performance, modular frequency-domain signal processing engine. It ingests continuous time-domain PCM audio streams, segments them into overlapping analysis frames, transforms them into discrete frequency bins via multi-channel 32-bit Fast Fourier Transforms (FFTs), executes frequency-domain filtering or polar transformations, and reconstructs continuous time-domain waveforms via Inverse Fast Fourier Transforms (iFFTs) and Overlap-Add (OLA) synthesis. + +Frequency-domain processing is essential for modern embedded audio subsystems. While time-domain finite impulse response (FIR) and infinite impulse response (IIR) filters excel at static equalization, frequency-domain architectures allow per-bin spectral gain modification, acoustic echo suppression, non-stationary noise reduction, dynamic spectral shaping, psychoacoustic masking, and feature extraction for machine learning models (such as keyword detection and speech recognition). + +The SOF STFT Process component is engineered specifically for hard real-time DSP constraints. It incorporates a single-allocation contiguous heap memory layout, zero-copy buffer sharing between Cartesian and polar representations, pre-computed window gain compensation, and Cadence Tensilica HiFi SIMD vector acceleration. It is fully integrated with the Intel IPC4 control plane, the Zephyr Loadable Linkable Extension (LLEXT) dynamic module loader, and ALSA Topology 2. + +.. contents:: Table of Contents + :local: + :depth: 2 + +------------------------------------------------------------------------------- + +Architectural Overview & Frequency-Domain Processing Principles +--------------------------------------------------------------- + +Continuous acoustic signals are inherently non-stationary; their spectral properties vary continuously over time. The Short-Time Fourier Transform resolves this by partitioning the time-domain signal into short, overlapping quasi-stationary windows where the signal's spectral properties can be assumed constant. + +Theoretical Foundations of Frequency-Domain Processing +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Processing audio signals in the frequency domain leverages fundamental mathematical properties: + +1. **The Convolution Theorem**: + Time-domain circular convolution corresponds to point-wise multiplication in the frequency domain: + + .. math:: + + x[n] * h[n] \quad \Longleftrightarrow \quad X[k] \cdot H[k] + + For long impulse responses (such as room reverberation suppression or high-order linear-phase filters), executing a fast FFT, point-wise complex multiplication, and iFFT reduces computational complexity from :math:`\mathcal{O}(N^2)` to :math:`\mathcal{O}(N \log_2 N)`. + +2. **Per-Bin Spectral Masking & Subtraction**: + Nonlinear and dynamic algorithms (such as spectral noise subtraction and speech enhancement) apply independent attenuation factors :math:`G[k]` to individual frequency bins based on estimated Signal-to-Noise Ratios (SNR): + + .. math:: + + Y[k] = G[k] \cdot X[k] + + Implementing equivalent sharp, dynamic, multi-band notch filters in the time domain would require hundreds of cascaded biquads with severe phase distortion and high computational overhead. + +3. **Decoupled Magnitude and Phase Control**: + By transforming Fourier coefficients into polar coordinates, algorithms can manipulate signal magnitude (gain, dynamic range, spectral shaping) while preserving or independently modeling phase trajectories. + +.. list-table:: Architectural Comparison: Audio Processing Paradigms + :widths: 22 26 26 26 + :header-rows: 1 + + * - Parameter + - Time-Domain Filters (IIR/FIR) + - SOF STFT Process + - SOF Phase Vocoder + * - **Processing Domain** + - Time domain (samples) + - Frequency domain (FFT bins) + - Frequency domain (Polar STFT) + * - **Hop Geometry** + - Single sample (:math:`R = 1`) + - Fixed hop (:math:`R_{\text{hop}} = N/2, N/4`) + - Variable hop (:math:`R_s = R_a / \text{speed}`) + * - **Primary Purpose** + - Static EQ, DC blocking, basic crossovers + - Spectral filtering, noise gating, speech enhancement + - Time-Scale Modification (0.5x to 2.0x playback) + * - **Phase Behavior** + - Minimum or linear phase + - Frame-dependent phase preservation + - Active phase unwrapping & accumulation + * - **Algorithmic Latency** + - Sub-millisecond (tap delay) + - Frame hop size (1 ms to 32 ms) + - Window hop size (2.7 ms to 5.3 ms) + * - **Memory Footprint** + - Minimal (< 1 KB state) + - Moderate (8 KB to 32 KB buffers) + - Moderate (8 KB to 32 KB buffers) + +.. _figure_237: + +.. graphviz:: + :align: center + :caption: SOF STFT Processing Architecture: Ingress, Framing, Analysis Windowing, Complex/Polar Transform, Synthesis Windowing & Overlap-Add Core + + digraph stft_architecture { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="rounded,filled", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#94a3b8", penwidth=1.2]; + + subgraph cluster_ingress { + label = "Time-Domain Ingress"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + src_pcm [label="Source PCM Stream\n(S16_LE / S32_LE)\nInterleaved Audio", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + ibuf [label="Per-Channel Ring Buffer\nstate->ibuf[ch]\n(Circular Input Queue)", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + } + + subgraph cluster_analysis { + label = "STFT Analysis Engine"; + style = "solid"; + color = "#1e3a8a"; + bgcolor = "#17255455"; + + prev_data [label="Overlap History Buffer\nstate->prev_data[ch]\n(N - R_hop Samples)", fillcolor="#1d4ed8", fontcolor="#ffffff", color="#60a5fa"]; + fft_in [label="FFT Input Assembly\nfft->fft_buf\n(Real Data + Zero Imag)", fillcolor="#2563eb", fontcolor="#ffffff", color="#93c5fd"]; + win_analysis [label="Analysis Windowing\nstft_process_apply_window()\nw[n] * real (HiFi SIMD)", fillcolor="#4f46e5", fontcolor="#ffffff", color="#a5b4fc"]; + fwd_fft [label="Forward 32-Bit FFT\nfft_multi_execute_32()\nTime -> Frequency", fillcolor="#6366f1", fontcolor="#ffffff", color="#c7d2fe"]; + } + + subgraph cluster_freq_domain { + label = "Frequency-Domain Core"; + style = "solid"; + color = "#701a75"; + bgcolor = "#4a044e33"; + + cartesian [label="Cartesian Complex\nfft->fft_out[k]\n(Real + j * Imag)", fillcolor="#9333ea", fontcolor="#ffffff", color="#d8b4fe"]; + polar [label="Optional Polar Transform\nCONFIG_STFT_PROCESS_MAG_PHASE\n(Q2.30 Mag, Q5.27 Phase)", fillcolor="#c026d3", fontcolor="#ffffff", color="#f0abfc", style="dashed,filled"]; + algo_hook [label="Frequency-Domain User Hook\nSpectral Masking / Filtering\nHermitian Symmetry Restoration", fillcolor="#db2777", fontcolor="#ffffff", color="#f472b6"]; + } + + subgraph cluster_synthesis { + label = "STFT Synthesis Engine (iSTFT)"; + style = "solid"; + color = "#064e3b"; + bgcolor = "#022c2255"; + + inv_fft [label="Inverse 32-Bit iFFT\nfft_multi_execute_32(inv=true)\nFrequency -> Time", fillcolor="#059669", fontcolor="#ffffff", color="#6ee7b7"]; + win_synthesis [label="Synthesis Windowing\nstft_process_apply_window()\nw[n] * real (HiFi SIMD)", fillcolor="#047857", fontcolor="#ffffff", color="#a7f3d0"]; + ola [label="Overlap-Add Accumulation\nstft_process_overlap_add()\nGain Comp * Real + obuf", fillcolor="#0f766e", fontcolor="#ffffff", color="#5eead4"]; + obuf [label="Per-Channel Output Ring\nstate->obuf[ch]\n(Circular Output Queue)", fillcolor="#0e7490", fontcolor="#ffffff", color="#67e8f9"]; + } + + subgraph cluster_egress { + label = "Time-Domain Egress"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + sink_pcm [label="Sink PCM Stream\n(S16_LE / S32_LE)\nReconstructed Audio", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + src_pcm -> ibuf [label="De-interleave"]; + ibuf -> fft_in [label="R_hop Samples"]; + prev_data -> fft_in [label="Concatenate Overlap"]; + fft_in -> win_analysis [label="Aligned Buffer"]; + win_analysis -> fwd_fft [label="Windowed Time Series"]; + fwd_fft -> cartesian [label="Complex Spectra"]; + cartesian -> polar [label="sofm_icomplex32_to_polar()"]; + polar -> algo_hook [label="Process Mag/Phase"]; + algo_hook -> polar [label="Updated Bins"]; + polar -> cartesian [label="sofm_ipolar32_to_complex()"]; + cartesian -> inv_fft [label="Conjugate Symmetric"]; + inv_fft -> win_synthesis [label="Raw Time Window"]; + win_synthesis -> ola [label="Windowed Window"]; + ola -> obuf [label="Saturating Add"]; + obuf -> sink_pcm [label="Interleave & Commit"]; + + fft_in -> prev_data [style="dashed", label="Update History"]; + } + +------------------------------------------------------------------------------- + +Circular Buffer Ingress, Hop Geometry & Overlap Assembly +-------------------------------------------------------- + +The STFT Process component bridges the continuous, frame-by-frame streaming nature of the SOF pipeline with the block-based nature of Fourier transforms. Incoming audio frames typically arrive in scheduling intervals of 1 ms (e.g., 48 samples at 48 kHz or 16 samples at 16 kHz), whereas Fourier analysis frames are significantly larger (typically 192 to 1536 samples). + +Ring Buffer Architecture & Pointer Arithmetic +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To manage the rate difference between pipeline ticks and FFT block executions, the component maintains dedicated per-channel circular buffers for both ingress (:c:member:`stft_process_state.ibuf`) and egress (:c:member:`stft_process_state.obuf`). + +Each buffer is managed by :c:struct:`stft_process_buffer`: + +* :c:member:`stft_process_buffer.addr`: Base memory address of the buffer. +* :c:member:`stft_process_buffer.end_addr`: Pointer to the byte immediately following the buffer. +* :c:member:`stft_process_buffer.r_ptr`: Read pointer advancing as samples are consumed. +* :c:member:`stft_process_buffer.w_ptr`: Write pointer advancing as samples are produced. +* :c:member:`stft_process_buffer.s_avail`: Count of available valid samples. +* :c:member:`stft_process_buffer.s_free`: Count of free space remaining in samples. +* :c:member:`stft_process_buffer.s_length`: Total capacity in samples. + +When read or write pointers reach :c:member:`stft_process_buffer.end_addr`, they wrap back to :c:member:`stft_process_buffer.addr` via :c:func:`stft_process_buffer_wrap`: + +.. code-block:: c + + static inline int32_t *stft_process_buffer_wrap(struct stft_process_buffer *buffer, int32_t *ptr) + { + if (ptr >= buffer->end_addr) + ptr -= buffer->s_length; + return ptr; + } + +Hop Sizing & Overlap Geometry +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The relationship between the analysis frame length :math:`N` and the hop size :math:`R_{\text{hop}}` defines the temporal and spectral characteristics of the STFT: + +* **Frame Length** (:math:`N`): Number of samples in each FFT analysis window (:c:member:`stft_process_fft.fft_size`). Must be a multiple of 4 for Xtensa HiFi SIMD vector operations. +* **Hop Size** (:math:`R_{\text{hop}}`): Number of new samples advanced between consecutive FFTs (:c:member:`stft_process_fft.fft_hop_size`). Must be a multiple of 2. +* **Overlap History Size** (:math:`L_{\text{prev}}`): Number of samples retained from the previous frame: + + .. math:: + + L_{\text{prev}} = N - R_{\text{hop}} + + These samples are preserved in ``state->prev_data[ch]``. + +.. _figure_238: + +.. graphviz:: + :align: center + :caption: STFT Framing, Overlap Buffer Geometry & Circular Ring Buffer Timeline (R_hop vs N_frame) + + digraph framing_timeline { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, pad=0.4, nodesep=0.4, ranksep=0.5]; + node [shape=rect, style="rounded,filled", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#94a3b8", penwidth=1.2]; + + subgraph cluster_ring { + label = "Circular Input Ring Buffer (state->ibuf[ch])"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + stream_in [label="Continuous Pipeline Streaming Ingress (Pipeline Ticks @ 1 ms)", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + ibuf_samples [label="Ring Buffer Storage: [ Old Samples | Available Hop Samples (R_hop) | Free Space ]", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + stream_in -> ibuf_samples [label="De-interleaved Ingestion"]; + } + + subgraph cluster_frame_assembly { + label = "Assembly of Analysis Frame m"; + style = "solid"; + color = "#1e3a8a"; + bgcolor = "#17255455"; + + subgraph cluster_parts { + rank = same; + history_part [label="History Overlap Segment\nprev_data[0 .. L_prev - 1]\n(N - R_hop Samples)", fillcolor="#1d4ed8", fontcolor="#ffffff", color="#60a5fa"]; + hop_part [label="New Data Segment\nibuf[0 .. R_hop - 1]\n(R_hop Samples)", fillcolor="#2563eb", fontcolor="#ffffff", color="#93c5fd"]; + } + + full_frame [label="Complete Composite Analysis Frame (fft->fft_buf)\n[ 0 ........................................ N - 1 ]\nLength N = L_prev + R_hop Samples", fillcolor="#4338ca", fontcolor="#ffffff", color="#818cf8"]; + + history_part -> full_frame [label="Copied from State"]; + hop_part -> full_frame [label="Drained from Ring"]; + } + + subgraph cluster_history_update { + label = "State Preservation for Frame m + 1"; + style = "solid"; + color = "#064e3b"; + bgcolor = "#022c2255"; + + next_history [label="New Overlap History (prev_data[ch])\nSamples [R_hop .. N - 1] of Current Frame\nBecomes History for Next Analysis Hop", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + } + + ibuf_samples -> hop_part [label="When s_avail >= R_hop"]; + full_frame -> next_history [style="dashed", color="#34d399", label="Retain Trailing Samples"]; + } + +When :c:func:`stft_process_fill_fft_buffer` executes: + +1. The first :math:`L_{\text{prev}}` complex samples of :c:member:`stft_process_fft.fft_buf` receive the overlap history from ``state->prev_data[ch]``, with imaginary components set to 0. +2. The next :math:`R_{\text{hop}}` complex samples are dequeued from ``state->ibuf[ch]``, with imaginary components set to 0. +3. The trailing :math:`L_{\text{prev}}` samples of the assembled buffer (indices :math:`R_{\text{hop}}` to :math:`N-1`) are copied back to ``state->prev_data[ch]`` to serve as the overlap history for the subsequent hop. + +------------------------------------------------------------------------------- + +Window Functions, Spectral Leakage & Constant Overlap-Add (COLA) +---------------------------------------------------------------- + +Multiplying a finite-duration signal by an abrupt rectangular window causes sharp discontinuities at the boundaries, generating broad sidelobe energy across all Fourier bins (spectral leakage). To isolate narrow spectral peaks and prevent adjacent bin interference, the analysis frame is multiplied by a smooth taper window :math:`w[n]`. + +Configurable Window Types +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The SOF STFT Process component supports five distinct window functions defined in :c:enum:`sof_stft_process_fft_window_type`: + +1. **Rectangular Window (`STFT_RECTANGULAR_WINDOW = 0`)**: + Uniform weighting (:math:`w[n] = 1`). Yields the narrowest main lobe (:math:`\Delta \omega = 4\pi / N`) for maximum frequency resolution, but severe first sidelobe leakage (:math:`-13\text{ dB}` attenuation), causing high inter-bin spectral interference. + +2. **Blackman Window (`STFT_BLACKMAN_WINDOW = 1`)**: + A three-term cosine window providing extreme sidelobe suppression (:math:`-58\text{ dB}`), virtually eliminating cross-bin leakage at the expense of a wider main lobe (:math:`\Delta \omega = 12\pi / N`): + + .. math:: + + w[n] = a_0 - a_1 \cos\left(\frac{2\pi n}{N}\right) + a_2 \cos\left(\frac{4\pi n}{N}\right) + + Implemented with exact Q1.31 coefficients defined by :c:macro:`WIN_BLACKMAN_A0_Q31`. + +3. **Hamming Window (`STFT_HAMMING_WINDOW = 2`)**: + Optimized raised cosine window designed to cancel the first sidelobe, achieving :math:`-43\text{ dB}` first sidelobe attenuation: + + .. math:: + + w[n] = 0.54 - 0.46 \cos\left(\frac{2\pi n}{N}\right) + +4. **Hann Window (`STFT_HANN_WINDOW = 3`, Standard Default)**: + A standard raised cosine window tapering smoothly to zero at both boundaries: + + .. math:: + + w[n] = 0.5 - 0.5 \cos\left(\frac{2\pi n}{N}\right), \quad 0 \le n < N + + Achieves :math:`-31.5\text{ dB}` first sidelobe attenuation with a rapid asymptotic decay of :math:`-18\text{ dB/octave}`. + +5. **Povey Window (`STFT_POVEY_WINDOW = 4`)**: + A specialized window widely adopted in automatic speech recognition (ASR) feature extraction pipelines (such as Kaldi): + + .. math:: + + w[n] = \left(\frac{1 - \cos\left(\frac{2\pi n}{N}\right)}{2}\right)^{0.85}, \quad 0 \le n < N + + By raising the standard Hann raised-cosine curve to the power of :math:`0.85`, the Povey window broadens the effective analysis center while preserving smooth, zero-endpoint boundary transitions, optimizing the trade-off between spectral resolution and time localization for human speech formants. + +.. _figure_239: + +.. graphviz:: + :align: center + :caption: Analysis & Synthesis Window Functions and Frequency Responses (Rectangular, Hann, Hamming, Blackman, Povey) + + digraph window_profiles { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.4, ranksep=0.5]; + node [shape=rect, style="rounded,filled", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#94a3b8", penwidth=1.2]; + + subgraph cluster_time_domain { + label = "Time-Domain Window Envelope Profiles (w[n])"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + rect [label="Rectangular Window\nFlat (1.0), Abrupt Step\nMax Frequency Resolution", fillcolor="#1e293b", fontcolor="#f87171", color="#ef4444"]; + hann [label="Hann Window (Default)\n0.5 - 0.5*cos(2*pi*n/N)\nSmooth Zero Boundaries", fillcolor="#1e293b", fontcolor="#38bdf8", color="#0284c7"]; + hamming [label="Hamming Window\n0.54 - 0.46*cos(2*pi*n/N)\nNon-Zero Pedestal (0.08)", fillcolor="#1e293b", fontcolor="#818cf8", color="#4f46e5"]; + blackman [label="Blackman Window\n3-Term Cosine Sum\nUltra-Smooth Roll-off", fillcolor="#1e293b", fontcolor="#c084fc", color="#9333ea"]; + povey [label="Povey Window (ASR)\n[Hann(n)]^0.85\nSpeech-Optimized Center", fillcolor="#1e293b", fontcolor="#34d399", color="#059669"]; + } + + subgraph cluster_freq_domain { + label = "Spectral Response & Sidelobe Attenuation"; + style = "solid"; + color = "#1e3a8a"; + bgcolor = "#17255455"; + + rect_spec [label="Main Lobe: 4*pi/N (Narrowest)\nSidelobe Atten: -13 dB\nAsymptotic Decay: -6 dB/oct", fillcolor="#7f1d1d", fontcolor="#ffffff", color="#f87171"]; + hann_spec [label="Main Lobe: 8*pi/N\nSidelobe Atten: -31.5 dB\nAsymptotic Decay: -18 dB/oct", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + hamming_spec [label="Main Lobe: 8*pi/N\nSidelobe Atten: -43 dB (First)\nAsymptotic Decay: -6 dB/oct", fillcolor="#3730a3", fontcolor="#ffffff", color="#818cf8"]; + blackman_spec [label="Main Lobe: 12*pi/N (Broadest)\nSidelobe Atten: -58 dB\nAsymptotic Decay: -18 dB/oct", fillcolor="#6b21a8", fontcolor="#ffffff", color="#c084fc"]; + povey_spec [label="Main Lobe: ~7.5*pi/N\nSidelobe Atten: -33 dB\nKaldi ASR Feature Extract", fillcolor="#065f46", fontcolor="#ffffff", color="#34d399"]; + } + + rect -> rect_spec; + hann -> hann_spec; + hamming -> hamming_spec; + blackman -> blackman_spec; + povey -> povey_spec; + } + +Constant Overlap-Add (COLA) Condition & Gain Compensation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When an audio signal is windowed during analysis and windowed again during synthesis, the cumulative gain across overlapping frames must sum to a constant across time to prevent amplitude modulation (tremolo) at the frame hop rate: + +.. math:: + + \sum_{m=-\infty}^{+\infty} w[n - m R_{\text{hop}}] = \text{constant} \quad \forall n + +In a cascaded STFT / iSTFT processing system where both the forward FFT and the inverse FFT apply the window :math:`w[n]`, the effective overlap-add weight is the squared window :math:`w[n]^2`. + +To guarantee exact unity gain (:math:`0\text{ dBFS}`) reconstruction without amplitude ripple, SOF computes a 32-bit Q1.31 window gain compensation factor :c:member:`stft_process_state.gain_comp`: + +.. math:: + + g_{\text{comp}} = \frac{R_{\text{hop}}}{\sum_{n=0}^{N-1} w[n]^2} + +During synthesis overlap-add in :c:func:`stft_process_overlap_add_ifft_buffer`, each real sample from the inverse FFT is multiplied by :math:`g_{\text{comp}}` before being accumulated into the output buffer: + +.. code-block:: c + + sample = Q_MULTSR_32X32((int64_t)state->gain_comp, fft->fft_buf[idx].real, 31, 31, 31); + *w = sat_int32((int64_t)*w + sample); + +------------------------------------------------------------------------------- + +Dual-Domain Processing: Cartesian Complex vs Polar Magnitude/Phase +------------------------------------------------------------------ + +The STFT Process component supports dual processing representations: Cartesian complex coordinates (:math:`\text{real} + j \cdot \text{imag}`) and polar coordinates (magnitude :math:`M_k` and phase angle :math:`\theta_k`). + +Cartesian Complex Pipeline (Default Fast Path) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In the default configuration, the 32-bit forward FFT generates complex coefficients directly into :c:member:`stft_process_fft.fft_out`: + +.. math:: + + X[k] = \text{Re}\{X[k]\} + j \cdot \text{Im}\{X[k]\}, \quad k = 0 \dots N-1 + +For linear filtering and spectral convolution, algorithms operate directly on Cartesian complex numbers: + +.. math:: + + Y[k] = X[k] \cdot H[k] = \left(\text{Re}\{X\} \text{Re}\{H\} - \text{Im}\{X\} \text{Im}\{H\}\right) + j \left(\text{Re}\{X\} \text{Im}\{H\} + \text{Im}\{X\} \text{Re}\{H\}\right) + +Polar Coordinate Pipeline (`CONFIG_STFT_PROCESS_MAGNITUDE_PHASE`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When enabled via Kconfig, the component automatically transforms complex Fourier bins into polar coordinates: + +1. **Complex to Polar Conversion**: + Using :c:func:`sofm_icomplex32_to_polar`, the complex bin is converted to magnitude and phase: + + .. math:: + + M_k = \sqrt{\text{Re}\{X[k]\}^2 + \text{Im}\{X[k]\}^2} \quad (\text{Q2.30}) + + .. math:: + + \theta_k = \text{atan2}\left(\text{Im}\{X[k]\}, \text{Re}\{X[k]\}\right) \quad (\text{Q5.27}) + + Because the time-domain signal is strictly real, the frequency spectrum exhibits Hermitian symmetry (:math:`X[N-k] = X^*[k]`). Therefore, polar conversion is only performed on the non-redundant lower half-spectrum: + + .. math:: + + k = 0 \dots \left(\frac{N}{2}\right) + +2. **Algorithm Hook Execution**: + The application algorithm modifies magnitude :math:`M_k` (e.g., applying Wiener gain masks, dynamic range compression, or noise reduction gains) and/or phase :math:`\theta_k`. + +3. **Polar to Complex Reconstruction**: + The modified polar coordinates are converted back to Cartesian form via :c:func:`sofm_ipolar32_to_complex`: + + .. math:: + + \text{Re}\{X[k]\} = M_k \cdot \cos(\theta_k), \quad \text{Im}\{X[k]\} = M_k \cdot \sin(\theta_k) + +4. **Conjugate Hermitian Symmetry Restoration**: + To guarantee that the subsequent inverse FFT yields a purely real time-domain signal (with zero imaginary component), the upper half-spectrum is reconstructed via :c:func:`stft_apply_fft_symmetry`: + + .. math:: + + \text{Re}\{X[N-k]\} = \text{Re}\{X[k]\}, \quad \text{Im}\{X[N-k]\} = -\text{Im}\{X[k]\}, \quad k = 1 \dots \frac{N}{2}-1 + +.. _figure_240: + +.. graphviz:: + :align: center + :caption: Dual-Domain Processing Pipeline: Cartesian Complex vs Polar Magnitude/Phase with Hermitian Symmetry Reconstruction + + digraph dual_domain { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="rounded,filled", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#94a3b8", penwidth=1.2]; + + subgraph cluster_fwd { + label = "Analysis FFT"; + style = "solid"; + color = "#1e3a8a"; + bgcolor = "#17255455"; + + fft_in [label="Time Window\n(Real Data)", fillcolor="#1d4ed8", fontcolor="#ffffff", color="#60a5fa"]; + r2c_fft [label="32-Bit Forward FFT\nfft_multi_execute_32()", fillcolor="#2563eb", fontcolor="#ffffff", color="#93c5fd"]; + fft_in -> r2c_fft; + } + + subgraph cluster_domains { + label = "Domain Representation"; + style = "solid"; + color = "#701a75"; + bgcolor = "#4a044e33"; + + cart_in [label="Cartesian Complex\nX[0 .. N-1]\n(Real + j*Imag)", fillcolor="#9333ea", fontcolor="#ffffff", color="#d8b4fe"]; + c2p [label="sofm_icomplex32_to_polar()\nk = 0 .. N/2", fillcolor="#c026d3", fontcolor="#ffffff", color="#f0abfc"]; + polar_repr [label="Polar Domain\nMag: Q2.30\nPhase: Q5.27", fillcolor="#db2777", fontcolor="#ffffff", color="#f472b6"]; + user_dsp [label="Spectral Gain Mask / Filter\nMagnitude Attenuation G[k]\nPhase Modification", fillcolor="#e11d48", fontcolor="#ffffff", color="#fb7185"]; + p2c [label="sofm_ipolar32_to_complex()\nk = 0 .. N/2", fillcolor="#c026d3", fontcolor="#ffffff", color="#f0abfc"]; + sym_restore [label="stft_apply_fft_symmetry()\nRe[N-k] = Re[k]\nIm[N-k] = -Im[k]", fillcolor="#9333ea", fontcolor="#ffffff", color="#d8b4fe"]; + + cart_in -> c2p [label="Half FFT"]; + c2p -> polar_repr; + polar_repr -> user_dsp [label="Process Bins"]; + user_dsp -> polar_repr [label="Updated"]; + polar_repr -> p2c; + p2c -> sym_restore [label="Lower Half"]; + sym_restore -> cart_in [style="dashed", label="Reconstruct Full N"]; + } + + subgraph cluster_inv { + label = "Synthesis iFFT"; + style = "solid"; + color = "#064e3b"; + bgcolor = "#022c2255"; + + c2r_ifft [label="32-Bit Inverse iFFT\nfft_multi_execute_32(inv=true)", fillcolor="#059669", fontcolor="#ffffff", color="#6ee7b7"]; + ola_out [label="Overlap-Add Accumulation\nWindow Gain Comp\nOutput to Ring Buffer", fillcolor="#047857", fontcolor="#ffffff", color="#a7f3d0"]; + c2r_ifft -> ola_out; + } + + r2c_fft -> cart_in [label="N Complex Bins"]; + cart_in -> c2r_ifft [label="Hermitian Symmetric"]; + } + +------------------------------------------------------------------------------- + +Single Contiguous Memory Block & Zero-Copy Polar Overlay +-------------------------------------------------------- + +In hard real-time audio firmware, dynamic heap allocation during runtime is prohibited, and excessive memory fragmentation must be prevented. The STFT Process subsystem employs an optimized memory architecture that consolidates all circular sample buffers, overlap arrays, and window tables into a single contiguous allocation. + +Single Contiguous Buffer Partitioning +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +During component initialization in :c:func:`stft_process_setup`, the total memory requirement across all channels is calculated and allocated as a single aligned buffer: + +.. math:: + + \text{Total Sample RAM} = \text{sizeof}(\text{int32\_t}) \times \left[ C \times (L_{\text{ibuf}} + L_{\text{obuf}} + L_{\text{prev}}) + N \right] + +Where: + +* :math:`C` is the number of active audio channels (:c:member:`stft_comp_data.channels`). +* :math:`L_{\text{ibuf}} = R_{\text{hop}} + \text{max\_frames}` (input circular buffer length). +* :math:`L_{\text{obuf}} = N + \text{max\_frames}` (output circular buffer length). +* :math:`L_{\text{prev}} = N - R_{\text{hop}}` (overlap history length). +* :math:`N` is the FFT frame size (window coefficient table length). + +The component verifies that :math:`\text{Total Sample RAM} \le \text{STFT\_MAX\_ALLOC\_SIZE}` (65,536 bytes / 64 KB) to protect against memory exhaustion. The buffer is allocated via :c:func:`mod_balloc_align` with 64-bit alignment (:math:`2 \times \text{sizeof}(\text{int32\_t})`), satisfying Xtensa SIMD load requirements. + +Zero-Copy Polar Buffer Overlay +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To support polar magnitude and phase processing without allocating an additional 32-bit array, the component overlays the polar coordinate structure directly on top of the FFT output buffer: + +.. code-block:: c + + /* Share the fft_out buffer for polar format */ + fft->fft_polar = (struct ipolar32 *)fft->fft_out; + +Because both :c:struct:`icomplex32` (two 32-bit integers: `real` and `imag`) and :c:struct:`ipolar32` (two 32-bit integers: `magnitude` and `angle`) have identical 64-bit sizes and alignments, in-place conversion from Cartesian to polar format consumes **zero additional heap memory**! + +.. _figure_241: + +.. graphviz:: + :align: center + :caption: Single Contiguous Memory Block Partitioning & Zero-Copy Polar Overlay Architecture + + digraph memory_layout { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.4, ranksep=0.5]; + node [shape=rect, style="rounded,filled", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#94a3b8", penwidth=1.2]; + + subgraph cluster_sample_block { + label = "Contiguous Sample Allocation Block: state->buffers (Single mod_balloc_align)"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + subgraph cluster_ch0 { + label = "Channel 0 Buffers"; + style = "dashed"; + color = "#0369a1"; + bgcolor = "#082f4944"; + + ch0_ibuf [label="ibuf[0]\n(R_hop + max_frames)", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + ch0_obuf [label="obuf[0]\n(N + max_frames)", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + ch0_prev [label="prev_data[0]\n(N - R_hop)", fillcolor="#075985", fontcolor="#ffffff", color="#38bdf8"]; + } + + subgraph cluster_ch1 { + label = "Channel 1 Buffers (Stereo)"; + style = "dashed"; + color = "#4338ca"; + bgcolor = "#1e1b4b44"; + + ch1_ibuf [label="ibuf[1]\n(R_hop + max_frames)", fillcolor="#4f46e5", fontcolor="#ffffff", color="#818cf8"]; + ch1_obuf [label="obuf[1]\n(N + max_frames)", fillcolor="#4338ca", fontcolor="#ffffff", color="#818cf8"]; + ch1_prev [label="prev_data[1]\n(N - R_hop)", fillcolor="#3730a3", fontcolor="#ffffff", color="#818cf8"]; + } + + win_table [label="Window Coefficient Table (state->window)\nN * sizeof(int32_t) Q1.31 Coefficients", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + + ch0_ibuf -> ch0_obuf -> ch0_prev -> ch1_ibuf -> ch1_obuf -> ch1_prev -> win_table [style="invis"]; + } + + subgraph cluster_fft_block { + label = "Dedicated FFT Scratch & Zero-Copy Polar Overlay"; + style = "solid"; + color = "#701a75"; + bgcolor = "#4a044e33"; + + fft_in_buf [label="fft->fft_buf\nN * sizeof(struct icomplex32)\nInput Time Window / iFFT Time Output", fillcolor="#9333ea", fontcolor="#ffffff", color="#d8b4fe"]; + + subgraph cluster_overlay { + label = "Shared Physical RAM Block (fft->fft_out)"; + style = "dotted"; + color = "#f43f5e"; + bgcolor = "#88133733"; + + fft_out_cart [label="fft->fft_out (Cartesian)\nN * struct icomplex32\n{ int32_t real; int32_t imag; }", fillcolor="#be185d", fontcolor="#ffffff", color="#f472b6"]; + fft_out_polar [label="fft->fft_polar (Polar Overlay)\nN/2 * struct ipolar32\n{ int32_t magnitude; int32_t angle; }", fillcolor="#9f1239", fontcolor="#ffffff", color="#fb7185", style="dashed,filled"]; + } + } + } + +------------------------------------------------------------------------------- + +Tensilica HiFi3 SIMD Vector Acceleration +---------------------------------------- + +The critical inner loops of the STFT Process subsystem—analysis windowing and synthesis overlap-add—are accelerated using Cadence Tensilica HiFi3 SIMD vector intrinsics. + +Parallel Analysis Windowing (`stft_process_apply_window`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The input FFT buffer stores interleaved 32-bit complex numbers (`real` and `imag`), but the analysis window applies only to the real audio samples (the imaginary component represents time-domain zero). + +In :c:func:`stft_process_apply_window` on HiFi3: + +1. Four complex samples are loaded per iteration using :c:macro:`AE_L32_I`. +2. The real components are unpacked and gathered into vector registers :c:type:`ae_f32x2` using :c:macro:`AE_SEL32_HH`. +3. Four Q1.31 window coefficients are loaded in parallel via :c:macro:`AE_L32X2_IP`. +4. Fractional vector multiplication with symmetric rounding is executed via :c:macro:`AE_MULFP32X2RS`: + + .. math:: + + \text{real}' = \text{real} \times w[n] \quad (\text{Q1.31} \times \text{Q1.31} \to \text{Q1.31}) + +5. The updated real parts are stored back into the complex buffer using :c:macro:`AE_S32_L_IP`, leaving the imaginary parts unaltered. + +Parallel Overlap-Add Synthesis (`stft_process_overlap_add_ifft_buffer`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +During synthesis overlap-add, the real output samples from the inverse FFT must be scaled by the window gain compensation factor :math:`g_{\text{comp}}` and added with 32-bit saturation to the existing content of the circular output buffer: + +1. The scalar gain compensation factor :c:member:`stft_process_state.gain_comp` is broadcast into a dual 32-bit vector register via :c:macro:`AE_MOVDA32`. +2. Two IFFT real output samples are loaded and packed into :c:type:`ae_f32x2` via :c:macro:`AE_L32_IP` and :c:macro:`AE_SEL32_HH`. +3. Two samples from the circular output buffer are loaded via :c:macro:`AE_L32X2_I`. +4. A fused multiply-accumulate with saturation (:c:macro:`AE_MULAFP32X2RS`) computes: + + .. math:: + + \text{obuf}[n] = \text{sat}_{32}\left( \text{obuf}[n] + (\text{ifft\_real}[n] \times g_{\text{comp}}) \right) + +5. The accumulated samples are written back to the circular buffer via aligned vector store :c:macro:`AE_S32X2_IP`. + +.. _figure_242: + +.. graphviz:: + :align: center + :caption: Cadence Tensilica HiFi3 SIMD Vector Execution: Parallel Analysis Windowing & Overlap-Add SIMD MACs + + digraph hifi_simd { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.4, ranksep=0.5]; + node [shape=rect, style="rounded,filled", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#94a3b8", penwidth=1.2]; + + subgraph cluster_window_simd { + label = "Parallel Analysis Windowing: 4 Samples / Iteration (HiFi3)"; + style = "solid"; + color = "#1e3a8a"; + bgcolor = "#17255455"; + + load_complex [label="Load 4 Complex Pairs\nAE_L32_I(buf, 0..3)\nExtract Reals via AE_SEL32_HH", fillcolor="#1d4ed8", fontcolor="#ffffff", color="#60a5fa"]; + load_win [label="Load 4 Q1.31 Window Coeffs\nAE_L32X2_IP(win01)\nAE_L32X2_IP(win23)", fillcolor="#2563eb", fontcolor="#ffffff", color="#93c5fd"]; + vec_mul [label="Vector Fractional Multiply\nAE_MULFP32X2RS(data01, win01)\nAE_MULFP32X2RS(data23, win23)", fillcolor="#4338ca", fontcolor="#ffffff", color="#818cf8"]; + store_complex [label="Store Back Updated Reals\nAE_S32_L_IP(data, buf)\nImag Parts Untouched", fillcolor="#4f46e5", fontcolor="#ffffff", color="#a5b4fc"]; + + load_complex -> vec_mul; + load_win -> vec_mul; + vec_mul -> store_complex; + } + + subgraph cluster_ola_simd { + label = "Parallel Synthesis Overlap-Add: 2 Samples / Iteration (HiFi3)"; + style = "solid"; + color = "#064e3b"; + bgcolor = "#022c2255"; + + bcast_gain [label="Broadcast Gain Factor\ngain = AE_MOVDA32(gain_comp)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + load_ifft [label="Load 2 IFFT Reals\nAE_L32_IP & AE_SEL32_HH", fillcolor="#059669", fontcolor="#ffffff", color="#6ee7b7"]; + load_obuf [label="Load 2 obuf Samples\nbuffer_data = AE_L32X2_I(w, 0)", fillcolor="#0e7490", fontcolor="#ffffff", color="#67e8f9"]; + fused_mac [label="Fused Saturating Multiply-Accumulate\nAE_MULAFP32X2RS(buffer_data, fft_data, gain)", fillcolor="#0f766e", fontcolor="#ffffff", color="#5eead4"]; + store_obuf [label="Aligned 64-Bit Store\nAE_S32X2_IP(buffer_data, w, 8)", fillcolor="#047857", fontcolor="#ffffff", color="#a7f3d0"]; + + bcast_gain -> fused_mac; + load_ifft -> fused_mac; + load_obuf -> fused_mac; + fused_mac -> store_obuf; + } + } + +------------------------------------------------------------------------------- + +IPC4 Control Plane, LLEXT Packaging & ALSA Topology 2 Graph +----------------------------------------------------------- + +The STFT Process component complies fully with the Intel IPC4 audio architecture and can be built either statically into the base firmware image or dynamically packaged as a Zephyr Loadable Linkable Extension (LLEXT). + +Configuration Payload Structure +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Runtime parameters are delivered via IPC4 large configuration messages as a 64-byte structured blob defined by :c:struct:`sof_stft_process_config`: + +.. code-block:: c + + struct sof_stft_process_config { + uint32_t size; /**< Size of this struct (64 bytes) */ + uint32_t reserved[8]; + int32_t sample_frequency; /**< Sample rate in Hz (e.g., 16000, 48000) */ + int32_t window_gain_comp; /**< Q1.31 gain compensation for iSTFT */ + int32_t reserved_32; + int16_t channel; /**< Channel select (-1=all, 0=left, 1=right) */ + int16_t frame_length; /**< Frame length N (samples, e.g., 512, 1024) */ + int16_t frame_shift; /**< Hop size R_hop (samples, e.g., 128, 256) */ + int16_t reserved_16; + enum sof_stft_process_fft_pad_type pad; /**< Padding type (PAD_END, PAD_CENTER) */ + enum sof_stft_process_fft_window_type window; /**< Window enum (RECT, BLACKMAN, HAMM, HANN, POVEY) */ + } __attribute__((packed)); + +Zephyr LLEXT Dynamic Module Packaging +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When built with ``CONFIG_COMP_STFT_PROCESS_MODULE=y``, the component is compiled into a standalone relocatable ELF extension (`stft_process.llext`): + +* **Module Name**: ``STFT_PROCESS`` (manifest alias ``STFTPROC``) +* **Component UUID**: ``a6:6e:11:0d:50:91:de:46:98:b8:b2:b3:a7:91:da:29`` +* **Topology GUID**: ``a66e110d-5091-de46-98b8-b2b3a791da29`` + +ALSA Topology 2 Pipeline Definition +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In ALSA Topology 2, the widget is declared in ``tools/topology/topology2/include/components/stft_process.conf`` as type ``effect``. Standard benchmark pipelines include pre-configured binary configurations: + +* ``hann_192_48.conf``: 4 ms window (192 samples), 1 ms hop (48 samples) @ 48 kHz +* ``hann_512_128.conf``: 10.7 ms window (512 samples), 2.7 ms hop (128 samples) @ 48 kHz +* ``hann_768_120.conf``: 16 ms window (768 samples), 2.5 ms hop (120 samples) @ 48 kHz +* ``hann_1024_256.conf``: 21.3 ms window (1024 samples), 5.3 ms hop (256 samples) @ 48 kHz +* ``hann_1536_240.conf``: 32 ms window (1536 samples), 5 ms hop (240 samples) @ 48 kHz + +.. _figure_243: + +.. graphviz:: + :align: center + :caption: ALSA Topology 2 STFT Process Audio Pipeline Graph & IPC4 Configuration Dispatch + + digraph topology_pipeline { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="rounded,filled", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#94a3b8", penwidth=1.2]; + + host_copier [label="Host Copier Gateway\n(PCM Playback Stream)\nhost-copier.0", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + stft_widget [label="STFT Process Component\n(Spectral Filter / Engine)\nstft_process.1\nUUID: a6:6e:11:0d...", fillcolor="#6366f1", fontcolor="#ffffff", color="#a5b4fc"]; + vol [label="Main Volume Control\nvolume.2", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + dai_copier [label="DAI Copier Gateway\n(Physical Output / I2S / HDA)\ndai-copier.3", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + + blob_ctl [label="Bytes Mixer Control\n'$PCM STFT_PROCESS bytes'\n(64-Byte Config Blob)", fillcolor="#334155", fontcolor="#93c5fd", color="#60a5fa", style="dashed,filled"]; + octave_tool [label="Tuning Script\nsetup_stft_process.m\n(Octave Blob Exporter)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + + host_copier -> stft_widget [label="Audio Stream\n(48 kHz, Stereo)"]; + stft_widget -> vol [label="Spectrally Processed PCM\n(Reconstructed Time Series)"]; + vol -> dai_copier [label="Volume Scaled Audio"]; + + octave_tool -> blob_ctl [label="Generates Blob (.conf)"]; + blob_ctl -> stft_widget [style="dotted", color="#60a5fa", label="IPC4 LARGE_CONFIG"]; + } + +------------------------------------------------------------------------------- + +Factory Bringup, Verification & Test Runbook +-------------------------------------------- + +This runbook outlines procedures to generate configuration blobs, compile topologies, verify standalone testbench processing, and confirm audio quality on physical targets. + +1. Binary Blob Generation via GNU Octave +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Generate standard pre-computed configuration blobs: + +.. code-block:: bash + + # Step 1: Navigate to the tuning directory + cd src/audio/stft_process/tune + + # Step 2: Run Octave to produce topology configuration files + octave --no-gui setup_stft_process.m + + # Output files generated in tools/topology/topology2/include/components/stft_process/: + # - hann_192_48.conf + # - hann_512_128.conf + # - hann_768_120.conf + # - hann_1024_256.conf + # - hann_1536_240.conf + +2. Standalone Testbench Loopback Verification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Test the component within the SOF testbench without requiring target hardware: + +.. code-block:: bash + + # Define workspace root + export SOF_WORKSPACE=/home/lrg/work + + # Execute testbench with 16-bit PCM configuration + sof-testbench -p -i input_16k.wav -o output_stft_16k.wav \ + -t tools/topology/topology2/build/topology1/stft_process_s16.tplg + + # Execute testbench with 32-bit PCM configuration + sof-testbench -p -i input_48k.wav -o output_stft_48k.wav \ + -t tools/topology/topology2/build/topology1/stft_process_s32.tplg + +3. Real-Time Hardware Mixer Verification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On physical development hardware (such as Panther Lake, Arrow Lake, or Tiger Lake), verify runtime parameter delivery via SSH: + +.. code-block:: bash + + # Step 1: Query available mixer controls + ssh root@ "amixer -c0 controls | grep -i 'STFT_PROCESS'" + + # Step 2: Push a new window configuration blob + ssh root@ "amixer -c0 cset name='Analog Playback STFT_PROCESS bytes' < hann_1024_256.blob" + +4. Acoustic Reconstruction & Linearity Validation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When the STFT Process component runs in passthrough (zero spectral modification), the analysis and synthesis stages must achieve mathematically transparent reconstruction: + +1. **Sine Wave Invariance Test**: + Feed a pure 1000 Hz full-scale sine wave into the pipeline. +2. **Harmonic Distortion (THD+N)**: + Measure the reconstructed output using Audio Precision or Python FFT scripts. Total Harmonic Distortion must remain below :math:`-90\text{ dBFS}`, proving Constant Overlap-Add (COLA) compliance. +3. **Transient Response Test**: + Feed a single-sample unit impulse (:math:`\delta[n]`). The output waveform must reconstruct the exact impulse with zero pre-ringing, phase smearing, or amplitude attenuation beyond the inherent frame delay. diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 70b2c05d..d75e90b1 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -62,6 +62,7 @@ Audio Processing Modules & Algorithms * :ref:`aria` (High-level architecture; also see upstream `Aria README `_) * :ref:`level_multiplier` (High-level architecture; also see upstream `Level Multiplier README `_) * :ref:`phase_vocoder` (High-level architecture; also see upstream `Phase Vocoder source tree `_) +* :ref:`stft_process` (High-level architecture; also see upstream `STFT Process README `_) .. _algorithm-specific-information: @@ -113,6 +114,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/aria firmware/level_multiplier firmware/phase_vocoder + firmware/stft_process rimage/index.rst firmware/llext_modules firmware/hostless_firmware From b446deb6cb28a73a3e8c58ba867f4208b31c1562 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 15:11:14 +0100 Subject: [PATCH 30/64] doc: developer_guides: add comprehensive media codecs architecture guide Author an in-depth, authoritative architectural and developer guide for the SOF Media Codecs: Audio Encoders & Decoders subsystem: - Architectural foundations: DSP compress-offload, host CPU C10/D3 deep sleep residency, deep buffer DMA, and low-power audio pipeline comparison. - Cadence Xtensa Audio (XA) API standard and lifecycle state machine: size query, pre/post-config, memory negotiation, and execution protocol. - Standard memory management: 4 XA memory classes (PERSIST, SCRATCH, INPUT, OUTPUT), two-phase allocation, and circular buffer linearization. - Supported codecs: MP3 dec/enc (1152 samples), AAC ADTS dec (1024 samples), Vorbis, SBC, in-tree PCM reference (xa_pcm_dec.c with EOS safety counter), and DTS Audio spatializer integration (dts.c). - Control plane integration: legacy IPC3 snd_codec vs modern IPC4 unified module architecture, DP scheduling domain, and asynchronous unsolicited compress EOS notifications (SOF_IPC4_NOTIFY_MODULE_EVENTID_COMPR_MAGIC_VAL). - ALSA Topology 2 deep-buffer pipeline graph: decoder.conf, encoder.conf, compr-playback.conf, format adaptors, and mixin lp_mode 1. - Factory bringup runbook: tinycompress (cplay, crecord), capabilities query, turbostat C10 power telemetry verification, and testbench validation. - 7 native vector Graphviz SVG architecture diagrams (Figures 244-250). Signed-off-by: Liam Girdwood --- data/modules.yaml | 12 + developer_guides/firmware/media_codecs.rst | 764 +++++++++++++++++++++ developer_guides/index.rst | 2 + 3 files changed, 778 insertions(+) create mode 100644 developer_guides/firmware/media_codecs.rst diff --git a/data/modules.yaml b/data/modules.yaml index 2820031e..1b2c6d8f 100644 --- a/data/modules.yaml +++ b/data/modules.yaml @@ -341,6 +341,18 @@ modules: - "Low false-reject and false-alarm rates" # --- Codecs & Compression --- + - id: media_codecs + name: "Media Codecs (Cadence XA & Compress-Offload)" + source: "SOF / Cadence" + category: "Codecs & Compression" + status: "Upstream" + description: "Hardware-accelerated compressed audio offload decoders and encoders using the Cadence Xtensa Audio (XA) standard." + simd: ["HiFi 3", "HiFi 4", "HiFi 5", "Scalar C"] + key_features: + - "ALSA compress-offload playback (MP3, AAC, Vorbis, PCM passthrough) and capture (MP3 enc)" + - "Standardized Cadence Xtensa Audio (XA) four-class memory tables and state machine" + - "Deep-buffer DMA host wakeup suppression enabling prolonged C10 deep sleep" + - id: aac_dec name: "AAC Decoder" source: "FFmpeg" diff --git a/developer_guides/firmware/media_codecs.rst b/developer_guides/firmware/media_codecs.rst new file mode 100644 index 00000000..dc55f620 --- /dev/null +++ b/developer_guides/firmware/media_codecs.rst @@ -0,0 +1,764 @@ +.. _media_codecs: + +Media Codecs: Audio Encoders & Decoders Architecture +#################################################### + +The Sound Open Firmware (SOF) Media Codec subsystem enables hardware-accelerated, ultra-low-power compressed audio streaming directly within the digital signal processor (DSP). By offloading bitstream decoding and encoding tasks from the host central processing unit (CPU) to the audio DSP, the subsystem eliminates frequent host wakeups, allowing mobile and desktop host platforms to maintain prolonged deep sleep power states (:math:`C10 / D3`). + +The architecture standardizes on the industry-proven **Cadence Xtensa Audio (XA) API**, providing a hardware-abstracted, modular wrapper that interfaces seamlessly with Tensilica HiFi DSPs (HiFi 3, HiFi 4, and HiFi 5). The subsystem supports both compressed media playback (decoding MPEG-1 Layer 3 MP3, Advanced Audio Coding AAC, Ogg Vorbis, Bluetooth SBC, and uncompressed PCM passthrough) and compressed media capture (real-time MP3 encoding and compressed feature streaming), complemented by third-party post-processing codecs such as DTS Interactive audio. + +.. contents:: + :local: + :depth: 2 + +Foundations of DSP Compress-Offload & Low-Power Audio +***************************************************** + +The Energy Bottleneck in Traditional Audio Playback +=================================================== + +In conventional Pulse Code Modulation (PCM) audio pipelines, the host CPU decodes compressed media files (e.g., MP3 or AAC bitstreams) in user space and feeds raw uncompressed PCM samples to the audio device driver. Because uncompressed PCM data streams consume high memory bandwidth (typically 1.411 Mbps for standard 44.1 kHz 16-bit stereo), host Direct Memory Access (DMA) ring buffers can buffer only a few milliseconds of audio (typically 1 ms to 20 ms). + +This architectural constraint forces the host CPU to wake up dozens or hundreds of times per second to replenish DMA ring buffers. In battery-powered mobile devices and modern laptops, these recurring wakeups prevent the application processor and its system-on-chip (SoC) power planes from entering ultra-low-power residency states (:math:`C8/C10` CPU package states and :math:`D3` device states), burning tens to hundreds of milliwatts of unnecessary battery power. + +ALSA Compress-Offload Mechanics +=============================== + +The Sound Open Firmware Media Codec subsystem resolves this bottleneck through **ALSA Compress-Offload** (:c:struct:`snd_compress_ops`). Rather than decoding audio on the host CPU: + +1. **Massive Host Transfer Chunks**: The host operating system offloads raw, compressed bitstreams to the DSP in massive chunks (spanning 10 to 30 seconds of compressed playback per transfer). +2. **Deep-Sleep Host Residency**: After bursting the compressed bitstream across the host interface via deep-buffer DMA, the host CPU immediately enters a deep C-state (:math:`C10`). The host remains completely asleep while the DSP executes autonomous decoding. +3. **Autonomous DSP Streaming**: The DSP receives the bitstream in local SRAM, executes frame-by-frame bitstream parsing and synthesis, writes synthesized PCM samples into internal pipeline ring buffers, applies post-processing (sample rate conversion, channel mixing, and volume adjustment), and streams the final samples to the Digital Audio Interface (DAI) without host intervention. +4. **Asynchronous Replenishment**: Only when the DSP input ring buffer approaches an empty threshold does the DSP emit an interrupt or IPC message to wake the host CPU for the next bitstream burst. + +Architectural Comparison: Streaming Paradigms +============================================= + +.. table:: Architectural Comparison: Audio Playback Paradigms + :widths: 22 26 26 26 + + +-----------------------+-----------------------------+-----------------------------+-----------------------------+ + | Architectural Vector | Traditional Host PCM Stream | DSP Fast-Decode Streaming | DSP Compress-Offload | + +=======================+=============================+=============================+=============================+ + | **Host CPU State** | High-frequency wakeups | Intermittent wakeups | Extended deep sleep | + | | (1 ms - 10 ms ticks; C0/C1) | (every 100 ms to 500 ms) | (C10 package state 10s-30s) | + +-----------------------+-----------------------------+-----------------------------+-----------------------------+ + | **Data Transferred** | Uncompressed PCM | Partially decoded frames | Raw compressed bitstream | + | | (1.411 Mbps to 9.2 Mbps) | (variable bandwidth) | (128 kbps to 320 kbps) | + +-----------------------+-----------------------------+-----------------------------+-----------------------------+ + | **Host DMA Bursts** | Continuous trickle DMA | Periodic medium bursts | High-throughput deep burst | + | | (sub-millisecond intervals) | (100 ms buffer chunks) | (2 MB to 8 MB every 30s) | + +-----------------------+-----------------------------+-----------------------------+-----------------------------+ + | **Decoding Engine** | Host CPU (SW user space) | Host or DSP co-processor | DSP Tensilica HiFi Core | + | | | | (Cadence NatureDSP / XA) | + +-----------------------+-----------------------------+-----------------------------+-----------------------------+ + | **DSP Memory RAM** | Minimal (1 KB - 4 KB ring) | Moderate (8 KB - 16 KB) | High (16 KB - 64 KB SRAM) | + | | | | (Bitstream and State) | + +-----------------------+-----------------------------+-----------------------------+-----------------------------+ + | **System Power** | High (150 mW - 350 mW) | Moderate (80 mW - 150 mW) | Ultra-Low (< 25 mW - 45 mW) | + +-----------------------+-----------------------------+-----------------------------+-----------------------------+ + +.. graphviz:: + :caption: SOF Compress-Offload Architecture: Host CPU Power-Down Timeline, Deep Buffer DMA, and DSP Autonomous Decoding Core + :alt: SOF Compress-Offload Architecture and Power Saving Mechanics + + digraph SOF_Compress_Offload { + graph [bgcolor="#0A192F", fontname="DejaVu Sans", fontsize=11, rankdir=TB, splines=spline, pad=0.3]; + node [shape=box, style="filled,rounded", fontname="DejaVu Sans", fontsize=10, penwidth=1.5]; + edge [fontname="DejaVu Sans", fontsize=9, penwidth=1.2, color="#38BDF8"]; + + subgraph cluster_host { + label="Host Linux Operating System (ALSA Compress-Offload Layer)"; + style="filled,rounded"; + color="#1E3A8A"; + fillcolor="#0F172A"; + fontcolor="#93C5FD"; + + user_app [label="Audio Player Application\n(tinycompress / cplay / PipeWire)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + snd_compr [label="ALSA Compress Core\n(snd_compress_ops / /dev/snd/comprC*D*)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + host_power [label="Host Power Plane State\nCPU Package C10 (Deep Sleep)", fillcolor="#065F46", fontcolor="#34D399", color="#10B981"]; + deep_dma [label="Host Copier DMA Controller\nBurst Transfers (2 MB - 8 MB Chunks)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + } + + subgraph cluster_dsp { + label="DSP Firmware (Sound Open Firmware Runtime)"; + style="filled,rounded"; + color="#047857"; + fillcolor="#064E3B"; + fontcolor="#A7F3D0"; + + in_ring [label="Compressed Bitstream Ring Buffer\n(Deep Buffer DMA Ingress: 16 KB - 64 KB)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#34D399"]; + xa_wrapper [label="Cadence Codec Adapter Layer\n(cadence.c / XA API Dispatcher)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#34D399"]; + + subgraph cluster_codecs { + label="Tensilica HiFi NatureDSP Codec Binaries"; + style="filled,rounded"; + color="#059669"; + fillcolor="#022C22"; + fontcolor="#6EE7B7"; + + mp3_dec [label="MP3 Decoder (xa_mp3_dec)\n1152 Samples/Frame", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + aac_dec [label="AAC Decoder (xa_aac_dec)\n1024 Samples/Frame (ADTS)", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + vorbis_dec [label="Vorbis Decoder (xa_vorbis_dec)\nVBR / Packed Codebooks", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + pcm_dec [label="PCM Ref Decoder (xa_pcm_dec)\nIn-Tree Open-Source Fallback", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + } + + out_ring [label="Synthesized PCM Ring Buffer\n(Uncompressed Interleaved S16/S32)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#34D399"]; + post_proc [label="Post-Processing Pipeline\n(Module-Copier -> SRC -> Selector -> Gain -> Mixin)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#34D399"]; + dai_copier [label="DAI Copier Output Gateway\n(I2S / SoundWire / HDA Endpoint)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#34D399"]; + } + + user_app -> snd_compr [label="cplay bitstream"]; + snd_compr -> deep_dma [label="Burst DMA Write"]; + deep_dma -> host_power [label="Trigger C10 Entry", style=dashed, color="#10B981"]; + deep_dma -> in_ring [label="PCIe / HDA Bus DMA", color="#F59E0B", penwidth=2.0]; + + in_ring -> xa_wrapper [label="Circular Unpack"]; + xa_wrapper -> mp3_dec [label="API Command"]; + xa_wrapper -> aac_dec; + xa_wrapper -> vorbis_dec; + xa_wrapper -> pcm_dec; + + mp3_dec -> out_ring [label="Linear Pack"]; + aac_dec -> out_ring; + vorbis_dec -> out_ring; + pcm_dec -> out_ring; + + out_ring -> post_proc [label="Audio Frames"]; + post_proc -> dai_copier [label="48 kHz S32_LE"]; + in_ring -> snd_compr [label="Low Watermark IPC Wakeup", style=dashed, color="#F43F5E", constraint=false]; + } + +Cadence Xtensa Audio (XA) API Standard & State Machine +****************************************************** + +NatureDSP Abstraction Architecture +================================== + +Cadence Tensilica HiFi DSPs execute proprietary, highly vectorized audio codec libraries optimized with hand-crafted SIMD assembly. To prevent tight coupling between the SOF audio infrastructure and vendor-specific codec binaries, SOF adopts the standardized **Cadence Xtensa Audio (XA) API** (:c:type:`xa_codec_func_t`). + +The XA standard enforces a unified function prototype across every codec family: + +.. math:: + + \text{XA\_ERRORCODE}\quad \text{api\_func}(\text{xa\_codec\_handle\_t}\; \text{handle},\; \text{WORD32}\; \text{cmd},\; \text{WORD32}\; \text{idx},\; \text{pVOID}\; \text{value}) + +This abstraction guarantees that the SOF module adapter (:file:`cadence.c`, :file:`cadence_ipc3.c`, :file:`cadence_ipc4.c`) interacts with every decoder and encoder through a clean, uniform command protocol regardless of internal algorithm complexity. + +Standardized Lifecycle Commands & Protocol Execution +==================================================== + +The XA execution lifecycle progresses through four deterministic phases: + +1. **Size Query & Identification**: + - ``XA_API_CMD_GET_API_SIZE``: Returns the exact byte count required for the persistent codec instance object (``cd->self``). + - ``XA_API_CMD_GET_LIB_ID_STRINGS`` (with sub-command ``XA_CMD_TYPE_LIB_NAME``): Queries the human-readable ASCII name of the underlying library for logging and diagnostics. +2. **Pre-Configuration & Memory Table Negotiation**: + - ``XA_API_CMD_INIT`` (sub-command ``XA_CMD_TYPE_INIT_API_PRE_CONFIG_PARAMS``): Initializes internal codec state variables to compile-time defaults. + - ``XA_API_CMD_INIT`` (sub-command ``XA_CMD_TYPE_INIT_API_POST_CONFIG_PARAMS``): Calculates the required sizes and alignment constraints of all external memory tables. + - ``XA_API_CMD_GET_N_MEMTABS``: Queries the total number of distinct memory tables required by the codec algorithm. + - ``XA_API_CMD_GET_MEM_INFO_TYPE`` / ``SIZE`` / ``ALIGNMENT``: Iterates across all memory tables to inspect usage types and alignment boundaries. + - ``XA_API_CMD_SET_MEM_PTR``: Binds allocated physical DSP SRAM blocks back to the codec handle. +3. **Runtime Configuration & Process Initialization**: + - ``XA_API_CMD_SET_CONFIG_PARAM``: Configures bitstream properties (e.g., bit depth, sampling frequency, channel count, and bitstream format such as ADTS). + - ``XA_API_CMD_SET_INPUT_BYTES``: Informs the codec of the exact number of valid encoded bytes staged in the input buffer. + - ``XA_API_CMD_INIT`` (sub-command ``XA_CMD_TYPE_INIT_PROCESS``): Consumes the initial bitstream header (e.g., ID3 tags, ADTS headers, or sync words) to initialize the parsing engine. + - ``XA_API_CMD_INIT`` (sub-command ``XA_CMD_TYPE_INIT_DONE_QUERY``): Queries whether the codec has completed stream synchronization and is prepared to output synthesized audio. +4. **Execution & End-of-Stream Handling**: + - ``XA_API_CMD_EXECUTE`` (sub-command ``XA_CMD_TYPE_DO_EXECUTE``): Executes the primary mathematical decoding/encoding transform over one audio frame. + - ``XA_API_CMD_EXECUTE`` (sub-command ``XA_CMD_TYPE_DONE_QUERY``): Verifies whether the current frame processing completed successfully. + - ``XA_API_CMD_GET_OUTPUT_BYTES``: Queries the count of valid uncompressed PCM bytes generated in the output buffer. + - ``XA_API_CMD_GET_CURIDX_INPUT_BUF``: Queries the byte offset in the input buffer indicating how many encoded bytes were consumed. + - ``XA_API_CMD_INPUT_OVER``: Explicitly signals to the codec that the upstream stream has ended, enabling proper flushing of synthesis filterbanks without truncation. + +.. graphviz:: + :caption: Cadence Xtensa Audio (XA) Codec Lifecycle State Machine & Execution Handshake + :alt: Cadence XA Codec Lifecycle State Machine + + digraph XA_State_Machine { + graph [bgcolor="#0A192F", fontname="DejaVu Sans", fontsize=11, rankdir=TB, splines=ortho, pad=0.3]; + node [shape=box, style="filled,rounded", fontname="DejaVu Sans", fontsize=10, penwidth=1.5]; + edge [fontname="DejaVu Sans", fontsize=9, penwidth=1.2, color="#38BDF8"]; + + init_uninit [label="STATE: UNINITIALIZED\n(DSP SRAM allocated)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + get_size [label="XA_API_CMD_GET_API_SIZE\nAllocate cd->self Handle", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + pre_cfg [label="INIT_API_PRE_CONFIG_PARAMS\nReset Default State", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + post_cfg [label="INIT_API_POST_CONFIG_PARAMS\nCalculate Memtab Requirements", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + mem_alloc [label="Iterate GET_N_MEMTABS\nmod_alloc_align(type, size, align)\nSET_MEM_PTR(i, ptr)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#34D399"]; + set_params [label="SET_CONFIG_PARAM\n(Bit depth, Channels, Rate, Format)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + init_proc [label="INIT_PROCESS\nConsume Bitstream Header / Sync", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#F59E0B"]; + query_done [label="INIT_DONE_QUERY\nIs Parser Ready?", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B", shape=diamond]; + exec_loop [label="EXECUTE_DO_EXECUTE\nDecode / Encode Frame Transform", fillcolor="#065F46", fontcolor="#34D399", color="#10B981"]; + query_exec [label="GET_OUTPUT_BYTES / GET_CURIDX\nUpdate PCM Produced & Bitstream Consumed", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#34D399"]; + input_over [label="XA_API_CMD_INPUT_OVER\nPipeline Signals End of Stream", fillcolor="#831843", fontcolor="#FDA4AF", color="#F43F5E"]; + eos_drain [label="Drain Trailing Filterbank Samples\nEmit IPC4 EOS Notification", fillcolor="#831843", fontcolor="#FDA4AF", color="#F43F5E"]; + + init_uninit -> get_size -> pre_cfg -> post_cfg -> mem_alloc -> set_params -> init_proc -> query_done; + query_done -> init_proc [label="Not Ready (More Data)", color="#F59E0B"]; + query_done -> exec_loop [label="Ready (init_done == 1)", color="#34D399"]; + exec_loop -> query_exec; + query_exec -> exec_loop [label="Next Audio Frame", color="#38BDF8"]; + query_exec -> input_over [label="expect_eos == true", color="#F43F5E"]; + input_over -> eos_drain; + } + +Standard Memory Management & Buffer Partitioning +************************************************ + +The Four XA Memory Classes +========================== + +To achieve deterministic memory safety and eliminate run-time heap allocations in real-time execution, Cadence XA codecs categorize all required memory blocks into four standardized usage classes: + +.. table:: Cadence Xtensa Audio Memory Classes + :widths: 20 20 60 + + +-----------------------+-------------------------+-------------------------------------------------------------+ + | Memory Class Macro | Storage Scope | Architectural Purpose & Lifetime | + +=======================+=========================+=============================================================+ + | ``XA_MEMTYPE_PERSIST``| Persistent DSP Memory | Holds internal filterbank delay lines, Huffman decode trees,| + | | | quantization tables, and channel inter-frame state. Must | + | | | remain untouched across consecutive frame processing calls. | + +-----------------------+-------------------------+-------------------------------------------------------------+ + | ``XA_MEMTYPE_SCRATCH``| Scratchpad Working RAM | Temporary calculation workspace used during FFTs, IMDCTs, | + | | | subband filter evaluations, and bitstream unpacking. Reused | + | | | safely by other modules when this codec is not executing. | + +-----------------------+-------------------------+-------------------------------------------------------------+ + | ``XA_MEMTYPE_INPUT`` | Bitstream Input Staging | Contiguous linear memory block holding incoming compressed | + | | | audio bytes presented to the codec parser. | + +-----------------------+-------------------------+-------------------------------------------------------------+ + | ``XA_MEMTYPE_OUTPUT`` | Synthesized PCM Output | Contiguous linear memory block where the codec writes raw | + | | | reconstructed PCM sample words before commitment to sink. | + +-----------------------+-------------------------+-------------------------------------------------------------+ + +Two-Phase Dynamic Memory Allocation +=================================== + +During component initialization in :c:func:`cadence_codec_init_memory_tables`, SOF executes a strict two-phase memory negotiation: + +1. **Table Metadata Query**: The component queries ``XA_API_CMD_GET_N_MEMTABS``, allocates an array of tracking pointers (``cd->mem_to_be_freed``), and iterates through each table index: + + .. code-block:: c + + API_CALL(cd, XA_API_CMD_GET_MEM_INFO_TYPE, i, &mem_type, ret); + API_CALL(cd, XA_API_CMD_GET_MEM_INFO_SIZE, i, &mem_size, ret); + API_CALL(cd, XA_API_CMD_GET_MEM_INFO_ALIGNMENT, i, &mem_alignment, ret); + +2. **Aligned Allocation & Binding**: Memory is allocated via SOF's aligned allocator (:c:func:`mod_alloc_align`), ensuring strict SIMD data alignment (typically 8-byte, 16-byte, or 64-byte boundaries for 128-bit Tensilica vector loads). The allocated pointer is then assigned back to the codec: + + .. code-block:: c + + ptr = mod_alloc_align(mod, mem_size, mem_alignment); + API_CALL(cd, XA_API_CMD_SET_MEM_PTR, i, ptr, ret); + +Circular Buffer Boundary Resolution (Linearization) +=================================================== + +The Sound Open Firmware audio pipeline operates natively on **circular ring buffers** (:c:struct:`sof_audio_buffer`), where read and write pointers advance modulo the buffer boundary. However, external codec binaries (such as MP3 and AAC decoders) require strictly **linear contiguous buffers** for bitstream parsing and PCM generation. + +To resolve this impedance mismatch without expensive heap allocations or copying overhead, SOF implements split-copy linearization functions (:c:func:`cadence_copy_data_from_buffer` and :c:func:`cadence_copy_data_to_buffer`): + +.. math:: + + \text{bytes\_to\_end} = \text{buffer\_start} + \text{buffer\_size} - \text{buffer\_ptr} + +- **Non-Wrapping Case** (:math:`\text{bytes\_to\_end} \ge \text{bytes\_to\_copy}`): The entire quantum is transferred in a single direct contiguous copy (:c:func:`memcpy_s`). +- **Wrapping Case** (:math:`\text{bytes\_to\_end} < \text{bytes\_to\_copy}`): The transfer is segmented into two sub-copies: + 1. Transfer :math:`\text{bytes\_to\_end}` from the current pointer up to the ring buffer boundary. + 2. Transfer the remaining :math:`\text{bytes\_to\_copy} - \text{bytes\_to\_end}` from the base address of the ring buffer. + +This guarantees that external codec engines always observe linear contiguous input and output arrays while preserving zero copy-buffer fragmentation across the SOF circular audio graph. + +.. graphviz:: + :caption: Cadence Codec Memory Architecture: Four-Class Allocation Tables & Circular Buffer Linearization Engine + :alt: Cadence Codec Memory Allocation and Linearization + + digraph Codec_Memory { + graph [bgcolor="#0A192F", fontname="DejaVu Sans", fontsize=11, rankdir=LR, splines=spline, pad=0.3]; + node [shape=box, style="filled,rounded", fontname="DejaVu Sans", fontsize=10, penwidth=1.5]; + edge [fontname="DejaVu Sans", fontsize=9, penwidth=1.2, color="#38BDF8"]; + + subgraph cluster_ring_in { + label="SOF Circular Input Buffer (Host Ingress)"; + style="filled,rounded"; + color="#1E3A8A"; + fillcolor="#0F172A"; + fontcolor="#93C5FD"; + + ring_head [label="Tail Unread Data\n[wrap_addr .. end_addr]", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + ring_wrap [label="Head New Data\n[base_addr .. wrap_len]", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + } + + subgraph cluster_linearizer { + label="Split-Copy Linearization Engine"; + style="filled,rounded"; + color="#047857"; + fillcolor="#064E3B"; + fontcolor="#A7F3D0"; + + split_unpack [label="cadence_copy_data_from_buffer()\nPart 1: bytes_to_end -> linear[0]\nPart 2: remaining -> linear[bytes_to_end]", fillcolor="#1E293B", fontcolor="#34D399", color="#10B981"]; + split_pack [label="cadence_copy_data_to_buffer()\nUnpack Linear Out Buff -> Circular Sink", fillcolor="#1E293B", fontcolor="#34D399", color="#10B981"]; + } + + subgraph cluster_memtabs { + label="Cadence XA Memory Tables (mod_alloc_align)"; + style="filled,rounded"; + color="#D97706"; + fillcolor="#451A03"; + fontcolor="#FDE68A"; + + tab_persist [label="XA_MEMTYPE_PERSIST\nFilterbank History / IMDCT State\n(Dedicated DSP L2 SRAM)", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + tab_scratch [label="XA_MEMTYPE_SCRATCH\nSubband Scratchpad Workspace\n(Shared / Overlay DSP RAM)", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + tab_input [label="XA_MEMTYPE_INPUT (mpd.in_buff)\nLinearized Bitstream Chunk\n(16 KB Alignment)", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + tab_output [label="XA_MEMTYPE_OUTPUT (mpd.out_buff)\nLinear PCM Output Frame\n(1152 / 1024 Samples)", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + } + + subgraph cluster_ring_out { + label="SOF Circular Output Buffer (Sink Egress)"; + style="filled,rounded"; + color="#1E3A8A"; + fillcolor="#0F172A"; + fontcolor="#93C5FD"; + + sink_buf [label="Downstream Circular Audio Ring\n(SRC / Selector / Gain Ingress)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + } + + ring_head -> split_unpack [label="Pass 1"]; + ring_wrap -> split_unpack [label="Pass 2"]; + split_unpack -> tab_input [label="Contiguous Linear Bitstream"]; + + tab_input -> tab_persist [style=invis]; + tab_persist -> tab_scratch [style=invis]; + + tab_output -> split_pack [label="Contiguous PCM Samples"]; + split_pack -> sink_buf [label="Committed Audio Frames"]; + } + +Supported Codecs, Encoders & In-Tree Reference Modules +****************************************************** + +Codec Family Dispatch Architecture +================================== + +The SOF Media Codec subsystem uses a unified registry table (``cadence_api_table[]`` of :c:struct:`cadence_api`) mapping ALSA compression codec identifiers (``SND_AUDIOCODEC_*``) and direction flags to concrete NatureDSP API dispatch pointers: + +.. table:: Supported Codec Matrix & Framing Specifications + :widths: 15 15 15 20 35 + + +-----------------------+-------------------+-----------------+-----------------------+-------------------------------------------------------------+ + | Codec Standard | API Identifier | Direction | Frame Size (Samples) | Algorithmic Characteristics & Bitstream Formatting | + +=======================+===================+=================+=======================+=============================================================+ + | **MPEG-1 Layer 3** | ``MP3_DEC_ID`` | Playback | 1152 samples / frame | Subband hybrid filterbank (32 bands), MDCT, Huffman coding, | + | **(MP3 Decoder)** | (``0x06``) | (Decoding) | (at 44.1/48 kHz) | bit reservoir, 16/24-bit PCM output. | + +-----------------------+-------------------+-----------------+-----------------------+-------------------------------------------------------------+ + | **MPEG-1 Layer 3** | ``MP3_ENC_ID`` | Capture | 1152 samples / frame | Real-time psychoacoustic masking model, bit reservoir, | + | **(MP3 Encoder)** | (``0x0A``) | (Encoding) | | configurable bitrates (default 320 kbps), 16-bit PCM input. | + +-----------------------+-------------------+-----------------+-----------------------+-------------------------------------------------------------+ + | **Advanced Audio** | ``AAC_DEC_ID`` | Playback | 1024 samples / frame | MPEG-4 Audio Data Transport Stream (ADTS) bitstream format, | + | **Coding (AAC)** | (``0x02``) | (Decoding) | (960 in LD mode) | temporal noise shaping (TNS), spectral band replication. | + +-----------------------+-------------------+-----------------+-----------------------+-------------------------------------------------------------+ + | **Ogg Vorbis** | ``VORBIS_DEC_ID`` | Playback | Dynamic block sizes | Variable Bitrate (VBR), MDCT filterbanks, vector | + | **(Vorbis Decoder)** | (``0x08``) | (Decoding) | (64 to 8192 samples) | quantization codebooks packed in bitstream headers. | + +-----------------------+-------------------+-----------------+-----------------------+-------------------------------------------------------------+ + | **Bluetooth SBC** | ``SBC_DEC_ID`` | Playback | 4, 8, 12, 16 blocks | Subband coding, 4 or 8 subbands, loudness/SNR bit allocation| + | **(SBC Decoder)** | (``0x07``) | (Decoding) | (up to 128 samples) | for A2DP Bluetooth audio sinks. | + +-----------------------+-------------------+-----------------+-----------------------+-------------------------------------------------------------+ + | **PCM Reference** | ``PCM_DEC_ID`` | Playback | Configurable buffer | In-tree open-source reference module implementing Cadence | + | **(Passthrough Dec)** | (``0xC0``) | (Decoding) | (up to 16 KB output) | XA API for uncompressed compress-offload & CI regression. | + +-----------------------+-------------------+-----------------+-----------------------+-------------------------------------------------------------+ + | **DTS Interactive** | Dedicated UUID | Playback | Frame aligned | Multi-channel surround virtualization, dynamic dialog | + | **(DTS Virtual:X)** | (``0x4fc3...``) | (Effect / Proc) | (2048-byte byte-ctl) | enhancement, and psychoacoustic speaker tuning. | + +-----------------------+-------------------+-----------------+-----------------------+-------------------------------------------------------------+ + +In-Tree Open-Source PCM Decoder Reference (:file:`xa_pcm_dec.c`) +================================================================ + +To allow development, continuous integration (CI) testing, and automated unit testing without requiring proprietary NatureDSP static binary blobs, SOF includes a reference in-tree implementation of the Cadence XA API: **PCM Decoder** (:file:`xa_pcm_dec.c`). + +The PCM decoder advertises complete conformance to the XA command standard: +- Implements :c:func:`xa_pcm_dec` responding to all commands (``GET_API_SIZE``, ``INIT``, ``EXECUTE``, ``SET_CONFIG_PARAM``). +- Manages an internal state machine (:c:struct:`struct xa_pcm_dec_state`) with 16 KB input and output buffers (``PCM_DEC_IN_BUF_SIZE = 16384``). +- Implements a dedicated **End-of-Stream Safety Counter** (``PCM_DEC_EOS_FULL_BUF_COUNT = 12``): Because raw uncompressed PCM bitstreams contain no internal syntactic markers (such as MP3 frame syncs or AAC ADTS headers) to denote the end of valid data, the fallback counter detects trailing repeated buffers following an ``input_over`` command, preventing infinite decode loops and cleanly triggering pipeline EOS termination. + +Third-Party Audio Codec Integration: DTS Audio (:file:`dts.c`) +============================================================== + +Beyond standard lossy bitstream decoders, the SOF codec subsystem integrates specialized post-processing and spatializer codecs, exemplified by the **DTS Audio Processing** module (:file:`src/audio/codec/dts/dts.c`). + +The DTS integration adheres to the module adapter framework: +- Wraps the vendor interface (:c:struct:`DtsSofInterface`) with standard SOF component callbacks (:c:func:`dts_effect_init`, :c:func:`dts_effect_prepare`, :c:func:`dts_effect_process`). +- Operates as an audio effect widget (:file:`dts.conf`, UUID ``4f:c3:5f:d9:0f:37:c7:4a:bc:86:bf:dc:5b:e2:41:e6``). +- Exposes a 2048-byte runtime byte control (``extctl``, get/put handler ``258``) for dynamic sound profile switching, virtual surround configuration, and speaker calibration parameters. +- Supports both static compilation and dynamic relocatable module packaging via Zephyr Loadable Linkable Extensions (LLEXT). + +.. graphviz:: + :caption: Codec Engine Architecture: Multi-Format Dispatcher, Frame Sizing & In-Tree PCM Reference Wrapper + :alt: Codec Engine Multi-Format Architecture + + digraph Codec_Engines { + graph [bgcolor="#0A192F", fontname="DejaVu Sans", fontsize=11, rankdir=TB, splines=spline, pad=0.3]; + node [shape=box, style="filled,rounded", fontname="DejaVu Sans", fontsize=10, penwidth=1.5]; + edge [fontname="DejaVu Sans", fontsize=9, penwidth=1.2, color="#38BDF8"]; + + compr_id [label="Incoming ALSA Stream Config\n(snd_codec.id & direction)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + api_resolver [label="API Resolver: cadence_codec_get_api_id()\nDirection Demux: Playback (Dec) vs Capture (Enc)", fillcolor="#1E293B", fontcolor="#34D399", color="#10B981"]; + + subgraph cluster_dispatch { + label="Cadence API Registry (cadence_api_table[])"; + style="filled,rounded"; + color="#3B82F6"; + fillcolor="#1E3A8A"; + fontcolor="#DBEAFE"; + + disp_mp3_dec [label="CADENCE_CODEC_MP3_DEC_ID\n(xa_mp3_dec) | 1152 Samples", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + disp_mp3_enc [label="CADENCE_CODEC_MP3_ENC_ID\n(xa_mp3_enc) | 1152 Samples / 320 kbps", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + disp_aac_dec [label="CADENCE_CODEC_AAC_DEC_ID\n(xa_aac_dec) | 1024 Samples / ADTS", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + disp_vorbis [label="CADENCE_CODEC_VORBIS_DEC_ID\n(xa_vorbis_dec) | Dynamic VBR", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + disp_pcm_dec [label="SOF_COMPRESS_CODEC_PCM_DEC_ID\n(xa_pcm_dec) | In-Tree Open Source", fillcolor="#1E293B", fontcolor="#6EE7B7", color="#34D399"]; + disp_dts [label="DTS Audio Processing\n(dts.c / DtsSofInterface) | Virtual:X", fillcolor="#1E293B", fontcolor="#F472B6", color="#EC4899"]; + } + + subgraph cluster_out_modes { + label="Output Synthesis & Packaging"; + style="filled,rounded"; + color="#047857"; + fillcolor="#064E3B"; + fontcolor="#A7F3D0"; + + pcm_16 [label="16-Bit Signed Integer PCM\n(S16_LE / S24_4LE container)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#34D399"]; + pcm_32 [label="32-Bit Signed Integer PCM\n(S32_LE / 48 kHz post-resample)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#34D399"]; + enc_bitstream [label="Compressed Capture Stream\n(MP3 Bitstream / MFCC AI Features)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#F59E0B"]; + } + + compr_id -> api_resolver; + api_resolver -> disp_mp3_dec [label="SND_AUDIOCODEC_MP3 + Playback"]; + api_resolver -> disp_mp3_enc [label="SND_AUDIOCODEC_MP3 + Capture"]; + api_resolver -> disp_aac_dec [label="SND_AUDIOCODEC_AAC"]; + api_resolver -> disp_vorbis [label="SND_AUDIOCODEC_VORBIS"]; + api_resolver -> disp_pcm_dec [label="SND_AUDIOCODEC_PCM / Stubs"]; + api_resolver -> disp_dts [label="DTS Effect UUID"]; + + disp_mp3_dec -> pcm_16; + disp_aac_dec -> pcm_16; + disp_vorbis -> pcm_16; + disp_pcm_dec -> pcm_16; + disp_dts -> pcm_32; + disp_mp3_enc -> enc_bitstream; + } + +Control Plane Integration (IPC3 vs IPC4) & Asynchronous Notifications +********************************************************************** + +IPC3 Compress Interface Model +============================= + +In the legacy SOF IPC3 protocol, compressed audio parameters are delivered during stream initialization via the stream configuration blob (:c:struct:`sof_ipc_stream_params`). The extended data payload (``stream_params->ext_data``) conveys the raw Linux :c:struct:`snd_codec` structure: + +- Codec ID selection occurs statically during stream preparation (:c:func:`cadence_codec_resolve_api`). +- IPC3 compress streaming is restricted exclusively to **playback** directions (:c:macro:`SOF_IPC_STREAM_PLAYBACK`). +- Control adjustments (volume, mute) are handled by separate downstream volume components rather than direct codec parameter updates. + +IPC4 Unified Module Architecture +================================ + +Under the modern Intel IPC4 architecture, media codecs are treated as first-class processing modules adhering to the unified IPC4 lifecycle: + +1. **Initialization Payload**: The host passes initialization metadata via :c:struct:`module_ext_init_data`. The payload packs the complete :c:struct:`snd_codec` structure immediately followed by a 32-bit stream direction word (``cd->direction``). +2. **Direction Flexibility**: Full support for both playback (:c:macro:`SOF_IPC_STREAM_PLAYBACK`) and capture (:c:macro:`SOF_IPC_STREAM_CAPTURE`), enabling real-time on-DSP encoding pipelines. +3. **Runtime Parameter Updates**: Runtime bitrate or channel mode updates are delivered via Large Config Set messages, parsed and dispatched through :c:func:`cadence_codec_apply_params`. +4. **Data Processing (DP) Scheduling Domain**: To ensure that computationally heavy decompression does not jitter ultra-low-latency real-time pipeline tasks (such as microphone beamforming), decoders are scheduled within the Data Processing (``"DP"``) domain, running cooperatively on secondary DSP cores or lower thread priorities. + +Asynchronous End-of-Stream (EOS) Notification Model +=================================================== + +A critical challenge in compressed playback is determining when the stream has terminated. In PCM streams, the host driver tracks exact sample playback positions. In compressed streams, however, because frame byte lengths vary dynamically, the host CPU cannot know when the last bitstream packet has been decoded without continuous polling. + +To solve this, SOF implements an **Asynchronous Unsolicited Notification Pipeline**: + +1. **Pre-Allocated Notification Template**: During module initialization (:c:func:`cadence_codec_notification_init`), SOF pre-allocates an IPC message container: + + .. code-block:: c + + primary.r.notif_type = SOF_IPC4_MODULE_NOTIFICATION; + primary.r.type = SOF_IPC4_GLB_NOTIFICATION; + primary.r.msg_tgt = SOF_IPC4_MESSAGE_TARGET_FW_GEN_MSG; + +2. **Event Magic Value**: The message payload binds the unique component ID with the compressed audio termination event: + + .. code-block:: c + + msg_module_data->event_id = SOF_IPC4_NOTIFY_MODULE_EVENTID_COMPR_MAGIC_VAL; + +3. **Autonomous Firing**: When the pipeline flags ``dev->pipeline->expect_eos`` and the codec signals completion (either via ``codec->mpd.produced == 0`` or ``XA_API_CMD_EXECUTE_DONE_QUERY``), SOF transmits the notification asynchronously (:c:func:`ipc_msg_send`). +4. **Pipeline EOS Propagation**: Simultaneously, SOF asserts the end-of-stream flag on the downstream sink buffer (:c:func:`audio_buffer_set_eos`), ensuring trailing samples flush through downstream SRC, volume, and mixer components without truncation or underrun clicks. + +.. graphviz:: + :caption: IPC4 Control Architecture, Codec Configuration Dispatch & Asynchronous EOS Event Pipeline + :alt: IPC4 Codec Control and Notification Architecture + + digraph IPC4_Control { + graph [bgcolor="#0A192F", fontname="DejaVu Sans", fontsize=11, rankdir=TB, splines=spline, pad=0.3]; + node [shape=box, style="filled,rounded", fontname="DejaVu Sans", fontsize=10, penwidth=1.5]; + edge [fontname="DejaVu Sans", fontsize=9, penwidth=1.2, color="#38BDF8"]; + + subgraph cluster_host_ipc { + label="Host ALSA Driver (sound/soc/sof/compress.c)"; + style="filled,rounded"; + color="#1E3A8A"; + fillcolor="#0F172A"; + fontcolor="#93C5FD"; + + host_init [label="INIT_INSTANCE IPC4 Msg\n(snd_codec + Direction Word)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + host_large_cfg [label="LARGE_CONFIG_SET IPC4 Msg\n(Runtime Bitrate / Mode Adjust)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + host_eos_handler [label="Unsolicited Notification Handler\nWake ALSA cplay / Trigger Drain Done", fillcolor="#065F46", fontcolor="#34D399", color="#10B981"]; + } + + subgraph cluster_dsp_ipc { + label="SOF DSP Module Framework"; + style="filled,rounded"; + color="#047857"; + fillcolor="#064E3B"; + fontcolor="#A7F3D0"; + + mod_init [label="cadence_codec_init()\nUnpack snd_codec & Resolve API", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#34D399"]; + notif_init [label="cadence_codec_notification_init()\nPre-allocate SOF_IPC4_GLB_NOTIFICATION\nMagic: EVENTID_COMPR_MAGIC_VAL", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#34D399"]; + cfg_apply [label="cadence_codec_apply_config()\nMap Parameters to XA Config IDs", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#34D399"]; + eos_detector [label="cadence_codec_process()\nDetects expect_eos && (done || produced == 0)", fillcolor="#831843", fontcolor="#FDA4AF", color="#F43F5E"]; + notif_sender [label="ipc_msg_send(cd->msg)\nAsynchronous Unsolicited Push to Host", fillcolor="#831843", fontcolor="#FDA4AF", color="#F43F5E"]; + buf_eos [label="audio_buffer_set_eos(sink)\nFlush Trailing Pipeline Samples", fillcolor="#065F46", fontcolor="#34D399", color="#10B981"]; + } + + host_init -> mod_init [label="IPC4 Pipeline Init", color="#38BDF8"]; + mod_init -> notif_init; + host_large_cfg -> cfg_apply [label="Large Config Blob", color="#F59E0B"]; + cfg_apply -> eos_detector [style=invis]; + + eos_detector -> notif_sender [label="Bitstream Exhausted", color="#F43F5E"]; + eos_detector -> buf_eos [label="Set Sink EOS", color="#10B981"]; + notif_sender -> host_eos_handler [label="GLB_NOTIFICATION Message", color="#F43F5E", constraint=false]; + } + +ALSA Topology 2 Integration & Deep Buffer Playback Pipeline +*********************************************************** + +Topology 2 Widget Definitions +============================= + +ALSA Topology 2 modularizes codec components through dedicated Class.Widget definitions: + +- **Decoder Widget** (:file:`tools/topology/topology2/include/components/decoder.conf`): Declares the primary decompression block with type ``"decoder"`` and UUID ``43:84:21:d8:f3:5f:4c:4a:b3:88:6c:fe:07:b9:56:aa``. Configures 1 input pin and 1 output pin, disabling dynamic power management (``no_pm "true"``) to preserve persistent state. +- **Encoder Widget** (:file:`tools/topology/topology2/include/components/encoder.conf`): Declares the real-time compression block with type ``"encoder"``, sharing the Cadence codec UUID to invoke the capture path. +- **DTS Codec Widget** (:file:`tools/topology/topology2/include/components/dts.conf`): Declares the DTS post-processing engine (UUID ``4f:c3:5f:d9:0f:37:c7:4a:bc:86:bf:dc:5b:e2:41:e6``), binding external byte controls with handler ID ``258``. + +Low-Power Deep-Buffer Pipeline Architecture +=========================================== + +In production topologies (such as :file:`tools/topology/topology2/include/pipelines/cavs/compr-playback.conf` and :file:`platform/intel/compr.conf`), the decoder is assembled into a specialized multi-stage, low-power playback graph: + +1. **Host Copier Ingress**: Configured with deep-buffer DMA (``$COMPR_DEEPBUFFER_MS``, typically 2000 ms to 4000 ms), accommodating massive compressed bitstream bursts. +2. **Decoder Engine**: Bound to the Data Processing (``"DP"``) scheduling domain and assigned to secondary DSP Core 1, isolating high-compute decompression from latency-critical audio mixing. +3. **Module Copier (Format Adaptor)**: Normalizes output PCM samples into standard 32-bit signed containers (:math:`S32\_LE`). +4. **Sample Rate Converter (SRC)**: Resamples variable decoded rates (e.g. 44.1 kHz CD audio) to the system-wide fixed hardware mixing frequency (48 kHz or 96 kHz). +5. **Channel Selector / Matrix**: Remaps audio channels or executes stereo/mono up/downmixing (``stereo_endpoint_playback_updownmix``). +6. **Pre-Mixer Volume / Gain**: Applies individual stream attenuation before merging into the main mixer. +7. **Mixin Endpoint**: Ingests the decoded, volume-scaled stream into the primary mixing pipeline (``lp_mode 1``), where it combines with standard system sounds, alerts, and notifications. + +.. graphviz:: + :caption: ALSA Topology 2 Deep-Buffer Compressed Playback Pipeline Graph + :alt: ALSA Topology 2 Compressed Playback Pipeline Graph + + digraph Topology_Graph { + graph [bgcolor="#0A192F", fontname="DejaVu Sans", fontsize=11, rankdir=LR, splines=ortho, pad=0.3]; + node [shape=box, style="filled,rounded", fontname="DejaVu Sans", fontsize=10, penwidth=1.5]; + edge [fontname="DejaVu Sans", fontsize=9, penwidth=1.2, color="#38BDF8"]; + + subgraph cluster_fe { + label="Frontend Compress Pipeline (compr-playback.N / lp_mode 1)"; + style="filled,rounded"; + color="#1E3A8A"; + fillcolor="#0F172A"; + fontcolor="#93C5FD"; + + host_fe [label="Host Copier (host-copier.1)\nDeep Buffer DMA: 2000 ms\nPCIe Ingress", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + dec_widget [label="Cadence Decoder (decoder.1)\nDomain: DP | Core: 1\nUUID: 43:84:21:d8:...", fillcolor="#065F46", fontcolor="#34D399", color="#10B981"]; + copier_s32 [label="Module Copier (module-copier.2)\nFormat Convert: S16 -> S32_LE", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + src_resample [label="SRC Resampler (src.1)\nResample: 44.1 kHz -> 48.0 kHz", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + micsel_ch [label="Selector / Remap (micsel.1)\nStereo Channel Re-alignment", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + gain_widget [label="Volume / Gain (gain.1)\nPre-Mixer Stream Attenuation", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + mixin_widget [label="Mixin Ingress (mixin.1)\nShared Mixing Ingress Pin", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + } + + subgraph cluster_be { + label="Backend Mixing & DAI Pipeline (Core 0)"; + style="filled,rounded"; + color="#047857"; + fillcolor="#064E3B"; + fontcolor="#A7F3D0"; + + mixout_widget [label="Mixout Core (mixout.1)\nMain Stream Aggregator", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + main_vol [label="Main Volume (volume.1)\nGlobal Hardware Sliders", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#34D399"]; + dai_gateway [label="DAI Copier (dai-copier.1)\nHardware Bus: I2S / SoundWire", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + } + + host_fe -> dec_widget [label="Compressed Bitstream"]; + dec_widget -> copier_s32 [label="Raw PCM"]; + copier_s32 -> src_resample [label="S32_LE"]; + src_resample -> micsel_ch [label="48 kHz S32_LE"]; + micsel_ch -> gain_widget [label="Stereo"]; + gain_widget -> mixin_widget [label="Attenuated Audio"]; + + mixin_widget -> mixout_widget [label="Inter-Pipeline Buffer", color="#F59E0B", penwidth=2.0]; + mixout_widget -> main_vol; + main_vol -> dai_gateway [label="To Speakers / Headphones"]; + } + +Factory Bringup, User-Space Offload & Verification Runbook +********************************************************** + +This runbook outlines procedures to verify compressed audio offload pipelines, test standalone decoders, query capabilities, and measure host power savings. + +1. Capabilities Query via ALSA Compress-Offload +=============================================== + +Verify that the kernel and DSP firmware correctly advertise compressed codec support: + +.. code-block:: bash + + # Step 1: Query ALSA compress device nodes + ls -la /dev/snd/compr* + + # Step 2: Query supported codecs and formats via tinycompress utility + cplay -k -d 0 -c 1 + + # Expected Output: + # Number of codecs supported: 3 + # Codec 0: ID 2 (SND_AUDIOCODEC_MP3) + # Sample Rates: 8000, 11025, 12000, 16000, 22050, 24000, 32000, 44100, 48000 Hz + # Bitrates: 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320 kbps + # Codec 1: ID 6 (SND_AUDIOCODEC_AAC) + # Bitstream Formats: ADTS (MPEG-4) + # Codec 2: ID 1 (SND_AUDIOCODEC_PCM) + +2. Compressed Playback Streaming via tinycompress +================================================= + +Stream an encoded MP3 or AAC file directly to the DSP offload hardware: + +.. code-block:: bash + + # Step 1: Play MP3 audio via ALSA compress offload + cplay -d 0 -c 1 /usr/share/sounds/test_audio_44k_320kbps.mp3 + + # Step 2: Verify live DSP log traces via mtrace or probe server + # Look for Cadence XA initialization and frame consumption: + # [DSP] cadence_codec_init() done + # [DSP] cadence_codec_prepare() period set to 24000 usec + # [DSP] cadence_codec_process() decoded 1152 samples, consumed 1045 bytes + +3. Compressed Capture & Encoding Validation +=========================================== + +Validate real-time compressed capture offload using the MP3 encoder: + +.. code-block:: bash + + # Step 1: Record 10 seconds of compressed MP3 capture from microphone + crecord -d 0 -c 2 -b 320 -s 48000 -r 10 /tmp/dsp_encoded_capture.mp3 + + # Step 2: Validate the generated MP3 bitstream integrity + ffprobe /tmp/dsp_encoded_capture.mp3 + + # Expected: + # Input #0, mp3, from '/tmp/dsp_encoded_capture.mp3': + # Duration: 00:00:10.00, bitrate: 320 kb/s + # Stream #0:0: Audio: mp3, 48000 Hz, stereo, fltp, 320 kb/s + +4. Power Telemetry & Host Deep-Sleep Verification +================================================= + +Measure the host CPU power residency during compressed offload versus standard PCM playback to confirm the power-saving benefit: + +.. code-block:: bash + + # Step 1: Monitor CPU Package C-State residency using turbostat + sudo turbostat --quiet --interval 5 --show Pkg_%pc10,PkgWatt + + # Test Case A: Standard PCM Playback (aplay -D plughw:0,0 test.wav) + # Pkg_%pc10: 12.4% | PkgWatt: 2.85 W (High host wakeup overhead) + + # Test Case B: Compress-Offload Playback (cplay -d 0 -c 1 test.mp3) + # Pkg_%pc10: 94.8% | PkgWatt: 0.38 W (Near-complete host C10 residency) + +5. Standalone Testbench Loopback & Bit-Exactness Testing +======================================================== + +Run the SOF standalone testbench to verify decoding linearity without hardware: + +.. code-block:: bash + + # Run testbench with reference PCM decoder + sof-testbench -p -i test_input.raw -o pcm_decoded_output.raw \ + -t tools/topology/topology2/build/topology1/compr_playback_test.tplg + + # Validate output against reference golden vector + diff -q pcm_decoded_output.raw test_golden_reference.raw + +.. graphviz:: + :caption: Comprehensive Verification & Bringup Workflow: tinycompress Offload, DSP Decoding & Power Telemetry + :alt: Media Codec Verification Workflow + + digraph Verification_Workflow { + graph [bgcolor="#0A192F", fontname="DejaVu Sans", fontsize=11, rankdir=LR, splines=spline, pad=0.3]; + node [shape=box, style="filled,rounded", fontname="DejaVu Sans", fontsize=10, penwidth=1.5]; + edge [fontname="DejaVu Sans", fontsize=9, penwidth=1.2, color="#38BDF8"]; + + subgraph cluster_step1 { + label="1. Capabilities & Query"; + style="filled,rounded"; + color="#1E3A8A"; + fillcolor="#0F172A"; + fontcolor="#93C5FD"; + + cplay_caps [label="cplay -k /dev/snd/compr*\nQuery MP3/AAC Descriptors", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8"]; + } + + subgraph cluster_step2 { + label="2. Bitstream Offload Streaming"; + style="filled,rounded"; + color="#047857"; + fillcolor="#064E3B"; + fontcolor="#A7F3D0"; + + cplay_run [label="cplay -d 0 -c 1 music.mp3\nBurst 2 MB Chunks to DSP", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#34D399"]; + crecord_run [label="crecord -d 0 -c 2 out.mp3\nCapture Real-Time 320 kbps", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#34D399"]; + } + + subgraph cluster_step3 { + label="3. Firmware Tracing & Telemetry"; + style="filled,rounded"; + color="#D97706"; + fillcolor="#451A03"; + fontcolor="#FDE68A"; + + trace_mon [label="mtrace / sof-logger\nVerify XA API Init & Consumed Bytes", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + turbostat [label="turbostat --show Pkg_%pc10\nValidate > 90% Host C10 Residency", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + } + + subgraph cluster_step4 { + label="4. Acoustic & EOS Validation"; + style="filled,rounded"; + color="#831843"; + fillcolor="#4C0519"; + fontcolor="#FECDD3"; + + eos_check [label="Verify Asynchronous EOS Event\nEVENTID_COMPR_MAGIC_VAL", fillcolor="#1E293B", fontcolor="#FDA4AF", color="#F43F5E"]; + thd_check [label="THD+N & Gapless Looping\nZero Audio Clicks or Dropouts", fillcolor="#1E293B", fontcolor="#FDA4AF", color="#F43F5E"]; + } + + cplay_caps -> cplay_run; + cplay_run -> trace_mon; + cplay_run -> turbostat; + crecord_run -> trace_mon; + trace_mon -> eos_check; + turbostat -> thd_check; + } diff --git a/developer_guides/index.rst b/developer_guides/index.rst index d75e90b1..ad077a05 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -63,6 +63,7 @@ Audio Processing Modules & Algorithms * :ref:`level_multiplier` (High-level architecture; also see upstream `Level Multiplier README `_) * :ref:`phase_vocoder` (High-level architecture; also see upstream `Phase Vocoder source tree `_) * :ref:`stft_process` (High-level architecture; also see upstream `STFT Process README `_) +* :ref:`media_codecs` (High-level architecture; also see upstream `Cadence Codec module adapter `_ & `Codec README `_) .. _algorithm-specific-information: @@ -115,6 +116,7 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/level_multiplier firmware/phase_vocoder firmware/stft_process + firmware/media_codecs rimage/index.rst firmware/llext_modules firmware/hostless_firmware From ddf1009c17b5b35f52ec217ee0ff11541e303c95 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 16:24:41 +0100 Subject: [PATCH 31/64] doc: developer_guides: add runtime tuning and control blobs architecture guide Signed-off-by: Liam Girdwood --- data/modules.yaml | 72 ++ developer_guides/index.rst | 73 +- .../tuning/runtime_tuning_sof_ctl.rst | 659 ++++++++++++++++++ developer_guides/tuning/sof-ctl.rst | 127 ---- 4 files changed, 776 insertions(+), 155 deletions(-) create mode 100644 developer_guides/tuning/runtime_tuning_sof_ctl.rst delete mode 100644 developer_guides/tuning/sof-ctl.rst diff --git a/data/modules.yaml b/data/modules.yaml index 1b2c6d8f..c277b4c9 100644 --- a/data/modules.yaml +++ b/data/modules.yaml @@ -50,6 +50,18 @@ modules: - "Continuous clock drift tracking" - "Decoupled clock domain bridging" + - id: dmic + name: "Digital Microphone (DMIC) Decimation" + source: "SOF" + category: "Foundational DSP" + status: "Upstream" + description: "Hardware PDM receiver, multi-stage CIC decimation, and FIR compensator filter." + simd: ["Hardware Accelerator", "Scalar C"] + key_features: + - "High-performance PDM clock divider matching dual FIFOs" + - "Cascaded Integrator-Comb (CIC) decimation with FIR compensation" + - "Octave tuning tool for passband ripple (<0.1 dB) and stopband (>95 dB)" + - id: demux name: "Audio Demux" source: "SOF" @@ -98,6 +110,18 @@ modules: - "Zero-overhead fast-path bypass when configured for unity gain (0 dB)" - "Runtime IPC4 calibration and LLEXT dynamic module packaging" + - id: up_down_mixer + name: "Up/Down Mixer" + source: "SOF" + category: "Basic Routing & Level" + status: "Upstream" + description: "Configurable matrix-based channel upmixer and downmixer with per-coefficient attenuation." + simd: ["HiFi 3", "HiFi 4", "HiFi 5", "Scalar C"] + key_features: + - "Matrix coefficients for stereo, surround 5.1, and 7.1 mapping" + - "Channel energy normalization and clipping prevention" + - "Zero-copy passthrough when channel geometry matches" + - id: tone name: "Tone Generator" source: "SOF" @@ -231,6 +255,18 @@ modules: - "Membrane excursion limiting" - "Maximizes acoustic output without damage" + - id: sound_dose + name: "Sound Dose & Exposure" + source: "SOF" + category: "Speaker Protection" + status: "Upstream" + description: "Hearing health monitoring and cumulative sound exposure limiter complying with EN 50332 and IEC 62368-1." + simd: ["HiFi 3", "Scalar C"] + key_features: + - "Continuous equivalent sound level (Leq) running integration" + - "Digital A-weighting and C-weighting IIR filter profiles" + - "Configurable Cumulative Sound Dose (CSD) threshold triggers and attenuation" + # --- Voice, Telephony & Speech --- - id: tdfb name: "Beamformer (TDFB)" @@ -340,6 +376,42 @@ modules: - "TFLite Micro runtime execution" - "Low false-reject and false-alarm rates" + - id: mfcc + name: "Mel-Frequency Cepstral Coefficients (MFCC)" + source: "SOF" + category: "Voice & Telephony" + status: "Upstream" + description: "Speech feature extraction engine computing triangular Mel filterbank energies and DCT-II cepstra." + simd: ["HiFi 3", "Scalar C"] + key_features: + - "Configurable triangular Mel filterbanks (100 Hz to 8 kHz)" + - "Discrete Cosine Transform (DCT-II) with cepstral liftering" + - "Direct integration with wake word spotting and ASR frontends" + + - id: mic_privacy_manager + name: "Microphone Privacy Manager" + source: "SOF" + category: "Voice & Telephony" + status: "Upstream" + description: "Hardware-enforced microphone capture mute and privacy state management." + simd: ["Scalar C"] + key_features: + - "Zero-sample hardware mute interlock" + - "GPIO privacy LED synchronization" + - "Host-independent privacy state enforcement" + + - id: rtnr + name: "Realtek Neural Noise Reduction (RTNR)" + source: "Realtek" + category: "Voice & Telephony" + status: "Upstream" + description: "Deep neural network noise suppression engine isolating speech from non-stationary background noise." + simd: ["HiFi 4", "Scalar C"] + key_features: + - "Neural network recurrent inference" + - "Non-stationary transient acoustic noise suppression" + - "Dual-microphone directional voice enhancement" + # --- Codecs & Compression --- - id: media_codecs name: "Media Codecs (Cadence XA & Compress-Offload)" diff --git a/developer_guides/index.rst b/developer_guides/index.rst index ad077a05..ae518b88 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -6,13 +6,14 @@ Developer Guides Sound Open Firmware (SOF) provides comprehensive architectural specifications, developer runbooks, and implementation guides covering the entire audio stack: from low-level DSP firmware and Zephyr RTOS integration to mainline Linux kernel drivers, embedded microcontroller audio bridges, and automated verification suites. -The developer documentation is organized into five core technical pillars: +The developer documentation is organized into six core technical pillars: 1. :ref:`fw_development_pillar` -2. :ref:`kernel_driver_pillar` -3. :ref:`hardware_bringup_pillar` -4. :ref:`testing_simulation_pillar` -5. :ref:`telemetry_diagnostics_pillar` +2. :ref:`algorithm_tuning_pillar` +3. :ref:`kernel_driver_pillar` +4. :ref:`hardware_bringup_pillar` +5. :ref:`testing_simulation_pillar` +6. :ref:`telemetry_diagnostics_pillar` --- @@ -65,23 +66,8 @@ Audio Processing Modules & Algorithms * :ref:`stft_process` (High-level architecture; also see upstream `STFT Process README `_) * :ref:`media_codecs` (High-level architecture; also see upstream `Cadence Codec module adapter `_ & `Codec README `_) -.. _algorithm-specific-information: - -Algorithm Tuning & Implementation Guides -======================================== - -Detailed filter design, coefficient generation, and tuning workflows: - -.. toctree:: - :maxdepth: 1 - - algorithms/demux/demux.rst - algorithms/eq/equalizers_tuning - algorithms/src/sample_rate_conversion - algorithms/tdfb/time_domain_fixed_beamformer - -Pipeline Architecture, Packaging & Modules -========================================== +Firmware Architecture, Packaging & Core Subsystems +================================================== Core pipeline architecture, firmware image packaging, cryptographic signing, loadable modules, and standalone hostless embedded firmware: @@ -123,12 +109,44 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa --- +.. _algorithm_tuning_pillar: +.. _algorithm-specific-information: + +2. Audio Algorithm Tuning, Calibration & Runtime Control (Tuning) +***************************************************************** + +Comprehensive workflows, filter coefficient synthesis, offline tuning tools (GNU Octave, MATLAB, Python), ALSA byte control packaging, Topology 2 and UCM2 integration, and live parameter injection via ``sof-ctl``, ``amixer``, and the Linux kernel ALSA subsystem: + +Core Runtime Tuning & Control Infrastructure +============================================ + +* :ref:`runtime_tuning_sof_ctl` (Authoritative runtime parameter injection, ABI serialization, and ``sof-ctl`` guide) + +Acoustic, Transducer & Array Tuning +=================================== + +* :ref:`equalizers_tuning` (Parametric FIR & IIR equalizers, MLS acoustical measurement, and speaker tuning) +* :ref:`time-domain-fixed-beamformer` (Time-Domain Fixed Beamformer array geometry and spatial filter design) +* :ref:`sample_rate_conversion` (Polyphase FIR filter design and multi-stage resampling) +* :ref:`demux` (Multi-channel routing matrix configuration) + +.. toctree:: + :maxdepth: 1 + + tuning/runtime_tuning_sof_ctl + algorithms/eq/equalizers_tuning + algorithms/tdfb/time_domain_fixed_beamformer + algorithms/src/sample_rate_conversion + algorithms/demux/demux.rst + +--- + .. _kernel_driver_pillar: -2. Kernel & Host Driver Development (Kernel) +3. Kernel & Host Driver Development (Kernel) ******************************************** -Guides for Linux ASoC kernel driver developers, machine drivers, DMI quirk authoring, topology configurations, virtualization environments, and host tuning utilities. +Guides for Linux ASoC kernel driver developers, machine drivers, DMI quirk authoring, topology configurations, virtualization environments, and host testing utilities. .. toctree:: :maxdepth: 1 @@ -138,14 +156,13 @@ Guides for Linux ASoC kernel driver developers, machine drivers, DMI quirk autho topology/topology virtualization/virtualization virtualization/running - tuning/sof-ctl ktest/setup_ktest_environment --- .. _hardware_bringup_pillar: -3. Hardware & Platform Bringup (HW) +4. Hardware & Platform Bringup (HW) *********************************** Hardware integration, platform memory layouts, boot architectures, and bringup checklists across silicon vendors and embedded development boards. @@ -161,7 +178,7 @@ For embedded microcontroller audio bridges and hostless targets (Teensy 4.1, ESP .. _testing_simulation_pillar: -4. Testing, Simulation & Toolchains (SDK & Test) +5. Testing, Simulation & Toolchains (SDK & Test) ************************************************ Unit testing with Zephyr Ztest and Twister runner, host audio pipeline simulation, automated hardware loopback verification, Zephyr CMake build flags, and fuzzing. @@ -180,7 +197,7 @@ Unit testing with Zephyr Ztest and Twister runner, host audio pipeline simulatio .. _telemetry_diagnostics_pillar: -5. DSP Telemetry, Logging & Diagnostics (Debug) +6. DSP Telemetry, Logging & Diagnostics (Debug) *********************************************** Real-time DSP trace streaming over network probes, Zephyr structured logging, compile-time string dictionary extraction (`smex`), `sof-logger`, interactive Zephyr shell, and kernel debug probes. diff --git a/developer_guides/tuning/runtime_tuning_sof_ctl.rst b/developer_guides/tuning/runtime_tuning_sof_ctl.rst new file mode 100644 index 00000000..ec4adad5 --- /dev/null +++ b/developer_guides/tuning/runtime_tuning_sof_ctl.rst @@ -0,0 +1,659 @@ +.. _runtime_tuning_sof_ctl: +.. _runtime_tuning: +.. _sof_ctl: + +Runtime Tuning, Control Blobs & Parameter Injection Architecture +################################################################ + +Sound Open Firmware (SOF) provides a unified, cross-platform architecture for audio algorithm tuning, acoustic calibration, and runtime parameter control. This architecture bridges offline numerical modeling tools (GNU Octave, MATLAB, Python) with the real-time DSP execution environment via standardized Application Binary Interface (ABI) headers, ALSA control abstractions, and high-performance Inter-Processor Communication (IPC) mailboxes. + +Whether deploying static factory acoustic corrections during boot via ALSA Topology 2, activating use-case profiles via ALSA Use Case Manager (UCM2), or interactively modifying filter coefficients at runtime using ``sof-ctl``, the SOF tuning subsystem ensures bit-exact parameter delivery without interrupting active audio streams or causing audible artifacts. + +.. contents:: + :local: + :depth: 3 + +--- + +End-to-End Tuning Lifecycle & System Architecture +************************************************* + +Audio DSP tuning in SOF operates across two distinct domains: + +1. **Offline Acoustic Modeling & Filter Synthesis**: Acoustic engineers measure transducer characteristics (microphones, speakers, enclosures, and rooms) in an anechoic chamber or listening room. Mathematical computing environments (such as GNU Octave, MATLAB, or SciPy) synthesize optimal filter coefficients, compression curves, beamforming steering matrices, and protection thresholds. +2. **Online Dynamic Parameter Injection & Verification**: The synthesized parameters are serialized into binary control blobs wrapped in standard SOF ABI headers. These blobs are delivered into the live Linux kernel ALSA subsystem, dispatched across the host-DSP IPC transport, and applied to active algorithm state structures within the DSP firmware. + +The complete tuning lifecycle progresses across six discrete stages: + +.. graphviz:: + :align: center + :caption: Figure 251: End-to-End SOF Audio Tuning & Calibration Lifecycle + + digraph tuning_lifecycle { + rankdir=TB; + compound=true; + node [shape=box, style="filled,rounded", fontname="DejaVu Sans", fontsize=10, penwidth=1.5]; + edge [fontname="DejaVu Sans", fontsize=9, color="#94A3B8", penwidth=1.2]; + + subgraph cluster_stage1 { + label="Stage 1: Offline Modeling & Synthesis (Host PC)"; + style="filled,rounded"; + color="#3B82F6"; + fillcolor="#1E3A8A"; + fontcolor="#93C5FD"; + + design_tool [label="Acoustic Measurement & Filter Design\nGNU Octave / MATLAB / Python\n(MLS, Swept Sine, Thiele-Small, Biquads)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#60A5FA"]; + raw_params [label="Mathematical Parameter Extraction\nTarget Curves, Poles/Zeros, Excursion Limits", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#93C5FD"]; + design_tool -> raw_params; + } + + subgraph cluster_stage2 { + label="Stage 2: Serialization & ABI Wrapping"; + style="filled,rounded"; + color="#8B5CF6"; + fillcolor="#4C1D95"; + fontcolor="#DDD6FE"; + + abi_gen [label="ABI Header Construction\nsof_get_abi() / tools/tune/common/\n(Magic: 0x00464f53, Size, Version, Type)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#A78BFA"]; + blob_pack [label="Quantization & Word Packing\nFixed-Point Conversion (Q1.31, Q9.23, Q2.30)\nPadding & 64-bit Alignment", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#C4B5FD"]; + abi_gen -> blob_pack; + } + + subgraph cluster_stage3 { + label="Stage 3: Multi-Target Storage & Packaging"; + style="filled,rounded"; + color="#10B981"; + fillcolor="#064E3B"; + fontcolor="#A7F3D0"; + + tplg2_target [label="ALSA Topology 2 (.conf)\nObject.Base.data.comp_config\nEmbedded in Boot ROM Topology", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#34D399"]; + ucm2_target [label="ALSA UCM2 Profile (.bin)\ncset-tlv Scenario Blobs\n(/lib/firmware/intel/sof-ipc4/)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#6EE7B7"]; + alsa_state [label="ALSA State File (.txt / .state)\nComma-Separated 32-bit Integers\n(/var/lib/alsa/asound.state)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#A7F3D0"]; + } + + subgraph cluster_stage4 { + label="Stage 4: Runtime Transport & Injection"; + style="filled,rounded"; + color="#F59E0B"; + fillcolor="#78350F"; + fontcolor="#FDE68A"; + + user_tools [label="User-Space Control Utilities\nsof-ctl / amixer / alsactl\n(Live Parameter Injection over Lab SSH)", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + kernel_asoc [label="Linux Kernel ALSA / ASoC\nsnd_soc_bytes / snd_ctl_elem_value\nRouting to snd-sof Driver", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#FBBF24"]; + user_tools -> kernel_asoc [label="ALSA ioctl / cset"]; + } + + subgraph cluster_stage5 { + label="Stage 5: Inter-Processor Communication (IPC)"; + style="filled,rounded"; + color="#EC4899"; + fillcolor="#831843"; + fontcolor="#FBCFE8"; + + ipc_transport [label="Host-DSP IPC Mailbox\nIPC4: Large Config Set (param_id)\nIPC3: SOF_IPC_COMP_SET_DATA (type)", fillcolor="#1E293B", fontcolor="#F472B6", color="#F472B6"]; + dsp_module [label="Target DSP Processing Module\nAtomic Pointer Swap / Cross-Fade\n(EQ, DRC, Crossover, Beamformer, Smart Amp)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#FBCFE8"]; + ipc_transport -> dsp_module [label="DMA / Mailbox Dispatch"]; + } + + subgraph cluster_stage6 { + label="Stage 6: Acoustic & Telemetry Verification"; + style="filled,rounded"; + color="#06B6D4"; + fillcolor="#164E63"; + fontcolor="#A5F3FC"; + + dsp_telemetry [label="DSP Telemetry & Firmware Traces\nmtrace / TCP Probe Server (Port 9999)\nValidation of Applied Config", fillcolor="#1E293B", fontcolor="#38BDF8", color="#38BDF8"]; + acoustic_eval [label="Acoustic & Electrical Verification\nReference Measurement Microphone\nLoopback FFT / THD+N / Frequency Response", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#67E8F9"]; + } + + raw_params -> abi_gen [label="Target Parameters"]; + blob_pack -> tplg2_target [label="sof_tplg2_write.m"]; + blob_pack -> ucm2_target [label="sof_ucm_blob_write.m"]; + blob_pack -> alsa_state [label="sof_alsactl_write.m"]; + blob_pack -> user_tools [label="Live sof-ctl -s"]; + + tplg2_target -> kernel_asoc [label="Boot Initialization", style="dashed"]; + ucm2_target -> kernel_asoc [label="Profile Switch", style="dashed"]; + alsa_state -> kernel_asoc [label="alsactl restore", style="dashed"]; + + kernel_asoc -> ipc_transport [label="IPC Payload Transfer"]; + dsp_module -> dsp_telemetry [label="Trace Logs"]; + dsp_module -> acoustic_eval [label="Audio Output Stream"]; + } + +Delivery Mechanisms in SOF +========================== + +SOF supports four distinct delivery vectors for audio tuning blobs, each addressing a specific stage in the system lifecycle: + +.. table:: Table 29: Parameter Delivery Mechanisms in SOF + :widths: 18 20 22 20 20 + :class: tight-table + + +-----------------------+---------------------+-----------------------+---------------------+---------------------+ + | Delivery Mechanism | Primary Use Case | File Format | Invocation Method | Persistence Model | + +=======================+=====================+=======================+=====================+=====================+ + | **ALSA Topology 2** | Static factory- | Text bytes in ALSA | Compiled into | Persistent across | + | | calibrated default | topology ``.conf`` | ``.tplg`` binary; | reboots and OS | + | | processing settings | (hex string format) | loaded by kernel | reinstallations | + | | | | at boot time | | + +-----------------------+---------------------+-----------------------+---------------------+---------------------+ + | **ALSA UCM2** | Scenario-dependent | Binary blob file | Dispatched via | Persistent per user | + | | profile switching | (``.bin``) in rootfs | ``cset-tlv`` when | session / audio | + | | (handset, speaker, | or firmware directory | routing use case | profile transition | + | | docking station) | | changes | | + +-----------------------+---------------------+-----------------------+---------------------+---------------------+ + | **ALSA State File** | Systemd service | ASCII comma-separated | Restored via | Persistent across | + | | state restoration | 32-bit unsigned ints | ``alsactl restore`` | normal system | + | | across boots | (``asound.state``) | during system boot | power cycles | + +-----------------------+---------------------+-----------------------+---------------------+---------------------+ + | **Interactive** | Real-time acoustic | Binary (``.bin``) or | Direct command | Transient (active | + | **sof-ctl** | calibration, filter | ASCII CSV (``.txt``) | execution over SSH | until next reboot | + | | tuning, and lab R&D | injected via ALSA ctl | or local terminal | or topology reload) | + +-----------------------+---------------------+-----------------------+---------------------+---------------------+ + +--- + +The SOF ABI Header Structure & Memory Layout +******************************************** + +Every configuration payload delivered to an SOF processing component must be encapsulated within a standardized Application Binary Interface (ABI) header. The ABI header serves four critical purposes: + +1. **Architecture Neutrality**: Guarantees identical binary parsing across 32-bit and 64-bit host processors and Xtensa / ARM / RISC-V DSP cores. +2. **Version Handshake & Compatibility**: Prevents mismatched user-space tools or stale firmware blobs from injecting corrupt structures by validating major, minor, and build ABI version numbers. +3. **Payload Demultiplexing & Sizing**: Explicitly conveys the exact payload length in bytes, shielding the DSP memory manager from buffer overflows. +4. **Command & Parameter Routing**: Conveys component-specific type selectors (IPC3) or parameter IDs (IPC4) to route data to the intended internal algorithm subsystem. + +ABI Header Definition +===================== + +The ABI header is defined in ``src/include/kernel/header.h`` and ``tools/tune/common/sof_get_abi.m``: + +.. code-block:: c + + #define SOF_ABI_MAGIC 0x00464f53 /* "SOF\0" in Little Endian */ + + struct sof_abi_hdr { + uint32_t magic; /* SOF_ABI_MAGIC */ + uint32_t type; /* Component-specific type (IPC3) or param_id (IPC4) */ + uint32_t size; /* Size in bytes of payload following this header */ + uint32_t abi_version; /* SOF ABI version encoded as SOF_ABI_VER(major, minor, build) */ + uint32_t reserved[4]; /* Reserved for future expansion, must be zero */ + uint32_t data[]; /* Flexible array member containing component payload */ + } __attribute__((packed)); + +Memory Serialization Datapath +============================= + +When serialized for ALSA control transport, the buffer layout differs depending on whether the payload is transported via the legacy ALSA TLV byte interface or modern binary containers: + +.. graphviz:: + :align: center + :caption: Figure 252: SOF ABI Header Structure & Binary Payload Serialization Datapath + + digraph abi_structure { + rankdir=LR; + node [shape=record, fontname="DejaVu Sans", fontsize=10, penwidth=1.5]; + edge [fontname="DejaVu Sans", fontsize=9, color="#94A3B8"]; + + memory_layout [label=" ALSA TLV Container Header\n(8 Bytes, Optional In Binary Mode) | SOF ABI Header (struct sof_abi_hdr)\n(32 Bytes Mandatory Envelope) | Module-Specific Configuration Payload\n(Variable Length, Multiple of 4 Bytes)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#38BDF8", style="filled,rounded"]; + + tlv_detail [label="{ tag: 0x0041534C (ALSA TLV) | length: Total Payload Bytes }", fillcolor="#0F172A", fontcolor="#93C5FD", color="#3B82F6", style="filled"]; + + abi_detail [label="{ magic: 0x00464f53 ('SOF\\0') | type / param_id: Subtype (0-255) | size: Payload Size (Bytes) | abi_version: Major.Minor.Build | reserved[4]: Zero Padding (16B) }", fillcolor="#0F172A", fontcolor="#DDD6FE", color="#8B5CF6", style="filled"]; + + payload_detail [label="{ Filter Coefficients (IIR / FIR) | Dynamic Range Constants (Threshold, Knee) | Speaker Model & Thiele-Small Parameters | FFT Windowing & Gain Normalization }", fillcolor="#0F172A", fontcolor="#A7F3D0", color="#10B981", style="filled"]; + + memory_layout:tlv -> tlv_detail; + memory_layout:abi -> abi_detail; + memory_layout:payload -> payload_detail; + } + +Two-Phase ABI Generation via ``sof-ctl`` +======================================== + +To eliminate manual maintenance of version numbers across external tuning scripts, the host utility ``tools/ctl/ctl.c`` provides an ABI header synthesis command: + +* **IPC3 ABI Synthesis**: + + .. code-block:: bash + + sof-ctl -g -t -b -o abi_header.bin + +* **IPC4 ABI Synthesis**: + + .. code-block:: bash + + sof-ctl -i 4 -g -p -b -o abi_header.bin + +The Octave helper ``tools/tune/common/sof_get_abi.m`` invokes this mechanism dynamically: + +.. code-block:: octave + + function [bytes, nbytes] = sof_get_abi(setsize, ipc_ver, type, param_id) + abifn = 'eq_get_abi.bin'; + if ipc_ver == 4 + cmd = sprintf('sof-ctl -i 4 -g %d -p %d -b -o %s', setsize, param_id, abifn); + else + cmd = sprintf('sof-ctl -g %d -t %d -b -o %s', setsize, type, abifn); + end + system(cmd); + fh = fopen(abifn, 'r'); + bytes = fread(fh, inf, 'uint8'); + fclose(fh); + delete(abifn); + nbytes = length(bytes); + end + +--- + +IPC Control Plane Architectures: IPC3 vs IPC4 +********************************************** + +SOF supports two major control protocols between the host Linux kernel and the DSP firmware. The choice of IPC architecture fundamentally dictates how tuning data is packed, routed, and applied. + +.. graphviz:: + :align: center + :caption: Figure 253: IPC3 vs IPC4 Parameter Transport & Large Config Set Architecture + + digraph ipc_architecture { + rankdir=TB; + compound=true; + node [shape=box, style="filled,rounded", fontname="DejaVu Sans", fontsize=10, penwidth=1.5]; + edge [fontname="DejaVu Sans", fontsize=9, color="#94A3B8", penwidth=1.2]; + + subgraph cluster_ipc3 { + label="Legacy IPC3 Parameter Flow"; + style="filled,rounded"; + color="#3B82F6"; + fillcolor="#1E3A8A"; + fontcolor="#93C5FD"; + + asoc_ipc3 [label="ALSA snd_soc_bytes_ext\n(Fixed 8-byte TLV + ABI Header)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#60A5FA"]; + msg_ipc3 [label="SOF_IPC_COMP_SET_DATA\nSingle Monolithic Mailbox Transfer\n(Stream Must Be Paused / Idle)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#93C5FD"]; + dsp_ipc3 [label="Component set_data() Callback\nDirect Memory Copy into State\nStatic Array Boundaries", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#BFDBFE"]; + + asoc_ipc3 -> msg_ipc3 -> dsp_ipc3; + } + + subgraph cluster_ipc4 { + label="Modern Intel IPC4 Unified Module Flow"; + style="filled,rounded"; + color="#10B981"; + fillcolor="#064E3B"; + fontcolor="#A7F3D0"; + + asoc_ipc4 [label="ALSA Control Byte Stream\nTargeted via Module Instance ID", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#34D399"]; + msg_ipc4 [label="Large Config Set (Type 2)\nFragmented DMA / Mailbox Payload\n(Multi-Chunk Streaming Support)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#6EE7B7"]; + dsp_ipc4 [label="Module set_large_config() Handler\nIndexed by Semantic param_id (0-255)\nLive Atomic Swap While Streaming", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#A7F3D0"]; + + asoc_ipc4 -> msg_ipc4 -> dsp_ipc4; + } + + note_contrast [label="Key Architectural Difference:\nIPC4 decouples parameter transfer from pipeline state, supports\nsemantic parameter IDs, and enables live coefficient updates while streaming.", shape=note, fillcolor="#0F172A", fontcolor="#FCD34D", color="#F59E0B"]; + } + +Detailed Protocol Comparison +============================ + +.. table:: Table 30: Architectural Comparison: IPC3 vs IPC4 Parameter Control Plane + :widths: 22 38 40 + :class: tight-table + + +------------------------+------------------------------------+------------------------------------+ + | Architectural Feature | Legacy SOF IPC3 | Modern Intel IPC4 | + +========================+====================================+====================================+ + | **Primary Command** | ``SOF_IPC_COMP_SET_DATA`` / | ``SOF_IPC4_MOD_LARGE_CONFIG_SET`` | + | | ``SOF_IPC_COMP_GET_DATA`` | (Type 2 Global Module Message) | + +------------------------+------------------------------------+------------------------------------+ + | **Parameter Routing** | Tagged by 32-bit ``type`` field | Indexed by standardized 8-bit | + | | in ``sof_abi_hdr`` | ``param_id`` (range 0 to 255) | + +------------------------+------------------------------------+------------------------------------+ + | **Payload Sizing** | Monolithic buffer, restricted to | Fragmented multi-chunk streaming | + | | maximum IPC mailbox window size | over DMA for arbitrarily large | + | | (typically 4 KB) | filter tables (e.g. 64 KB) | + +------------------------+------------------------------------+------------------------------------+ + | **Streaming State** | Requires stream to be paused or in | Fully asynchronous; coefficients | + | **Compatibility** | idle state; hot swapping can fail | update atomically on active audio | + | | with ``-EBUSY`` | frames without underruns | + +------------------------+------------------------------------+------------------------------------+ + | **Module Target ID** | Identified by pipeline and | Identified by 32-bit Module ID | + | | component ID (e.g. ``EQIIR1.0``) | and Instance ID (e.g. ``0x10001``) | + +------------------------+------------------------------------+------------------------------------+ + | **Fast-Path Initial** | Carried within stream PCM params | Delivered via ``INIT_INSTANCE`` | + | **Configuration** | payload (``ext_data``) | initialization blob | + +------------------------+------------------------------------+------------------------------------+ + +--- + +Static Deployment Packaging: Topology 2, UCM2 & ALSA State +********************************************************** + +Offline tuning scripts in SOF automate the generation of production artifacts for all three major deployment mechanisms: + +.. graphviz:: + :align: center + :caption: Figure 254: Topology 2 & UCM2 Static Blob Packaging Architecture + + digraph static_packaging { + rankdir=LR; + node [shape=box, style="filled,rounded", fontname="DejaVu Sans", fontsize=10, penwidth=1.5]; + edge [fontname="DejaVu Sans", fontsize=9, color="#94A3B8"]; + + octave_script [label="Octave Tuning Script\n(e.g. sof_example_drc.m)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#3B82F6"]; + + subgraph cluster_outputs { + label="Automated Synthesis Outputs"; + style="filled,rounded"; + color="#64748B"; + fillcolor="#0F172A"; + fontcolor="#CBD5E1"; + + fn_tplg2 [label="sof_tplg2_write()\nTopology 2 Data Block (.conf)\nObject.Base.data.\"comp\" { bytes ... }", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#10B981"]; + fn_ucm [label="sof_ucm_blob_write()\nUCM2 cset-tlv Binary (.bin)\nPure uint8 Binary Stream", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#8B5CF6"]; + fn_alsa [label="sof_alsactl_write()\nALSA State CSV Text (.txt)\nComma-Separated 32-bit Words", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#F59E0B"]; + } + + octave_script -> fn_tplg2; + octave_script -> fn_ucm; + octave_script -> fn_alsa; + } + +1. Topology 2 Data Blocks (``sof_tplg2_write.m``) +================================================= + +The helper ``sof_tplg2_write.m`` converts binary blobs into ALSA Topology 2 configuration syntax: + +1. Validates the ABI header sanity using ``sof_check_blob_header()``. +2. Strips the 8-byte ALSA TLV container header, retaining only the clean ABI header and payload. +3. Formats bytes into an 8-column hexadecimal text block conforming to ``Object.Base.data`` syntax: + +.. code-block:: text + + # Exported with script sof_example_drc.m + # cd tools/tune/drc; octave --no-window-system sof_example_drc.m + Object.Base.data."drc_config" { + bytes " + 0x53,0x4f,0x46,0x00,0x01,0x00,0x00,0x00, + 0x80,0x00,0x00,0x00,0x00,0x00,0x01,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x01,0x00,0x00,0x00,0xe8,0xff,0xff,0xff" + } + +This block is included directly into component topology files under ``tools/topology/topology2/include/components//``. + +2. ALSA Use Case Manager (UCM2) Binary Files (``sof_ucm_blob_write.m``) +======================================================================= + +For runtime profile switching without recompiling firmware or topology, ``sof_ucm_blob_write.m`` exports raw binary files (``.bin``). In an ALSA UCM configuration (e.g. ``HiFi.conf``), these blobs are referenced dynamically: + +.. code-block:: text + + SectionDevice."Speaker" { + Value { + PlaybackChannels "2" + } + EnableSequence [ + cset-tlv "name='DRC1.0 DRC' file='/lib/firmware/intel/sof-ipc4/drc/speaker_default.bin'" + cset-tlv "name='EQIIR1.0 EQIIR' file='/lib/firmware/intel/sof-ipc4/eq_iir/speaker_profile.bin'" + ] + } + +3. ALSA State Format (``sof_alsactl_write.m``) +============================================== + +To enable systemd state persistence via ``alsactl``, ``sof_alsactl_write.m`` packages configuration data as comma-separated 32-bit decimal words: + +.. code-block:: text + + 1414418259,1,128,65536,0,0,0,0,1,-24,1966080,786432,196608,16384000,393216,... + +These values can be loaded directly into active mixer controls or merged into ``/var/lib/alsa/asound.state``. + +--- + +Host User-Space Control Tools (``sof-ctl``, ``amixer``, ``alsactl``) +******************************************************************** + +SOF provides dedicated host utilities to discover, inspect, and update component configuration controls on live target devices. + +.. graphviz:: + :align: center + :caption: Figure 255: Runtime Parameter Injection Architecture: sof-ctl, ALSA Byte Controls & SOF DSP Driver + + digraph injection_architecture { + rankdir=TB; + compound=true; + node [shape=box, style="filled,rounded", fontname="DejaVu Sans", fontsize=10, penwidth=1.5]; + edge [fontname="DejaVu Sans", fontsize=9, color="#94A3B8", penwidth=1.2]; + + subgraph cluster_userspace { + label="Host User Space"; + style="filled,rounded"; + color="#3B82F6"; + fillcolor="#1E3A8A"; + fontcolor="#93C5FD"; + + sof_ctl_bin [label="sof-ctl Utility\n(Direct Binary & CSV Control Injection)", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + amixer_bin [label="amixer / alsamixer\n(ALSA Native Command-Line Client)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#60A5FA"]; + alsactl_bin [label="alsactl store / restore\n(Systemd State Persistence)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#93C5FD"]; + } + + subgraph cluster_kernel { + label="Linux Kernel ALSA Subsystem"; + style="filled,rounded"; + color="#10B981"; + fillcolor="#064E3B"; + fontcolor="#A7F3D0"; + + alsa_core [label="ALSA Core ctl_ioctl()\nSNDRV_CTL_IOCTL_ELEM_WRITE / READ", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#34D399"]; + snd_sof [label="snd-sof Host Driver (sound/soc/sof/)\nsnd_sof_bytes_ext_put() Handler\nValidation of ABI Magic & Sizing", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#6EE7B7"]; + alsa_core -> snd_sof; + } + + subgraph cluster_dsp { + label="DSP Firmware Execution Domain"; + style="filled,rounded"; + color="#8B5CF6"; + fillcolor="#4C1D95"; + fontcolor="#DDD6FE"; + + ipc_handler [label="IPC Message Dispatcher\nDecodes Module ID & Parameter ID", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#A78BFA"]; + active_algo [label="Active Audio Processing Algorithm\nAtomic Cross-Fading & Buffer Update\n(Zero Interruption to Audio Stream)", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#C4B5FD"]; + ipc_handler -> active_algo; + } + + sof_ctl_bin -> alsa_core [label="Direct Control I/O"]; + amixer_bin -> alsa_core [label="cset / cget"]; + alsactl_bin -> alsa_core [label="state write"]; + + snd_sof -> ipc_handler [label="Host Mailbox / DMA Stream"]; + } + +The ``sof-ctl`` Command-Line Interface +====================================== + +``sof-ctl`` is located in ``tools/ctl/ctl.c`` and compiled alongside host tools (``build-tools.sh -A``). It provides comprehensive control over ALSA byte controls: + +.. table:: Table 31: ``sof-ctl`` Command-Line Flag Reference + :widths: 15 15 70 + :class: tight-table + + +---------------+-------------------+----------------------------------------------------------------------+ + | Flag | Argument | Functional Description | + +===============+===================+======================================================================+ + | ``-D`` | ```` | Specifies the ALSA sound card device name (default is ``hw:0``) | + +---------------+-------------------+----------------------------------------------------------------------+ + | ``-n`` | ```` | Targets an ALSA control by numeric control ID (e.g. ``-n 22``) | + +---------------+-------------------+----------------------------------------------------------------------+ + | ``-c`` | ```` | Targets an ALSA control by exact string name | + | | | (e.g. ``-c "name='EQIIR1.0 EQIIR'"``) | + +---------------+-------------------+----------------------------------------------------------------------+ + | ``-i`` | ``{3|4}`` | Selects the IPC protocol version; defaults to ``3`` (IPC3) | + +---------------+-------------------+----------------------------------------------------------------------+ + | ``-s`` | ```` | Injects configuration data into the targeted control from file | + +---------------+-------------------+----------------------------------------------------------------------+ + | ``-b`` | *(None)* | Enables binary mode (uses raw binary files instead of CSV) | + +---------------+-------------------+----------------------------------------------------------------------+ + | ``-r`` | *(None)* | Raw mode: Omits ABI header on input/output operations | + +---------------+-------------------+----------------------------------------------------------------------+ + | ``-o`` | ```` | Specifies output file for dumping readback control data | + +---------------+-------------------+----------------------------------------------------------------------+ + | ``-p`` | ```` | Specifies the IPC4 parameter ID (range 0 to 255) | + +---------------+-------------------+----------------------------------------------------------------------+ + | ``-t`` | ```` | Specifies the component-specific configuration type (IPC3) | + +---------------+-------------------+----------------------------------------------------------------------+ + | ``-g`` | ```` | Generates a standalone valid ABI header of specified payload | + | | | size and writes it to stdout or file | + +---------------+-------------------+----------------------------------------------------------------------+ + +--- + +Interactive Tuning & Verification Runbook +***************************************** + +This runbook outlines the exact step-by-step procedure to inspect live controls, synthesize custom tuning parameters, inject them into an active DSP pipeline, and verify the acoustic result. + +.. graphviz:: + :align: center + :caption: Figure 256: Interactive Tuning & Acoustic Verification Workflow over Lab Network + + digraph verification_workflow { + rankdir=LR; + node [shape=box, style="filled,rounded", fontname="DejaVu Sans", fontsize=10, penwidth=1.5]; + edge [fontname="DejaVu Sans", fontsize=9, color="#94A3B8"]; + + step1 [label="1. Enumerate Controls\namixer controls | grep EQ\nIdentify Target numid", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#3B82F6"]; + step2 [label="2. Synthesize Filter\nGNU Octave / MATLAB\nCompute Target Biquads", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#8B5CF6"]; + step3 [label="3. Wrap with ABI\nsof_get_abi() Packing\nProduce .bin / .txt Blob", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#10B981"]; + step4 [label="4. Live Injection\nsof-ctl -Dhw:0 -i 4\n-n -b -s blob.bin", fillcolor="#1E293B", fontcolor="#FCD34D", color="#F59E0B"]; + step5 [label="5. Verify in Traces\nmtrace / probe server\nCheck Parameter Handshake", fillcolor="#1E293B", fontcolor="#F8FAFC", color="#EC4899"]; + step6 [label="6. Measure Acoustic Output\nReference Measurement Mic\nValidate Target Curve", fillcolor="#1E293B", fontcolor="#38BDF8", color="#06B6D4"]; + + step1 -> step2 -> step3 -> step4 -> step5 -> step6; + } + +Step 1: Enumerate ALSA Controls on Target DUT +============================================= + +Log into the target DUT over SSH and list the available ALSA control elements: + +.. code-block:: bash + + # Query available processing controls + ssh root@ "amixer -Dhw:0 controls | grep -E 'EQ|DRC|CROSSOVER|LEVEL'" + + # Expected Output Example (IPC4): + # numid=18,iface=MIXER,name='EQIIR1.0 18 EQIIR' + # numid=19,iface=MIXER,name='DRC1.0 19 DRC' + # numid=20,iface=MIXER,name='level_multiplier.1.1.extctl' + +Step 2: Synthesize Filter Coefficients in GNU Octave +==================================================== + +Launch GNU Octave on the host development machine and calculate the desired filter response: + +.. code-block:: octave + + % Example: Design an Equalizer Notch Filter at 1 kHz with Q=10 + fs = 48000; + f0 = 1000; + q = 10.0; + gain_db = -18.0; + + % Calculate biquad coefficients + [b, a] = sof_eq_notch(f0, q, gain_db, fs); + + % Quantize coefficients to 32-bit signed fixed point + bqs = sof_eq_coef_quant(b, a); + + % Wrap into SOF EQ configuration structure + config = sof_eq_iir_generate_config(bqs); + +Step 3: Construct Binary Blob Wrapped with ABI Header +===================================================== + +Serialize the configuration structure and prepend the SOF ABI header: + +.. code-block:: octave + + % Build binary blob for IPC4 (param_id = 1) + ipc_version = 4; + endian = "little"; + blob8_ipc4 = sof_eq_iir_build_blob(config, endian, ipc_version); + + % Export to binary and text formats + sof_ucm_blob_write("notch_1khz.bin", blob8_ipc4); + sof_alsactl_write("notch_1khz.txt", blob8_ipc4); + +Step 4: Live Injection via ``sof-ctl`` While Audio is Streaming +=============================================================== + +Transfer the binary blob to the target DUT and apply it to the active pipeline without stopping playback: + +.. code-block:: bash + + # Step 4a: Copy blob to DUT + scp notch_1khz.bin root@:/tmp/ + + # Step 4b: Start background playback stream (if not already running) + ssh root@ "aplay -Dplughw:0,0 /usr/share/sounds/test_audio_48k.wav &" + + # Step 4c: Inject configuration dynamically using sof-ctl + ssh root@ "sof-ctl -Dhw:0 -i 4 -n 18 -p 1 -b -s /tmp/notch_1khz.bin" + + # Expected Output: + # Applying configuration "/tmp/notch_1khz.bin" into device hw:0 control numid=18. + # Success. + +Step 5: Verify Readback and DSP Firmware Execution +================================================== + +Verify that the DSP accepted the coefficients by reading the active configuration back from the hardware control: + +.. code-block:: bash + + # Step 5a: Read back active coefficients from DSP memory + ssh root@ "sof-ctl -Dhw:0 -i 4 -n 18 -p 1 -b -o /tmp/active_dump.bin" + + # Step 5b: Verify exact byte-level match with synthesized blob + ssh root@ "cmp /tmp/notch_1khz.bin /tmp/active_dump.bin && echo 'VERIFIED: Bit-exact match in DSP RAM!'" + + # Step 5c: Inspect DSP firmware logs for parameter update confirmation + ssh root@ "mtrace" | grep -i "eq_iir" + # Look for: [DSP] eq_iir_set_config(): 1 biquads updated, atomic swap complete. + +--- + +Troubleshooting & Protocol Diagnostics +************************************** + +When tuning parameters fail to take effect or trigger errors, consult the following diagnostic matrix: + +.. table:: Table 32: Common Tuning Errors, Root Causes & Remediation + :widths: 18 32 50 + :class: tight-table + + +--------------------+--------------------------------+-------------------------------------------------------------+ + | Return Code / Log | Primary Root Cause | Engineering Remediation & Action Required | + +====================+================================+=============================================================+ + | ``-EINVAL`` | Invalid ABI header or | Check that ``magic == 0x00464f53``. Verify that payload | + | (Invalid argument) | payload size mismatch | size matches exact structure byte length without TLV header.| + | | | Ensure all structures align to 32-bit word boundaries. | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | ``-EBUSY`` | Stream state conflict | Under IPC3, dynamic parameter updates are disallowed during | + | (Device busy) | during runtime injection | active streaming. Stop or pause the PCM stream before | + | | | re-sending, or upgrade pipeline to modern IPC4 architecture.| + +--------------------+--------------------------------+-------------------------------------------------------------+ + | ``-ENOENT`` | Invalid parameter ID or | Ensure ``-p `` matches the module's supported | + | (No such file) | control target mismatch | handler (e.g. param 1 for base config). Verify ``numid`` | + | | | using ``amixer controls``. | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | ``-EIO`` | Inter-Processor Communication | DSP firmware crashed or task hung. Inspect DSP logs via | + | (I/O error) | mailbox timeout | ``mtrace`` or TCP probe server. Verify DSP core is running. | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | Audible Zipper | Missing cross-fade or | Verify that algorithm implements atomic parameter swapping | + | Noise / Clicks | unquantized coefficient jumps | with sample-level linear gain interpolation or waits for | + | | | zero-crossing events before applying discontinuous filters. | + +--------------------+--------------------------------+-------------------------------------------------------------+ diff --git a/developer_guides/tuning/sof-ctl.rst b/developer_guides/tuning/sof-ctl.rst deleted file mode 100644 index 63973510..00000000 --- a/developer_guides/tuning/sof-ctl.rst +++ /dev/null @@ -1,127 +0,0 @@ -.. _runtime_tuning: - -Runtime Tuning -############## - -Runtime tuning of components and pipelines can be achieved using the sof-ctl -tool. - -The tool is available in SOF repository in directory tools/ctl. It performs -runtime IO access using the ext bytes or tlv bytes control of |SOF| components -like eq_iir and eq_fir. This tool is used to upload runtime data to alter the -performance or processing characteristsics at runtime of a audio component. -e.g Capability to change EQ response in runtime is useful for transducer tuning -and for scenario of having equalizers under control of user space service. - -This document mainly focuses on examples of the sof-ctl around the EQ FIR and -IIR components since the tool was developed alongside these components. The -concepts outlined here for EQs will equally applt to other component types -that support updating runtime data. - -Please find other document(s) in this section how to setup persistently -equalizers via topology in boot. There will be also general documentation -about IIR and FIR and tuning. - -Find out effect numids -********************** - -To access the right instance the numid of the equalizer needs to be -known. As example with topology sof-apl-eq-pcm512x.tplg the numids are -as follows: - -.. code-block:: bash - - amixer -Dhw:0 controls | grep EQ - #numid=23,iface=MIXER,name='EQFIR1.0 EQFIR' - #numid=22,iface=MIXER,name='EQIIR1.0 EQIIR' - -Therefore to control the IIR instance use numid=22 and to control the -FIR EQ instance use numid=23. Note that this depends on topology and -varies. In case there are even more equalizers in the topology the -numbers x.y in e.g. EQFIR1.0 help to navigate to pipeline and instance -number. In this example the equalizers are in the same pipeline 1 in -cascade. - - -Example equalizer settings -************************** - -This directory contains some simple example setups for -convenience. The used file format for txt format files is comma -separated unsigned 32 bit decimal integers. Though the files are -single line, additional blanks and line feeds are tolerated. The -trailing comma seen is not mandatory. The data format for filter -coefficients and other embedded control is described in uapi/eq.h. - -Creating equalizer configurations requires GNU Octave or Matlab(R) -numerical computing software with signal toolbox. The equalizer tuning -tool is found in tools/tune/eq directory. - -===================== ================================================ -File name Explanation ---------------------- ------------------------------------------------ -eq_iir_flat.txt Recursive filter with one as transfer function -eq_iir_bandpass.txt Simple bandpass response -eq_iir_bassboost.txt Simple high-pass and low-shelf -eq_iir_loudness.txt Loudness effect from example_iir_eq.m -===================== ================================================ - -===================== ================================================ -File name Explanation ---------------------- ------------------------------------------------ -eq_fir_flat.txt One tap filter with coefficient one -eq_fir_mid.txt Simple mid boost response -eq_fir_loudness.txt Loudness effect from example_fir_eq.m -===================== ================================================ - -Code to generate IIR and FIR loudness effects is available in in -skripts example_iir_eq.m and example_fir_eq.m in SOFT/tune/eq. The -flat response generation is also demonstrated in these example. The -flat responses are there embedded to previous responses as selectable -options to demonstrate the preset EQ capability. However the kernel -does not yet support preset switching without re-uploading the whole -configuration. - -The equalizer can be updated only when SOF is idle. Update during -playback is not currently supported (until SOF v1.4) and when attempted the -playback will continue with existing setting. The driver will re-send to -configuration when DSP is not busy. - -E.g. to switch the IIR equalizer to bandpass use command: - -.. code-block:: bash - - sof-ctl -Dhw:0 -n 22 -s eq_iir_bandpass.txt - -Succesfull execution will produce next output. - -.. code-block:: bash - - #Applying configuration "eq_iir_bandpass.txt" into device hw:0 control numid=22. - #84,2,1,0,0,2,2,3316150158,2048164275,513807534,3267352229,513807534,0,16384, - #3867454526,1191025347,38870735,77741469,38870735,4294967294,16458 - #Success. - -After this command the playback sound will have all lowest and highest -frequencies suppressed and sound very thin. You may experiment with -responses "flat" and "bassboost" to hear other examples of -manipulating spectral characteristics of playback audio. - -To check what has been applied to DSP the equalizer coefficients can -be read back by omitting the -s switch. - -.. code-block:: bash - - sof-ctl -Dhw:0 -n 22 - #Retrieving configuration for device hw:0 control numid=22. - #Success. - #84,2,1,0,0,2,2,3316150158,2048164275,513807534,3267352229,513807534,0,16384, - #3867454526,1191025347,38870735,77741469,38870735,4294967294,16458 - -Help -**** - -For completeness the command line options are described with -h switch. - -Mail list sound-open-firmware@alsa-project.org is recommended contact for -technical discussion about equalizers and tuning. From d33d2ee8292673097fb102df8b9b945b7e4e768a Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 16:33:19 +0100 Subject: [PATCH 32/64] doc: developer_guides: add dynamic range compression & multiband drc tuning guide Signed-off-by: Liam Girdwood --- developer_guides/index.rst | 7 + developer_guides/tuning/drc_tuning.rst | 727 ++++++++++++++++++ .../images/drc_tuning_adaptive_release.svg | 50 ++ .../images/drc_tuning_detector_arch.svg | 190 +++++ .../tuning/images/drc_tuning_static_curve.svg | 59 ++ .../images/drc_tuning_toolchain_workflow.svg | 197 +++++ .../images/multiband_drc_tuning_pipeline.svg | 177 +++++ 7 files changed, 1407 insertions(+) create mode 100644 developer_guides/tuning/drc_tuning.rst create mode 100644 developer_guides/tuning/images/drc_tuning_adaptive_release.svg create mode 100644 developer_guides/tuning/images/drc_tuning_detector_arch.svg create mode 100644 developer_guides/tuning/images/drc_tuning_static_curve.svg create mode 100644 developer_guides/tuning/images/drc_tuning_toolchain_workflow.svg create mode 100644 developer_guides/tuning/images/multiband_drc_tuning_pipeline.svg diff --git a/developer_guides/index.rst b/developer_guides/index.rst index ae518b88..9aeff0af 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -122,6 +122,11 @@ Core Runtime Tuning & Control Infrastructure * :ref:`runtime_tuning_sof_ctl` (Authoritative runtime parameter injection, ABI serialization, and ``sof-ctl`` guide) +Dynamics & Transducer Protection Tuning +======================================= + +* :ref:`drc_tuning` (Single-band DRC and Multiband DRC compression curves, adaptive ballistics, and speaker protection) + Acoustic, Transducer & Array Tuning =================================== @@ -134,11 +139,13 @@ Acoustic, Transducer & Array Tuning :maxdepth: 1 tuning/runtime_tuning_sof_ctl + tuning/drc_tuning algorithms/eq/equalizers_tuning algorithms/tdfb/time_domain_fixed_beamformer algorithms/src/sample_rate_conversion algorithms/demux/demux.rst + --- .. _kernel_driver_pillar: diff --git a/developer_guides/tuning/drc_tuning.rst b/developer_guides/tuning/drc_tuning.rst new file mode 100644 index 00000000..f8cda8c6 --- /dev/null +++ b/developer_guides/tuning/drc_tuning.rst @@ -0,0 +1,727 @@ +.. _drc_tuning: + +Dynamic Range Compression & Multiband DRC Tuning Guide +###################################################### + +Sound Open Firmware (SOF) provides advanced dynamics processing algorithms designed to manage audio dynamics, protect micro-speaker transducers from physical and thermal damage, maximize loudness and speech intelligibility, and prevent ADC clipping during capture. + +This guide details the complete tuning lifecycle for both the **Single-Band Dynamic Range Compressor (DRC)** and the **Multiband Dynamic Range Compressor (Multiband DRC)**: from mathematical curve synthesis and adaptive envelope ballistics to Linkwitz-Riley crossover splitting, offline GNU Octave / MATLAB calibration, Topology 2 / UCM2 packaging, and live in-system injection via ``sof-ctl``. + +--- + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +Theoretical Foundations of Dynamic Range Compression +***************************************************** + +The dynamic range of an audio signal represents the ratio between the loudest peak and the quietest nuance or ambient noise floor. In modern consumer audio devices—especially thin-profile laptops, tablets, smart displays, and conference speakerphones—loudspeakers have small enclosures, low coil mass, and limited cone excursion limits (:math:`X_{\text{max}}`). Uncontrolled audio dynamics present two major hazards: + +1. **Physical & Thermal Transducer Damage**: High-amplitude low-frequency peaks drive voice coils beyond linear excursion limits, bottoming out against magnet back plates or overheating voice coils (:math:`P = I^2 R`). +2. **Dynamic Clutter & Loss of Intelligibility**: In noisy acoustic environments, quiet whispers become inaudible while sudden movie explosions or notification chimes sound harsh and jarring. + +A Dynamic Range Compressor acts as an automated, program-dependent gain control that narrows the dynamic span of an audio stream by attenuating signals that exceed a specified threshold, while leaving lower-level signals untouched or boosted via makeup gain. + +.. _figure_drc_detector_arch: + +.. figure:: images/drc_tuning_detector_arch.svg + :align: center + :alt: Single-Band DRC Processing Architecture + + Figure 1: Single-Band DRC Processing Architecture: Sidechain Detector, Lookahead Circular Delay, and Division-Based Gain Interpolation + +Static Transfer Curve & Soft-Knee Geometry +========================================== + +The static behavior of a compressor defines the relationship between the input signal level (:math:`x_{\text{dB}}`) and the output signal level (:math:`y_{\text{dB}}`) under steady-state conditions: + +* **Threshold** (:math:`T_{\text{dB}}`): The signal level above which compression begins. Levels below :math:`T_{\text{dB}}` traverse the compressor with unity gain (1:1 input-to-output slope). +* **Compression Ratio** (:math:`R:1`): The degree of attenuation applied once the signal enters the compression region. A ratio of 4:1 means that a 4 dB increase in input level yields only a 1 dB increase in output level (a slope of :math:`s = 1/R = 0.25`). A ratio of :math:`\infty:1` represents a brickwall limiter. +* **Knee Width** (:math:`W_{\text{dB}}`): The decibel range over which the transition from linear 1:1 passthrough to the compressed slope occurs. +* **Makeup Gain / Post-Gain** (:math:`M_{\text{dB}}`): A static gain boost applied to the compressed stream to restore subjective loudness after peak attenuation. + +.. _figure_drc_static_curve: + +.. figure:: images/drc_tuning_static_curve.svg + :align: center + :alt: Static Decibel Transfer Curve with Soft-Knee Geometry + + Figure 2: Static Transfer Curve & Soft-Knee Geometry: Linear Region, Exponential Knee, and Post-Gain Offset + +Hard-Knee vs Soft-Knee Transitions +---------------------------------- + +In a conventional **hard-knee** compressor, the transition between the linear region (:math:`s = 1.0`) and the compressed region (:math:`s = 1/R`) is instantaneous at :math:`x_{\text{dB}} = T_{\text{dB}}`. This sharp derivative discontinuity creates rapid gain modulation, introducing audible high-frequency harmonic distortion (clipping artifacts) during transient crossings. + +SOF implements an **exponential soft knee** in the linear domain to eliminate slope discontinuities. The transfer function across the three distinct regions is formulated as: + +1. **Linear Region** (:math:`x < x_{\text{th}}`, where :math:`x_{\text{th}} = 10^{T_{\text{dB}} / 20}`): + + .. math:: + + y = x + +2. **Soft-Knee Region** (:math:`x_{\text{th}} \le x \le x_{\text{knee}}`, where :math:`x_{\text{knee}} = 10^{(T_{\text{dB}} + W_{\text{dB}}) / 20}`): + + .. math:: + + y = x_{\text{th}} + \frac{1 - e^{-k (x - x_{\text{th}})}}{k} + + The constant :math:`k` governs the rate of curvature. It is calculated numerically using a bisection root solver to ensure that the slope at the upper boundary of the knee (:math:`x = x_{\text{knee}}`) matches the desired compressed slope (:math:`s = 1/R`): + + .. math:: + + \left. \frac{d y_{\text{dB}}}{d x_{\text{dB}}} \right|_{x = x_{\text{knee}}} = \frac{1}{R} + +3. **Compressed Region** (:math:`x > x_{\text{knee}}`): + + .. math:: + + y = \text{ratio\_base} \cdot x^{1/R} + + where :math:`\text{ratio\_base}` is computed to ensure continuity at :math:`x_{\text{knee}}`: + + .. math:: + + \text{ratio\_base} = y(x_{\text{knee}}) \cdot \left( x_{\text{knee}} \right)^{-1/R} + +Perceptual Full-Range Makeup Gain +--------------------------------- + +Attenuating peaks reduces the total root-mean-square (RMS) energy of the stream. To compensate, SOF provides an empirical perceptual makeup gain calculation alongside user-defined post-gain: + +.. math:: + + G_{\text{makeup}} = \left( \frac{1}{\text{ratio\_base}} \right)^{0.6} + +The total linear output multiplier applied to the processed audio is: + +.. math:: + + G_{\text{out}} = 10^{M_{\text{dB}} / 20} \cdot G_{\text{makeup}} + +--- + +Detector Ballistics & Adaptive Release Dynamics +*********************************************** + +Static compression curves only determine steady-state attenuation. In real-time audio streams, how fast the compressor attenuates upon loud transients (**attack**) and how smoothly it recovers gain when the input drops (**release**) determine acoustic transparency. + +.. _figure_drc_adaptive_release: + +.. figure:: images/drc_tuning_adaptive_release.svg + :align: center + :alt: Adaptive 4-Zone Release Curve Ballistics + + Figure 3: Adaptive 4-Zone Release Curve vs Single Exponential Decay: Transient Protection without Acoustic Pumping + +Attack Ballistics & Circular Lookahead Delay +============================================ + +When an abrupt transient occurs (such as a snare drum hit or a microphone pop), the compressor must attenuate gain rapidly to protect the downstream amplifier and speaker. + +* **Attack Time Constant** (:math:`t_{\text{att}}`): The time required for the compressor gain to reduce by :math:`10\text{ dB}`. SOF enforces a minimum attack time of :math:`1\text{ ms}` (:math:`0.001\text{ s}`). The per-sample attack rate is governed by: + + .. math:: + + \text{one\_over\_attack\_frames} = \frac{1}{t_{\text{att}} \cdot f_s} + +* **Lookahead Pre-Delay Buffer** (``pre_delay_time``): Even with an attack time of :math:`1\text{ ms}`, an instantaneous step transient will leak through during the initial frames before the envelope detector ramps down gain. To solve this, SOF routes the main audio signal through a circular pre-delay buffer (``pre_delay_buffers``, up to 512 frames / ~6–10 ms), while feeding the sidechain detector without delay. + The detector senses the upcoming peak and begins ramping down gain *before* the transient emerges from the pre-delay buffer. This eliminates transient overshoot without introducing harsh waveshaping distortion. + +The Hazard of Static Release Times +================================== + +Setting the release time constant presents a fundamental engineering dilemma in audio engineering: + +* **Fast Release (< 50 ms)**: Restores dynamic clarity quickly following brief transients, but modulates low-frequency audio waveforms. When a 50 Hz sine wave passes through a fast-releasing compressor, the gain ramps up and down within each cycle, creating severe low-frequency harmonic distortion known as **breathing**. +* **Slow Release (> 500 ms)**: Cleanly avoids waveform distortion on bass, but causes noticeable **pumping**. A single loud transient causes the entire stream to duck and remain suppressed for hundreds of milliseconds, muffling trailing dialogue or background details. + +SOF Adaptive 4-Zone Release Curve +================================= + +SOF resolves this compromise through a non-linear, adaptive multi-segment release curve. Rather than using a single static exponential decay, SOF defines four release zones that adjust recovery speed based on how far below the envelope threshold the signal has dropped: + +.. math:: + + \mathbf{rz} = [rz_1, rz_2, rz_3, rz_4] = [0.09, 0.16, 0.42, 0.98] + +These zones scale the baseline release frames (:math:`N_{\text{rel}} = t_{\text{rel}} \cdot f_s`). A 4th-order polynomial is fitted across the four zones: + +.. math:: + + y(x) = k_A + k_B x + k_C x^2 + k_D x^3 + k_E x^4 + +where :math:`x \in \{0, 1, 2, 3\}` corresponds to the release zones. + +* When a short transient occurs and the signal level drops slightly, the compressor operates in the low zones (:math:`rz_1 = 0.09`), releasing rapidly to preserve dialogue clarity and room ambiance. +* During sustained loud passages, the compressor transitions toward the higher zones (:math:`rz_4 = 0.98`), smoothly decelerating gain recovery to prevent low-frequency harmonic distortion. + +Division-Based Sub-Block Processing +=================================== + +Logarithmic decibel conversion and exponential curve evaluation require significant DSP cycle budgets. To optimize real-time performance on embedded DSP architectures (Tensilica Xtensa HiFi 3, HiFi 4, HiFi 5, and RISC-V), SOF employs **sub-block division processing**: + +* Audio processing divides into sub-blocks of ``DRC_DIVISION_FRAMES = 32`` frames. +* Heavy logarithmic, exponential, and polynomial calculations execute once per sub-block. +* Individual samples within the 32-frame window are scaled using lightweight linear interpolation between boundary gain values. + +--- + +Multiband DRC Architecture & Compound Pipeline +********************************************** + +While single-band DRC provides effective protection for broadband signals, it suffers from **spectral pumping**: a high-energy kick drum (60 Hz) pulls down the wideband gain multiplier, causing midrange vocals (1 kHz) and high-frequency cymbals (10 kHz) to duck in unison. + +The **Multiband Dynamic Range Compressor (Multiband DRC)** eliminates spectral pumping by splitting the audio spectrum into up to four independent frequency bands, applying tailored compression curves to each band, and recombining them into a coherent output. + +.. _figure_multiband_drc_pipeline: + +.. figure:: images/multiband_drc_tuning_pipeline.svg + :align: center + :alt: Multiband DRC 4-Stage Compound Processing Pipeline + + Figure 4: Multiband DRC 4-Stage Compound Processing Pipeline: Emphasis, Linkwitz-Riley Crossover, N-Way DRC, and De-Emphasis + +The 4-Stage Compound Processing Pipeline +======================================== + +Multiband DRC operates as a single-source-single-sink compound component comprising four integrated processing stages: + +Stage 1: Pre-Emphasis Equalizer +------------------------------- + +Human hearing sensitivity decreases at extreme high frequencies, and high-frequency content typically carries less energy than bass. Multiband DRC incorporates a 2-biquad IIR pre-emphasis filter (`emp_coef[2]`) that shapes the incoming audio before crossover splitting. + +* **Anchor Frequency**: Typically set to :math:`15\text{ kHz}`. +* **Stage Gain**: User-defined pre-emphasis boost (e.g. :math:`0.01`). +* **Stage Ratio**: Progressive frequency ratio between cascaded stages (:math:`2.0`). + +Stage 2: Linkwitz-Riley 4th-Order (LR4) Crossover Filter Bank +------------------------------------------------------------- + +To divide the audio spectrum into frequency bands without introducing phase distortion or amplitude irregularities, SOF utilizes cascaded Linkwitz-Riley 4th-order (LR4) filter networks (`crossover_coef[6]`): + +* **Flat Magnitude Summation**: Each LR4 crossover section consists of two cascaded 2nd-order Butterworth filters (:math:`Q = 0.7071`, combined :math:`Q = 0.5`). The lowpass and highpass outputs sum with a mathematically flat :math:`0\text{ dB}` magnitude response across the crossover frequency :math:`F_c`. +* **Zero Phase Difference**: The lowpass and highpass branches exhibit a :math:`360^\circ` (:math:`0^\circ`) phase alignment at :math:`F_c`, preventing destructive comb filtering when the bands recombine. +* **Band Counts**: Supports 2-band, 3-band, or 4-band frequency splits using up to six LR4 biquad pairs. + +Stage 3: Per-Band Independent DRC Engines +----------------------------------------- + +Each split band feeds an independent instance of the single-band DRC engine: + +* **Band 0 (Low / Bass)**: Tuned with a low threshold, high ratio (:math:`10:1` to :math:`\infty:1`), fast attack (:math:`1\text{ ms} - 3\text{ ms}`), and moderate lookahead to clamp woofer cone excursion and voice coil thermal dissipation. +* **Band 1 (Mid / Dialogue)**: Tuned with a moderate threshold, gentle ratio (:math:`2:1` to :math:`4:1`), and slower attack (:math:`5\text{ ms} - 10\text{ ms}`) to ensure natural, intelligible vocals without pumping. +* **Band 2 (High / Treble)**: Tuned with an elevated threshold and fast release to control high-frequency sibilance (de-essing) and protect micro-tweeters. + +Stage 4: Summation & De-Emphasis Equalizer +------------------------------------------ + +The outputs of all active per-band DRC engines sum into a single composite stream, which passes through a 2-biquad IIR de-emphasis filter (`deemp_coef[2]`). The de-emphasis filter applies the exact inverse frequency response of the Stage 1 pre-emphasis filter, restoring the original acoustic spectral balance. + +--- + +Fixed-Point Data Structures & Quantization Formats +************************************************** + +Firmware components operate on fixed-point DSP hardware without native floating-point units. All tuning parameters are converted into packed, 32-bit fixed-point representations before transmission to the DSP. + +Single-Band DRC Parameter Structure (`struct sof_drc_params`) +============================================================== + +The Single-Band DRC configuration is defined in ``src/audio/drc/drc_user.h``: + +.. code-block:: c + + struct sof_drc_params { + int32_t enabled; /* 1 = enable, 0 = disable */ + int32_t db_threshold; /* Q8.24 - Threshold in dB */ + int32_t db_knee; /* Q8.24 - Knee width in dB */ + int32_t ratio; /* Q8.24 - Compression ratio */ + int32_t pre_delay_time; /* Q2.30 - Lookahead time in seconds */ + int32_t linear_threshold; /* Q2.30 - Threshold in linear scale */ + int32_t slope; /* Q2.30 - Inverse ratio (1 / ratio) */ + int32_t K; /* Q12.20 - Knee curvature coefficient */ + int32_t knee_alpha; /* Q8.24 - Pre-calculated knee parameter */ + int32_t knee_beta; /* Q8.24 - Pre-calculated knee parameter */ + int32_t knee_threshold; /* Q8.24 - Threshold + knee in linear dB */ + int32_t ratio_base; /* Q2.30 - Linear base multiplier */ + int32_t output_linear_gain; /* Q8.24 - Total post-makeup linear gain */ + int32_t one_over_attack_frames; /* Q2.30 - Attack rate per frame */ + int32_t sat_release_frames_inv_neg; /* Q2.30 - Saturation release rate */ + int32_t sat_release_rate_at_neg_two_db;/* Q2.30 - Rate at -2 dB */ + int32_t kSpacingDb; /* Q32.0 - Decibel spacing per step */ + int32_t kA; /* Q20.12 - Polynomial coefficient A */ + int32_t kB; /* Q20.12 - Polynomial coefficient B */ + int32_t kC; /* Q20.12 - Polynomial coefficient C */ + int32_t kD; /* Q20.12 - Polynomial coefficient D */ + int32_t kE; /* Q20.12 - Polynomial coefficient E */ + } __attribute__((packed)); + +Fixed-Point Q-Format Specifications (Table 33) +---------------------------------------------- + +.. list-table:: Table 33: DRC Parameter Fixed-Point Formats & Quantization Rules + :header-rows: 1 + :widths: 30 15 20 35 + :class: tight-table + + * - Struct Field + - Q-Format + - Range + - Resolution / Multiplier + * - ``db_threshold``, ``db_knee`` + - Q8.24 + - :math:`[-128.0, +127.99]` + - :math:`2^{-24} \approx 5.96 \times 10^{-8}` + * - ``ratio``, ``output_linear_gain`` + - Q8.24 + - :math:`[0.0, +127.99]` + - :math:`2^{-24} \approx 5.96 \times 10^{-8}` + * - ``pre_delay_time``, ``slope`` + - Q2.30 + - :math:`[0.0, +1.999]` + - :math:`2^{-30} \approx 9.31 \times 10^{-10}` + * - ``linear_threshold``, ``ratio_base`` + - Q2.30 + - :math:`[0.0, +1.999]` + - :math:`2^{-30} \approx 9.31 \times 10^{-10}` + * - ``one_over_attack_frames`` + - Q2.30 + - :math:`[0.0, +1.999]` + - :math:`2^{-30} \approx 9.31 \times 10^{-10}` + * - ``K`` (Knee Curvature) + - Q12.20 + - :math:`[0.0, +2047.99]` + - :math:`2^{-20} \approx 9.54 \times 10^{-7}` + * - ``kA``, ``kB``, ``kC``, ``kD``, ``kE`` + - Q20.12 + - :math:`[-524288, +524287]` + - :math:`2^{-12} \approx 2.44 \times 10^{-4}` + * - ``kSpacingDb`` + - Q32.0 + - :math:`[-2^{31}, 2^{31}-1]` + - Integer decibels + +Multiband DRC Parameter Structure (`struct sof_multiband_drc_config`) +===================================================================== + +Multiband DRC encapsulates all crossover and per-band parameters into a single compound structure defined in ``src/audio/multiband_drc/user/multiband_drc.h``: + +.. code-block:: c + + struct sof_multiband_drc_config { + uint32_t size; /* Total payload size in bytes */ + uint32_t num_bands; /* Number of active bands: 1 to 4 */ + uint32_t enable_emp_deemp; /* 1 = enable emphasis, 0 = bypass */ + uint32_t reserved[8]; /* Reserved for future expansion */ + + /* 2-Biquad Pre-Emphasis Equalizer */ + struct sof_eq_iir_biquad emp_coef[2]; + + /* 2-Biquad De-Emphasis Equalizer */ + struct sof_eq_iir_biquad deemp_coef[2]; + + /* Linkwitz-Riley LR4 Crossover Filter Bank (up to 6 biquad pairs) */ + struct sof_eq_iir_biquad crossover_coef[6]; + + /* Flexible array member: num_bands * struct sof_drc_params */ + struct sof_drc_params drc_coef[]; + }; + +--- + +Offline Calibration & Synthesis Workflows +***************************************** + +The SOF firmware tree includes comprehensive GNU Octave and MATLAB tuning scripts under ``src/audio/drc/tune/`` and ``src/audio/multiband_drc/tune/`` to synthesize filter coefficients, plot static curves, and export binary blobs wrapped with ABI headers. + +.. _figure_drc_toolchain_workflow: + +.. figure:: images/drc_tuning_toolchain_workflow.svg + :align: center + :alt: End-to-End DRC Tuning, Calibration and Injection Toolchain + + Figure 5: End-to-End Tuning Toolchain: Octave Synthesis, ABI Serialization, and Multi-Target Packaging + +Single-Band DRC Synthesis Workflow +================================== + +The top-level script for Single-Band DRC tuning is ``src/audio/drc/tune/sof_example_drc.m``. + +Step 1: Define Tuning Parameters +-------------------------------- + +Edit ``sof_example_drc.m`` or create a custom tuning script defining acoustic parameters: + +.. code-block:: octave + + % Small micro-speaker excursion protection preset + params.enabled = 1; + params.threshold = -30; % Threshold: -30 dBFS + params.knee = 20; % Soft-knee width: 20 dB + params.ratio = 10; % High ratio: 10:1 limiting + params.attack = 0.003; % Attack time: 3 ms + params.release = 0.25; % Base release: 250 ms + params.pre_delay = 0.006; % Lookahead delay: 6 ms + params.release_zone = [0.09 0.16 0.42 0.98]; % 4 adaptive release zones + params.release_spacing = 5; % Spacing per step: 5 dB + params.post_gain = 3; % Makeup post-gain: +3 dB + +Step 2: Synthesize Fixed-Point Coefficients +------------------------------------------- + +Invoke ``sof_drc_gen_coefs.m`` to generate the fixed-point parameter struct: + +.. code-block:: octave + + sample_rate = 48000; + coefs = sof_drc_gen_coefs(params, sample_rate); + + % Convert coefficients into struct sof_drc_config + config = sof_drc_generate_config(coefs); + +Step 3: Plot and Inspect Static Decibel Curve +--------------------------------------------- + +Visualize the synthesized static compression curve to verify threshold, knee shape, and slope: + +.. code-block:: octave + + sof_drc_plot_db_curve(coefs); + +The script displays input decibels versus output decibels, rendering the linear passthrough region, the smooth quadratic knee curvature, and the post-gain makeup offset. + +Step 4: Serialize and Export Blobs +---------------------------------- + +Serialize the configuration struct into binary and text formats wrapped with the SOF ABI header: + +.. code-block:: octave + + endian = "little"; + + % IPC3 Binary and ALSA CSV format + blob_ipc3 = sof_drc_build_blob(config, endian, 3); + sof_ucm_blob_write("drc_speaker_ipc3.bin", blob_ipc3); + sof_alsactl_write("drc_speaker_ipc3.txt", blob_ipc3); + + % IPC4 Binary and ALSA CSV format + blob_ipc4 = sof_drc_build_blob(config, endian, 4); + sof_ucm_blob_write("drc_speaker_ipc4.bin", blob_ipc4); + sof_alsactl_write("drc_speaker_ipc4.txt", blob_ipc4); + + % Topology 2 Component Definition (.conf) + sof_tplg2_write("drc_speaker.conf", blob_ipc4, "drc_config", ... + "Exported with sof_example_drc.m", "octave sof_example_drc.m"); + +Multiband DRC Synthesis Workflow +================================ + +The top-level script for Multiband DRC tuning is ``src/audio/multiband_drc/tune/sof_example_multiband_drc.m``. + +Step 1: Configure Multi-Band Frequency Splits and Dynamics +---------------------------------------------------------- + +Define band boundaries, pre-emphasis filters, and per-band dynamics curves: + +.. code-block:: octave + + rz1 = [0.09 0.16 0.42 0.98]; + + prm.name = "multimedia_3band"; + prm.sample_rate = 48000; + prm.num_bands = 3; % 3-Way Frequency Division + prm.enable_emp_deemp = 1; % Enable pre/de-emphasis filters + prm.stage_gain = 0.01; + prm.stage_ratio = 2.0; + + % Lower frequency boundaries for bands [Low, Mid, High, Ultra-High] + prm.band_lower_freq = [ 0 2000 8000 16000 ]; + prm.enable_bands = [ 1 2 3 0 ]; + + % Per-Band Dynamics + prm.threshold = [ -32 -28 -24 -24 ]; % Stricter on bass + prm.knee = [ 20 18 16 16 ]; + prm.ratio = [ 12 6 4 4 ]; % Heavy limiting on bass + prm.attack = [ 0.002 0.005 0.008 0.008 ]; % Fast attack on bass + prm.release = [ 0.2 0.25 0.3 0.3 ]; + prm.pre_delay = [ 0.006 0.004 0.002 0.002 ]; % Longer lookahead on bass + prm.release_spacing = [ 5 5 5 5 ]; + prm.post_gain = [ 2 1 0 0 ]; + prm.release_zone = [ rz1' rz1' rz1' rz1' ]; + +Step 2: Generate Quantized Compound Blobs +----------------------------------------- + +Invoke ``sof_example_multiband_drc.m`` or run the subroutines: + +.. code-block:: octave + + % 1. Synthesize Emphasis / De-emphasis biquads + [emp_coefs, deemp_coefs] = sof_iir_gen_quant_coefs(iir_params, sample_rate, prm.enable_emp_deemp); + + % 2. Synthesize Linkwitz-Riley LR4 Crossover biquads + crossover_coefs = sof_crossover_gen_quant_coefs(prm.num_bands, sample_rate, ... + prm.band_lower_freq(2) / (sample_rate / 2), ... + prm.band_lower_freq(3) / (sample_rate / 2), ... + 0); + + % 3. Synthesize per-band DRC quantized parameters + drc_coefs = sof_drc_gen_quant_coefs(prm.num_bands, sample_rate, drc_params); + + % 4. Build composite binary blobs wrapped with ABI header + blob_ipc4 = sof_multiband_drc_build_blob(prm.num_bands, prm.enable_emp_deemp, ... + emp_coefs, deemp_coefs, crossover_coefs, ... + drc_coefs, "little", 4); + + % 5. Export to files + sof_tplg2_write("multiband_drc_3band.conf", blob_ipc4, "multiband_drc_config"); + sof_ucm_blob_write("multiband_drc_3band.bin", blob_ipc4); + sof_alsactl_write("multiband_drc_3band.txt", blob_ipc4); + +--- + +Production Preset Profiles & Practical Tuning Recipes +***************************************************** + +Different acoustic applications require distinct dynamics strategies. The following recipes provide validated starting configurations: + +Preset 1: Micro-Speaker Excursion Protection (Table 34) +======================================================= + +* **Target**: Thin laptop and tablet speakers susceptible to cone bottoming out and voice coil thermal damage. +* **Strategy**: Aggressive threshold, high ratio, fast attack, and extended lookahead. + +.. list-table:: Table 34: Micro-Speaker Excursion Protection Recipe + :header-rows: 1 + :widths: 30 20 50 + :class: tight-table + + * - Parameter + - Recommended + - Acoustic Rationale + * - ``threshold`` + - :math:`-30\text{ dBFS}` + - Catches high-energy passages before mechanical limits + * - ``knee`` + - :math:`20\text{ dB}` + - Smooth entry avoiding audible distortion + * - ``ratio`` + - :math:`10:1` to :math:`12:1` + - Near-brickwall limiting preventing speaker blowout + * - ``attack`` + - :math:`2\text{ ms}` + - Clamps sharp transients immediately + * - ``release`` + - :math:`200\text{ ms}` + - Balanced recovery avoiding rapid breathing + * - ``pre_delay`` + - :math:`6\text{ ms}` + - Allows gain ramp-down before transient emerges + * - ``post_gain`` + - :math:`+3\text{ dB}` + - Restores perceived loudness in compact enclosures + +Preset 2: Voice Capture & Digital Microphone Leveling (Table 35) +================================================================ + +* **Target**: Conference microphone arrays, DMIC voice capture, and automatic speech recognition (ASR). +* **Strategy**: Transparent leveling of quiet talkers while preventing ADC clipping. + +.. list-table:: Table 35: Digital Microphone Capture Leveling Recipe + :header-rows: 1 + :widths: 30 20 50 + :class: tight-table + + * - Parameter + - Recommended + - Acoustic Rationale + * - ``threshold`` + - :math:`-35\text{ dBFS}` + - Active across normal speaking levels + * - ``knee`` + - :math:`25\text{ dB}` + - Ultra-wide soft knee for imperceptible compression + * - ``ratio`` + - :math:`4:1` to :math:`6:1` + - Transparent leveling preserving speech naturalness + * - ``attack`` + - :math:`5\text{ ms}` + - Lets natural consonant attack pass without dulling + * - ``release`` + - :math:`350\text{ ms}` + - Slow recovery preventing room reverberation amplification + * - ``pre_delay`` + - :math:`2\text{ ms}` + - Minimal lookahead to keep capture latency ultralow + * - ``post_gain`` + - :math:`0\text{ dB}` + - Headroom maintained for downstream beamforming / AEC + +Preset 3: Multimedia 3-Band Acoustic Tuning (Table 36) +======================================================= + +* **Target**: Premium laptop audio playback, soundbars, and gaming headphones. +* **Strategy**: Multiband frequency isolation preventing bass kicks from ducking vocals or treble. + +.. list-table:: Table 36: Multimedia 3-Band DRC Frequency & Parameter Matrix + :header-rows: 1 + :widths: 25 20 20 35 + :class: tight-table + + * - Band Specification + - Crossover Range + - Dynamics Parameters + - Functional Objective + * - **Band 0 (Bass)** + - :math:`0 - 2\text{ kHz}` + - :math:`T = -32\text{ dBFS}`, :math:`R = 12:1`, :math:`A = 2\text{ ms}` + - Strict excursion control on woofers; prevents bass resonance buzzing + * - **Band 1 (Midrange)** + - :math:`2\text{ kHz} - 8\text{ kHz}` + - :math:`T = -28\text{ dBFS}`, :math:`R = 6:1`, :math:`A = 5\text{ ms}` + - Transparent vocal presence leveling; enhances dialogue intelligibility + * - **Band 2 (Treble)** + - :math:`8\text{ kHz} - 20\text{ kHz}` + - :math:`T = -24\text{ dBFS}`, :math:`R = 4:1`, :math:`A = 8\text{ ms}` + - Gentle sibilance control (de-esser); protects sensitive micro-tweeters + +--- + +Runtime Parameter Injection & Live In-System Verification +********************************************************* + +Modern SOF topologies expose DRC parameters as ALSA byte controls, allowing acoustic engineers to inject new calibration profiles into active streaming pipelines without recompiling firmware or rebooting the target device. + +Step 1: Enumerate Target ALSA Controls on DUT +============================================= + +Connect to the target Device Under Test (DUT) over SSH and locate the active DRC controls: + +.. code-block:: bash + + # Query ALSA controls matching DRC or Multiband DRC + ssh root@ "amixer -Dhw:0 controls | grep -i drc" + + # Expected Output (IPC4 Example): + # numid=19,iface=MIXER,name='DRC1.0 19 DRC' + # numid=24,iface=MIXER,name='MULTIBAND_DRC1.0 24 MULTIBAND_DRC' + +Step 2: Transfer and Inject New Calibration Blob +================================================ + +Copy the synthesized binary blob to the target DUT and inject it into the active DSP pipeline using ``sof-ctl``: + +.. code-block:: bash + + # Copy blob to DUT + scp drc_speaker_ipc4.bin root@:/tmp/drc_tuning.bin + + # Inject blob into control numid=19 (IPC4 param_id=1, binary mode) + ssh root@ "sof-ctl -Dhw:0 -i 4 -n 19 -p 1 -b -s /tmp/drc_tuning.bin" + + # Expected Output: + # Applying configuration "/tmp/drc_tuning.bin" into device hw:0 control numid=19. + # Success. + +Step 3: Read Back and Bit-Verify Active Coefficients +==================================================== + +Confirm that the DSP accepted and installed the new parameters into memory by reading back the active control payload: + +.. code-block:: bash + + # Read back active parameters from DSP + ssh root@ "sof-ctl -Dhw:0 -i 4 -n 19 -p 1 -o /tmp/drc_readback.bin" + + # Verify bit-exact match against source blob + ssh root@ "cmp /tmp/drc_tuning.bin /tmp/drc_readback.bin && echo 'VERIFIED: Bit-exact match!'" + +Step 4: Monitor Real-Time DSP Execution via Traces +================================================== + +Monitor DSP firmware trace logs during parameter injection to verify configuration confirmation: + +.. code-block:: bash + + # Inspect trace logs over SSH + ssh root@ "mtrace | grep -i drc" + + # Expected Trace Log: + # [DSP] drc_set_config(): DRC configuration updated, num_channels=2, sample_rate=48000 + # [DSP] drc_set_config(): threshold=-30 dB, knee=20 dB, ratio=10:1, makeup=3 dB + +Step 5: Acoustic Verification with Stepped Amplitude Tones +========================================================== + +To verify compression behavior acoustically: + +1. Generate a stepped 1 kHz sine wave with amplitude steps at :math:`-40\text{ dBFS}, -30\text{ dBFS}, -20\text{ dBFS}, -10\text{ dBFS}, \text{and } 0\text{ dBFS}`. +2. Play the tone through the playback pipeline while capturing audio from the speaker output via a calibrated microphone or loopback card: + + .. code-block:: bash + + # Play stepped test tone on DUT + ssh root@ "aplay -Dplughw:0,0 /usr/share/sounds/stepped_tone_1khz.wav" + +3. Measure output levels at each step to plot the physical transfer curve. Verify that below :math:`-30\text{ dBFS}`, output increases linearly (:math:`1\text{ dB}` per input step), and above :math:`-30\text{ dBFS}`, output increases by only :math:`0.1\text{ dB}` per :math:`1\text{ dB}` input step (:math:`10:1` ratio). + +--- + +Tuning Diagnostics & Acoustic Artifact Matrix +********************************************* + +During the tuning process, incorrect parameter combinations can introduce audible artifacts or cause driver rejection. Table 37 provides diagnostic remedies: + +.. list-table:: Table 37: DRC Tuning Diagnostics & Acoustic Artifact Matrix + :header-rows: 1 + :widths: 20 30 50 + :class: tight-table + + * - Symptom / Artifact + - Root Cause + - Corrective Tuning Action + * - **Pumping / Breathing** + - Release time too fast on broadband signals; baseline noise rises audibly between speech syllables. + - Increase ``release`` time constant (:math:`\ge 200\text{ ms}`) or deploy Multiband DRC to isolate low frequencies from vocals. + * - **Transient Distortion** + - Attack time too slow; sharp peaks punch through unattenuated and clip downstream DAC/amplifier. + - Decrease ``attack`` time (:math:`1 - 3\text{ ms}`) and increase lookahead ``pre_delay`` (:math:`6\text{ ms}`) to ramp gain down ahead of peak arrival. + * - **Dull / Muffled Treble** + - Single-band compressor ducks entire spectrum during heavy bass notes. + - Migrate to 3-band Multiband DRC. Set woofer crossover to :math:`2\text{ kHz}` so low-frequency excursion clamping leaves treble unaffected. + * - **Crossover Phasing** + - Inverted band polarities or non-Linkwitz-Riley filter poles causing nulls at :math:`F_c`. + - Verify that crossover biquad coefficients adhere to Linkwitz-Riley 4th-order (LR4) specification (:math:`Q = 0.7071` cascaded). + * - **Driver Error -EINVAL** + - ABI magic header mismatch or payload size does not match packed struct size. + - Ensure blob is serialized with ``sof_drc_build_blob.m`` or ``sof_multiband_drc_build_blob.m`` matching active ABI version. + * - **Driver Error -EBUSY** + - Parameter update attempted while pipeline is in an active DMA lock. + - Trigger playback stream briefly or ensure pipeline state is ``RUNNING`` or ``PREPARED`` during control update. + +--- + +Upstream References & Code Links +******************************** + +* **Firmware Implementation**: + - DRC Core: `thesofproject/sof: src/audio/drc/drc.c `_ + - DRC Generic Implementation: `thesofproject/sof: src/audio/drc/drc_generic.c `_ + - DRC SIMD HiFi4 Vector Engine: `thesofproject/sof: src/audio/drc/drc_hifi4.c `_ + - Multiband DRC Core: `thesofproject/sof: src/audio/multiband_drc/multiband_drc.c `_ + - Multiband DRC Generic: `thesofproject/sof: src/audio/multiband_drc/multiband_drc_generic.c `_ +* **Data Structures & Headers**: + - `thesofproject/sof: src/audio/drc/drc_user.h `_ + - `thesofproject/sof: src/audio/multiband_drc/user/multiband_drc.h `_ +* **Tuning Scripts**: + - `thesofproject/sof: src/audio/drc/tune/ `_ + - `thesofproject/sof: src/audio/multiband_drc/tune/ `_ +* **Topology 2 Definitions**: + - DRC Widget: `thesofproject/sof: tools/topology/topology2/include/components/drc.conf `_ + - Multiband DRC Widget: `thesofproject/sof: tools/topology/topology2/include/components/multiband_drc.conf `_ diff --git a/developer_guides/tuning/images/drc_tuning_adaptive_release.svg b/developer_guides/tuning/images/drc_tuning_adaptive_release.svg new file mode 100644 index 00000000..2af09205 --- /dev/null +++ b/developer_guides/tuning/images/drc_tuning_adaptive_release.svg @@ -0,0 +1,50 @@ + +Adaptive 4-Zone Release Ballistics vs Static Decay + +0 ms + +50 ms + +100 ms + +150 ms + +200 ms + +250 ms + +300 ms + +-15 dB + +-12 dB + +-9 dB + +-6 dB + +-3 dB + +0 dB + +Zone 1 (Fast Transient Exit: rz1 = 0.09) + +Zone 2 (Mid-Level Recovery: rz2 = 0.16) + +Zone 3 (Decelerating Smooth Decay: rz3 = 0.42) + +Zone 4 (Harmonic Preservation: rz4 = 0.98) + + +Fast Initial Recovery (Eliminates Pumping) +Smooth Asymptotic Tail (Zero Harmonic Distortion) + + +Time Elapsed (ms) +Gain Reduction (dB) + + +SOF Adaptive 4-Zone Release + +Single Static Exponential Decay + \ No newline at end of file diff --git a/developer_guides/tuning/images/drc_tuning_detector_arch.svg b/developer_guides/tuning/images/drc_tuning_detector_arch.svg new file mode 100644 index 00000000..6c10ad06 --- /dev/null +++ b/developer_guides/tuning/images/drc_tuning_detector_arch.svg @@ -0,0 +1,190 @@ + + + + + + +DRC_Detector + + +cluster_main_path + +Main Audio Delay Path + + +cluster_sidechain + +Sidechain Envelope & Gain Computer Path + + + +in + +Audio In +x[n] +(PCM 32/24/16-bit) + + + +split + + + + +in->split + + + + + +lookahead + +Lookahead Circular Buffer +(pre_delay_time, up to 512 frames) +Compensates for Attack Ramp-Down + + + +split->lookahead + + + Audio Signal + + + +abs_rect + +Peak Level Detector +|x[n]| Absolute Rectification + + + +split->abs_rect + + + Sidechain Tap + + + +mult + +Gain Multiplier +Linear Scaling: x_delayed * g[n] + + + +lookahead->mult + + + Delayed Audio (x_delayed) + + + +out + +Audio Out +y[n] +(Protected / Leveled Stream) + + + +mult->out + + + + + +db_conv + +dB Domain Conversion +x_dB = 20 * log10(|x|) + + + +abs_rect->db_conv + + + + + +curve_comp + +Static Transfer Computer +Soft-Knee Polynomial +Ratio (1/R) & Threshold + + + +db_conv->curve_comp + + + + + +diff + +Gain Reduction (dB) +GR_dB = f(x_dB) - x_dB + + + +curve_comp->diff + + + + + +ballistics + +Envelope Ballistics Engine +Fast Attack (1-3 ms) +Adaptive 4-Zone Release (kA..kE) + + + +diff->ballistics + + + Target Attenuation + + + +lin_conv + +Linear Gain & Post-Gain +g_target = db2mag(GR_dB + post_gain) +Empirical Makeup Gain + + + +ballistics->lin_conv + + + + + +div_interp + +32-Frame Sub-Block Interpolator +Linear Interpolation per Sample +Smooth Parameter Transitions + + + +lin_conv->div_interp + + + Sub-block Gain (every 32 frames) + + + +div_interp->mult + + + Per-Sample Gain g[n] + + + diff --git a/developer_guides/tuning/images/drc_tuning_static_curve.svg b/developer_guides/tuning/images/drc_tuning_static_curve.svg new file mode 100644 index 00000000..23adb59c --- /dev/null +++ b/developer_guides/tuning/images/drc_tuning_static_curve.svg @@ -0,0 +1,59 @@ + +DRC Static Transfer Function & Soft-Knee Geometry + +-60 + +-60 + +-50 + +-50 + +-40 + +-40 + +-30 + +-30 + +-20 + +-20 + +-10 + +-10 + +0 + +0 + +1:1 Unity Gain (Passthrough) + +Soft-Knee (W = 20 dB) + +Compressed (R = 10:1) + + + + +Threshold T = -30 dBFS + + +T + W = -10 dBFS +Slope s = 1/R = 0.10 + +Post-Gain (+4 dB) + + +Input Level (dBFS) +Output Level (dBFS) + + +DRC Transfer Curve (0 dB Post) + +DRC with +4 dB Post-Gain + +Linear Passthrough (No Compression) + \ No newline at end of file diff --git a/developer_guides/tuning/images/drc_tuning_toolchain_workflow.svg b/developer_guides/tuning/images/drc_tuning_toolchain_workflow.svg new file mode 100644 index 00000000..32c0b6c8 --- /dev/null +++ b/developer_guides/tuning/images/drc_tuning_toolchain_workflow.svg @@ -0,0 +1,197 @@ + + + + + + +DRC_Toolchain + + +cluster_stage1 + +Stage 1: Offline Mathematical Synthesis (Host Workstation) + + +cluster_stage2 + +Stage 2: Quantization, Packaging & ABI Serialization + + +cluster_targets + +Production Packaging Targets + + +cluster_stage3 + +Stage 3: Live In-System Injection & DSP Verification (Target DUT) + + + +spec + +Acoustic Measurement & Requirements +• Speaker Excursion Limit (Xmax) +• Voice Coil Thermal Dissipation +• Microphone Sensitivity & Dynamic Span + + + +octave + +GNU Octave / MATLAB Tuning Suite +• sof_example_drc.m / sof_example_multiband_drc.m +• Threshold, Knee, Ratio, Attack, Release, Post-Gain +• 4-Zone Release Fitting (kA..kE) + + + +spec->octave + + + + + +plot + +Curve Inspection & Visualization +• sof_drc_plot_db_curve.m +• Decibel Transfer Verification (Slope 1/R) + + + +octave->plot + + + + + +quant + +Fixed-Point Struct Quantization +• Q8.24, Q2.30, Q12.20, Q20.12 +• sof_drc_generate_config.m + + + +octave->quant + + + + + +abi + +SOF ABI Header Packaging +• struct sof_abi_hdr (magic: 0x00464f53) +• IPC3 vs IPC4 Serialization (sof_drc_build_blob.m) + + + +quant->abi + + + + + +tplg + +ALSA Topology 2 +(.conf / .tplg) +Build-Time Driver Defaults + + + +abi->tplg + + + + + +ucm + +ALSA UCM2 Profile +(/usr/share/alsa/ucm2/) +Per-Device Acoustic Config + + + +abi->ucm + + + + + +state + +ALSA asound.state +(/var/lib/alsa/asound.state) +systemd Boot Restoration + + + +abi->state + + + + + +scp + +Secure Copy to Target DUT +scp drc_tuning.bin root@<dut>:/tmp/ + + + +abi->scp + + + Binary Blob (.bin) + + + +ctl + +Live Parameter Injection (sof-ctl) +sof-ctl -Dhw:0 -i 4 -n <numid> -p 1 -b -s /tmp/drc_tuning.bin +Zero Interruption to Active Audio Stream + + + +scp->ctl + + + + + +verify + +DSP Memory Readback & bit-exact Verification +sof-ctl -o /tmp/drc_readback.bin && cmp ... +mtrace Firmware Trace Log Confirmation + + + +ctl->verify + + + + + +acoustic + +Acoustic Measurement Validation +Stepped Sine Wave Test Tone Playback +Measurement Mic / Analyzer Verification + + + +verify->acoustic + + + + + diff --git a/developer_guides/tuning/images/multiband_drc_tuning_pipeline.svg b/developer_guides/tuning/images/multiband_drc_tuning_pipeline.svg new file mode 100644 index 00000000..e1b139e9 --- /dev/null +++ b/developer_guides/tuning/images/multiband_drc_tuning_pipeline.svg @@ -0,0 +1,177 @@ + + + + + + +Multiband_DRC_Pipeline + + +cluster_stage1 + +Stage 1: Pre-Emphasis Equalizer + + +cluster_stage2 + +Stage 2: LR4 Crossover Filter Bank + + +cluster_stage3 + +Stage 3: Per-Band Independent DRC Engines + + +cluster_stage4 + +Stage 4: Recombination & De-Emphasis + + + +in + +Audio In +x[n] +(48 kHz PCM) + + + +emp + +2-Biquad Pre-Emphasis +(emp_coef[2]) +Anchor: 15 kHz +High-Frequency Sensitivity Shaping + + + +in->emp + + + + + +xover + +Linkwitz-Riley 4th-Order (LR4) +(crossover_coef[6]) +24 dB/oct Steep Rolloff +0 dB Flat Sum / 0° Phase Shift + + + +emp->xover + + + + + +drc_low + +Band 0: Low / Bass (0 - 2 kHz) +• Threshold: -32 dBFS +• Ratio: 12:1 (Excursion Limiting) +• Attack: 2 ms, Pre-Delay: 6 ms + + + +xover->drc_low + + + Low Band (< 2 kHz) + + + +drc_mid + +Band 1: Midrange (2 - 8 kHz) +• Threshold: -28 dBFS +• Ratio: 6:1 (Vocal Leveling) +• Attack: 5 ms, Pre-Delay: 4 ms + + + +xover->drc_mid + + + Mid Band (2-8 kHz) + + + +drc_high + +Band 2: Treble (8 - 20 kHz) +• Threshold: -24 dBFS +• Ratio: 4:1 (De-Esser / Sibilance) +• Attack: 8 ms, Pre-Delay: 2 ms + + + +xover->drc_high + + + High Band (> 8 kHz) + + + +sum + +Subband Summation +(+) +32-bit Saturated Accumulator + + + +drc_low->sum + + + Protected Bass + + + +drc_mid->sum + + + Leveled Vocals + + + +drc_high->sum + + + De-Essed Treble + + + +deemp + +2-Biquad De-Emphasis +(deemp_coef[2]) +Exact Inverse of Pre-Emphasis +Restores Acoustic Naturalness + + + +sum->deemp + + + + + +out + +Audio Out +y[n] +(Coherent Multi-Band Stream) + + + +deemp->out + + + + + From 7ddf2ef41b7c79a089313d2091d1885545faf9b9 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 16:42:36 +0100 Subject: [PATCH 33/64] doc: developer_guides: add crossover filter design & multi-driver speaker tuning guide Signed-off-by: Liam Girdwood --- developer_guides/index.rst | 2 + developer_guides/tuning/crossover_tuning.rst | 793 ++++++++++++++++++ .../crossover_tuning_firmware_engine.svg | 122 +++ .../images/crossover_tuning_lr4_acoustics.svg | 178 ++++ .../crossover_tuning_toolchain_workflow.svg | 193 +++++ .../crossover_tuning_topology_routing.svg | 154 ++++ .../crossover_tuning_tree_topologies.svg | 243 ++++++ 7 files changed, 1685 insertions(+) create mode 100644 developer_guides/tuning/crossover_tuning.rst create mode 100644 developer_guides/tuning/images/crossover_tuning_firmware_engine.svg create mode 100644 developer_guides/tuning/images/crossover_tuning_lr4_acoustics.svg create mode 100644 developer_guides/tuning/images/crossover_tuning_toolchain_workflow.svg create mode 100644 developer_guides/tuning/images/crossover_tuning_topology_routing.svg create mode 100644 developer_guides/tuning/images/crossover_tuning_tree_topologies.svg diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 9aeff0af..9c882cb4 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -130,6 +130,7 @@ Dynamics & Transducer Protection Tuning Acoustic, Transducer & Array Tuning =================================== +* :ref:`crossover_tuning` (Linkwitz-Riley LR4 2-way, 3-way, and 4-way crossover filter design, phase-alignment merge, and multi-driver speaker tuning) * :ref:`equalizers_tuning` (Parametric FIR & IIR equalizers, MLS acoustical measurement, and speaker tuning) * :ref:`time-domain-fixed-beamformer` (Time-Domain Fixed Beamformer array geometry and spatial filter design) * :ref:`sample_rate_conversion` (Polyphase FIR filter design and multi-stage resampling) @@ -140,6 +141,7 @@ Acoustic, Transducer & Array Tuning tuning/runtime_tuning_sof_ctl tuning/drc_tuning + tuning/crossover_tuning algorithms/eq/equalizers_tuning algorithms/tdfb/time_domain_fixed_beamformer algorithms/src/sample_rate_conversion diff --git a/developer_guides/tuning/crossover_tuning.rst b/developer_guides/tuning/crossover_tuning.rst new file mode 100644 index 00000000..8a65fc3c --- /dev/null +++ b/developer_guides/tuning/crossover_tuning.rst @@ -0,0 +1,793 @@ +.. _crossover_tuning: + +Crossover Filter Design & Multi-Driver Speaker Tuning +##################################################### + +The Sound Open Firmware (**SOF**) Crossover component provides high-precision digital frequency-splitting filters designed to partition full-bandwidth audio into discrete acoustic bands for multi-driver loudspeaker systems (such as subwoofers, woofers, midranges, squawkers, and tweeters). By separating frequencies before power amplification, the crossover filter enables active multi-amplification architectures, eliminates bulky and lossy passive analog crossover networks, drastically reduces intermodulation distortion, and protects delicate high-frequency transducers from damaging low-frequency excursion. + +This guide details the mathematical foundations of SOF's Linkwitz-Riley 4th-order (**LR4**) filters, the recursive filter tree architectures for 2-way, 3-way, and 4-way systems, the innovative all-pass phase-alignment merge mechanism, the fixed-point Q2.30 DSP implementation, Topology 2 multi-sink pipeline routing, offline GNU Octave calibration workflows, live runtime parameter injection via ``sof-ctl``, and acoustic diagnostic troubleshooting. + +.. contents:: + :local: + :depth: 2 + +Theoretical Foundations of Crossover Filters +******************************************** + +Active digital crossovers divide an input signal :math:`x(n)` into :math:`N` frequency bands such that the acoustic summation of all driver outputs reproduces the original input signal with minimal magnitude ripple, linear phase behavior, and zero polar lobing tilt in the listening window. + +Butterworth vs. Linkwitz-Riley Topologies +========================================= + +Standard Butterworth filters of order :math:`N` exhibit maximally flat passbands with a :math:`-3.01\text{ dB}` attenuation at their cutoff frequency :math:`\omega_c`: + +.. math:: + + |H_{\text{Butter}}(j\omega_c)| = \frac{1}{\sqrt{1 + 1}} = \frac{1}{\sqrt{2}} \approx -3.01\text{ dB} + +When two complementary Butterworth filters (low-pass and high-pass) are summed in phase, their power response sums to unity, but their voltage magnitude response exhibits a :math:`+3\text{ dB}` resonant peak at the crossover frequency: + +.. math:: + + |H_{\text{LP,Butter}}(j\omega_c) + H_{\text{HP,Butter}}(j\omega_c)| = \frac{1}{\sqrt{2}} + \frac{1}{\sqrt{2}} = \sqrt{2} \approx +3.01\text{ dB} + +This :math:`+3\text{ dB}` peak produces audible acoustic boominess, coloration, and severe driver strain at the crossover boundary. + +To eliminate magnitude peaking, **Linkwitz and Riley** demonstrated that cascading two identical Butterworth filters of order :math:`N/2` in series creates an even-order filter whose cutoff attenuation is exactly :math:`-6.02\text{ dB}`: + +.. math:: + + |H_{\text{LR}}(j\omega_c)| = |H_{\text{Butter}}(j\omega_c)|^2 = \left(\frac{1}{\sqrt{2}}\right)^2 = \frac{1}{2} = -6.02\text{ dB} + +When summed in phase, the combined magnitude response is perfectly flat across the entire audio spectrum: + +.. math:: + + H_{\text{sum}}(j\omega) = H_{\text{LP,LR4}}(j\omega) + H_{\text{HP,LR4}}(j\omega) \equiv 1.00 \quad (0.00\text{ dB ripple}) + +Table 38 compares common crossover filter topologies. + +.. list-table:: Crossover Filter Topologies & Theoretical Acoustic Specifications + :header-rows: 1 + :widths: 20 12 15 18 18 17 + :class: tight-table + + * - Filter Type & Order + - Roll-Off Slope + - Cutoff Level (:math:`F_c`) + - Summed Magnitude + - Phase at :math:`F_c` + - Polar Lobe Tilt + * - **Butterworth 2nd (BW2)** + - -12 dB/oct + - -3.01 dB + - Flat (if inverted) + - :math:`180^\circ` shift + - Off-axis tilt + * - **Linkwitz-Riley 2nd (LR2)** + - -12 dB/oct + - -6.02 dB + - Flat (if inverted) + - :math:`180^\circ` shift + - On-axis symmetric + * - **Butterworth 3rd (BW3)** + - -18 dB/oct + - -3.01 dB + - Flat magnitude + - :math:`90^\circ` quadrature + - Asymmetric tilt + * - **Butterworth 4th (BW4)** + - -24 dB/oct + - -3.01 dB + - **+3.01 dB peak** + - :math:`180^\circ` shift + - Severe peaking + * - **Linkwitz-Riley 4th (LR4)** + - **-24 dB/oct** + - **-6.02 dB** + - **0.00 dB (Flat)** + - **In-Phase** (:math:`0^\circ / 360^\circ`) + - **Zero tilt (On-axis)** + +Mathematical Formulation of Linkwitz-Riley 4th-Order (LR4) +========================================================== + +The SOF Crossover engine exclusively implements Linkwitz-Riley 4th-order (**LR4**) filters because they provide three indispensable electro-acoustic properties: + +1. **Zero Magnitude Peaking**: The summed voltage transfer function forms an all-pass network with an exact 0.00 dB magnitude response: + + .. math:: + + |H_{\text{LP,LR4}}(j\omega) + H_{\text{HP,LR4}}(j\omega)| = 1.00 \quad \forall \omega + +2. **Phase Coherence & Zero Lobing Tilt**: The low-pass and high-pass outputs are precisely in phase (:math:`\Delta\phi = 0^\circ` or :math:`360^\circ`) at all frequencies. As a consequence, the constructive acoustic interference lobe is directed exactly perpendicular to the speaker baffle (on-axis) without vertical acoustic tilt. +3. **Steep Transducer Isolation (-24 dB/octave)**: An attenuation rate of :math:`-24\text{ dB/octave}` (or :math:`-80\text{ dB/decade}`) rapidly suppresses out-of-band energy, protecting fragile micro-tweeter voice coils from low-frequency excursion damage while mitigating cone breakup resonance modes in woofers. + +In the continuous Laplace domain (:math:`s`), an LR4 low-pass filter with angular cutoff frequency :math:`\omega_c = 2\pi F_c` is defined as the square of a 2nd-order Butterworth filter: + +.. math:: + + H_{\text{LP,LR4}}(s) = \left[ \frac{\omega_c^2}{s^2 + \sqrt{2}\omega_c s + \omega_c^2} \right]^2 = \frac{\omega_c^4}{\left(s^2 + \sqrt{2}\omega_c s + \omega_c^2\right)^2} + +Similarly, the complementary LR4 high-pass filter is: + +.. math:: + + H_{\text{HP,LR4}}(s) = \left[ \frac{s^2}{s^2 + \sqrt{2}\omega_c s + \omega_c^2} \right]^2 = \frac{s^4}{\left(s^2 + \sqrt{2}\omega_c s + \omega_c^2\right)^2} + +Summing the two transfer functions yields: + +.. math:: + + H_{\text{sum}}(s) = H_{\text{LP,LR4}}(s) + H_{\text{HP,LR4}}(s) = \frac{s^4 + \omega_c^4}{\left(s^2 + \sqrt{2}\omega_c s + \omega_c^2\right)^2} = \frac{s^2 - \sqrt{2}\omega_c s + \omega_c^2}{s^2 + \sqrt{2}\omega_c s + \omega_c^2} + +Evaluating along the imaginary axis :math:`s = j\omega`: + +.. math:: + + |H_{\text{sum}}(j\omega)| = \left| \frac{-\omega^2 - j\sqrt{2}\omega_c\omega + \omega_c^2}{-\omega^2 + j\sqrt{2}\omega_c\omega + \omega_c^2} \right| = \frac{\sqrt{(\omega_c^2 - \omega^2)^2 + 2\omega_c^2\omega^2}}{\sqrt{(\omega_c^2 - \omega^2)^2 + 2\omega_c^2\omega^2}} \equiv 1.00 + +The sum is an all-pass network with unity gain and a phase rotation of :math:`-360^\circ` across the frequency spectrum. + +Bilinear Transform & Digital Biquad Realization +----------------------------------------------- + +To execute the LR4 filter inside the SOF DSP audio processing pipeline at sampling frequency :math:`f_s`, the continuous transfer function is mapped to the discrete :math:`z`-domain via the **Bilinear Transform** with frequency pre-warping: + +.. math:: + + s = \frac{2}{T_s} \frac{1 - z^{-1}}{1 + z^{-1}} = 2 f_s \frac{1 - z^{-1}}{1 + z^{-1}} + +To preserve the exact analog cutoff frequency :math:`F_c` in the digital domain, the continuous cutoff is pre-warped: + +.. math:: + + \omega_a = 2 f_s \tan\left( \frac{\pi F_c}{f_s} \right) + +Each 4th-order filter is factored into a cascade of **two identical 2nd-order Direct-Form I (DF1) biquad sections**: + +.. math:: + + H_{\text{biquad}}(z) = \frac{b_0 + b_1 z^{-1} + b_2 z^{-2}}{1 + a_1 z^{-1} + a_2 z^{-2}} + +For a 2nd-order Butterworth low-pass stage with resonance :math:`Q = 1/\sqrt{2} \approx 0.7071`, let normalized cutoff :math:`\theta = \frac{\pi F_c}{f_s}`: + +.. math:: + + d &= \sqrt{2} \approx 1.41421356 \\ + s_n &= \frac{1}{2} d \sin(\theta) \\ + \beta &= \frac{1}{2} \frac{1 - s_n}{1 + s_n} \\ + \gamma &= (0.5 + \beta) \cos(\theta) \\ + \alpha_{\text{LP}} &= \frac{1}{4} (0.5 + \beta - \gamma) + +The resulting discrete low-pass biquad coefficients are: + +.. math:: + + b_0 = 2\alpha_{\text{LP}}, \quad b_1 = 4\alpha_{\text{LP}}, \quad b_2 = 2\alpha_{\text{LP}}, \quad a_1 = -2\gamma, \quad a_2 = 2\beta + +For the complementary high-pass biquad stage, let :math:`\alpha_{\text{HP}} = \frac{1}{4} (0.5 + \beta + \gamma)`: + +.. math:: + + b_0 = 2\alpha_{\text{HP}}, \quad b_1 = -4\alpha_{\text{HP}}, \quad b_2 = 2\alpha_{\text{HP}}, \quad a_1 = -2\gamma, \quad a_2 = 2\beta + +.. figure:: images/crossover_tuning_lr4_acoustics.svg + :alt: Linkwitz-Riley LR4 Acoustic Response and Summation + :width: 100% + :align: center + + Figure 263: Linkwitz-Riley LR4 (-24 dB/oct) Acoustic Magnitude and Phase Response. Note the exact -6.02 dB level at :math:`F_c`, the 0.00 dB flat acoustic summation across the spectrum, and the absence of the +3.01 dB peak characteristic of Butterworth 4th-order filters. + +Filter Tree Topologies & The Phase Asymmetry Solution +***************************************************** + +SOF supports 2-way, 3-way, and 4-way crossover configurations. While 2-way and 4-way systems possess natural structural symmetry, 3-way systems present an inherent acoustic phase challenge that SOF resolves through an innovative all-pass merge block. + +.. figure:: images/crossover_tuning_tree_topologies.svg + :alt: SOF Crossover Filter Tree Topologies + :width: 100% + :align: center + + Figure 262: SOF Crossover Filter Tree Topologies. (1) Two-way symmetrical split; (2) Three-way split with all-pass LR4 phase-alignment merge block on the LOW branch; (3) Four-way symmetrical binary tree with natural 8-pole group delay parity. + +Two-Way Crossover Topology +========================== + +In a 2-way configuration (such as a separate woofer and tweeter), the input signal :math:`x(n)` is split by a single LR4 filter pair at crossover frequency :math:`F_{c,\text{low}}`: + +* **LOW Output** (``assign_sink[0]``): Filtered by LR4 LP0 (:math:`F_{c,\text{low}}`), routing bass and lower-midrange energy to the woofer. +* **HIGH Output** (``assign_sink[1]``): Filtered by LR4 HP0 (:math:`F_{c,\text{low}}`), routing upper-midrange and treble energy to the tweeter. + +Because both outputs traverse exactly one LR4 filter (consisting of two cascaded biquads, or 4 poles), both paths experience identical group delay and phase lag (:math:`-360^\circ` rotation across the band). When acoustically radiated, the outputs sum with zero phase cancellation. + +Three-Way Crossover & The Phase Alignment Problem +================================================= + +In a traditional 3-way crossover (woofer, midrange, tweeter), the input is partitioned using two cutoff frequencies: :math:`F_{c,\text{low}}` (e.g. 250 Hz) and :math:`F_{c,\text{high}}` (e.g. 2500 Hz). + +In an intuitive asymmetric tree: + +1. The input :math:`x(n)` is first split at :math:`F_{c,\text{low}}` into a low-frequency signal :math:`z_1` (via LP0) and a high-frequency signal :math:`z_2` (via HP0). +2. The high signal :math:`z_2` is then split at :math:`F_{c,\text{high}}` into a midrange signal (via LP2) and a treble signal (via HP2). + +The Phase Asymmetry Flaw +------------------------ + +Under this naive architecture: + +* The **Midrange** and **Tweeter** branches have traversed **two sequential LR4 filters** (LP0 followed by LP2/HP2), totaling **8 poles** and experiencing a :math:`-720^\circ` phase rotation. +* The **Woofer** branch :math:`z_1` has traversed **only one LR4 filter** (LP0), totaling **4 poles** and experiencing only :math:`-360^\circ` of phase rotation. + +At the crossover boundary :math:`F_{c,\text{low}}`, the woofer output and the midrange output are **out of phase by** :math:`360^\circ` relative to higher bands, creating severe group delay disparity, smearing transients, and causing deep destructive notches in the acoustic radiation pattern if listener alignment is slightly off-axis. + +The SOF LR4 Merge Solution +-------------------------- + +To eliminate this phase asymmetry, SOF introduces an **All-Pass Phase-Equalization Merge Block** on the LOW branch (implemented in ``crossover_generic_split_3way()`` in ``crossover_generic.c``). + +The low-frequency signal :math:`z_1` is routed through a secondary pair of LR4 filters tuned to :math:`F_{c,\text{high}}` (LP1 and HP1), and their outputs are algebraically summed back together before reaching the woofer sink: + +.. math:: + + y_{\text{low}}(n) = \text{LP1}(z_1(n)) + \text{HP1}(z_1(n)) + +Because the sum of an LR4 low-pass and high-pass filter is an all-pass network: + +.. math:: + + |H_{\text{LP1}}(j\omega) + H_{\text{HP1}}(j\omega)| = 1.00 \quad \forall \omega + +The magnitude spectrum of the woofer signal is **completely unaffected** (0.00 dB alteration). However, passing through the LP1/HP1 network injects an exact **4-pole phase lag and group delay** matching the secondary filter stage of the midrange and tweeter paths! + +As a result, all three outputs (LOW, MID, HIGH) traverse exactly 8 poles (4 biquads), achieving **perfect group delay parity and phase alignment** across all three drivers. + +Four-Way Crossover Topology +=========================== + +In a 4-way system (subwoofer, woofer, squawker/midrange, tweeter), SOF implements a symmetrical binary tree with three crossover frequencies: :math:`F_{c,\text{low}}`, :math:`F_{c,\text{mid}}`, and :math:`F_{c,\text{high}}`: + +1. **Root Split**: Input :math:`x(n)` is split at :math:`F_{c,\text{mid}}` by LR4 LP1 and HP1 into low group :math:`z_1` and high group :math:`z_2`. +2. **Low-Band Split**: :math:`z_1` is split at :math:`F_{c,\text{low}}` by LR4 LP0 and HP0 into ``assign_sink[0]`` (Subwoofer) and ``assign_sink[1]`` (Woofer). +3. **High-Band Split**: :math:`z_2` is split at :math:`F_{c,\text{high}}` by LR4 LP2 and HP2 into ``assign_sink[2]`` (Midrange) and ``assign_sink[3]`` (Tweeter). + +Every output path naturally traverses exactly two LR4 stages (8 poles, 4 biquads). No merge block is required, and all 4 paths maintain identical group delay. + +Firmware Architecture & Fixed-Point Implementation +************************************************** + +The SOF Crossover component (``src/audio/crossover/``) operates as a multi-sink module adapter plugin. + +Firmware Data Structures +======================== + +The user configuration blob is defined in ``include/user/crossover.h``: + +.. code-block:: c + + #define SOF_CROSSOVER_MAX_STREAMS 4 + + struct sof_crossover_config { + uint32_t size; /* Total blob size in bytes */ + uint32_t num_sinks; /* Number of output streams (2, 3, or 4) */ + uint32_t reserved[4]; /* Reserved for 64-bit alignment */ + uint32_t assign_sink[SOF_CROSSOVER_MAX_STREAMS];/* Sink pipeline ID (IPC3) or output pin index (IPC4) */ + struct sof_eq_iir_biquad coef[]; /* Cascaded LR4 biquad coefficients */ + } __attribute__((packed)); + +Each biquad is defined by ``struct sof_eq_iir_biquad`` (from ``include/user/eq.h``): + +.. code-block:: c + + struct sof_eq_iir_biquad { + int32_t a2; /* Q2.30 - Recursive feedback coefficient y[n-2] */ + int32_t a1; /* Q2.30 - Recursive feedback coefficient y[n-1] */ + int32_t b2; /* Q2.30 - Feedforward coefficient x[n-2] */ + int32_t b1; /* Q2.30 - Feedforward coefficient x[n-1] */ + int32_t b0; /* Q2.30 - Feedforward coefficient x[n] */ + int32_t output_shift; /* Right-shift count (negative indicates left shift) */ + int32_t output_gain; /* Q2.14 - Linear output post-gain multiplier (16384 = unity 1.0) */ + } __attribute__((packed)); + +Fixed-Point Q-Format Specifications (Table 39) +============================================== + +To ensure deterministic real-time processing on fixed-point DSP architectures (such as Tensilica HiFi 3, HiFi 4, and HiFi 5), all coefficients are quantized according to Table 39. + +.. list-table:: ``sof_crossover_config`` Memory Layout & Q-Format Quantization Rules + :header-rows: 1 + :widths: 25 15 20 40 + :class: tight-table + + * - Struct Field + - Q-Format + - Dynamic Range + - Resolution & Description + * - ``size`` + - Q32.0 (uint) + - :math:`[0, 1024]` + - Total payload size in bytes including headers + * - ``num_sinks`` + - Q32.0 (uint) + - :math:`2, 3, \text{or } 4` + - Number of split frequency sinks enabled + * - ``assign_sink[i]`` + - Q32.0 (uint) + - :math:`[0, 3]` + - Output pin index (IPC4) or pipeline ID (IPC3) + * - ``b0, b1, b2`` + - Q2.30 (signed) + - :math:`[-2.0, +1.999999998]` + - Feedforward numerator biquad coefficients; :math:`\text{LSB} = 2^{-30} \approx 9.31 \times 10^{-10}` + * - ``a1, a2`` + - Q2.30 (signed) + - :math:`[-2.0, +1.999999998]` + - Feedback denominator biquad coefficients (negated in firmware difference equation) + * - ``output_shift`` + - Q32.0 (signed) + - :math:`0` (nominal) + - Post-biquad bit shift count for headroom management + * - ``output_gain`` + - Q2.14 (signed) + - :math:`[0, 32767]` + - Post-biquad linear scaling factor; :math:`16384 = 1.000` (unity gain) + +Firmware Execution Engine +========================= + +.. figure:: images/crossover_tuning_firmware_engine.svg + :alt: SOF Crossover Firmware Execution Engine + :width: 100% + :align: center + + Figure 265: SOF Crossover Firmware Execution Engine. Per-channel Direct-Form I biquad processing loop with 64-bit accumulators, saturation protection, and multi-sink buffer scatter. + +Each channel maintains an independent filter state struct: + +.. code-block:: c + + struct crossover_state { + struct iir_state_df1 lowpass[CROSSOVER_MAX_LR4]; /* Low-pass filter delay lines */ + struct iir_state_df1 highpass[CROSSOVER_MAX_LR4]; /* High-pass filter delay lines */ + }; + +During each processing period: + +1. Input samples are read from the upstream buffer in 32-bit Q1.31 format. +2. The dynamic split function pointer (``cd->crossover_split``) routes the sample through the configured LR4 tree. +3. In each LR4 biquad stage, the Direct-Form I difference equation is computed using 64-bit precision: + + .. math:: + + y(n) = \text{sat}_{32}\left( \left[ \sum_{k=0}^2 b_k x(n-k) - \sum_{k=1}^2 a_k y(n-k) \right] \gg 30 \right) + +4. The resulting split samples :math:`\text{out}[0 \dots N-1]` are written to their respective downstream sink buffers (``bsinks[j]``) with appropriate bit-depth scaling and rounding. +5. Inactive or disconnected sinks are gracefully bypassed without pipeline stalls. + +Topology 2 & IPC4 Multi-Sink Architecture +***************************************** + +In modern SOF systems running IPC4, the Crossover component is defined as an effect widget with **1 input pin** and **up to 4 output pins**. + +Topology 2 Widget Definition (``crossover.conf``) +================================================== + +.. code-block:: text + + Class.Widget."crossover" { + DefineAttribute."index" { type "integer" } + DefineAttribute."instance" { type "integer" } + + + + attributes { + !constructor [ "index" "instance" ] + !mandatory [ + "num_input_pins" + "num_output_pins" + "num_input_audio_formats" + "num_output_audio_formats" + ] + !immutable [ "uuid" "type" ] + unique "instance" + } + + uuid "d1:9a:8c:94:6a:80:31:41:ad:6c:b2:bd:a9:e3:5a:9f" + type "effect" + no_pm "true" + num_input_pins 1 + } + +IPC4 Multi-Output Binding Pipeline +================================== + +Figure 264 shows a representative 2-way playback topology with independent power amplifier DAIs (SSP0 for Woofer and SSP2 for Tweeter). + +.. figure:: images/crossover_tuning_topology_routing.svg + :alt: Topology 2 Multi-Sink Pipeline Architecture + :width: 100% + :align: center + + Figure 264: Topology 2 Multi-Sink Pipeline Architecture and IPC4 Output Pin Binding. Wideband PCM playback is ingested by Pipeline 1, split into Woofer and Tweeter streams by ``crossover.1.1``, and routed via Output Pins 0 and 1 to downstream DAI copiers SSP0 and SSP2. + +In ``cavs-nocodec-crossover.conf``, the crossover widget declares two output pins bound to independent sink pipelines: + +.. code-block:: text + + Object.Widget.crossover [ + { + index 1 + name "crossover.1.1" + + num_input_audio_formats 1 + Object.Base.input_audio_format.1 { + input_pin_index 0 + in_bit_depth 32 + in_valid_bit_depth 32 + } + + num_output_pins 2 + num_output_audio_formats 2 + Object.Base.output_audio_format { + 1 { + output_pin_index 0 + out_bit_depth 32 + out_valid_bit_depth 32 + } + 2 { + output_pin_index 1 + out_bit_depth 32 + out_valid_bit_depth 32 + } + } + + Object.Base.output_pin_binding.1 { + output_pin_binding_name "dai-copier.SSP.NoCodec-0.playback" + } + + Object.Base.output_pin_binding.2 { + output_pin_binding_name "dai-copier.SSP.NoCodec-2.playback" + } + + Object.Control.bytes."1" { + name "crossover.1.1_bytes_control" + IncludeByKey.EFX_CROSSOVER_PARAMS { + "2way" "include/components/crossover/coef_2way_48000_200_0_1.conf" + "3way" "include/components/crossover/coef_3way_48000_200_1000_0_1_2.conf" + "4way" "include/components/crossover/coef_4way_48000_200_1000_3000_0_1_2_3.conf" + } + } + } + ] + +.. note:: + + In IPC4 firmware manifests (``crossover.toml``), ``init_config = 1`` is required so that the kernel driver appends ``base_cfg_ext`` to the IPC initialization message. This extension supplies ``nb_output_pins`` and the pin index array, allowing the firmware to bind sink channels correctly. + +Offline Filter Synthesis & Calibration Toolchain +************************************************ + +SOF provides an in-tree GNU Octave / MATLAB toolchain in ``src/audio/crossover/tune/`` that automates the calculation of continuous analog Butterworth filters, bilinear transformation with pre-warping, fixed-point coefficient quantization, all-pass phase-merge insertion, and multi-target serialization. + +.. figure:: images/crossover_tuning_toolchain_workflow.svg + :alt: End-to-End Crossover Tuning Toolchain + :width: 100% + :align: center + + Figure 266: End-to-End Crossover Tuning, Offline Synthesis, and Live Injection Toolchain. Complete toolchain from physical acoustic profiling to GNU Octave synthesis, multi-target packaging, and runtime parameter injection via ``sof-ctl``. + +Octave Script Walkthrough +========================= + +The central synthesis driver is ``sof_example_crossover.m``: + +.. code-block:: octave + + function sof_example_crossover() + cr.fs = 48e3; % Sample rate: 48 kHz + cr.fc_low = 2200; % Crossover frequency: 2200 Hz + cr.num_sinks = 2; % 2-way crossover + cr.sinks = [0 1]; % Output pin 0 (Woofer), Output pin 1 (Tweeter) + + export_crossover(cr); + end + +The synthesis pipeline executes the following stages: + +1. **Coefficient Synthesis** (``sof_crossover_gen_coefs.m``): + Computes the analog 2nd-order Butterworth coefficients and applies the bilinear transform with frequency pre-warping: + + .. code-block:: octave + + crossover = sof_crossover_gen_coefs(cr.fs, cr.fc_low); + +2. **Fixed-Point Quantization** (``sof_crossover_coef_quant.m``): + Quantizes continuous coefficients into 32-bit Q2.30 integers and appends shift/gain values: + + .. code-block:: octave + + crossover_bqs = sof_crossover_coef_quant(crossover.lp, crossover.hp); + +3. **Configuration Struct Assembly** (``sof_crossover_generate_config.m``): + Interleaves low-pass and high-pass biquads into the canonical sequence: + ``[LP0, HP0, LP1, HP1, LP2, HP2]``. +4. **Binary Blob Construction** (``sof_crossover_build_blob.m``): + Packs the ``sof_abi_hdr`` and configuration struct into little-endian binary bytes. +5. **Multi-Target Artifact Export**: + Exports configuration data into three production targets: + + * **Topology 2**: ``sof_tplg2_write(conf_path, blob, 'crossover_config', 'Exported Control Bytes')`` + * **UCM2 Profile**: ``sof_ucm_blob_write(bin_path, blob)`` + * **ALSA State**: ``sof_alsactl_write(txt_path, blob)`` +6. **Frequency & Phase Plot Verification** (``sof_crossover_plot_freq.m``): + Plots magnitude and unwrapped phase responses to visually confirm 0.00 dB acoustic summation. + +Production Acoustic Tuning Presets +********************************** + +Below are three validated production preset configurations designed for real-world acoustic hardware. + +Preset 1: 2-Way Laptop Woofer/Tweeter Split +============================================== + +* **Target Hardware**: Ultra-thin laptop with separate bottom-firing micro-woofers and top-firing silk dome tweeters. +* **Acoustic Rationale**: Micro-tweeters suffer non-linear distortion and thermal runaway if driven below 1800 Hz. The 2200 Hz crossover protects the tweeter while maintaining smooth off-axis dispersion. + +.. list-table:: 2-Way Laptop Woofer/Tweeter Crossover Calibration Preset + :header-rows: 1 + :widths: 25 20 20 35 + :class: tight-table + + * - Parameter + - Value + - Unit / Format + - Acoustic Justification + * - ``Sampling Rate (Fs)`` + - 48000 + - Hz + - Standard high-definition audio clock + * - ``num_sinks`` + - 2 + - Integer + - 2-way split (Woofer + Tweeter) + * - ``Cutoff (Fc_low)`` + - 2200 + - Hz + - :math:`> 2 \times F_s` of tweeter (resonance :math:`\approx 1000\text{ Hz}`) + * - ``Filter Topology`` + - LR4 (-24 dB/oct) + - Architecture + - Cascaded DF1 biquads with -6.02 dB crossover point + * - ``assign_sink[0]`` + - 0 + - Pin Index + - Low band to Woofer amplifier (SSP0) + * - ``assign_sink[1]`` + - 1 + - Pin Index + - High band to Tweeter amplifier (SSP2) + * - ``biquads allocated`` + - 2 + - Sections + - LP0 (2 biquads) and HP0 (2 biquads) + +Preset 2: 3-Way Conference Speaker / Soundbar +=============================================== + +* **Target Hardware**: Smart conference speakerphone or television soundbar with dedicated subwoofer, stereo midrange drivers, and dual high-frequency tweeters. +* **Acoustic Rationale**: Low crossover at 250 Hz isolates bass vibrations from vocal clarity; high crossover at 2800 Hz keeps directional treble crisp. All-pass merge block preserves perfect speech intelligibility across the band. + +.. list-table:: 3-Way Soundbar / Conference Speaker Crossover Calibration Preset + :header-rows: 1 + :widths: 25 20 20 35 + :class: tight-table + + * - Parameter + - Value + - Unit / Format + - Acoustic Justification + * - ``Sampling Rate (Fs)`` + - 48000 + - Hz + - Standard teleconferencing clock + * - ``num_sinks`` + - 3 + - Integer + - 3-way split (Bass + Vocal Mid + Treble) + * - ``Fc_low`` + - 250 + - Hz + - Subwoofer/Woofer acoustic boundary + * - ``Fc_high`` + - 2800 + - Hz + - Midrange/Tweeter acoustic boundary + * - ``Phase Merge Block`` + - Enabled + - Architecture + - LP1/HP1 at 2800 Hz merged on LOW path (8-pole parity) + * - ``assign_sink[0]`` + - 0 + - Pin Index + - Subwoofer amplifier sink + * - ``assign_sink[1]`` + - 1 + - Pin Index + - Midrange driver amplifier sink + * - ``assign_sink[2]`` + - 2 + - Pin Index + - High-frequency tweeter amplifier sink + +Preset 3: 4-Way High-Fidelity Studio Monitor +============================================= + +* **Target Hardware**: Active 4-way reference studio monitor (Subwoofer + Mid-Bass Woofer + Dome Midrange + Ribbon Tweeter). +* **Acoustic Rationale**: Full-range linear reproduction from 20 Hz to 20 kHz. Symmetrical binary tree provides identical 8-pole group delay across all four acoustic bands. + +.. list-table:: 4-Way High-Fidelity Studio Monitor Crossover Calibration Preset + :header-rows: 1 + :widths: 25 20 20 35 + :class: tight-table + + * - Parameter + - Value + - Unit / Format + - Acoustic Justification + * - ``Sampling Rate (Fs)`` + - 48000 + - Hz + - Studio reference clock + * - ``num_sinks`` + - 4 + - Integer + - Subwoofer, Low-Mid, High-Mid, Tweeter + * - ``Fc_low`` + - 100 + - Hz + - Sub-bass excursion boundary + * - ``Fc_mid`` + - 800 + - Hz + - Primary vocal fundamental boundary + * - ``Fc_high`` + - 3500 + - Hz + - Ribbon tweeter protection boundary + * - ``assign_sink[0..3]`` + - 0, 1, 2, 3 + - Pin Indices + - Mapped to DAIs 0, 1, 2, 3 respectively + * - ``Group Delay Parity`` + - Exactly 8 poles + - Verification + - Zero group delay skew between any driver pair + +Live Runtime Injection & Calibration Runbook +******************************************** + +The SOF Crossover component allows live, glitch-free coefficient updating over the network without stopping the audio stream or rebooting the device under test (**DUT**). + +Step 1: Enumerate Active Crossover Byte Controls +================================================ + +Query the ALSA mixer on the target DUT to locate the Crossover byte control: + +.. code-block:: bash + + # Connect to DUT over SSH and inspect controls + ssh root@spider "amixer -Dhw:0 controls | grep -i crossover" + +Expected output: + +.. code-block:: text + + numid=14,iface=MIXER,name='crossover.1.1_bytes_control' + +Step 2: Synthesize New Coefficients in GNU Octave +================================================= + +On the host development workstation, launch GNU Octave and synthesize a new crossover profile (e.g. testing an alternative 2500 Hz cutoff): + +.. code-block:: octave + + cd ~/work/sof/src/audio/crossover/tune/ + cr.fs = 48e3; + cr.fc_low = 2500; + cr.num_sinks = 2; + cr.sinks = [0 1]; + export_crossover(cr); + +This generates ``../../../../tools/ctl/ipc4/crossover/coef_2way.bin``. + +Step 3: Inject the Binary Blob via ``sof-ctl`` +============================================== + +Transfer and inject the binary blob directly into the running DSP pipeline: + +.. code-block:: bash + + # Copy blob to DUT + scp tools/ctl/ipc4/crossover/coef_2way.bin root@spider:/tmp/crossover_new.bin + + # Inject blob into ALSA control numid 14 using IPC4 Large Config Set + ssh root@spider "sof-ctl -Dhw:0 -i 4 -n 14 -p 0 -b -s /tmp/crossover_new.bin" + +Step 4: Verify Active Coefficients & DSP Trace Logs +=================================================== + +Verify that the DSP processed and applied the new coefficients: + +.. code-block:: bash + + # Read back active coefficients from DSP memory + ssh root@spider "sof-ctl -Dhw:0 -i 4 -n 14 -p 0 -r -o /tmp/crossover_readback.bin" + + # Verify byte-level integrity + ssh root@spider "cmp /tmp/crossover_new.bin /tmp/crossover_readback.bin && echo 'COEFFICIENTS MATCH'" + +Inspect the firmware trace stream: + +.. code-block:: bash + + ssh root@spider "mtrace | grep -i crossover" + +Expected DSP log output: + +.. code-block:: text + + crossover crossover.1.1: crossover_init_coef_ch: num_sinks = 2 + crossover crossover.1.1: LR4 LP0 b0=0x00a12b40 b1=0x01425680 b2=0x00a12b40 a1=0x831a2c00 a2=0x38b29000 + crossover crossover.1.1: configuration applied successfully (no stream interruption) + +Acoustic Diagnostics & Troubleshooting Matrix +********************************************* + +Table 43 provides diagnostic procedures for resolving acoustic anomalies and runtime configuration issues during crossover tuning. + +.. list-table:: Crossover Acoustic Artifact & Diagnostic Troubleshooting Matrix + :header-rows: 1 + :widths: 22 25 25 28 + :class: tight-table + + * - Symptom / Artifact + - Root Cause + - Diagnostic Check + - Corrective Action + * - **Deep acoustic notch at** :math:`F_c` (:math:`-12\text{ to } -30\text{ dB}`) + - Drivers wired with inverted acoustic polarity or off-axis phase cancellation + - Measure on-axis acoustic frequency sweep with measurement mic + - Invert electrical polarity of one driver or ensure LR4 filters are used (which are in-phase). + * - **Tweeter distortion / premature failure** + - Crossover frequency :math:`F_c` set too close to tweeter mechanical resonance :math:`F_s` + - Impedance sweep showing resonance peak :math:`F_s` + - Increase :math:`F_c` such that :math:`F_c \ge 2 \times F_s`. Verify -24 dB/oct slope. + * - **Vocal smearing in 3-way system** + - Phase asymmetry between Woofer and Midrange branches + - Group delay calculation in GNU Octave + - Verify all-pass merge block is enabled (``crossover_generic_split_3way``) to ensure 8-pole delay parity. + * - **High-frequency harshness / breakup** + - Woofer cone breakup frequencies leaking past crossover + - Measure individual woofer near-field frequency response + - Lower :math:`F_c` or insert a notch filter in upstream Parametric EQ (``eq_iir``) at breakup resonance. + * - **Audio glitch / zipper noise on parameter injection** + - Instantaneous coefficient update causing state buffer discontinuity + - Check ``mtrace`` for buffer resets + - Ensure coefficient injection occurs with ramped gain or when signal is quiescent. + * - **DSP pipeline underflow / XRUN** + - Downstream sink pipelines consuming samples at mismatched rates + - Inspect ALSA XRUN counters (``/proc/asound/card0/pcm*p/sub*/status``) + - Ensure all output DAIs are synchronized to the same common hardware clock domain. + +Upstream Firmware References +**************************** + +The SOF Crossover implementation and tuning scripts reside in the upstream repository: + +* **DSP Processing Core**: `src/audio/crossover/crossover.c `_ +* **Filter Kernels & Split Logic**: `src/audio/crossover/crossover_generic.c `_ +* **IPC4 Adapter & Pin Initialization**: `src/audio/crossover/crossover_ipc4.c `_ +* **User Configuration Header**: `src/include/user/crossover.h `_ +* **Common Crossover Header**: `src/include/module/crossover/crossover_common.h `_ +* **GNU Octave Tuning Scripts**: `src/audio/crossover/tune/ `_ +* **Topology 2 Component Definition**: `tools/topology/topology2/include/components/crossover.conf `_ diff --git a/developer_guides/tuning/images/crossover_tuning_firmware_engine.svg b/developer_guides/tuning/images/crossover_tuning_firmware_engine.svg new file mode 100644 index 00000000..8cbafc89 --- /dev/null +++ b/developer_guides/tuning/images/crossover_tuning_firmware_engine.svg @@ -0,0 +1,122 @@ + + + + + + + + + + + + + + + SOF Crossover Firmware Execution Engine & Direct-Form Biquad Core + Fixed-Point Q2.30 Arithmetic, 64-Bit Accumulators, and Cascaded LR4 Biquad Filter States + + + + + Per-Channel Crossover Processing Loop (crossover_generic.c) + Processes block of N frames across each audio channel independently + + + + for (ch = 0; ch < nch; ch++) { + state = &cd->state[ch]; /* Lowpass[3] & Highpass[3] */ + + + + for (i = 0; i < frames; i++) { + + + + 1. Read Sample: + x = audio_stream_read_frag(source, idx); + S16/S24/S32 normalized to 32-bit Q1.31 format + + + + 2. Dynamic Split Dispatch: + cd->crossover_split(x_in, out, state); + • 2-way: crossover_generic_split_2way() → out[0..1] + + + + 3. Multi-Sink Scatter / Write: + for (j = 0; j < num_sinks; j++) { + if (!bsinks[j]) continue; /* skip inactive pins */ + y = audio_stream_write_frag(bsinks[j], idx); + *y = sat_int32(Q_SHIFT_RND(out[j], 31, valid_bits)); + } + + } /* end frames */ + } /* end channels */ + + + + + + LR4 Biquad Cascade & State Memory + One LR4 Filter = 2 Identical 2nd-Order Biquads in Series + + + + LR4 Filter Architecture (4-Pole, -24 dB/oct) + + + + Biquad 1 (DF1) + 2nd Order Butterworth + + + w(n) + + + + Biquad 2 (DF1) + Identical Coefficients + + + y(n) + + + + Biquad Difference Equation: + y[n] = b0*x[n] + b1*x[n-1] + b2*x[n-2] + - a1*y[n-1] - a2*y[n-2] + + Accumulator: 64-bit signed int64_t + Coefficients: Q2.30 format (±2.0 range) + Shift / Gain: Q2.14 post-scale multiplier + + + + State Allocation (struct crossover_state): + • lowpass[3]: 3 LR4 stages (delay x[n-1,2], y[n-1,2]) + • highpass[3]: 3 LR4 stages (delay history) + • Delay slots per LR4: 4 words per channel + • Total delay RAM per stereo channel: 48 words + • Reset on STREAM_STOP via crossover_reset_state() + Prevents residual ringing / pops on subsequent stream start + + diff --git a/developer_guides/tuning/images/crossover_tuning_lr4_acoustics.svg b/developer_guides/tuning/images/crossover_tuning_lr4_acoustics.svg new file mode 100644 index 00000000..59f51cb1 --- /dev/null +++ b/developer_guides/tuning/images/crossover_tuning_lr4_acoustics.svg @@ -0,0 +1,178 @@ + + + + + + + + + Linkwitz-Riley LR4 (-24 dB/oct) Acoustic Response & Summation + Comparison Against Butterworth 4th-Order (BW4) Showing Magnitude Flatness & Phase Coherence + + + + + + + + + + + + + + +6 dB + + + 0 dB + + + -6 dB + + + -12 dB + + + -24 dB + + + -36 dB + + + + + 0.1 Fc + + + 0.5 Fc + + + Fc (Crossover) + + + 2 Fc + + + 10 Fc + + + + + + + + + + + + + + + + + -6.02 dB + + + +3 dB (BW4 Peak) + + + A. Frequency Magnitude Response (dB) + + + + + + + + + + + + +180° + + + 0° + + + -180° + + + + + + + + + + + + + + + + + + Δφ = 0° (In-Phase Coherence) + + + B. Unwrapped Acoustic Phase Shift (Degrees) + Low-pass and High-pass outputs have zero phase difference at Fc (zero acoustic lobe tilt) + + + + + + + Legend & Curves + + + LR4 Acoustic Sum (0 dB Flat) + + + LR4 Low-Pass (-24 dB/oct) + + + LR4 High-Pass (+24 dB/oct) + + + Butterworth 4 (+3 dB Peak) + + + + Why LR4 in SOF? + + 1. Zero Peaking: + Summed output magnitude + is exactly 1.0 (0.00 dB ripple). + + 2. Phase Coherence: + Outputs are 360° (0°) in phase; + lobing points on-axis. + + 3. Steep Attenuation: + 24 dB/oct roll-off protects + tweeters from bass excursions. + + + + Transfer Function: + H_LR4(s) = [H_BW2(s)]² + |H_LP(jωc)| = 0.5 (-6.02 dB) + |H_LP|² + |H_HP|² = 1.00 + + diff --git a/developer_guides/tuning/images/crossover_tuning_toolchain_workflow.svg b/developer_guides/tuning/images/crossover_tuning_toolchain_workflow.svg new file mode 100644 index 00000000..b0387ab5 --- /dev/null +++ b/developer_guides/tuning/images/crossover_tuning_toolchain_workflow.svg @@ -0,0 +1,193 @@ + + + + + + + + + + + + End-to-End Crossover Tuning, Offline Synthesis & Live Injection Toolchain + From Acoustic Transducer Characterization to GNU Octave Synthesis, Multi-Target Packaging, and Live DSP Injection + + + + + + + 1 + Acoustic Profiling + Speaker Measurement + + + + Measurement Mic + Anechoic / Near-field + Chirp / Stepped Sine / MLS + SoundCheck / Klippel / REW + + + + + Transducer Limits + Woofer: High-freq breakup + Tweeter: Resonance Fs & Xmax + Harmonic Distortion (THD) + Directivity Index (DI) + + + + + Crossover Decisions + • Select Cutoffs: + Fc_low = 2200 Hz + (or 3-way/4-way) + • Ensure Tweeter Fc > 2*Fs + • Check off-axis polar tilt + • Sampling rate: 48 kHz + + + + + + + + + + 2 + Octave Filter Engine + src/audio/crossover/tune/ + + + + sof_crossover_gen_coefs.m + • Continuous Butterworth 2nd + • Bilinear transform (s → z) + • Warp freq: 2*fs*tan(wd/(2*fs)) + + + + + + sof_crossover_coef_quant.m + • Quantize a1,a2,b0,b1,b2 + • Format: 32-bit Q2.30 + • Gain: 16384 (1.0 in Q2.14) + + + + + + sof_crossover_generate_config + • Interleave LP & HP biquads + • Set assign_sinks[0..3] + • Insert LR4 merge block + • 2-way: 2 bqs; 3/4-way: 6 bqs + + + + + + sof_crossover_plot_freq.m + • Verify magnitude sum = 0dB + • Check phase alignment Δφ=0° + • Confirm -24 dB/oct slope + + + + + + + + + + 3 + Packaging Formats + Multi-Target Serializers + + + + Topology 2 Conf + • sof_tplg2_write.m + • coef_2way_48000_200.conf + • Compiled into .tplg binary + • Boot-time default filters + + + + UCM2 Binary Blob + • sof_ucm_blob_write.m + • coef_2way.bin + • Exported to /usr/share/ucm2 + • Per-device acoustic curves + + + + ALSA State (asound.state) + • sof_alsactl_write.m + • coef_2way.txt + • Comma-separated hex words + • systemd alsa-restore + + + + + + + + + + 4 + Live Injection + Lab DUT SSH / sof-ctl + + + + 1. Enumerate Controls + amixer -Dhw:0 controls | \ + grep -i crossover + Locate control numid (e.g. 14) + + + + 2. Inject via sof-ctl + sof-ctl -Dhw:0 -i 4 \ + -n 14 -p 0 -b \ + -s coef_2way.bin + Live update without stream stop + + + + 3. Firmware Trace Log + mtrace | grep crossover + "crossover_init_coef_ch" + "num_sinks = 2, status OK" + + + + 4. Acoustic Verification + • Run acoustic sweep + • Check on-axis flat sum + • Measure woofer excursion + • Verify zero notch at Fc + + diff --git a/developer_guides/tuning/images/crossover_tuning_topology_routing.svg b/developer_guides/tuning/images/crossover_tuning_topology_routing.svg new file mode 100644 index 00000000..eb557eae --- /dev/null +++ b/developer_guides/tuning/images/crossover_tuning_topology_routing.svg @@ -0,0 +1,154 @@ + + + + + + + + + + + + + + + Topology 2 Multi-Sink Pipeline Architecture & IPC4 Output Pin Binding + One-to-Many Multi-DAI Routing for Independent Woofer and Tweeter Power Amplifiers + + + + + + Pipeline 1 (Host Ingest & Crossover Split) + Stream Domain: 48 kHz / 32-bit LE / Stereo / Period: 48 frames (1 ms) + + + + Host Copier + PCM 0 Playback + host-gateway-playback + + + + + + + Buffer: 2 Periods + 96 frames / 32-bit + + + + + + Widget: crossover.1.1 + UUID: d1:9a:8c:94:6a:80:31:41:ad:6c:b2:bd:a9:e3:5a:9f + + + + Input Pin 0 + Stereo Wideband + + + + LR4 Split Engine + Direct-Form 1 Biquads + + + + Pin 0 + Woofer + + + Pin 1 + Tweeter + + + + Object.Control.bytes."1" + name "crossover.1.1_bytes_control" + IncludeByKey: "2way" / "3way" / "4way" conf + IPC4 Large Config: param_id = 0 + + + + + + + + + Pipeline 2: Woofer Playback Path (Low Frequencies) + Bound via output_pin_binding.1: dai-copier.SSP.NoCodec-0.playback + + + + Pin 0 Out + + + + DAI Copier (SSP0) + dai-copier.SSP.NoCodec-0 + Format: 32-bit / 48 kHz / 2ch + + + + + + Woofer Power Amplifier + I2S / SoundWire / SSP0 + Left/Right Bass Transducers + + + + • Passband: DC (0 Hz) up to Fc (e.g. 2200 Hz) + • High-frequency roll-off: -24 dB/oct (-48 dB at 2*Fc) + + + + + + Pipeline 3: Tweeter Playback Path (High Frequencies) + Bound via output_pin_binding.2: dai-copier.SSP.NoCodec-2.playback + + + + Pin 1 Out + + + + DAI Copier (SSP2) + dai-copier.SSP.NoCodec-2 + Format: 32-bit / 48 kHz / 2ch + + + + + + Tweeter Power Amplifier + I2S / SoundWire / SSP2 + Left/Right Silk Dome Tweeters + + + + • Passband: Fc (e.g. 2200 Hz) up to Nyquist (24 kHz) + • Low-frequency attenuation: -24 dB/oct protects voice coils + + diff --git a/developer_guides/tuning/images/crossover_tuning_tree_topologies.svg b/developer_guides/tuning/images/crossover_tuning_tree_topologies.svg new file mode 100644 index 00000000..9d17a1ab --- /dev/null +++ b/developer_guides/tuning/images/crossover_tuning_tree_topologies.svg @@ -0,0 +1,243 @@ + + + + + + + + + + + + + + + + SOF Crossover Filter Tree Topologies & Phase-Alignment Architecture + Recursive Linkwitz-Riley 4th-Order (LR4, -24 dB/oct) Splitting with All-Pass Phase Equalization + + + + + 1. Two-Way Crossover (Woofer / Tweeter Split) + Single crossover frequency (Fc_low). Symmetrical 4-pole phase shift (360° rotation) on both branches. + + + + x(n) In + Full Bandwidth + + + + + + + + + LR4 LP0 (Fc_low) + 2x Biquad Low-Pass (-24dB/oct) + + + + + LR4 HP0 (Fc_low) + 2x Biquad High-Pass (+24dB/oct) + + + + + assign_sink[0]: LOW + Woofer Driver (0 to Fc_low) + + + + assign_sink[1]: HIGH + Tweeter Driver (Fc_low to Fs/2) + + + + Acoustic Properties: + • -6 dB at Fc_low; in-phase sum is 0 dB flat + • Total biquads: 2 (LP0, HP0) = 14 words + + + + + + 2. Three-Way Crossover (Woofer / Midrange / Tweeter) with Phase-Alignment Merge + Solves acoustic phase cancellation: LOW path traverses an all-pass LR4 merge block matching the 8-pole latency of MID/HIGH. + + + + x(n) In + Wideband + + + + + + + + + LR4 LP0 (Fc_low) + Stage 1 Split (z1) + + + + + LR4 HP0 (Fc_low) + Stage 1 Split (z2) + + + + + + + LR4 Merge: LP1(Fc_high) + HP1(Fc_high) + All-Pass Acoustic Equalization (360° phase lag added) + y_low = sat_int32(LP1(z1) + HP1(z1)) + + + + + + + + + LR4 LP2 (Fc_high) + Extracts Midrange + + + + + LR4 HP2 (Fc_high) + Extracts Tweeter Band + + + + + assign_sink[0]: LOW + Woofer (0 to Fc_low) + + + + assign_sink[1]: MID + Midrange (Fc_low to Fc_high) + + + + assign_sink[2]: HIGH + Tweeter (Fc_high to Fs/2) + + + + Phase Balance + LOW: 8 poles + (LP0 + Merge) + MID: 8 poles + HIGH: 8 poles + Zero Tilt + + + + + + 3. Four-Way Symmetrical Binary Tree (Subwoofer / Low-Mid / High-Mid / Tweeter) + Dual-stage binary tree. Every acoustic branch naturally traverses exactly two LR4 stages (8 poles), perfectly phase-balanced. + + + + x(n) In + Full Band + + + + + + + + LR4 LP1 (Fc_mid) + Low Group (z1) + + + + LR4 HP1 (Fc_mid) + High Group (z2) + + + + + + + + LR4 LP0 (Fc_low) + Subwoofer Split + + + + LR4 HP0 (Fc_low) + Mid-Bass Split + + + + + + + + LR4 LP2 (Fc_high) + Mid-High Split + + + + LR4 HP2 (Fc_high) + Ultra-High Tweeter + + + + + assign_sink[0]: SUB / LOW + Subwoofer (0 to Fc_low) + + + + assign_sink[1]: MID_LOW + Woofer (Fc_low to Fc_mid) + + + + assign_sink[2]: MID_HIGH + Squawker (Fc_mid to Fc_high) + + + + assign_sink[3]: HIGH + Tweeter (Fc_high to Fs/2) + + + + Memory & Pipeline Cost: + • 6 total LR4 filters (12 biquads) + • 42 words in coef[] array + • Exact 8-pole group delay parity + • Zero acoustic summing ripple + + From c2d3a70578a610e7b1b878f0728ee71824c0548a Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 16:53:03 +0100 Subject: [PATCH 34/64] doc: developer_guides: add smart amplifier (dsm) & transducer protection calibration guide Signed-off-by: Liam Girdwood --- data/modules.yaml | 12 +- developer_guides/index.rst | 2 + .../smart_amp_tuning_excursion_thermal.svg | 165 ++++ .../images/smart_amp_tuning_iv_sense_loop.svg | 193 ++++ .../smart_amp_tuning_toolchain_workflow.svg | 232 +++++ .../smart_amp_tuning_transducer_model.svg | 202 +++++ .../smart_amp_tuning_two_layer_arch.svg | 122 +++ developer_guides/tuning/smart_amp_tuning.rst | 841 ++++++++++++++++++ 8 files changed, 1764 insertions(+), 5 deletions(-) create mode 100644 developer_guides/tuning/images/smart_amp_tuning_excursion_thermal.svg create mode 100644 developer_guides/tuning/images/smart_amp_tuning_iv_sense_loop.svg create mode 100644 developer_guides/tuning/images/smart_amp_tuning_toolchain_workflow.svg create mode 100644 developer_guides/tuning/images/smart_amp_tuning_transducer_model.svg create mode 100644 developer_guides/tuning/images/smart_amp_tuning_two_layer_arch.svg create mode 100644 developer_guides/tuning/smart_amp_tuning.rst diff --git a/data/modules.yaml b/data/modules.yaml index c277b4c9..fc856acf 100644 --- a/data/modules.yaml +++ b/data/modules.yaml @@ -244,16 +244,18 @@ modules: - "Single contiguous buffer layout and zero-copy polar memory overlay" - id: smart_amp - name: "Smart Amp Protection" + name: "Smart Amp Protection (DSM)" source: "SOF" category: "Speaker Protection" status: "Upstream" - description: "Speaker protection algorithm monitoring voltage/current feedback to maximize loudness safely." + description: "Closed-loop dynamic speaker management monitoring real-time voltage/current (I/V) feedback to maximize loudness and prevent mechanical and thermal destruction." simd: ["HiFi 3", "HiFi 4", "Scalar C"] key_features: - - "Real-time voice coil temperature estimation" - - "Membrane excursion limiting" - - "Maximizes acoustic output without damage" + - "Real-time voice coil temperature estimation via continuous Re(t) tracking" + - "Nonlinear membrane excursion prediction and adaptive high-pass limiting" + - "Closed-loop hardware I/V sense feedback via SoundWire and I2S/TDM" + - "Two-layer modular architecture supporting Maxim DSM and vendor engines" + - "Live runtime parameter injection and telemetry readback via sof-ctl" - id: sound_dose name: "Sound Dose & Exposure" diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 9c882cb4..d92465e6 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -126,6 +126,7 @@ Dynamics & Transducer Protection Tuning ======================================= * :ref:`drc_tuning` (Single-band DRC and Multiband DRC compression curves, adaptive ballistics, and speaker protection) +* :ref:`smart_amp_tuning` (Smart Amplifier Dynamic Speaker Management, I/V sense feedback calibration, Thiele-Small modeling, thermal and excursion protection) Acoustic, Transducer & Array Tuning =================================== @@ -141,6 +142,7 @@ Acoustic, Transducer & Array Tuning tuning/runtime_tuning_sof_ctl tuning/drc_tuning + tuning/smart_amp_tuning tuning/crossover_tuning algorithms/eq/equalizers_tuning algorithms/tdfb/time_domain_fixed_beamformer diff --git a/developer_guides/tuning/images/smart_amp_tuning_excursion_thermal.svg b/developer_guides/tuning/images/smart_amp_tuning_excursion_thermal.svg new file mode 100644 index 00000000..f24c4264 --- /dev/null +++ b/developer_guides/tuning/images/smart_amp_tuning_excursion_thermal.svg @@ -0,0 +1,165 @@ + + + + + + + + Dynamic Excursion Limiting and Voice Coil Thermal Protection Curves + Physics-Based Nonlinear Transducer Boundaries vs. Adaptive Frequency and Gain Attenuation + + + + + A. Mechanical Excursion vs. Frequency: X(f) + + + + + + + + + + + + + + + + + + + + + + + 0.80 + 0.50 + 0.30 + 0.15 + 0.00 + Excursion (mm pk) + + + 20 + 100 + 850 (Fs) + 3000 + 10k + Frequency (Hz) + + + + + Xmech (0.80 mm) - Mechanical Damage! + + + + Xmax (0.50 mm) - Distortion Threshold + + + + Unprotected (+12 dBFS) + + + + + DSM Protected (Adaptive HPF) + + + + Excursion Protection: + Adaptive cutoff shifts up + from 80 Hz --> 350 Hz + + + + + + B. Thermal Rise vs. Time: Tv(t) & Compression + + + + + + + + + + + + + + + + + + + + + + + 135°C + 115°C + 95°C + 60°C + 25°C + Voice Coil Temp Tv(t) + + + 0s + 10s + 30s + 60s + 180s + Time (s) + + + + + Tcutoff (135°C) - Coil Adhesive Breakdown + + + + Tlimit (115°C) - Regulated Ceiling + + + + Twarn (95°C) - Thermal Gain Reduction Active + + + + Unprotected Burnout + + + + + DSM Thermal Closed-Loop (Regulated at Tlimit) + + + + Thermal Limiter: + Dual-rate attenuation: + Smooth wideband -3 dB to -6 dB + + diff --git a/developer_guides/tuning/images/smart_amp_tuning_iv_sense_loop.svg b/developer_guides/tuning/images/smart_amp_tuning_iv_sense_loop.svg new file mode 100644 index 00000000..37391349 --- /dev/null +++ b/developer_guides/tuning/images/smart_amp_tuning_iv_sense_loop.svg @@ -0,0 +1,193 @@ + + + + + + + + + + + + + + + + + Closed-Loop Hardware I/V Sense Feedback Architecture + Feed-Forward Speaker Protection & Real-Time Feedback Current/Voltage Sensing via SoundWire / I2S + + + + + SOF DSP Firmware Pipeline Architecture + + + + Host Audio Stream + ALSA PCM Playback + + + + + + Volume / EQ + Equalizer / Gain + + + + + + + SOF Smart Amp Component (num_input_pins: 2, num_output_pins: 1) + + + + Pin 0: Feed-Forward (FF) + Playback (L, R) S32_LE + + + + Pin 1: Feedback (FB) + I/V Sense (IL, VL, IR, VR) + + + + Generic Layer + source_ch_map + feedback_ch_map + fmt_conv: S16/S24 + Memory Manager + + + + + + + + DSM Inner Model + Excursion Predict + Thermal Modeling + Adaptive High-Pass + Re(t) Impedance + Dynamic Limiter + Maxim DSM / Vendor + mod_ops->ff/fb_proc + + + + + + + + Pin 0 Out (Sink) + Protected Audio + + + + + + Runtime ABI Telemetry & Injection (SOF_SMART_AMP_MODEL / CONFIG) + IPC4 / ALSA Byte Control <--> Live voice coil temp Tv(t), Re(t), Xpeak, supply voltage + + + + SOF Capture Pipeline (I/V Sense Demux) + DAI In (SoundWire / SSP) --> Demux --> Feedback Buffer (sad->feedback_buf) + Extracts synchronized current I(t) and terminal voltage V(t) at 48 kHz / 24-bit + Direction: SOF_IPC_STREAM_CAPTURE | Invalidate & Writeback DMA cache + + + + + + + + + + + + + + Hardware Platform & Smart Codec IC + + + + Smart Amplifier Hardware IC (e.g. MAX98373 / CS35L41) + Digital Audio Interface: SoundWire (SDW) or Multi-slot TDM + + + + DAC & Class-D Stage + H-Bridge Output Stage + PVDD: 5V - 12V Boost + + + + On-Die I/V Sense ADCs + Current ADC: Rsense + Voltage ADC: Vout + 24-bit @ 48 kHz / 96 kHz + + + + Micro-Speaker + Voice Coil (Re, Le) + Cone / Diaphragm (Mms) + Suspension (Cms, Rms) + Excursion x(t) ≤ Xmax + Temp Tv(t) ≤ Tlimit + Acoustic Radiated Pressure + + + + Vout + + + + Kelvin Sensing: I_sense + V_sense + + + + + SDW Tx (Audio) + + + + SDW Rx (I/V) + + + + Key Closed-Loop Advantages in SOF: + 1. Dynamic Headroom Maximization: + Plays +6 dB to +10 dB louder than open-loop fixed limiters. + 2. Continuous Re(T) Tracking: + Tracks voice coil heat directly without external sensors. + 3. Acoustic Leak & Aging Adaptation: + Detects speaker gasket failure, broken mesh, and aging. + 4. Thermal Runaway Prevention: + Instantly attenuates output before voice coil burns out. + diff --git a/developer_guides/tuning/images/smart_amp_tuning_toolchain_workflow.svg b/developer_guides/tuning/images/smart_amp_tuning_toolchain_workflow.svg new file mode 100644 index 00000000..bc48b230 --- /dev/null +++ b/developer_guides/tuning/images/smart_amp_tuning_toolchain_workflow.svg @@ -0,0 +1,232 @@ + + + + + + + + + + + + + + End-to-End Smart Amplifier Acoustic Characterization and Telemetry Workflow + From Laser Transducer Metrology to Topology 2 Packaging, Runtime Injection, and Live Voice Coil Telemetry + + + + + + + + 1 + Transducer Metrology + + + + Klippel / Laser + Laser Vibrometer + Displacement X(f) + Xmax, Xmech limit + + + Thiele-Small Extr. + Fs, Qms, Qes, Qts + Bl, Mms, Cms, Rms + Sd, Vas volume + Re0 @ 25°C baseline + + + Thermal Soak Test + IEC 60268-5 noise + Rth,vc, Cth,vc (τ1) + Rth,mag, Cth,mag (τ2) + Thermal camera verify + + + Deliverables: + speaker_ts.json + thermal_params.csv + Safety boundary spec + + + + + + + + + + + 2 + Model Calibration + + + Python / Octave + sof_dsm_calib.py + Convert SI units to + Q-format DSM params + Delay & buffer sizing + + + Excursion Limiter + Kms(x) compensation + Adaptive HPF cutoff + Attack: 1.0 ms + Release: 50 ms + + + Thermal Limiter + Twarn = 95°C + Tlimit = 115°C + Tcutoff = 130°C + Smooth multiband gain + + + Deliverables: + dsm_params.bin + Raw calibration blob + + + + + + + + + + + 3 + Topology Packaging + + + Topology 2 Widget + Class.Widget."smart_amp" + num_input_pins 2 + num_output_pins 1 + cpc 5000 | is_pages 1 + + + Channel Routing + source_ch_map: + [0, 1, -1, -1, ...] + feedback_ch_map: + [-1, -1, 0, 1, ...] + + + ALSA Byte Control + Object.Control.bytes."1" + max 4096 bytes + SMART_AMP_UUID + IPC4 BaseCfgExt init + + + Deliverables: + sof-board-dsm.tplg + Compiled binary tplg + + + + + + + + + + + 4 + Runtime Injection + + + sof-ctl Tool + sof-ctl -D hw:0 + -n 'Smart Amp' + -s dsm_blob.bin + Live zero-glitch push + + + IPC ABI Dispatch + SOF_CTRL_CMD_BINARY + SOF_SMART_AMP_MODEL + ID + Value pairs (8B) + Stereo / Per-channel + + + Inner Model Set + mod_ops->set_config + dsm_api_set_params + Updates active state + No DSP reset required + + + Result: + Live Calibration Active + Audible in real-time + + + + + + + + + + + 5 + Live Telemetry & QA + + + Volatile Telemetry + sof-ctl -g telemetry + Cmd 0x10: Tv(t) temp + Cmd 0x11: Re(t) res + Cmd 0x12: Xpeak mm + + + Stress Testing + 0 dBFS Sine Sweeps + EIAJ Dynamic Noise + 100-hour soak test + Zero transducer burn + + + Acoustic QA + Sound Pressure (SPL) + THD+N < 5% Target + Rub & buzz detection + +8 dB loudness gain + + + Certification: + Production Ready + Sign-off for Golden Tplg + + + + + + + Iterative Acoustic Feedback: Online Telemetry Calibration Loop + Real-time thermal tracking and excursion measurements calibrate room-temperature baseline parameters + + diff --git a/developer_guides/tuning/images/smart_amp_tuning_transducer_model.svg b/developer_guides/tuning/images/smart_amp_tuning_transducer_model.svg new file mode 100644 index 00000000..8d4e6738 --- /dev/null +++ b/developer_guides/tuning/images/smart_amp_tuning_transducer_model.svg @@ -0,0 +1,202 @@ + + + + + + + + + + + + + + + + + + + + + + Electro-Acoustic Transducer Physics & Lumped Parameter Model + Coupled Electrical, Mechanical, Acoustical, and Thermal Domains for Micro-Speaker DSM Algorithms + + + + + 1. Electrical Domain (Voice Coil) + + + + Re(T) Voice Coil DC + Re(T) = Re0[1+αΔT] + + + Le(x) Inductance + Le(x) · di(t)/dt + + + Back-EMF (Electromotive Coupling) + e_emf(t) = Bl(x) · v(t) = Bl(x) · dx/dt + + + + Electrical Voltage Equation: + u(t) = Re(T)·i(t) + Le(x)·di/dt + Bl(x)·v(t) + Measured by Smart Amp I/V sense ADCs in real-time + + + + + 2. Mechanical Domain (Suspension & Mass) + + + Mms Moving Mass + Diaphragm + Coil + + + Cms(x) Compliance + Kms(x) = 1 / Cms(x) + + + Lorentz Driving Force + F_lorentz(t) = Bl(x) · i(t) + + + + Newton's 2nd Law (Equation of Motion): + F(t) = Mms·d²x/dt² + Rms·dx/dt + x(t)/Cms + Displacement x(t) strictly limited to Xmax + + + + + 3. Acoustic Radiation (Acoustics) + + + Sd Radiating Area + Effective Piston + + + Vas Volume + Equivalent Box + + + Acoustic Radiated Pressure + p(r, t) = (ρ0 · Sd / 2πr) · d²x/dt² + + + Efficiency & Acoustic Output: + η0 = (ρ0 · Bl² · Sd²) / (2πc · Mms² · Re) + Typically < 0.5% in smartphone/laptop micro-speakers + + + + Bl·i + + + Sd·v + + + + + 4. Thermal Dynamic Model (Voice Coil & Magnet Temperature Tracking) + + + + Voice Coil Node (Tv) + P_loss = i_rms² · Re(T) + C_th,vc: ~0.05 J/K + τ_vc = 1.0 - 2.5 sec + + + R_th,vc + + + Magnet Node (Tm) + Thermal Mass Dissipation + C_th,mag: ~2.0 J/K + τ_mag = 120 - 300 sec + + + R_th,amb + + + Chassis Ambient + T_ambient: 25 - 45°C + Infinite Heat Sink + Enclosure body + + + + Online Temperature Estimation from Real-Time Voice Coil Resistance: + Tv(t) = T0 + (1 / α_Cu) · [ ( Re(t) / Re0 ) - 1 ] + where α_Cu = 0.00393 / °C (Copper) + Safe Operating Limits: T_warn = 95°C | T_limit = 115°C | T_cutoff = 130°C + Micro-speakers convert >99% of electrical power to heat. Exceeding T_limit melts voice coil adhesives. + + + + + 5. Key Thiele-Small Parameters + + + Fs + Resonance Freq + 750 - 950 Hz + + Re + DC Resistance + 6.5 - 8.0 Ω + + Qts + Total Quality Factor + 1.2 - 2.5 + + Bl + Force Factor + 0.95 - 1.25 N/A + + Mms + Moving Mass + 45 - 80 mg + + Cms + Compliance + 0.3 - 0.7 mm/N + + Xmax + Max Safe Excursion + 0.40 - 0.55 mm + + Xmech + Damage Boundary + 0.65 - 0.80 mm + + Tmax + Max Coil Temp + 110 - 125 °C + + diff --git a/developer_guides/tuning/images/smart_amp_tuning_two_layer_arch.svg b/developer_guides/tuning/images/smart_amp_tuning_two_layer_arch.svg new file mode 100644 index 00000000..2ac67093 --- /dev/null +++ b/developer_guides/tuning/images/smart_amp_tuning_two_layer_arch.svg @@ -0,0 +1,122 @@ + + + + + + + + + + + + + + + + + SOF Smart Amplifier Two-Layer Modular Software Architecture + Decoupling Pipeline Audio Infrastructure (Generic Layer) from Vendor Protection Modeling (Inner Model Layer) + + + + SOF Pipeline & Module Framework Interface + + + comp_ops.create() / init() + + + comp_ops.prepare() + + + comp_ops.process() + + + set_config() / get_config() + + + comp_ops.reset() / free() + + + + + + Generic Layer (smart_amp.c & smart_amp_generic.c) - struct smart_amp_data + + + + Dynamic Memory Block Allocator + MOD_MEMBLK_PRIVATE + Allocated BEFORE inner model init (for handles/state) + MOD_MEMBLK_FRAME + Allocated AFTER init (for internal audio frame buffers) + MOD_MEMBLK_PARAM + Allocated AFTER init (for caldata & parameter tables) + Free on error / smart_amp_free_mod_memories() + + + + Channel Remapping Matrix + struct sof_smart_amp_config + source_ch_map[PLATFORM_MAX_CHANNELS] + Maps playback input channels to algorithm feeds + feedback_ch_map[PLATFORM_MAX_CHANNELS] + Maps capture channels to I_sense / V_sense inputs + Unmapped channels (-1): zeroed via bzero() + + + + Sample Format Conversion Matrix + Input / Stream Formats: S16_LE, S24_4LE, S32_LE + smart_amp_resolve_mod_fmt() + Enforces Inner Model bitdepth ≥ SOF stream bitdepth + remap_s16_to_b32() : left-shift 16 bits + remap_s24_to_s32() : left-shift 8 bits + feed_s32_to_s24() : Q_SHIFT_RND & sat_int24 + + + + Pluggable Interface: struct inner_model_ops (mod_ops) + init() | query_memblk_size() | set_memblk() | get_supported_fmts() | set_fmt() | ff_proc() | fb_proc() | get_config() | set_config() | reset() + + + + + Inner Model Layer (Solution-Specific Engine) - struct smart_amp_mod_data_base + + + + CONFIG_MAXIM_DSM (smart_amp_maxim_dsm.c) + UUID: 0cd84e80-ebd3-11ea-adc10242ac120002 + dsm_api_init() / dsm_api_ff_process() / dsm_api_fb_process() + circularbuffersize, delayedsamples, Thiele-Small predictor + Volatile Telemetry: maxim_dsm_get_volatile_param() [Tv, Re, Xpeak, Vbat] + + + + Passthrough / Custom Vendor Models + CONFIG_MAXIM_DSM=n --> passthru_smart_amp.c (UUID: 64a794f0...) + smart_amp_test_ipc4.c (UUID: 167a961e-8ae4-11ea-89f1000c29ce1635) + Enables proprietary 3rd-party vendor libraries (Cirrus, TI, NXP, Goodix) + Without modifying any SOF core audio buffering or pipeline code + diff --git a/developer_guides/tuning/smart_amp_tuning.rst b/developer_guides/tuning/smart_amp_tuning.rst new file mode 100644 index 00000000..90f4a061 --- /dev/null +++ b/developer_guides/tuning/smart_amp_tuning.rst @@ -0,0 +1,841 @@ +.. _smart_amp_tuning: + +Smart Amplifier (DSM) & Transducer Protection Calibration Guide +############################################################### + +Sound Open Firmware (SOF) incorporates an advanced, closed-loop **Smart Amplifier Dynamic Speaker Management (DSM)** processing architecture. This subsystem protects micro-acoustic transducers—such as smartphone micro-speakers, laptop downward/upward-firing drivers, tablets, and smart conference speakers—against catastrophic physical destruction while extracting up to **+6 dB to +10 dB of additional acoustic sound pressure level (SPL)** compared to conventional static brickwall limiters. + +This guide provides an authoritative, mathematically rigorous reference for acoustic engineers, audio systems architects, and DSP firmware developers. It details the coupled electro-mechanical-thermal physics of micro-transducers, the closed-loop hardware current and voltage (:math:`I/V`) sensing return path, the SOF two-layer modular component architecture, offline parameter synthesis, runtime binary control blobs, Topology 2 integration, and live telemetry extraction using ``sof-ctl``. + +--- + +Physical & Mathematical Foundations of Electro-Acoustic Transducers +******************************************************************* + +Modern micro-speakers operate at the thermodynamic and mechanical limits of material physics. To maximize acoustic loudness within strict industrial design constraints (sub-millimeter chassis thickness and tiny back-cavity volumes of :math:`0.5\text{ to }2.0\text{ cm}^3`), transducers are routinely driven with peak voltages far exceeding their continuous steady-state ratings. Without active, physics-based closed-loop management, continuous playback causes rapid destruction via mechanical over-excursion or voice coil thermal burnout. + +Coupled Electro-Mechanical-Acoustical Equations of Motion +========================================================= + +An electro-dynamic moving-coil loudspeaker is modeled as a coupled multi-domain physical system comprising electrical, mechanical, acoustical, and thermal dynamics: + +.. figure:: images/smart_amp_tuning_transducer_model.svg + :align: center + :width: 100% + :alt: Electro-acoustic transducer model and lumped parameter physical foundations for Smart Amp DSM + + Electro-Acoustic Transducer Physics & Lumped Parameter Dynamic Model + +1. Electrical Domain (Voice Coil Dynamics) +------------------------------------------ + +The electrical terminal behavior of the loudspeaker voice coil is governed by Kirchhoff's voltage law: + +.. math:: + + u(t) = R_e(T_v) \cdot i(t) + L_e(x) \cdot \frac{di(t)}{dt} + e_{\text{emf}}(t) + +where: + +* :math:`u(t)` is the instantaneous terminal voltage applied across the speaker voice coil (measured in Volts). +* :math:`i(t)` is the instantaneous electrical current traversing the voice coil (measured in Amperes). +* :math:`R_e(T_v)` is the voice coil direct current (DC) electrical resistance as a function of temperature :math:`T_v` (measured in Ohms). +* :math:`L_e(x)` is the voice coil electrical inductance, which varies nonlinearly with cone displacement :math:`x` (measured in Henries). +* :math:`e_{\text{emf}}(t)` is the counter-electromotive force (back-EMF) induced by the voice coil moving through the permanent magnetic flux field: + +.. math:: + + e_{\text{emf}}(t) = B\cdot l(x) \cdot v(t) = B\cdot l(x) \cdot \frac{dx(t)}{dt} + +where :math:`B\cdot l(x)` is the electro-mechanical force factor (the product of magnetic flux density :math:`B` in the magnetic gap and voice coil wire length :math:`l`, measured in Tesla-meters or N/A), and :math:`v(t) = \frac{dx}{dt}` is the voice coil instantaneous mechanical velocity. + +2. Mechanical Domain (Cone & Suspension Dynamics) +------------------------------------------------- + +The mechanical displacement of the speaker cone, diaphragm, and voice coil assembly is governed by Newton's second law of motion: + +.. math:: + + F_{\text{lorentz}}(t) = M_{ms} \cdot \frac{d^2 x(t)}{dt^2} + R_{ms} \cdot \frac{dx(t)}{dt} + \frac{x(t)}{C_{ms}(x)} + +where: + +* :math:`F_{\text{lorentz}}(t) = B\cdot l(x) \cdot i(t)` is the electro-dynamic Lorentz driving force generated in the voice coil. +* :math:`M_{ms}` is the total moving mass of the transducer assembly, including the diaphragm, voice coil, former, suspension air load, and surround (measured in kilograms or grams). +* :math:`R_{ms}` is the mechanical damping resistance of the suspension and surround dissipation (measured in mechanical Ohms, N·s/m). +* :math:`C_{ms}(x)` is the mechanical compliance of the suspension (spider, surround, and enclosed air spring), which is the inverse of suspension stiffness :math:`K_{ms}(x) = \frac{1}{C_{ms}(x)}` (measured in meters/Newton). +* :math:`x(t)` is the instantaneous axial displacement (excursion) of the voice coil relative to its resting center position (measured in millimeters or meters). + +3. Acoustical Domain (Sound Pressure Radiation) +----------------------------------------------- + +In the acoustic far-field at distance :math:`r` in half-space (:math:`2\pi` steradians), the acoustic pressure :math:`p(r, t)` radiated by a micro-speaker acting as an acoustic piston is directly proportional to the second derivative of displacement (cone acceleration): + +.. math:: + + p(r, t) = \frac{\rho_0 \cdot S_d}{2\pi r} \cdot \frac{d^2 x(t)}{dt^2} = \frac{\rho_0 \cdot S_d}{2\pi r} \cdot a(t) + +where :math:`\rho_0` is the density of air (:math:`\approx 1.204\text{ kg/m}^3` at :math:`20^\circ\text{C}`), and :math:`S_d` is the effective radiating surface area of the diaphragm (measured in :math:`\text{m}^2` or :math:`\text{cm}^2`). + +The fundamental electro-acoustic power conversion efficiency :math:`\eta_0` of a direct-radiator loudspeaker is notoriously low: + +.. math:: + + \eta_0 = \frac{\rho_0 \cdot (B\cdot l)^2 \cdot S_d^2}{2\pi c \cdot M_{ms}^2 \cdot R_e} + +In smartphone and laptop micro-speakers, :math:`\eta_0` is typically **under 0.5%**. Consequently, **over 99.5% of all electrical audio power supplied to the transducer is converted directly into Joule heat within the voice coil**. + +Transducer Damage Mechanisms: Mechanical & Thermal +================================================== + +Micro-speakers are subject to two distinct, catastrophic failure modes that define the operational envelope of the SOF Smart Amplifier component: + +.. list-table:: Transducer Damage Mechanisms and Operational Protection Envelopes + :header-rows: 1 + :widths: 20 25 25 30 + + * - Failure Mode + - Physical Mechanism + - Critical Threshold + - SOF Protection Mechanism + * - **Mechanical Excursion Boundary** + - Voice coil over-travel, bottoming against back plate, spider/surround plastic deformation or tearing, voice coil former rocking. + - :math:`X(t) > X_{\text{max}}` (distortion threshold); :math:`X(t) \ge X_{\text{mech}}` (destructive impact). + - Predictive mechanical displacement filter with dynamic adaptive high-pass frequency shifting. + * - **Thermal Voice Coil Overheating** + - Joule heating (:math:`P = I_{\text{rms}}^2 R_e`), melting of voice coil insulating enamel, decomposition of former bonding adhesive, open/short circuit. + - :math:`T_v(t) > 115^\circ\text{C}` (long-term adhesive degradation); :math:`T_v(t) \ge 135^\circ\text{C}` (instantaneous burnout). + - Continuous online :math:`R_e(t)` tracking via :math:`I/V` sensing, dual-rate thermal state predictor, dynamic gain attenuation. + * - **Acoustic Enclosure Breach** + - Degradation of speaker gasket seal, puncture of protective acoustic mesh, back-cavity air leak eliminating air-spring stiffness. + - Resonant frequency drop (:math:`F_s \to F_{s,\text{free-air}}`), sudden spike in low-frequency excursion. + - Real-time impedance curve tracking detecting :math:`F_s` downward migration and raising protective high-pass cutoff. + +Online Voice Coil Temperature Tracking via DC Resistance +======================================================== + +Because the voice coil is wound from pure electrolytic copper wire, its electrical resistance exhibits a linear positive temperature coefficient of resistance (PTCR): + +.. math:: + + R_e(T_v) = R_{e,0} \cdot \left[ 1 + \alpha_{\text{Cu}} \cdot (T_v - T_0) \right] + +where: + +* :math:`R_{e,0}` is the baseline voice coil DC resistance measured at reference ambient temperature :math:`T_0` (typically :math:`20^\circ\text{C}` or :math:`25^\circ\text{C}`). +* :math:`\alpha_{\text{Cu}} \approx 0.00393\text{ K}^{-1}` (or :math:`0.393\%\text{ / }^\circ\text{C}`) is the temperature coefficient of copper. +* :math:`T_v` is the instantaneous voice coil temperature. + +By continuously measuring the real-time voltage :math:`V(t)` and current :math:`I(t)` at the speaker terminals via hardware sensing ADCs, the DSP extracts the real-time DC resistance :math:`R_e(t)` during active playback. The voice coil temperature is calculated directly without requiring external thermal sensors: + +.. math:: + + T_v(t) = T_0 + \frac{1}{\alpha_{\text{Cu}}} \cdot \left( \frac{R_e(t)}{R_{e,0}} - 1 \right) + +For example, if a micro-speaker with a cold baseline resistance of :math:`R_{e,0} = 7.00\ \Omega` at :math:`25^\circ\text{C}` heats up during high-volume playback until the DSP measures :math:`R_e(t) = 9.48\ \Omega`, the voice coil temperature is calculated as: + +.. math:: + + T_v(t) = 25^\circ\text{C} + \frac{1}{0.00393} \cdot \left( \frac{9.48}{7.00} - 1 \right) = 25 + 254.45 \cdot (1.3543 - 1) \approx 115.1^\circ\text{C} + +The SOF Smart Amplifier inner model tracks this value at sub-second intervals, initiating gradual, smooth thermal gain reduction as :math:`T_v` approaches the designated warning threshold :math:`T_{\text{warn}}`. + +--- + +Hardware Current and Voltage (I/V) Sense Feedback Architecture +************************************************************** + +Closed-loop transducer protection requires synchronized, low-latency digitization of the analog electrical signals delivered to the speaker voice coil. + +.. figure:: images/smart_amp_tuning_iv_sense_loop.svg + :align: center + :width: 100% + :alt: Closed-loop hardware IV sense feedback architecture for SOF Smart Amplifier + + Closed-Loop Hardware Current and Voltage (:math:`I/V`) Sense Feedback Architecture + +Hardware Sense Implementation +============================= + +Modern smart amplifier integrated circuits (e.g., Maxim/Analog Devices MAX98373, MAX98390, MAX98396; Cirrus Logic CS35L41, CS35L45; Realtek ALC1308, ALC1318; Texas Instruments TAS2563, TAS2781) integrate dedicated on-die instrumentation ADCs: + +1. **Current Sensing** (:math:`I_{\text{sense}}`): An internal, precision low-ohmic current shunt resistor (typically :math:`20\text{ to }50\text{ m}\Omega`) placed in series with the Class-D H-bridge output stage measures the return current flowing through the voice coil. +2. **Voltage Sensing** (:math:`V_{\text{sense}}`): A differential voltage divider connected directly across the speaker positive and negative output terminals measures the true terminal voltage delivered to the load, eliminating PCB trace resistance drops. +3. **Synchronous Digital Transport**: The smart amplifier digitizes both signals using 16-bit or 24-bit delta-sigma ADCs sampled at :math:`48\text{ kHz}` or :math:`96\text{ kHz}`. The resulting :math:`I` and :math:`V` digital audio frames are formatted into multi-channel digital streams and transmitted back to the host DSP over **MIPI SoundWire (SDW)** or multi-slot **I2S/TDM**. + +DSP Audio & Feedback Pipeline Trajectory +======================================== + +In Sound Open Firmware, the Smart Amplifier operates across two synchronized pipeline streams: + +* **Feed-Forward Playback Path**: Host audio is decoded, equalized, and fed into the primary input pin of the ``smart_amp`` component (``smart_amp.c``). The inner model processes the audio (applying excursion limiting and thermal attenuation) and forwards the protected signal to the digital audio interface (DAI) for Class-D amplification. +* **Feedback Return Path**: The hardware :math:`I/V` sense stream is received on a dedicated capture DAI, routed through an audio demultiplexer, and delivered to the second input pin of the ``smart_amp`` component as a feedback buffer (``sad->feedback_buf``). +* **Cache Management & DMA Synchronization**: The generic layer invalidates the CPU/DSP data cache (``buffer_stream_invalidate``) over the feedback buffer before processing, remaps the channel order, executes feedback parameter estimation (``fb_proc``), and updates the buffer pointers (``comp_update_buffer_consume``). + +--- + +SOF Smart Amplifier Two-Layer Software Architecture +*************************************************** + +The SOF Smart Amplifier component is engineered as a clean, two-layer decoupled software framework: + +.. figure:: images/smart_amp_tuning_two_layer_arch.svg + :align: center + :width: 100% + :alt: SOF Smart Amplifier Two-Layer Modular Software Architecture + + SOF Smart Amplifier Two-Layer Modular Software Architecture + +1. The Generic Layer (smart_amp.c & smart_amp_generic.c) +======================================================== + +The generic layer acts as the universal architectural glue connecting the SOF pipeline framework to vendor-specific protection algorithms. It handles all common, infrastructure-level responsibilities: + +* **Triple Memory Block Management**: The generic layer allocates, tracks, and releases three distinct memory blocks on behalf of the inner model, strictly preventing heap leaks and double-free errors: + + .. list-table:: Dynamic Memory Block Allocations Managed by Generic Layer + :header-rows: 1 + :widths: 25 25 50 + + * - Memory Block Type + - Allocation Timing + - Purpose and Lifecycle + * - ``MOD_MEMBLK_PRIVATE`` + - Allocated **before** inner model initialization (``mod_ops->init()``). + - Houses inner model private data structures, vendor algorithm state handles, circular history buffers, and filter state variables. + * - ``MOD_MEMBLK_FRAME`` + - Allocated **after** inner model initialization. + - Intermediate audio sample frame buffers sized to match the algorithm internal processing block size (e.g., 48 samples for 1 ms periods). + * - ``MOD_MEMBLK_PARAM`` + - Allocated **after** inner model initialization. + - Large parameter calibration blobs, Thiele-Small lookup tables, and model coefficient storage. + +* **Channel Remapping Matrix**: Real-world hardware routing varies widely between single-speaker mono laptops, stereo notebooks, and multi-speaker tablet designs. The generic layer uses ``struct sof_smart_amp_config`` to remap feed-forward channels (``source_ch_map``) and feedback channels (``feedback_ch_map``). Unmapped channels indicated by ``-1`` are zeroed automatically via ``bzero()`` to guarantee clean state. + +* **Sample Format Conversion**: To maximize DSP efficiency and numerical headroom, inner models typically operate internally in 32-bit fixed-point format (``SOF_IPC_FRAME_S32_LE``). The generic layer implements optimized format converters (``smart_amp_generic.c``): + + * ``remap_s16_to_s32``: Zeroes unmapped channels and arithmetic left-shifts 16-bit input samples by 16 bits into 32-bit containers. + * ``remap_s24_to_s32``: Arithmetic left-shifts 24-bit samples by 8 bits into 32-bit containers. + * ``feed_s32_to_s24``: Converts 32-bit processed output back to 24-bit using rounding and saturation (``sat_int24(Q_SHIFT_RND(val, 31, 23))``). + +2. The Pluggable Inner Model Interface (struct inner_model_ops) +=============================================================== + +The boundary between the generic layer and the algorithm model is defined by ``struct inner_model_ops`` (``src/include/sof/audio/smart_amp/smart_amp.h``): + +.. code-block:: c + + struct inner_model_ops { + int (*init)(struct smart_amp_mod_data_base *mod); + int (*query_memblk_size)(struct smart_amp_mod_data_base *mod, + enum smart_amp_mod_memblk blk); + int (*set_memblk)(struct smart_amp_mod_data_base *mod, + enum smart_amp_mod_memblk blk, + struct smart_amp_buf *buf); + int (*get_supported_fmts)(struct smart_amp_mod_data_base *mod, + const uint16_t **mod_fmts, int *num_mod_fmts); + int (*set_fmt)(struct smart_amp_mod_data_base *mod, uint16_t mod_fmt); + int (*ff_proc)(struct smart_amp_mod_data_base *mod, + uint32_t frames, + struct smart_amp_mod_stream *in, + struct smart_amp_mod_stream *out); + int (*fb_proc)(struct smart_amp_mod_data_base *mod, + uint32_t frames, + struct smart_amp_mod_stream *in); + int (*get_config)(struct smart_amp_mod_data_base *mod, + struct sof_ipc_ctrl_data *cdata, uint32_t size); + int (*set_config)(struct smart_amp_mod_data_base *mod, + struct sof_ipc_ctrl_data *cdata); + int (*reset)(struct smart_amp_mod_data_base *mod); + }; + +3. Inner Model Implementations +============================== + +The SOF repository supports multiple build configurations: + +* **Maxim Dynamic Speaker Management** (``CONFIG_MAXIM_DSM=y``): Implemented in ``smart_amp_maxim_dsm.c`` with UUID ``0cd84e80-ebd3-11ea-adc10242ac120002``. Links against the production Maxim DSM library (``dsm_api_public.h``), providing comprehensive excursion prediction, thermal tracking, and adaptive acoustic filtering. +* **Passthrough Fallback** (``CONFIG_MAXIM_DSM=n``): Implemented in ``smart_amp_passthru.c`` with UUID ``64a794f0-55d3-4bca-9d5b7b588badd037``. Forwards audio cleanly without processing while maintaining pipeline and topology compatibility. +* **Test Verification Adapter**: Implemented in ``smart_amp_test_ipc4.c`` with UUID ``167a961e-8ae4-11ea-89f1000c29ce1635`` for automated CI/CD and verification suites. + +--- + +Dynamic Speaker Management (DSM) Processing & Protection Algorithms +******************************************************************* + +The core inner model executes two concurrent mathematical processing loops: feed-forward prediction and feedback parameter calibration. + +.. figure:: images/smart_amp_tuning_excursion_thermal.svg + :align: center + :width: 100% + :alt: Dynamic Excursion Limiting and Voice Coil Thermal Protection Curves + + Dynamic Excursion Limiting and Voice Coil Thermal Protection Curves + +Feed-Forward Excursion Prediction & Adaptive High-Pass Filtering +================================================================ + +Below the mechanical resonance frequency :math:`F_s`, transducer displacement :math:`x(t)` increases at a rate of **12 dB per octave** for a constant input voltage: + +.. math:: + + |X(j\omega)| \approx \frac{|U(j\omega)| \cdot B\cdot l}{\omega^2 \cdot M_{ms} \cdot R_e} \quad \text{for } \omega \ll \omega_s + +If high-amplitude bass energy is presented to a micro-speaker, cone excursion rapidly exceeds :math:`X_{\text{max}}` (causing severe acoustic distortion) and reaches :math:`X_{\text{mech}}` (causing destructive voice coil bottoming against the steel pole piece). + +Rather than applying a static, conservative high-pass filter that permanently strips all low-frequency bass from low-volume audio, the DSM engine employs an **Adaptive Dynamic High-Pass Filter (DHPF)**: + +1. **Continuous Excursion Estimation**: The inner model simulates the mechanical displacement transfer function :math:`H_x(s) = \frac{X(s)}{U(s)}` in real time using fixed-point IIR filter structures parameterized with the speaker Thiele-Small constants (:math:`F_s, Q_{ts}, B\cdot l, M_{ms}, C_{ms}`). +2. **Dynamic Cutoff Frequency Modulation**: + * Under low-to-medium drive levels (:math:`x(t) \ll X_{\text{max}}`), the high-pass cutoff frequency :math:`f_c` relaxes downward to its baseline setting (e.g., :math:`80\text{ Hz}` to :math:`120\text{ Hz}`), delivering rich bass response. + * As the audio signal approaches full-scale and predicted peak displacement nears :math:`X_{\text{max}}`, the cutoff frequency :math:`f_c` shifts upward dynamically (e.g., to :math:`250\text{ Hz}` or :math:`380\text{ Hz}`). + * The energy that would cause mechanical damage is attenuated, while mid-frequency and vocal band audio remains entirely uncompressed and clean. + +Thermal Dynamic Modeling & Gain Attenuation +=========================================== + +Voice coil heating is governed by a two-stage thermal equivalent network: + +1. **Voice Coil Thermal Node**: Possesses low thermal capacitance (:math:`C_{\text{th,vc}} \sim 0.05\text{ J/K}`) and small thermal resistance (:math:`R_{\text{th,vc}} \sim 15\text{ K/W}`), resulting in a very fast thermal time constant: + + .. math:: + + \tau_{\text{vc}} = R_{\text{th,vc}} \cdot C_{\text{th,vc}} \approx 1.0\text{ to }2.5\text{ seconds} + +2. **Magnet / Chassis Thermal Node**: Possesses large thermal capacitance (:math:`C_{\text{th,mag}} \sim 2.0\text{ J/K}`) and thermal resistance to ambient air (:math:`R_{\text{th,amb}} \sim 25\text{ K/W}`), resulting in a slow thermal time constant: + + .. math:: + + \tau_{\text{mag}} = R_{\text{th,amb}} \cdot C_{\text{th,mag}} \approx 120\text{ to }300\text{ seconds} + +The DSM thermal algorithm continuously calculates Joule power dissipation: + +.. math:: + + P_{\text{joule}}(t) = i^2(t) \cdot R_e(T_v) + +When the estimated voice coil temperature :math:`T_v(t)` rises above the warning threshold :math:`T_{\text{warn}}` (typically :math:`95^\circ\text{C}`), a smooth wideband thermal limiter introduces progressive attenuation: + +.. math:: + + G_{\text{thermal}}(t) = \min\left( 1.0,\ 1.0 - K_{\text{therm}} \cdot \frac{T_v(t) - T_{\text{warn}}}{T_{\text{limit}} - T_{\text{warn}}} \right) + +This dual-time-constant architecture prevents abrupt gain modulation or audible pumping artifacts, holding the voice coil steadily and safely at :math:`T_{\text{limit}}` even during indefinite high-volume music playback. + +--- + +SOF Control ABI, Topology 2, and Data Structures +************************************************ + +Communication between user-space tuning utilities (e.g., ``sof-ctl``, ALSA UCM) and the firmware Smart Amplifier component occurs through ALSA byte controls via two distinct configuration IDs. + +Static Configuration Structure (struct sof_smart_amp_config) +============================================================ + +Static pipeline routing and channel mapping are defined by ``struct sof_smart_amp_config`` (``src/include/user/smart_amp.h``): + +.. code-block:: c + + /* smart amp component configuration data (24 bytes total) */ + struct sof_smart_amp_config { + uint32_t size; /* Total config size in bytes (24) */ + uint32_t feedback_channels; /* Number of feedback channels (e.g., 2 or 4) */ + int8_t source_ch_map[PLATFORM_MAX_CHANNELS]; /* Channel map for audio playback source */ + int8_t feedback_ch_map[PLATFORM_MAX_CHANNELS]; /* Channel map for audio feedback sensing */ + }; + +* ``size``: Exact byte size of the structure (``sizeof(struct sof_smart_amp_config) = 24``). +* ``feedback_channels``: Total number of active feedback channels transmitted across SoundWire or I2S (e.g., :math:`2` for mono current/voltage, :math:`4` for stereo :math:`I_L, V_L, I_R, V_R`). +* ``source_ch_map``: Array of 8 signed 8-bit integers. Each entry maps an input stream channel index to the corresponding algorithm feed-forward channel. An entry of ``-1`` indicates an unmapped channel. +* ``feedback_ch_map``: Array of 8 signed 8-bit integers mapping capture stream channels to the internal algorithm feedback channels. + +Topology 2 Widget Definition (smart_amp.conf) +============================================= + +In Topology 2 (``tools/topology/topology2/include/components/smart_amp.conf``), the component is declared as an audio effect widget with two input pins and one output pin: + +.. code-block:: text + + Define { + SMART_AMP_UUID "0cd84e80-ebd3-11ea-adc10242ac120002" + } + + Class.Widget."smart_amp" { + DefineAttribute."index" {} + + + DefineAttribute."cpc" { + token_ref "comp.word" + } + DefineAttribute."is_pages" { + token_ref "comp.word" + } + + Object.Control.bytes."1" { + !access [ + tlv_read + tlv_callback + ] + Object.Base.extops.1 { + name "extctl" + get 258 + put 0 + } + max 4096 + Object.Base.data.1 { + IncludeByKey.SMART_AMP_UUID { + "0cd84e80-ebd3-11ea-adc10242ac120002" { + # ABI initialization header (SOF4 base_cfg_ext) followed by 24-byte config: + # size = 24 (0x18), feedback_channels = 2 (0x02) + # source_ch_map = [0, -1, -1, -1, -1, -1, -1, -1] + # feedback_ch_map = [-1, 0, -1, -1, -1, -1, -1, -1] + bytes "0x53, 0x4f, 0x46, 0x34, 0x02, 0x00, 0x00, 0x00, + 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x18, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff" + } + } + } + } + + uuid $SMART_AMP_UUID + type "effect" + no_pm "true" + cpc 5000 + is_pages 1 + num_input_pins 2 + num_output_pins 1 + } + +Binary Control Types: Configuration vs. Model +============================================= + +The component dispatches ALSA byte controls (``smart_amp_ctrl_set_bin_data`` and ``smart_amp_ctrl_get_bin_data``) using two data types: + +* **``SOF_SMART_AMP_CONFIG`` (Type 0)**: Reads or writes the static 24-byte ``struct sof_smart_amp_config`` routing structure. +* **``SOF_SMART_AMP_MODEL`` (Type 1)**: Reads or writes the inner model calibration and parameter database (``smart_amp_caldata``). + +Parameter Block Layout & Channel Bitmasks +----------------------------------------- + +In the Maxim DSM inner model (``smart_amp_maxim_dsm.c``), parameter data is serialized as a sequence of 8-byte tuples consisting of a 4-byte Parameter ID followed by a 4-byte Parameter Value: + +.. list-table:: DSM Binary Parameter Payload Entry Format + :header-rows: 1 + :widths: 20 20 60 + + * - Field + - Byte Width + - Description and Encoding + * - **Parameter ID** + - 4 Bytes (int32) + - Encodes target parameter command index and channel bitmask: + * ``0x01000000``: Channel 1 only (``DSM_CH1_BITMASK``) + * ``0x02000000``: Channel 2 only (``DSM_CH2_BITMASK``) + * ``0x03000000``: Stereo (both channels simultaneously) (``DSM_SET_STEREO_CMD_ID``) + * - **Parameter Value** + - 4 Bytes (int32) + - 32-bit signed fixed-point integer representing the calibrated parameter value in vendor Q-format. + +Volatile Telemetry Parameters +----------------------------- + +When a user reads the model control blob (``get_config``), the inner model queries its volatile diagnostic registers (``DSM_API_ADAPTIVE_PARAM_START`` to ``DSM_API_ADAPTIVE_PARAM_END``) and populates the return blob with live operational metrics: + +.. list-table:: Volatile Diagnostic Telemetry Registers Accessible via get_config + :header-rows: 1 + :widths: 15 25 20 40 + + * - Command ID + - Telemetry Parameter + - Engineering Units + - Description and Diagnostic Utility + * - ``0x10`` + - :math:`T_v(t)` (Voice Coil Temp) + - :math:`^\circ\text{C}` (Celsius) + - Instantaneous voice coil operating temperature. Critical for verifying thermal model safety margins. + * - ``0x11`` + - :math:`R_e(t)` (DC Resistance) + - :math:`\text{m}\Omega` (Milliohms) + - Measured voice coil electrical resistance. Verifies cold baseline calibration and detects voice coil short circuits. + * - ``0x12`` + - :math:`X_{\text{peak}}` (Peak Excursion) + - :math:`\mu\text{m}` (Micrometers) + - Peak mechanical cone displacement. Confirms that excursion remains strictly within :math:`X_{\text{max}}`. + * - ``0x13`` + - :math:`V_{\text{pvd}}` / :math:`V_{\text{bat}}` (Supply) + - :math:`\text{mV}` (Millivolts) + - Battery or booster rail voltage. Allows monitoring of power amplifier voltage droop under heavy transients. + * - ``0x14`` + - Thermal Attenuation Status + - :math:`\text{mdB}` (Milli-dB) + - Amount of active gain reduction currently applied by the thermal limiter (:math:`0\text{ to } -12000\text{ mdB}`). + +--- + +End-to-End Tuning and Calibration Workflow +****************************************** + +Acoustic calibration of a smart amplifier requires an empirical workflow combining physical laser metrology, thermal measurement, algorithm parameter synthesis, and live telemetry verification: + +.. figure:: images/smart_amp_tuning_toolchain_workflow.svg + :align: center + :width: 100% + :alt: End-to-End Smart Amplifier Acoustic Characterization and Telemetry Workflow + + End-to-End Smart Amplifier Acoustic Characterization and Telemetry Workflow + +Phase 1: Transducer Metrology & Thiele-Small Extraction +======================================================= + +Using an industry-standard acoustic laser vibrometer system (e.g., Klippel R&D Distortion Analyzer / Laser Displacement Sensor): + +1. **Laser Small-Signal Analysis**: Mount the raw driver in free air. Measure the complex electrical impedance curve :math:`Z(f)` from :math:`20\text{ Hz}` to :math:`20\text{ kHz}` at low drive levels (:math:`0.1\text{ V}_{\text{rms}}`). Extract linear Thiele-Small parameters: :math:`F_s, Q_{ms}, Q_{es}, Q_{ts}, B\cdot l, M_{ms}, C_{ms}, R_e, L_e`. +2. **Enclosure Loading Analysis**: Install the micro-speaker into the final target laptop or smartphone chassis. Re-measure the impedance curve. The resonant frequency will shift upward from :math:`F_{s,\text{free-air}}` to :math:`F_{s,\text{box}}` due to the pneumatic air-spring stiffness of the enclosed back-cavity volume: + + .. math:: + + F_{s,\text{box}} = F_{s,\text{free-air}} \cdot \sqrt{1 + \frac{V_{as}}{V_{\text{box}}}} + +3. **Large-Signal Nonlinear Characterization**: Drive the mounted speaker with high-amplitude multitone and sinusoidal sweeps up to rated peak voltage. Measure: + * :math:`B\cdot l(x)`: Force factor symmetry and roll-off as a function of displacement. + * :math:`K_{ms}(x)`: Suspension mechanical stiffness progressive hardening curve. + * :math:`X_{\text{max}}`: Excursion limit at which total harmonic distortion (THD) reaches 10% or :math:`B\cdot l(x)` drops to :math:`70\%` of baseline. + * :math:`X_{\text{mech}}`: Absolute mechanical destruction limit (voice coil bottoming against back plate). +4. **Thermal Step-Response Soak Test**: Apply continuous band-limited IEC 60268-5 pink noise. Log the voice coil temperature rise using an infrared thermal imaging camera and precision resistance measurement. Fit the dual-pole thermal network time constants (:math:`\tau_{\text{vc}}, R_{\text{th,vc}}, \tau_{\text{mag}}, R_{\text{th,mag}}`). + +Phase 2: Offline Parameter Synthesis via Python +=============================================== + +Below is a complete, production-grade Python calibration script (``sof_smart_amp_tool.py``) that converts physical transducer measurements into a validated SOF Smart Amplifier binary configuration blob: + +.. code-block:: python + + #!/usr/bin/env python3 + """ + Sound Open Firmware (SOF) Smart Amplifier Calibration Generator + Converts physical Thiele-Small and thermal parameters into SOF ABI binary blobs. + """ + + import struct + import sys + + # SOF ABI Constants + SOF_ABI_MAGIC = 0x0043544C # "CTL\0" + SOF_ABI_VERSION = 0x0314000 # ABI 3.20.0 + SOF_CTRL_CMD_BINARY = 0x100 + SOF_SMART_AMP_CONFIG = 0 + SOF_SMART_AMP_MODEL = 1 + + # DSM Command Masks + DSM_CH1_BITMASK = 0x01000000 + DSM_CH2_BITMASK = 0x02000000 + DSM_STEREO_CMD = 0x03000000 + + + def create_smart_amp_config( + feedback_channels=2, source_map=None, feedback_map=None + ): + """Builds the 24-byte struct sof_smart_amp_config binary payload.""" + if source_map is None: + source_map = [0, 1, -1, -1, -1, -1, -1, -1] + if feedback_map is None: + feedback_map = [-1, -1, 0, 1, -1, -1, -1, -1] + + # Format: uint32 size, uint32 feedback_channels, int8_t source[8], int8_t feedback[8] + blob_size = 24 + payload = struct.pack( + " Generated smart_amp_config.bin ({len(config_bytes)} bytes)") + + # 2. Generate Model Calibration Parameters (Example: Laptop Micro-Speaker) + # Values converted to fixed-point integer representations required by DSM API + dsm_params = { + 0x01: 880, # Fs (Resonant Frequency in Hz) + 0x02: 7200, # Re0 (DC Resistance in mOhm: 7.20 Ohm) + 0x03: 165, # Qts (Total Q-factor x 100: 1.65) + 0x04: 450, # Xmax (Excursion limit in um: 0.45 mm) + 0x05: 750, # Xmech (Mechanical damage limit in um: 0.75 mm) + 0x06: 95, # Twarn (Thermal warning threshold in deg C) + 0x07: 115, # Tlimit (Thermal maximum ceiling in deg C) + 0x08: 130, # Tcutoff (Emergency shutdown temp in deg C) + 0x09: 1500, # tau_vc (Coil thermal time constant in ms: 1.5 s) + 0x0A: 180, # tau_mag (Magnet thermal time constant in s: 180 s) + } + + model_bytes = create_dsm_model_blob(dsm_params) + with open("smart_amp_model.bin", "wb") as f: + f.write(model_bytes) + print( + f" -> Generated smart_amp_model.bin ({len(model_bytes)} bytes," + f" {len(dsm_params)} params)" + ) + + + if __name__ == "__main__": + main() + +--- + +Production Preset Recipes +************************* + +The table below provides production-ready parameter recipes for three distinct commercial hardware acoustic architectures: + +.. list-table:: Validated Production Preset Recipes for SOF Smart Amplifier Deployment + :header-rows: 1 + :widths: 22 26 26 26 + + * - Parameter / Attribute + - **Recipe A: Ultra-Thin Laptop** + - **Recipe B: Thin Multimedia Tablet** + - **Recipe C: Conference Smart Speaker** + * - **Target Hardware** + - 14-inch Laptop (Dual Micro-Speakers) + - 11-inch Tablet (Quad Micro-Speakers) + - Desktop Smart Speaker / Soundbar + * - **Smart Amp Codec IC** + - Maxim / ADI MAX98390 (SDW) + - Cirrus Logic CS35L41 (I2S/TDM) + - Texas Instruments TAS2781 (I2S) + * - **Resonance** (:math:`F_s`) + - 880 Hz (enclosed) + - 650 Hz (enclosed) + - 180 Hz (ported enclosure) + * - **Cold Resistance** (:math:`R_{e,0}`) + - 7.20 Ω (7200 mΩ) + - 6.80 Ω (6800 mΩ) + - 3.60 Ω (3600 mΩ) + * - **Excursion Limit** (:math:`X_{\text{max}}`) + - 0.45 mm (450 μm) + - 0.55 mm (550 μm) + - 2.20 mm (2200 μm) + * - **Mechanical Limit** (:math:`X_{\text{mech}}`) + - 0.70 mm (700 μm) + - 0.85 mm (850 μm) + - 3.50 mm (3500 μm) + * - **Thermal Warning** (:math:`T_{\text{warn}}`) + - 95°C + - 100°C + - 110°C + * - **Thermal Limit** (:math:`T_{\text{limit}}`) + - 115°C + - 120°C + - 135°C + * - **Shutdown Temp** (:math:`T_{\text{cutoff}}`) + - 130°C + - 135°C + - 150°C + * - **Coil Time Const** (:math:`\tau_{\text{vc}}`) + - 1.5 seconds + - 2.0 seconds + - 4.5 seconds + * - **Magnet Time Const** (:math:`\tau_{\text{mag}}`) + - 160 seconds + - 220 seconds + - 480 seconds + * - **Effective SPL Gain** + - **+8.5 dB** (unclipped peak SPL) + - **+7.2 dB** (unclipped peak SPL) + - **+6.0 dB** (unclipped peak SPL) + +--- + +Live Injection & Real-Time Telemetry Runbook via sof-ctl +******************************************************** + +SOF allows acoustic engineers to inject new calibration profiles, adjust safety limits, and query real-time voice coil temperature and excursion telemetry over SSH without interrupting active audio playback. + +Step 1: Discovering Smart Amplifier ALSA Controls on the DUT +============================================================ + +Log into the target DUT (Spider, Dragon Fly, or Aphid) over lab SSH and enumerate the available ALSA byte controls: + +.. code-block:: bash + + # Enumerate all byte controls matching Smart Amp on soundcard 0 + timeout 15 ssh -o ConnectTimeout=5 root@spider 'amixer -c 0 scontrols | grep -i "smart_amp\|dsm"' + +Expected output: + +.. code-block:: text + + Simple mixer control 'Smart Amp Config',0 + Simple mixer control 'Smart Amp Model',0 + +To inspect the raw control index and element numbers: + +.. code-block:: bash + + timeout 15 ssh -o ConnectTimeout=5 root@spider 'amixer -c 0 cget name="Smart Amp Model"' + +Step 2: Live Calibration Parameter Injection +============================================ + +Transmit the newly synthesized calibration binary blob (``smart_amp_model.bin``) directly into the running DSP firmware using ``sof-ctl``: + +.. code-block:: bash + + # Copy generated binary blob to target DUT + scp smart_amp_model.bin root@spider:/tmp/smart_amp_model.bin + + # Inject parameter blob via sof-ctl while playback is active + timeout 15 ssh -o ConnectTimeout=5 root@spider \ + 'sof-ctl -D hw:0 -n "Smart Amp Model" -s /tmp/smart_amp_model.bin' + +Verification in Kernel DSP Trace Logs +------------------------------------- + +Inspect the real-time DSP trace log buffer to confirm that the generic layer received the payload and updated the inner model: + +.. code-block:: bash + + timeout 15 ssh -o ConnectTimeout=5 root@spider 'dmesg | grep -i "smart_amp"' + +Expected firmware output: + +.. code-block:: text + + sof-audio-pci-intel-tgl 0000:00:1f.3: smart_amp_set_configuration: config_id 1 size 80 + sof-audio-pci-intel-tgl 0000:00:1f.3: [DSM] Parameter table updated: 10 parameters applied successfully. + sof-audio-pci-intel-tgl 0000:00:1f.3: [DSM] Fs=880Hz, Re0=7200mOhm, Xmax=450um, Tlimit=115C + +Step 3: Real-Time Telemetry Readback & Voice Coil Monitoring +============================================================ + +To read back live diagnostic metrics from the inner model during active audio playback: + +.. code-block:: bash + + # Dump the binary telemetry response from the running firmware + timeout 15 ssh -o ConnectTimeout=5 root@spider \ + 'sof-ctl -D hw:0 -n "Smart Amp Model" -g /tmp/dsm_telemetry.bin && od -tx4 /tmp/dsm_telemetry.bin | head -n 12' + +Automated Python Live Poller +---------------------------- + +Use the following monitoring snippet to stream real-time voice coil temperature and resistance over SSH during acoustic stress testing: + +.. code-block:: python + + #!/usr/bin/env python3 + """Streams real-time voice coil temperature and resistance from SOF DUT.""" + + import struct + import subprocess + import time + + + def poll_telemetry(dut_host="root@spider"): + cmd = f"ssh -o ConnectTimeout=5 {dut_host} 'sof-ctl -D hw:0 -n \"Smart Amp Model\" -g /tmp/live.bin && cat /tmp/live.bin'" + proc = subprocess.run( + cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=10, + ) + if proc.returncode != 0 or len(proc.stdout) < 40: + return None + + # Unpack volatile parameter registers + # Cmd 0x10 = Tv (deg C), Cmd 0x11 = Re (mOhm), Cmd 0x12 = Xpeak (um) + raw = proc.stdout + # Search for parameter entries in the returned blob + num_entries = len(raw) // 8 + telemetry = {} + for i in range(num_entries): + cmd_id, val = struct.unpack_from("8} | {'Temp (°C)':>10} | {'Re (Ω)':>8} | {'Xpeak (mm)':>10} |" + f" {'Status':>12}" + ) + print("-" * 60) + + for step in range(30): + t = poll_telemetry() + if t and 0x10 in t: + temp_c = t.get(0x10, 0) + re_ohm = t.get(0x11, 0) / 1000.0 + x_mm = t.get(0x12, 0) / 1000.0 + status = ( + "WARNING" + if temp_c >= 100 + else ("PROTECT" if x_mm >= 0.44 else "NOMINAL") + ) + print( + f"{step*0.5:7.1f}s | {temp_c:9d}°C | {re_ohm:7.2f}Ω | {x_mm:9.3f}mm" + f" | {status:>12}" + ) + time.sleep(0.5) + +--- + +Acoustic Diagnostics & Troubleshooting Matrix +********************************************* + +The diagnostic matrix below maps real-world acoustic anomalies, driver distortion symptoms, and hardware faults to their underlying root causes and corrective tuning procedures: + +.. list-table:: Smart Amplifier Acoustic Diagnostics, Failure Modes, and Corrective Actions + :header-rows: 1 + :widths: 20 25 25 30 + + * - Symptom + - Probable Root Cause + - Diagnostic Verification + - Corrective Action + * - **Harsh Clicking / Buzzing at High Volume** + - Cone excursion exceeding :math:`X_{\text{mech}}`; voice coil bottoming out against the back plate due to underestimated mechanical compliance or high-pass filter cutoff set too low. + - Inspect telemetry register ``0x12`` (:math:`X_{\text{peak}}`). If :math:`X_{\text{peak}} > X_{\text{max}}`, excursion limiting is failing to engage. + - Lower :math:`X_{\text{max}}` in the calibration blob by :math:`15\%`. Increase the adaptive high-pass filter attack rate or increase :math:`F_s` in the model if enclosure volume is smaller than modeled. + * - **Premature Muting / Severe Volume Pumping** + - Thermal model estimating excessive voice coil heating due to incorrect cold baseline resistance :math:`R_{e,0}` or inverted :math:`I/V` feedback channels. + - Read register ``0x10`` (:math:`T_v`) while the speaker is cold at room temperature. If :math:`T_v \gg 25^\circ\text{C}` at zero volume, baseline calibration is invalid. + - Measure true voice coil resistance with a 4-wire milliohm meter. Update :math:`R_{e,0}` in the calibration blob. Verify that ``source_ch_map`` and ``feedback_ch_map`` are not swapped in the topology. + * - **Catastrophic Voice Coil Burnout During Stress Testing** + - Thermal protection disabled, feedback capture pipeline dropped frames, or :math:`I_{\text{sense}}` ADC channel mapped to silence (zero current reported). + - Check kernel dmesg for ``[DSM] ZERO_I`` or ``IV_DATA_WARNING`` error codes. Verify capture pipeline DAI is running without XRUNs. + - Verify DAI SoundWire/TDM slot allocation. Ensure capture pipeline buffer writeback and cache invalidation are active. Set ``Tlimit`` conservatively (:math:`\le 115^\circ\text{C}`). + * - **Thin, Bass-Deficient Audio at Low Volumes** + - Baseline adaptive high-pass filter cutoff :math:`f_c` configured excessively high, or permanent thermal attenuation latched in active state. + - Check telemetry register ``0x14`` (thermal attenuation status). If negative dB attenuation is reported at idle, the limiter has latched. + - Lower the idle cutoff frequency :math:`f_c` to :math:`90\text{ Hz}`. Verify that the inner model receives ``COMP_TRIGGER_START`` and ``COMP_TRIGGER_RELEASE`` reset commands upon stream restart. + * - **False Acoustic Leak / Seal Failure Alarm** + - Production assembly tolerance variation in speaker enclosure gasket compression causing :math:`\pm 10\%` natural shift in resonant frequency :math:`F_{s,\text{box}}`. + - Log :math:`F_s` tracking across a 20-unit manufacturing sample in the acoustic chamber. + - Broaden the acceptable resonant frequency window in the leak detection threshold parameters before raising protective high-pass cutoff. From 5ef012b5bad765e4a09145e00555cc60e1dfb944 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 17:01:02 +0100 Subject: [PATCH 35/64] doc: developer_guides: add sound dose & hearing health calibration guide Signed-off-by: Liam Girdwood --- data/modules.yaml | 10 +- developer_guides/index.rst | 2 + ...d_dose_tuning_a_weight_filter_response.svg | 182 +++++ ...nd_dose_tuning_energy_integration_math.svg | 201 +++++ ...und_dose_tuning_hats_measurement_setup.svg | 197 +++++ ...sound_dose_tuning_host_dsp_closed_loop.svg | 155 ++++ ...sound_dose_tuning_standards_compliance.svg | 206 +++++ developer_guides/tuning/sound_dose_tuning.rst | 743 ++++++++++++++++++ 8 files changed, 1692 insertions(+), 4 deletions(-) create mode 100644 developer_guides/tuning/images/sound_dose_tuning_a_weight_filter_response.svg create mode 100644 developer_guides/tuning/images/sound_dose_tuning_energy_integration_math.svg create mode 100644 developer_guides/tuning/images/sound_dose_tuning_hats_measurement_setup.svg create mode 100644 developer_guides/tuning/images/sound_dose_tuning_host_dsp_closed_loop.svg create mode 100644 developer_guides/tuning/images/sound_dose_tuning_standards_compliance.svg create mode 100644 developer_guides/tuning/sound_dose_tuning.rst diff --git a/data/modules.yaml b/data/modules.yaml index fc856acf..69762c58 100644 --- a/data/modules.yaml +++ b/data/modules.yaml @@ -262,12 +262,14 @@ modules: source: "SOF" category: "Speaker Protection" status: "Upstream" - description: "Hearing health monitoring and cumulative sound exposure limiter complying with EN 50332 and IEC 62368-1." + description: "Auditory health monitoring and cumulative sound exposure limiter complying with IEC 62368-1 Clause 10.6, EN 50332-1/-2/-3, and WHO-ITU H.870." simd: ["HiFi 3", "Scalar C"] key_features: - - "Continuous equivalent sound level (Leq) running integration" - - "Digital A-weighting and C-weighting IIR filter profiles" - - "Configurable Cumulative Sound Dose (CSD) threshold triggers and attenuation" + - "IEC 61672-1 Class 1 A-weighting cascaded Direct Form I IIR biquad filtering" + - "Overflow-proof 64-bit real-time energy accumulation and integer base-2 logarithm decibel conversion" + - "Autonomous 1-second asynchronous IPC4 notification dispatch without host polling" + - "Smooth per-frame exponential slew gain limiter (0.05 dB/frame) eliminating clicks and pops" + - "Acoustic laboratory HATS calibration, rolling 7-day CSD tracking, and runtime control via sof-ctl" # --- Voice, Telephony & Speech --- - id: tdfb diff --git a/developer_guides/index.rst b/developer_guides/index.rst index d92465e6..a9a7cf09 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -127,6 +127,7 @@ Dynamics & Transducer Protection Tuning * :ref:`drc_tuning` (Single-band DRC and Multiband DRC compression curves, adaptive ballistics, and speaker protection) * :ref:`smart_amp_tuning` (Smart Amplifier Dynamic Speaker Management, I/V sense feedback calibration, Thiele-Small modeling, thermal and excursion protection) +* :ref:`sound_dose_tuning` (Sound Dose Evaluator, IEC 61672-1 Class 1 A-weighting, EN 50332 / IEC 62368-1 compliance, HATS acoustic sensitivity calibration, and closed-loop exposure regulation) Acoustic, Transducer & Array Tuning =================================== @@ -143,6 +144,7 @@ Acoustic, Transducer & Array Tuning tuning/runtime_tuning_sof_ctl tuning/drc_tuning tuning/smart_amp_tuning + tuning/sound_dose_tuning tuning/crossover_tuning algorithms/eq/equalizers_tuning algorithms/tdfb/time_domain_fixed_beamformer diff --git a/developer_guides/tuning/images/sound_dose_tuning_a_weight_filter_response.svg b/developer_guides/tuning/images/sound_dose_tuning_a_weight_filter_response.svg new file mode 100644 index 00000000..c7dca8de --- /dev/null +++ b/developer_guides/tuning/images/sound_dose_tuning_a_weight_filter_response.svg @@ -0,0 +1,182 @@ + + + + + + + + + + + + + + + + + + + + + IEC 61672-1 Class 1 A-Weighting Acoustic Filter Cascade + + + Continuous Theoretical Curve vs SOF Fixed-Point Cascaded DF1 Biquad Implementation (48 kHz / 44.1 kHz) + + + + + + + + + + +10 dB + + + + 0 dB + + + + -10 dB + + + + -20 dB + + + + -30 dB + + + + -40 dB + + + + -50 dB + + + + + 20 Hz + + + + 100 Hz + + + + 1 kHz + + + + 3.15 kHz + + + + 10 kHz + + + + 20 kHz + + + + ±1.0 dB Class 1 Mask + + + + + + + + + + + + + -19.1 dB + + + + 0.0 dB (Ref) + + + + +1.2 dB + + + + + + IEC 61672-1 Class 1 Target + + SOF Cascaded DF1 IIR (48k) + + + + + + + + SOF Cascaded DF1 Biquad Structure + + + Pre-computed 6-pole, 2-zero IIR Filter + + + + + + Biquad 1: Sub-Audible High-Pass + f_p1,2 = 20.6 Hz (HP1 + HP1 pair) + Eliminates DC bias & sub-acoustic rumble + + + + + + + + Biquad 2: Mid-Bass Transition + f_p3 = 107.7 Hz, f_p4 = 737.9 Hz + Forms +20 dB/dec rolloff slope + + + + + + + + Biquad 3: Ultrasonic Low-Pass + f_p5,6 = 12,194 Hz (LP2 pair) + Rolls off ultrasonic content above 12 kHz + + + + + + Normalization: -3.0 dB at 1 kHz in firmware + +3.0 dB offset added back during log conversion + + + + + + + + IEC 61672-1 Analytic Weighting Formula & Bilinear Transformation Pre-Warping + + + R_A(f) = (12194² · f⁴) / [ (f² + 20.6²) · √((f² + 107.7²)(f² + 737.9²)) · (f² + 12194²) ] + + + Continuous-to-discrete mapping uses Bilinear Transform with pre-warping: ω_a = 2·f_s · tan(π·F_c / f_s). + + + At 48 kHz, cascaded Direct Form I structures exhibit negligible frequency warping below 10 kHz, ensuring compliance within Class 1 bounds. + + + diff --git a/developer_guides/tuning/images/sound_dose_tuning_energy_integration_math.svg b/developer_guides/tuning/images/sound_dose_tuning_energy_integration_math.svg new file mode 100644 index 00000000..7744006e --- /dev/null +++ b/developer_guides/tuning/images/sound_dose_tuning_energy_integration_math.svg @@ -0,0 +1,201 @@ + + + + + + + + + + + + + + + + + + + + + + + + + Fixed-Point Real-Time Energy Accumulation & Logarithmic Math + + + Per-Sample Cascaded DF1 Filtering, 64-Bit Integration, Integer Log2 Conversion, and Timestamp Latching + + + + + + + + STAGE 1: Per-Sample Real-Time Inner Processing Loop (sound_dose_s16 / sound_dose_s32) + + + + + + Input Sample + x[n] + S16_LE (Q1.15) + S32_LE (Q1.31) + + + + + + + + + + Gain Slew Core + Q_MULTSR_32X32 + cd->gain (Q2.30) + ±0.05 dB / frame + + + + + + + + Sink Buffer + y[n] (Audio Out) + + + + + + + + + + A-Weight IIR DF1 + iir_df1(cd->iir[ch]) + + + + + + + + + + Power Squarer + y_A[n] · y_A[n] + Q1.15² → Q2.30 + + + + + + + + + + 64-Bit Energy Accum + cd->energy[ch] += ... + Capacity: ~5.15×10¹³ + Zero overflow risk (< 2⁶³-1) + + + + + + + + + + 1.0-SECOND PERIODIC TRIGGER (frames_count ≥ 48000) + + + + + + + + + STAGE 2: Periodic Logarithmic Conversion, MEL Calibration, and IPC Notification Dispatch + + + + + + 1. Scaling & Normalization + energy_sum = Σ cd->energy[ch] + log_arg = energy_sum >> 19 + • Fits within uint32_t + • Clamped: MAX(log_arg, 1) + + + → + + + + + 2. Integer Log2 Engine + tmp = base2_logarithm(arg) + tmp += LOG_FIXED_OFFSET + 65536 * (19 - 30) = -720,896 + tmp += log_offset_for_mean + + + → + + + + + 3. Decibel Scale & Offsets + mult = 10 / log2(10) · 2²⁹ + tmp = Q_MULTSR_32X32(tmp) + +3.00 dB (Filter Norm Offset) + +3.01 dB (Full-scale Sine Ref) + + + → + + + + + 4. Digital dBFS Level + cd->level_dbfs + Stereo Correction: + -1.5 dB × 2 = -3.0 dB + Binaural Acoustic Sum + + + + + + + Acoustic Calibration Equation (MEL Derivation in Centi-Decibels) + + + + MEL = dbfs_value + sens_dbfs_dbspl + volume_offset + + + • sens_dbfs_dbspl: Headphone sensitivity in dBSPL produced by 0 dBFS input (e.g. 10000 = 100.0 dB) + + + • volume_offset: Dynamic volume attenuation relative to maximum volume (e.g. -1200 = -12.0 dB) + + + + + + + + IPC4 Asynchronous Notification + + High-Precision 96-Bit Timestamp: + time_us = (frames * coef) >> 26 + Dispatched via SOF IPC4 Global Event: + SOF_IPC4_GLB_NOTIFICATION (Event 0xCA) + + + diff --git a/developer_guides/tuning/images/sound_dose_tuning_hats_measurement_setup.svg b/developer_guides/tuning/images/sound_dose_tuning_hats_measurement_setup.svg new file mode 100644 index 00000000..b137a65b --- /dev/null +++ b/developer_guides/tuning/images/sound_dose_tuning_hats_measurement_setup.svg @@ -0,0 +1,197 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Acoustic Laboratory Metrology & Headphone Sensitivity Calibration + + + EN 50332 / IEC 60318-4 Head and Torso Simulator (HATS) Test Rig & Parameter Derivation + + + + + + + + + + + ACOUSTIC ANALYZER + + + Audio Precision / SoundCheck + + + + + + Stimulus Generator + • IEC 60268-1 Pink Noise + • ITU-T P.50 Artificial Voice + Digital Level: 0 dBFS + + + + + Acoustic Acquisition + • Mic Conditioning Amp + • IEC 61672 A-Weighting + Measured Sound Level: + L_p,HATS (dBA SPL) + + + + + Calibration Script + sof_sound_dose_tool.py + + + + + + + PCM 0 dBFS + + + + + + + + SOF TARGET DUT + + + Spider (TGL) / Aphid (PTL) / Dragon Fly + + + + + + SOF Audio Pipeline + Host Copier → Volume + → Sound Dose Module → DAI + + + + + Sound Dose Module State + sens_dbfs_dbspl: 10040 + volume_offset: 0 (0 dB) + gain: 0 (0 dB) + + + + + Hardware Audio Codec + • High-SNR Headphone DAC + • Integrated Headphone Amp + Max V_out: ~150 mV RMS + + + + + + + Analog Out + + + + + + + + HATS ACOUSTIC FIXTURE + + + IEC 60318-4 / ITU-T P.57 Type 3.3 Artificial Ear + + + + + + Transducer Under Evaluation + • In-Ear / On-Ear / Over-Ear Headphone + • Mounted on anatomical pinna with sealed seal + + + + + + Occluded Ear Simulator (IEC 60318-4) + • Replicates human ear canal acoustic impedance + • Built-in 1/2" laboratory reference microphone + P_acoustic: Sound pressure at eardrum + + + + + + Microphone Signal Return + • Preamp output: Calibrated mV/Pa signal + • Connected directly to APx555 input channel + + + + + + + + + + Calibrated Mic Signal (mV/Pa) + + + + + + + + Acoustic Sensitivity Derivation & Live Calibration Protocol + + + + + + Mathematical Calibration Formula: + + sens_dbfs_dbspl = round( (L_p,HATS - 0.0) · 100 ) + + + Example: L_p,HATS = 100.42 dBA SPL → sens_dbfs_dbspl = 10042 (centi-decibels) + + + + + + + Verification & Deployment Sequence: + 1. Generate binary blob: sof_sound_dose_tool.py --sens 100.4 + 2. Inject to DUT over SSH: sof-ctl -i 4 -n <numid> -s setup.bin + 3. Verify reported MEL matches APx555 measured level within ±0.2 dBA + + + diff --git a/developer_guides/tuning/images/sound_dose_tuning_host_dsp_closed_loop.svg b/developer_guides/tuning/images/sound_dose_tuning_host_dsp_closed_loop.svg new file mode 100644 index 00000000..03e53be6 --- /dev/null +++ b/developer_guides/tuning/images/sound_dose_tuning_host_dsp_closed_loop.svg @@ -0,0 +1,155 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Host-DSP Closed-Loop Sound Dose Architecture & Regulation + + + Asynchronous IPC4 Notification Dispatch, 7-Day Rolling CSD Tracking, and Smooth Protective Attenuation + + + + + + + + + SOF FIRMWARE DSP DOMAIN + + + + + + + Real-Time Audio Pipeline + Input PCM → Slew Gain Mult → Sink PCM + cd->gain: ±0.05 dB/frame slew rate + + + + + + IEC 61672 Class 1 A-Weighting Core + • Cascaded DF1 Biquads (48k / 44.1k) + • 64-bit energy accumulator: cd->energy[ch] + • 1-second periodic tick (48,000 frames) + + + + + + Integer Log2 & MEL Calculator + MEL = dbfs + sens_dbfs + vol_offset + • Precision: Centi-decibels (0.01 dB) + • 96-bit microsecond clock latching + + + + + + Asynchronous IPC4 Dispatcher + SOF_IPC4_GLB_NOTIFICATION + Event: 0xCA (Bytes Control ID 3) + + No Host Polling Required: + Dispatched autonomously every 1.000 s + Transmits full struct sof_sound_dose + + + + + + + + + + IPC4 NOTIFY (1s) + + + + + + SET GAIN (dB) + + + + + + + + + HOST LINUX OPERATING SYSTEM & DAEMON + + + + + + Kernel ALSA / SOF Driver + sound/soc/sof/ipc4-topology.c + Dispatches snd_ctl_notify() change event + + + + + + User-Space Sound Dose Daemon + PipeWire / PulseAudio / CRAS Exposure Service + + + CSD_7d = Σ [ Δt / 40h · 10^((MEL - 80)/10) ] + + • Persistent SQLite/JSON storage across reboots + • Sliding 7-day circular exposure FIFO + + + + + + Policy Engine & User Protection Actions + + + + + CSD < 80%: Normal Operation + Full volume capability, passive logging + + + + + 80% ≤ CSD < 100%: Advisory Warning + System desktop notification displayed to user + + + + + CSD ≥ 100% or MEL > 100 dBA: Enforcement + • Injects SOF_SOUND_DOSE_GAIN_PARAM_ID = -12 dB + • Requires explicit user confirmation dialog to override + + + + diff --git a/developer_guides/tuning/images/sound_dose_tuning_standards_compliance.svg b/developer_guides/tuning/images/sound_dose_tuning_standards_compliance.svg new file mode 100644 index 00000000..98b28997 --- /dev/null +++ b/developer_guides/tuning/images/sound_dose_tuning_standards_compliance.svg @@ -0,0 +1,206 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + International Sound Exposure Standards & Dose Protection Boundaries + + + IEC 62368-1 Clause 10.6 • EN 50332-1/-2/-3 • WHO-ITU H.870 (3 dB Exchange Rate Formulation) + + + + + + + + + + SAFE EXPOSURE REGIME + + + < 80 dBA + + + Weekly Dose: < 100% CSD (1.6 Pa²h) + + + • Permissible: Up to 40 hrs / week + • Cochlear State: Normal homeostasis + • Outer hair cells (OHC) unperturbed + • SOF Action: Autonomous passive logging + ✔ Full user volume control allowed + + + + + + + + + ADVISORY WARNING REGIME + + + 80 to 89 dBA + + + Weekly Dose: 80% to 100% CSD + + + • Permissible: 5 to 40 hrs / week + • Cochlear State: Metabolic fatigue + • Temporary Threshold Shift (TTS) risk + • SOF Action: High-frequency notifications + ⚠ System issues advisory alert dialog + + + + + + + + + HAZARDOUS DOSE REGIME + + + >= 90 dBA or > 100% CSD + + + Mandatory Regulatory Intervention + + + • Permissible: < 23 min at 100 dBA + • Cochlear State: Irreversible apoptosis + • Permanent Threshold Shift (PTS) + • SOF Action: Smooth attenuation ramp + ⛔ Mandatory volume cap & user ACK + + + + + + + The 3 dB Equal-Energy Exchange Principle: Weekly Exposure Halving Schedule (WHO-ITU H.870 / IEC 62368-1) + + + T_safe(MEL) = 40 hours · 10^((80 - MEL) / 10) • Every +3 dB increase halves the permissible exposure window before reaching 100% CSD + + + + + + + 80 dBA + 40 Hours + Baseline (100% CSD) + + + → + + + + + 83 dBA + 20 Hours + 2x Energy Density + + → + + + + + 86 dBA + 10 Hours + 4x Energy Density + + → + + + + + 89 dBA + 5 Hours + 8x Energy Density + + → + + + + + 92 dBA + 2.5 Hours + 16x Energy Density + + → + + + + + 95 dBA + 75 Minutes + 32x Energy Density + + → + + + + + 100 dBA + 23.7 Min + EN 50332 Hard Cap + + + + + + + + Harmonized European Test Standards & Equipment Limits (EN 50332 Series) + + + + + EN 50332-1 (Packaged Sets) + Player + Headphone max output ≤ 100 dBA + + + + + EN 50332-2 (Standalone Player/Phone) + Max output voltage V_out ≤ 150 mV RMS + + + + + EN 50332-3 (Sound Dose Management) + Active CSD metering, rolling 7-day storage + + + diff --git a/developer_guides/tuning/sound_dose_tuning.rst b/developer_guides/tuning/sound_dose_tuning.rst new file mode 100644 index 00000000..a9a67546 --- /dev/null +++ b/developer_guides/tuning/sound_dose_tuning.rst @@ -0,0 +1,743 @@ +.. _sound_dose_tuning: + +Sound Dose / Hearing Health Calibration & Acoustic Protection Guide +################################################################### + +The **Sound Dose Evaluator** is an autonomous, real-time auditory safety subsystem in Sound Open Firmware (SOF). Designed to comply with international consumer audio health regulations—specifically **IEC 62368-1 Clause 10.6**, **EN 50332-1/-2/-3**, and **WHO-ITU H.870**—the Sound Dose module continuously analyzes audio streams routed to headphones, headsets, and personal listening devices. It computes spectral energy exposure in real time using an **IEC 61672-1 Class 1 A-weighting** biquad filter cascade, translates digital audio levels into physical sound pressure levels (:math:`\text{dBA SPL}`), tracks cumulative exposure across rolling temporal windows, and autonomously reports exposure metrics to the host operating system while providing artifact-free dynamic attenuation when safe exposure limits are exceeded. + +This guide provides the complete mathematical, electro-acoustic, and operational calibration methodology required to tune, measure, and deploy the Sound Dose module on Sound Open Firmware platforms. + +.. contents:: Table of Contents + :local: + :depth: 3 + +Theoretical Foundations & Regulatory Mandates +============================================= + +Auditory Physiology & Cellular Damage Mechanisms +------------------------------------------------ + +Prolonged exposure to high sound pressure levels induces irreversible physiological damage to the human auditory system. The human inner ear contains the **cochlea**, a fluid-filled, spiral-shaped cavity lined with the basilar membrane. Transduction of acoustic vibrations into neural impulses is performed by approximately 15,000 hair cells: + +* **Inner Hair Cells (IHCs)**: Primary sensory transducers that release neurotransmitters to auditory nerve fibers in response to stereocilia deflection. +* **Outer Hair Cells (OHCs)**: Electromotile amplifiers that actively alter their length via the motor protein prestin, providing up to 50 dB of mechanical amplification for quiet sounds and sharpening frequency selectivity. + +When exposed to excessive acoustic energy, outer hair cells undergo intense metabolic overload. This causes severe oxidative stress, marked accumulation of reactive oxygen species (ROS), intracellular calcium excitotoxicity, mitochondrial swelling, and structural rupture of stereocilia tip-links. While moderate over-exposure leads to a **Temporary Threshold Shift (TTS)** that recovers over several hours as cellular homeostasis is restored, repeated or severe acoustic trauma results in permanent hair cell apoptosis and spiral ganglion synaptic decoupling—causing irreversible **Permanent Threshold Shift (PTS)**, high-frequency sensorineural hearing loss, and chronic tinnitus. + +.. figure:: images/sound_dose_tuning_standards_compliance.svg + :alt: Acoustic Safety Standards and Dose Protection Boundaries showing exposure regimes, the 3 dB exchange rate halving schedule, and EN 50332 test limits. + + Acoustic Safety Standards & Calculated Sound Dose (CSD) Exposure Regimes + +International Regulatory Standards (IEC 62368-1 & EN 50332) +------------------------------------------------------------ + +To protect consumers against premature hearing loss, international regulatory bodies have enacted strict standards governing personal music players and mobile computing platforms: + +* **IEC 62368-1 Clause 10.6**: Audio energy safety standard establishing permissible listening duration, warning notifications, and mandatory attenuation thresholds for consumer audio equipment. +* **EN 50332-1**: Specifies test methods for packaged equipment (personal player bundled with manufacturer headphones). The maximum acoustic sound pressure level must not exceed **100 dBA SPL** with an input test signal of 0 dBFS pink noise. +* **EN 50332-2**: Specifies test methods for standalone players and headphones sold independently. The player maximum electrical output voltage must not exceed **150 mV RMS**, and the wideband headphone characteristic voltage (:math:`WBCV`) must be :math:`\ge 75\text{ mV}` to produce 94 dBA SPL. +* **EN 50332-3 & WHO-ITU H.870**: Standardizes **Calculated Sound Dose (CSD)** and exposure dose monitoring across rolling 7-day listening intervals. + +The 3 dB Equal Energy Exchange Principle +----------------------------------------- + +The human ear integrates acoustic power over time. The cumulative acoustic energy dose :math:`E_{\text{dose}}` is defined by: + +.. math:: + + E_{\text{dose}} = \int_0^T p_A^2(t)\,dt + +where :math:`p_A(t)` is the instantaneous A-weighted acoustic sound pressure in Pascals. Because acoustic sound intensity doubles with every :math:`+3\,\text{dB}` increase, the permissible exposure duration before reaching **100% CSD** halves with each +3 dB increase in sound pressure level: + +.. math:: + + T_{\text{safe}}(\text{MEL}) = 40\,\text{hours} \cdot 10^{\frac{80 - \text{MEL}}{10}} + +The international reference baseline for **100% CSD** corresponds to continuous exposure of **80 dBA for 40 hours per week**, representing an acoustic energy dosage of: + +.. math:: + + \text{Dose}_{\text{ref}} = (20\,\mu\text{Pa} \cdot 10^{80/20})^2 \cdot 40\,\text{hours} \approx 1.6\,\text{Pa}^2\text{h} + +.. table:: Sound Pressure Level vs Maximum Permissible Weekly Exposure Time (IEC 62368-1 / WHO-ITU H.870) + :widths: 15 25 35 25 + + +-----------------------+-----------------------------+-----------------------------------+------------------------------------+ + | Sound Level (dBA) | Permissible Time per Week | Relative Energy Density | Regulatory Action | + +=======================+=============================+===================================+====================================+ + | **< 80 dBA** | Unlimited (> 40 hours) | Baseline (1.0x) | Safe Zone (Normal playback) | + +-----------------------+-----------------------------+-----------------------------------+------------------------------------+ + | **83 dBA** | 20 hours | 2.0x | Normal playback | + +-----------------------+-----------------------------+-----------------------------------+------------------------------------+ + | **86 dBA** | 10 hours | 4.0x | Advisory tracking | + +-----------------------+-----------------------------+-----------------------------------+------------------------------------+ + | **89 dBA** | 5 hours | 8.0x | Advisory warning dialog | + +-----------------------+-----------------------------+-----------------------------------+------------------------------------+ + | **92 dBA** | 2.5 hours (150 min) | 16.0x | Warning threshold | + +-----------------------+-----------------------------+-----------------------------------+------------------------------------+ + | **95 dBA** | 1.25 hours (75 min) | 32.0x | Mandatory prompt | + +-----------------------+-----------------------------+-----------------------------------+------------------------------------+ + | **100 dBA** | 23.7 minutes | 100.0x | EN 50332 Maximum Volume Cap | + +-----------------------+-----------------------------+-----------------------------------+------------------------------------+ + | **>= 105 dBA** | < 7.5 minutes | 316.0x | Instantaneous trauma danger zone | + +-----------------------+-----------------------------+-----------------------------------+------------------------------------+ + +IEC 61672-1 Class 1 A-Weighting Acoustic Filter +=============================================== + +Continuous Weighting Formulation +-------------------------------- + +Human auditory sensitivity varies substantially across the audible spectrum, exhibiting peak sensitivity in the 2 kHz to 4 kHz region (corresponding to the acoustic resonance of the outer ear canal) and rolling off sharply below 500 Hz. The **IEC 61672-1:2013** standard defines the A-weighting frequency curve to approximate the inverse equal-loudness response of the human ear at low-to-moderate sound levels: + +.. math:: + + R_A(f) = \frac{12194^2 \cdot f^4}{(f^2 + 20.6^2) \cdot \sqrt{(f^2 + 107.7^2)(f^2 + 737.9^2)} \cdot (f^2 + 12194^2)} + +The relative decibel weighting :math:`A(f)` referenced to 1000 Hz is given by: + +.. math:: + + A(f) = 20 \log_{10}(R_A(f)) - 20 \log_{10}(R_A(1000)) + +.. figure:: images/sound_dose_tuning_a_weight_filter_response.svg + :alt: IEC 61672-1 Class 1 A-Weighting Acoustic Filter Cascade showing continuous curve vs discrete DF1 biquad realization. + + IEC 61672-1 Class 1 A-Weighting Acoustic Filter Cascade + +Discrete Cascaded Direct Form I (DF1) Realization +------------------------------------------------- + +To evaluate A-weighting in real time on fixed-point DSPs without excessive instruction overhead, SOF decomposes the 6-pole, 2-zero continuous analog prototype into three cascaded second-order Direct Form I (DF1) IIR biquad sections (`struct iir_state_df1`): + +1. **Biquad 1 (Sub-Audible High-Pass)**: Two real poles at 20.6 Hz and two zeros at DC (:math:`s=0`), eliminating DC offsets and sub-audible physical rumble. +2. **Biquad 2 (Mid-Bass Transition)**: Real poles at 107.7 Hz and 737.9 Hz shaping the rising slope between 100 Hz and 1 kHz. +3. **Biquad 3 (Ultrasonic Low-Pass)**: Conjugate pole pair at 12,194 Hz rolling off ultrasonic and high-frequency content above 12 kHz. + +The continuous poles and zeros are mapped to discrete :math:`z`-plane coefficients using the Bilinear Transformation with frequency pre-warping: + +.. math:: + + s \leftarrow \frac{2}{T_s} \frac{1 - z^{-1}}{1 + z^{-1}}, \quad \omega_a = \frac{2}{T_s} \tan\left(\frac{\omega_d T_s}{2}\right) + +Pre-computed coefficient sets are stored in ``sound_dose_iir_48k.h`` and ``sound_dose_iir_44k.h``. + +Firmware Normalization Strategy +------------------------------- + +.. note:: + In the filter synthesis toolchain (``sof_sound_dose_time_domain_filters.m``), the IIR filter cascade is intentionally normalized to **-3.0 dB at 1 kHz** (``eq.iir_norm_offs_db = -3``). + +Because the A-weighting transfer function exhibits a :math:`+1.2\text{ dB}` resonance peak around 3.15 kHz, normalizing to 0 dB at 1 kHz would cause digital full-scale sinusoidal signals at 3.15 kHz to exceed :math:`0\text{ dBFS}`, inducing clipping and saturation in 16-bit or 32-bit fixed-point arithmetic. By attenuating by -3.0 dB in the filter stage, SOF guarantees complete mathematical headroom. + +The :math:`+3.00\text{ dB}` normalization offset (``SOUND_DOSE_WEIGHT_FILTERS_OFFS_Q16 = 196608``) is algebraically restored during the logarithmic decibel conversion in ``sound_dose_calculate_mel()``. + +Real-Time 64-Bit Energy Integration & Logarithmic Math +====================================================== + +The Sound Dose module executes a two-stage evaluation pipeline: an inner per-sample filtering and energy accumulation loop, followed by a periodic 1-second logarithmic decibel conversion and timestamping step. + +.. figure:: images/sound_dose_tuning_energy_integration_math.svg + :alt: Fixed-Point Real-Time Energy Accumulation and Logarithmic Math flowchart showing sample filtering, 64-bit integration, integer log2, and MEL calculation. + + Fixed-Point Real-Time Energy Accumulation & Logarithmic Decibel Math + +Stage 1: Per-Sample Real-Time Processing Loop +--------------------------------------------- + +For each incoming audio frame, the module executes the following operations per channel: + +1. **Protective Gain Multiplier**: Multiplies the input sample by the internal dynamic gain :math:`g \in Q2.30`: + + .. code-block:: c + + sample = sat_int32(Q_MULTSR_32X32((int64_t)cd->gain, *x, + SOUND_DOSE_GAIN_Q, SOUND_DOSE_S32_Q, SOUND_DOSE_S32_Q)); + *y = sample; + +2. **A-Weighting Filtering**: Passes the scaled sample through the cascaded Direct Form I IIR filter: + + .. code-block:: c + + weighted = iir_df1(iir, sample) >> 16; + +3. **Power Squaring & Accumulation**: Computes instantaneous power and accumulates it into a per-channel 64-bit signed integer buffer: + + .. code-block:: c + + cd->energy[ch] += (int64_t)weighted * weighted; + +**Mathematical Headroom Analysis**: +Under full-scale 0 dBFS square-wave input, each sample squared yields :math:`2^{30}` in :math:`Q2.30` representation. Over 1 second at 48 kHz (48,000 frames), the maximum accumulated energy is: + +.. math:: + + E_{\text{max}} = 48000 \times 2^{30} \approx 5.15396 \times 10^{13} + +Because a 64-bit signed integer supports values up to :math:`2^{63}-1 \approx 9.22337 \times 10^{18}`, the accumulator maintains a margin of more than :math:`178,000\times` above full-scale saturation, completely preventing accumulator overflow. + +Stage 2: Periodic 1-Second Logarithmic Decibel Conversion +--------------------------------------------------------- + +When the accumulated frame count reaches the 1-second boundary (``cd->frames_count >= cd->report_count``), ``sound_dose_calculate_mel()`` converts the accumulated energy into Momentary Exposure Level (MEL): + +1. **Multichannel Summation**: Sums energy across all active channels: + + .. math:: + + E_{\text{sum}} = \sum_{ch=0}^{C-1} E_{ch} + +2. **Bit-Shift Normalization**: Scales :math:`E_{\text{sum}}` down by 19 bits (``SOUND_DOSE_ENERGY_SHIFT = 19``) so that the argument fits securely within a 32-bit unsigned integer: + + .. code-block:: c + + log_arg = (uint32_t)(energy_sum >> SOUND_DOSE_ENERGY_SHIFT); + log_arg = MAX(log_arg, 1); + +3. **Integer Base-2 Logarithm**: Computes base-2 logarithm using ``base2_logarithm(log_arg)``, returning a :math:`Q16.16` signed integer. + +4. **Fixed Offset & Mean Normalization**: + - Adds ``SOUND_DOSE_LOG_FIXED_OFFSET = 65536 * (19 - 30) = -720896`` to compensate for the :math:`Q2.30` scaling and the 19-bit right shift. + - Adds ``cd->log_offset_for_mean`` (:math:`\log_2(1/48000) \times 2^{16} = -1019134`), which divides total energy by frame count to compute mean acoustic power. + +5. **Decibel Conversion & Offsets**: Multiplies by :math:`\frac{10}{\log_2(10)} \cdot 2^{29}` (``SOUND_DOSE_TEN_OVER_LOG2_10_Q29 = 1616142483``) in :math:`Q29` fixed-point arithmetic: + + .. code-block:: c + + tmp = Q_MULTSR_32X32((int64_t)tmp, SOUND_DOSE_TEN_OVER_LOG2_10_Q29, + SOUND_DOSE_LOGOFFS_Q, SOUND_DOSE_LOGMULT_Q, SOUND_DOSE_LOGOFFS_Q); + cd->level_dbfs = tmp + SOUND_DOSE_WEIGHT_FILTERS_OFFS_Q16 + SOUND_DOSE_DFBS_OFFS_Q16; + + - ``SOUND_DOSE_WEIGHT_FILTERS_OFFS_Q16``: Adds back the :math:`+3.00\text{ dB}` normalization attenuation. + - ``SOUND_DOSE_DFBS_OFFS_Q16``: Adds :math:`+3.01\text{ dB}` (:math:`197263`) to calibrate against a full-scale sinusoidal peak-to-RMS reference. + +6. **Binaural Stereo Spatial Correction**: + For multichannel and stereo streams, sums the acoustic power delivered to both ears and subtracts :math:`-1.5\text{ dB}` per channel: + + .. code-block:: c + + if (cd->channels > 1) + cd->level_dbfs += cd->channels * SOUND_DOSE_MEL_CHANNELS_SUM_FIX; + + For a stereo headphone (2 channels), this subtracts :math:`-3.0\text{ dB}` (:math:`-1.5 \times 2`), aligning digital stereo power with binaural hearing threshold definitions. + +7. **Momentary Exposure Level (MEL) Derivation**: + Translates digital :math:`\text{dBFS}` into physical acoustic sound pressure level :math:`\text{dBA SPL}` (expressed in centi-decibels, :math:`0.01\text{ dB}`): + + .. math:: + + \text{MEL} = \text{dbfs\_value} + \text{sens\_dbfs\_dbspl} + \text{volume\_offset} + +8. **96-Bit Fixed-Point Microsecond Timestamping**: + Calculates exact stream presentation timestamp from continuous frame count without floating-point math: + + .. code-block:: c + + tmp_l = (cd->total_frames_count & 0xffffffff) * cd->rate_to_us_coef; + tmp_h = (cd->total_frames_count >> 32) * cd->rate_to_us_coef; + cd->feature->stream_time_us = (tmp_l >> 26) + ((tmp_h & ((1LL << 32) - 1)) << 6); + + where ``SOUND_DOSE_1M_OVER_48K_Q26 = 1398101333`` (:math:`\text{round}\left(\frac{1000000}{48000} \cdot 2^{26}\right)`). + +Acoustic Metrology & Headphone Sensitivity Calibration +====================================================== + +Measurement Laboratory Setup +---------------------------- + +Accurate Sound Dose monitoring requires precise acoustic calibration of the physical headphone output path. Transducer sensitivity must be measured using standardized acoustic laboratory metrology equipment. + +.. figure:: images/sound_dose_tuning_hats_measurement_setup.svg + :alt: Acoustic Laboratory Metrology and Headphone Sensitivity Calibration showing HATS test fixture, APx555 analyzer, and parameter derivation. + + Acoustic Laboratory Metrology & Headphone Sensitivity Calibration Test Rig + +The standardized measurement setup consists of: + +1. **Head and Torso Simulator (HATS)**: Brüel & Kjær Type 4128C / Type 5128 or GRAS KEMAR 45BB fitted with anatomically accurate anthropomorphic pinnae. +2. **Occluded Ear Simulator**: Conforming to **IEC 60318-4** (formerly IEC 60711) and **ITU-T P.57 Type 3.3 / Type 4.3**, replicating the acoustic transfer impedance of the human ear canal up to 10 kHz. +3. **Calibrated Pressure Microphone**: 1/2" laboratory reference microphone mounted at the eardrum reference point (DRP). +4. **Microphone Preamplifier**: Calibrated with an acoustic pistonphone (e.g. 94.0 dBA SPL at 1000 Hz). +5. **Precision Audio Analyzer**: Audio Precision APx555 or equivalent high-dynamic-range test instrument. + +Physical Calibration Runbook +---------------------------- + +Follow this step-by-step procedure to determine the acoustic sensitivity parameter ``sens_dbfs_dbspl`` for a specific device and headphone combination: + +1. **Acoustic Calibration Verification**: + Mount the acoustic sound calibrator (94.0 dBSPL at 1 kHz) onto the HATS ear simulator microphone. Verify the analyzer reads :math:`94.0 \pm 0.1\text{ dBA}`. + +2. **Transducer Mounting & Seal Inspection**: + Place the headphone or in-ear monitor onto the artificial ear. For over-ear headphones, apply the standardized 5 Newton headband clamping force. Verify acoustic seal integrity by injecting a 100 Hz test tone; improper sealing results in bass leakage and false low sensitivity readings. + +3. **Digital Stimulus Injection**: + Play standard **IEC 60268-1 Pink Noise** (crest factor 6 dB to 12 dB) at **0 dBFS digital peak** through the SOF audio pipeline. Ensure the ALSA user volume slider is set to maximum (0 dBFS digital gain). + +4. **Acoustic Sound Pressure Measurement**: + Record the unattenuated acoustic sound pressure level :math:`L_{p,\text{HATS}}` on the analyzer in dBA SPL (using 10-second :math:`L_{\text{eq}}` time averaging). + +5. **Sensitivity Parameter Calculation**: + Compute the sensitivity parameter in centi-decibels (:math:`100\text{ units} = 1\text{ dB}`): + + .. math:: + + \text{sens\_dbfs\_dbspl} = \text{round}\left( L_{p,\text{HATS}} \times 100 \right) + + *Example*: If a 0 dBFS pink noise stream generates :math:`100.42\text{ dBA SPL}` at the artificial eardrum, the parameter value is: + + .. math:: + + \text{sens\_dbfs\_dbspl} = 10042 + +6. **Linearity Verification**: + Reduce playback volume in 6 dB steps (-6 dBFS, -12 dBFS, -18 dBFS, -24 dBFS). Verify that the measured sound pressure level drops by exactly 6 dB at each step. + +Wideband Headphone Characteristic Voltage (WBCV) +------------------------------------------------ + +For standalone playback devices complying with **EN 50332-2**, measure the maximum electrical output voltage :math:`V_{\text{max}}` delivered into a standard :math:`32\,\Omega` resistive dummy load. Under EN 50332-2 Clause 4: + +.. math:: + + V_{\text{max}} \le 150\,\text{mV RMS} + +For standalone headphones, the Wideband Characteristic Voltage (:math:`WBCV`) is the electrical input voltage required to generate 94 dBA SPL at the artificial ear: + +.. math:: + + WBCV = V_{\text{test}} \cdot 10^{\frac{94 - L_{p,\text{measured}}}{20}} \ge 75\,\text{mV} + +Control Plane & ABI Specification +================================= + +Parameter Identifiers & Controls +-------------------------------- + +The Sound Dose module exposes four parameter IDs over the SOF control plane. + +.. table:: Sound Dose Control Parameter IDs & Functional Roles + :widths: 10 30 20 40 + + +----------+---------------------------------+-------------+--------------------------------------------------------------+ + | Param ID | Identifier | Direction | Description | + +==========+=================================+=============+==============================================================+ + | **0** | SOF_SOUND_DOSE_SETUP_PARAM_ID | Host -> DSP | Sets static transducer sensitivity (sens_dbfs_dbspl). | + +----------+---------------------------------+-------------+--------------------------------------------------------------+ + | **1** | SOF_SOUND_DOSE_VOLUME_PARAM_ID | Host -> DSP | Dynamic user volume attenuation offset (volume_offset). | + +----------+---------------------------------+-------------+--------------------------------------------------------------+ + | **2** | SOF_SOUND_DOSE_GAIN_PARAM_ID | Host -> DSP | Internal protective gain attenuation (gain). | + +----------+---------------------------------+-------------+--------------------------------------------------------------+ + | **3** | SOF_SOUND_DOSE_PAYLOAD_PARAM_ID | DSP -> Host | 1-second exposure telemetry payload (struct sof_sound_dose). | + +----------+---------------------------------+-------------+--------------------------------------------------------------+ + +ABI Data Structures & Memory Layouts +------------------------------------ + +All decibel parameters in the Sound Dose ABI are formatted as **16-bit signed integers in centi-decibels** (:math:`\text{dB} \times 100`). + +.. table:: Sound Dose Firmware Data Structures & Binary Memory Layout + :widths: 25 15 20 40 + + +-------------------------------------+----------+-----------------+----------------------------------------------------------+ + | Structure / Field | Type | Units / Format | Description & Bounds | + +=====================================+==========+=================+==========================================================+ + | **struct sound_dose_setup_config** | | | **Parameter ID 0 (4 bytes total)** | + +-------------------------------------+----------+-----------------+----------------------------------------------------------+ + | sens_dbfs_dbspl | int16_t | centi-dB (x100) | Transducer sensitivity: -1000 to +13000 (-10 to +130 dB) | + +-------------------------------------+----------+-----------------+----------------------------------------------------------+ + | reserved | int16_t | padding | Reserved for 32-bit alignment | + +-------------------------------------+----------+-----------------+----------------------------------------------------------+ + | **struct sound_dose_volume_config** | | | **Parameter ID 1 (4 bytes total)** | + +-------------------------------------+----------+-----------------+----------------------------------------------------------+ + | volume_offset | int16_t | centi-dB (x100) | Volume attenuation: -10000 to +4000 (-100 to +40 dB) | + +-------------------------------------+----------+-----------------+----------------------------------------------------------+ + | reserved | int16_t | padding | Reserved for 32-bit alignment | + +-------------------------------------+----------+-----------------+----------------------------------------------------------+ + | **struct sound_dose_gain_config** | | | **Parameter ID 2 (4 bytes total)** | + +-------------------------------------+----------+-----------------+----------------------------------------------------------+ + | gain | int16_t | centi-dB (x100) | Protective gain: -10000 to 0 (-100 to 0 dB) | + +-------------------------------------+----------+-----------------+----------------------------------------------------------+ + | reserved | int16_t | padding | Reserved for 32-bit alignment | + +-------------------------------------+----------+-----------------+----------------------------------------------------------+ + | **struct sof_sound_dose** | | | **Parameter ID 3 Container (28 bytes total)** | + +-------------------------------------+----------+-----------------+----------------------------------------------------------+ + | mel_value | int16_t | centi-dB (x100) | Calculated Momentary Exposure Level (dBA SPL) | + +-------------------------------------+----------+-----------------+----------------------------------------------------------+ + | dbfs_value | int16_t | centi-dB (x100) | Digital weighted signal level (dBFS x100) | + +-------------------------------------+----------+-----------------+----------------------------------------------------------+ + | current_sens_dbfs_dbspl | int16_t | centi-dB (x100) | Active sensitivity parameter readback | + +-------------------------------------+----------+-----------------+----------------------------------------------------------+ + | current_volume_offset | int16_t | centi-dB (x100) | Active volume offset readback | + +-------------------------------------+----------+-----------------+----------------------------------------------------------+ + | current_gain | int16_t | centi-dB (x100) | Active protective gain readback | + +-------------------------------------+----------+-----------------+----------------------------------------------------------+ + | reserved16 | uint16_t | padding | Reserved for alignment | + +-------------------------------------+----------+-----------------+----------------------------------------------------------+ + | reserved32[4] | uint32_t | padding | Reserved for future multi-band metrics (16 bytes) | + +-------------------------------------+----------+-----------------+----------------------------------------------------------+ + +Host-DSP Closed-Loop Exposure Regulation +======================================== + +Closed-Loop Regulation Architecture +----------------------------------- + +The Sound Dose module provides a true closed-loop regulation system spanning the SOF DSP firmware and the host operating system. + +.. figure:: images/sound_dose_tuning_host_dsp_closed_loop.svg + :alt: Host-DSP Closed-Loop Sound Dose Architecture showing asynchronous notification dispatch, 7-day rolling CSD tracking, and smooth protective attenuation. + + Host-DSP Closed-Loop Sound Dose Architecture & Regulation + +Asynchronous Notification Pipeline +---------------------------------- + +Rather than requiring the host driver to poll telemetry continuously across PCI/I2C buses, the Sound Dose module uses an autonomous event-driven architecture: + +1. Every 1.000 second, ``sound_dose_report_mel()`` formats an IPC notification message. +2. The message is transmitted to the host driver via ``SOF_IPC4_GLB_NOTIFICATION`` with event ID ``SOF_IPC4_NOTIFY_MODULE_EVENTID_ALSA_MAGIC_VAL | SOF_IPC4_BYTES_CONTROL_PARAM_ID``. +3. The Linux kernel SOF driver (``sound/soc/sof/ipc4-topology.c``) intercepts the notification and dispatches an ALSA control change event via ``snd_ctl_notify()``. +4. User-space daemons (such as PipeWire, PulseAudio, or ChromeOS CRAS) receive the event instantly without consuming host CPU polling cycles. + +Smooth Slew Gain Ramping +------------------------ + +When the host exposure daemon determines that the user has exceeded safe exposure limits, it injects a gain attenuation command via ``SOF_SOUND_DOSE_GAIN_PARAM_ID`` (e.g. :math:`-12.0\text{ dB}`). + +To prevent audible clicks, pops, or transient zipper noise, SOF implements an exponential per-frame gain slew rate: + +.. code-block:: c + + if (cd->new_gain < cd->gain) { + cd->gain = Q_MULTSR_32X32((int64_t)cd->gain, SOUND_DOSE_GAIN_DOWN_Q30, 30, 30, 30); + cd->gain = MAX(cd->gain, cd->new_gain); + } else if (cd->new_gain > cd->gain) { + cd->gain = Q_MULTSR_32X32((int64_t)cd->gain, SOUND_DOSE_GAIN_UP_Q30, 30, 30, 30); + cd->gain = MIN(cd->gain, SOUND_DOSE_GAIN_ONE_Q30); + } + +* **Down-Slew Rate**: ``SOUND_DOSE_GAIN_DOWN_Q30 = 1067578625`` (:math:`10^{-0.05/20} \cdot 2^{30}`), reducing gain by **0.05 dB per audio frame**. +* **Up-Slew Rate**: ``SOUND_DOSE_GAIN_UP_Q30 = 1079940603`` (:math:`10^{+0.05/20} \cdot 2^{30}`), restoring gain by **0.05 dB per audio frame**. + +At 48 kHz, a 6 dB attenuation executes smoothly over 120 frames (2.5 ms), ensuring rapid hearing protection while remaining imperceptible to the listener. + +.. note:: + The byte control ``SOF_SOUND_DOSE_GAIN_PARAM_ID`` is an internal kernel control not exposed in standard user-facing mixer interfaces (such as ``alsamixer``). This prevents users from trivially bypassing regulatory protection by moving the volume slider. + +Python Calibration Toolchain +============================ + +The Sound Open Firmware repository provides calibration and injection tooling to generate binary control blobs and monitor live telemetry over SSH. + +The following Python script (``sof_sound_dose_tool.py``) handles binary blob generation, parameter conversions, and remote execution: + +.. code-block:: python + + #!/usr/bin/env python3 + """Sound Open Firmware Sound Dose Calibration & Telemetry Tool. + + SPDX-License-Identifier: BSD-3-Clause + Copyright(c) 2026 Intel Corporation. + """ + + import argparse + import struct + import subprocess + import sys + + # SOF IPC4 ABI Constants + SOF_IPC4_ABI_MAGIC = 0x34435049 # 'IPC4' + SOF_ABI_VERSION = 0x00040000 + + # Param IDs + PARAM_ID_SETUP = 0 + PARAM_ID_VOLUME = 1 + PARAM_ID_GAIN = 2 + PARAM_ID_PAYLOAD = 3 + + def pack_ipc4_blob(param_id: int, payload: bytes) -> bytes: + """Encapsulate payload with SOF IPC4 control header.""" + header = struct.pack(" bytes: + """Build Parameter ID 0: Setup sensitivity blob.""" + sens_centidb = int(round(sens_dbspl * 100.0)) + if not (-1000 <= sens_centidb <= 13000): + raise ValueError(f"Sensitivity {sens_dbspl} dB out of bounds [-10, +130] dB") + payload = struct.pack(" bytes: + """Build Parameter ID 1: Volume offset blob.""" + vol_centidb = int(round(vol_db * 100.0)) + if not (-10000 <= vol_centidb <= 4000): + raise ValueError(f"Volume offset {vol_db} dB out of bounds [-100, +40] dB") + payload = struct.pack(" bytes: + """Build Parameter ID 2: Gain attenuation blob.""" + gain_centidb = int(round(gain_db * 100.0)) + if not (-10000 <= gain_centidb <= 0): + raise ValueError(f"Gain {gain_db} dB out of bounds [-100, 0] dB") + payload = struct.pack(" + + attributes { + !constructor [ "index" "instance" ] + !mandatory [ + "num_input_pins" + "num_output_pins" + "num_input_audio_formats" + "num_output_audio_formats" + ] + !immutable [ "uuid" "type" ] + unique "instance" + } + + uuid "7c:9d:3f:a4:75:ea:d5:44:94:2d:96:79:91:a3:38:09" + type "effect" + no_pm "true" + num_input_pins 1 + num_output_pins 1 + } + +Control Bindings +---------------- + +The module binds four byte controls in ``sound_dose_controls_playback.conf``: + +.. code-block:: text + + Object.Control { + bytes."1" { + name '$ANALOG_PLAYBACK_PCM Sound Dose setup bytes' + max 44 + IncludeByKey.BENCH_SOUND_DOSE_PARAMS { + "default" "include/components/sound_dose/setup_sens_100db.conf" + } + } + bytes."2" { + name '$ANALOG_PLAYBACK_PCM Sound Dose volume bytes' + max 44 + + } + bytes."3" { + name '$ANALOG_PLAYBACK_PCM Sound Dose gain bytes' + max 44 + + } + bytes."4" { + name '$ANALOG_PLAYBACK_PCM Sound Dose data bytes' + max 256 + + } + } + +Interactive Live Injection Runbook & Troubleshooting +==================================================== + +Target DUT SSH Deployment Sequence +---------------------------------- + +Execute the following commands on the host to configure, calibrate, and verify the Sound Dose module on a target DUT (e.g. Spider, Aphid, or Dragon Fly): + +1. **Synthesize Acoustic Setup Blob**: + Generate a binary sensitivity blob for an over-ear headset measured at 96.0 dBA SPL: + + .. code-block:: bash + + python3 sof_sound_dose_tool.py gen-setup --sens 96.0 --out setup_sens_96db.bin + +2. **Locate Mixer Controls on Target DUT**: + Query the ALSA control numbers on the target platform: + + .. code-block:: bash + + timeout 15 ssh root@spider "amixer controls | grep -i 'Sound Dose'" + + *Example Output*: + + .. code-block:: text + + numid=42,iface=MIXER,name='Analog Playback Sound Dose setup bytes' + numid=43,iface=MIXER,name='Analog Playback Sound Dose volume bytes' + numid=44,iface=MIXER,name='Analog Playback Sound Dose gain bytes' + numid=45,iface=MIXER,name='Analog Playback Sound Dose data bytes' + +3. **Inject Calibration Blob via sof-ctl**: + Inject the sensitivity configuration blob into the active audio pipeline: + + .. code-block:: bash + + scp setup_sens_96db.bin root@spider:/tmp/ + timeout 15 ssh root@spider "sof-ctl -i 4 -n 42 -p 0 -b -s /tmp/setup_sens_96db.bin" + +4. **Verify Live Exposure Telemetry**: + Read back the 1-second telemetry payload: + + .. code-block:: bash + + timeout 15 ssh root@spider "sof-ctl -i 4 -n 45 -p 0 -b -g /tmp/sound_dose_data.bin" + scp root@spider:/tmp/sound_dose_data.bin /tmp/ + python3 sof_sound_dose_tool.py parse /tmp/sound_dose_data.bin + +5. **Test Protective Gain Attenuation**: + Command a -10 dB protective gain reduction and verify smooth attenuation: + + .. code-block:: bash + + python3 sof_sound_dose_tool.py gen-gain --gain -10.0 --out gain_m10db.bin + scp gain_m10db.bin root@spider:/tmp/ + timeout 15 ssh root@spider "sof-ctl -i 4 -n 44 -p 0 -b -s /tmp/gain_m10db.bin" + +Diagnostic Troubleshooting Matrix +--------------------------------- + +.. table:: Sound Dose Acoustic Calibration & Firmware Diagnostics Matrix + :widths: 20 25 25 30 + + +-------------------------------------+---------------------------------------+------------------------------------+-------------------------------------------------------------+ + | Symptom / Anomaly | Root Cause Analysis | Acoustic Manifestation | Corrective Engineering Action | + +=====================================+=======================================+====================================+=============================================================+ + | **MEL reports higher than HATS** | Over-estimated transducer sensitivity | Premature regulatory intervention; | Re-measure transducer sensitivity on HATS using standard | + | | parameter (sens_dbfs_dbspl). | false 100% CSD warnings. | 0 dBFS pink noise. Re-inject calibrated centi-dB parameter. | + +-------------------------------------+---------------------------------------+------------------------------------+-------------------------------------------------------------+ + | **Premature CSD accumulation** | Volume offset parameter out of sync | Weekly sound dose accumulates at | Ensure host volume daemon transmits volume_offset updates | + | | with hardware mixer attenuation. | full volume rate even when quiet. | to Parameter ID 1 whenever main volume slider is adjusted. | + +-------------------------------------+---------------------------------------+------------------------------------+-------------------------------------------------------------+ + | **Clicks or pops on attenuation** | Direct step gain change bypassing | Audible zipper noise or transient | Verify cd->gain updates via Q_MULTSR_32X32 with | + | | smooth per-frame exponential slew. | pop artifact during regulation. | SOUND_DOSE_GAIN_DOWN_Q30 (0.05 dB/frame). | + +-------------------------------------+---------------------------------------+------------------------------------+-------------------------------------------------------------+ + | **Missing 1s IPC notifications** | IPC4 notification event ID mismatch | Host daemon fails to update CSD | Verify primary->r.notif_type = SOF_IPC4_MODULE_NOTIFICATION | + | | or disabled global notifications. | rolling accumulator; stays at 0%. | and kernel driver handles SOF_IPC4_GLB_NOTIFICATION. | + +-------------------------------------+---------------------------------------+------------------------------------+-------------------------------------------------------------+ + | **Asymmetric L/R exposure reading** | Acoustic seal leakage on one HATS | L/R channels report divergent MEL; | Inspect artificial pinna seating; check headband clamping | + | | pinna or unbalanced headphone driver. | false high stereo dose sum. | force (5 N); verify driver DC resistance balance. | + +-------------------------------------+---------------------------------------+------------------------------------+-------------------------------------------------------------+ + +Related Documentation +===================== + +* :ref:`sound_dose`: High-level Sound Dose Evaluator Architecture guide. +* :ref:`runtime_tuning_sof_ctl`: Unified runtime tuning and control blobs architecture guide. +* :ref:`smart_amp_tuning`: Smart Amplifier (DSM) & Transducer Protection calibration guide. +* :ref:`drc_tuning`: Dynamic Range Compression & Multiband DRC tuning guide. From ea3a2ebba007ba4c43670fede477a83d0b564370 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 17:14:12 +0100 Subject: [PATCH 36/64] doc: developer_guides: add digital microphone (dmic) tuning & calibration guide Signed-off-by: Liam Girdwood --- data/modules.yaml | 14 +- developer_guides/index.rst | 2 + developer_guides/tuning/dmic_tuning.rst | 807 ++++++++++++++++++ .../dmic_tuning_array_phase_matching.svg | 166 ++++ .../images/dmic_tuning_clocking_modes.svg | 168 ++++ .../dmic_tuning_pdm_decimation_pipeline.svg | 230 +++++ ...mic_tuning_sensitivity_calibration_rig.svg | 152 ++++ .../images/dmic_tuning_toolchain_workflow.svg | 225 +++++ 8 files changed, 1759 insertions(+), 5 deletions(-) create mode 100644 developer_guides/tuning/dmic_tuning.rst create mode 100644 developer_guides/tuning/images/dmic_tuning_array_phase_matching.svg create mode 100644 developer_guides/tuning/images/dmic_tuning_clocking_modes.svg create mode 100644 developer_guides/tuning/images/dmic_tuning_pdm_decimation_pipeline.svg create mode 100644 developer_guides/tuning/images/dmic_tuning_sensitivity_calibration_rig.svg create mode 100644 developer_guides/tuning/images/dmic_tuning_toolchain_workflow.svg diff --git a/data/modules.yaml b/data/modules.yaml index 69762c58..de4baf87 100644 --- a/data/modules.yaml +++ b/data/modules.yaml @@ -51,16 +51,20 @@ modules: - "Decoupled clock domain bridging" - id: dmic - name: "Digital Microphone (DMIC) Decimation" + name: "Digital Microphone (DMIC) Decimation & Array Tuning" source: "SOF" category: "Foundational DSP" status: "Upstream" - description: "Hardware PDM receiver, multi-stage CIC decimation, and FIR compensator filter." + tuning_guide: "developer_guides/tuning/dmic_tuning" + description: "Hardware PDM ingress, 5th-order CIC comb decimation, multirate FIR droop compensation, DC-offset compensation, and multichannel array acoustic calibration." simd: ["Hardware Accelerator", "Scalar C"] key_features: - - "High-performance PDM clock divider matching dual FIFOs" - - "Cascaded Integrator-Comb (CIC) decimation with FIR compensation" - - "Octave tuning tool for passband ripple (<0.1 dB) and stopband (>95 dB)" + - "5th-order Cascaded Integrator-Comb (CIC) filter with up to 31x decimation" + - "Multirate droop-compensating FIR filters with passband ripple < 0.1 dB and stopband > 90 dB" + - "Dual-FIFO mode matching for concurrent 48 kHz communications and 16 kHz wake-on-voice" + - "Acoustic sensitivity calibration and inter-channel gain trimming for beamforming arrays" + - "Automated logarithmic unmute gain ramping eliminating stream start pops" + - "Standalone Python calibration CLI (sof_dmic_tool.py) and ACPI NHLT / Topology 2 integration" - id: demux name: "Audio Demux" diff --git a/developer_guides/index.rst b/developer_guides/index.rst index a9a7cf09..00a3ad58 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -133,6 +133,7 @@ Acoustic, Transducer & Array Tuning =================================== * :ref:`crossover_tuning` (Linkwitz-Riley LR4 2-way, 3-way, and 4-way crossover filter design, phase-alignment merge, and multi-driver speaker tuning) +* :ref:`dmic_tuning` (Digital Microphone acoustic calibration, 5th-order CIC and FIR decimation design, dual-FIFO mode matching, and array sensitivity/phase alignment) * :ref:`equalizers_tuning` (Parametric FIR & IIR equalizers, MLS acoustical measurement, and speaker tuning) * :ref:`time-domain-fixed-beamformer` (Time-Domain Fixed Beamformer array geometry and spatial filter design) * :ref:`sample_rate_conversion` (Polyphase FIR filter design and multi-stage resampling) @@ -146,6 +147,7 @@ Acoustic, Transducer & Array Tuning tuning/smart_amp_tuning tuning/sound_dose_tuning tuning/crossover_tuning + tuning/dmic_tuning algorithms/eq/equalizers_tuning algorithms/tdfb/time_domain_fixed_beamformer algorithms/src/sample_rate_conversion diff --git a/developer_guides/tuning/dmic_tuning.rst b/developer_guides/tuning/dmic_tuning.rst new file mode 100644 index 00000000..03cc440d --- /dev/null +++ b/developer_guides/tuning/dmic_tuning.rst @@ -0,0 +1,807 @@ +.. _dmic_tuning: + +Digital Microphone (DMIC) Acoustic Calibration & Array Tuning Guide +################################################################### + +Digital Microphone (DMIC) interfaces represent the primary audio capture ingress for modern Intel-based +computing platforms, including Tiger Lake (cAVS 2.5), Arrow Lake (ACE 1.5), and Panther Lake (ACE 3.0). +Unlike conventional analog microphone inputs that rely on external codecs, digital MEMS (Micro-Electro-Mechanical +Systems) microphones integrate the acoustic transducer, preamplifier, and a 4th-order or 5th-order +Sigma-Delta (:math:`\Sigma\Delta`) modulator directly into a sub-millimeter silicon package. The transducers +stream 1-bit oversampled Pulse Density Modulation (PDM) data directly into the DSP hardware. + +The Sound Open Firmware (SOF) DMIC processing subsystem incorporates high-performance hardware decimation +engines, programmable clock generation, dual-FIFO multirate dispatchers, and acoustic sensitivity/phase +calibration filters. This guide provides an authoritative mathematical, architectural, and operational +reference for configuring DMIC hardware decimators, calculating clock dividers and duty cycles, aligning +microphone array phase and gain, compiling ACPI NHLT and ALSA Topology 2 binaries, and validating live +transducer performance on target Hardware Under Test (DUT). + +.. contents:: Table of Contents + :local: + :depth: 3 + +Transducer Physics & PDM Digital Ingress +======================================== + +MEMS Digital Transducer Architecture +------------------------------------ + +Modern digital microphones convert ambient acoustic pressure fluctuations :math:`P(t)` (measured in Pascals, +where :math:`1\text{ Pa} = 94\text{ dBSPL}`) into a high-rate 1-bit PDM pulse stream. The acoustic sensor consists +of a flexible conductive diaphragm suspended over a rigid perforated backplate, forming a variable capacitor. +Sound waves passing through the acoustic port deflect the diaphragm, modulating the capacitance. + +An on-chip Application-Specific Integrated Circuit (ASIC) amplifies the capacitive charge and converts the +continuous analog voltage into a 1-bit digital bitstream using an oversampled :math:`\Sigma\Delta` modulator. +The key acoustic metrology parameters governing digital microphones are summarized below: + +* **Acoustic Sensitivity** (:math:`S`): The electrical signal level output by the microphone when subjected to a + standard reference sound pressure level of :math:`1.0\text{ Pa}` (:math:`94\text{ dBSPL}`) at :math:`1\text{ kHz}`. For digital + microphones, sensitivity is expressed in decibels relative to full scale (**dBFS**). A standard digital + microphone has a nominal sensitivity of :math:`-26\text{ dBFS}` (with typical production distributions ranging + from :math:`-38\text{ dBFS}` to :math:`-18\text{ dBFS}`). + +* **Acoustic Overload Point (AOP)**: The maximum sound pressure level at which the total harmonic distortion + (THD) reaches :math:`10\%` (or :math:`1\%` depending on manufacturer rating). Standard mobile microphones offer an + AOP of :math:`120\text{ dBSPL}` to :math:`135\text{ dBSPL}`. Inputs exceeding the AOP cause severe clipping and + :math:`\Sigma\Delta` modulator saturation. + +* **Signal-to-Noise Ratio (SNR)**: The ratio between the nominal sensitivity level (:math:`94\text{ dBSPL}` at :math:`1\text{ kHz}`) + and the A-weighted acoustic noise floor of the microphone (:math:`V_{\text{noise}}`), measured in :math:`\text{dBA}`: + + .. math:: + + \text{SNR} = 94\text{ dBSPL} - \text{EIN} + + Where :math:`\text{EIN}` is the **Equivalent Input Noise** level in :math:`\text{dBSPL(A)}`. High-fidelity laptop arrays + typically utilize microphones with an SNR between :math:`64\text{ dBA}` and :math:`72\text{ dBA}` (:math:`\text{EIN} \approx 22 - 30\text{ dBSPL}`). + +Double-Data-Rate (DDR) Stereo Multiplexing +------------------------------------------ + +To minimize physical pin count and routing complexity across narrow laptop display hinges, digital microphones +use a shared two-wire interface consisting of a single clock line (``PDM_CLK``) and a single data line +(``PDM_DATA``). A pair of microphones (Mic A and Mic B) multiplexes onto this single data line using +Double-Data-Rate (DDR) timing governed by an external hardware ``SELECT`` pin: + +.. table:: Stereo PDM Transducer Pin Configuration & Multiplexing + :widths: 20 20 30 30 + + +---------------+------------------+------------------------------+------------------------------+ + | Component | SELECT Pin State | Clock Driving Edge | Bus Release Phase | + +===============+==================+==============================+==============================+ + | **Mic A (L)** | Tied to Ground | Rising Edge (Polarity 0) | High-Z on Falling Edge | + +---------------+------------------+------------------------------+------------------------------+ + | **Mic B (R)** | Tied to VDD | Falling Edge (Polarity 1) | High-Z on Rising Edge | + +---------------+------------------+------------------------------+------------------------------+ + +When ``PDM_CLK`` rises, Mic A latches its instantaneous 1-bit comparator state onto ``PDM_DATA`` while Mic B +maintains high-impedance (tri-state). When ``PDM_CLK`` falls, Mic A releases the bus into high-impedance, +and Mic B drives its 1-bit state onto ``PDM_DATA``. The SOF hardware receiver samples both edges, de-interleaving +the stream into independent Left and Right channels. + +Hardware Decimation Pipeline Architecture +========================================= + +The DSP DMIC hardware controller ingests raw 1-bit PDM streams from up to 4 physical PDM controllers +(supporting up to 8 microphone channels) and decimates them to linear 24-bit or 32-bit PCM audio samples. +The complete signal chain is illustrated in :numref:`fig_dmic_pdm_decimation_pipeline`. + +.. _fig_dmic_pdm_decimation_pipeline: +.. figure:: images/dmic_tuning_pdm_decimation_pipeline.svg + :alt: DMIC Hardware Decimation Signal Processing Pipeline + :width: 100% + :align: center + + Digital Microphone (DMIC) Hardware Decimation Signal Processing Pipeline + +The signal processing chain consists of five sequential hardware stages: + +1. **Cascaded Integrator-Comb (CIC) 5th-Order Filter**: High-ratio coarse decimation stage downsampling the + overclocked 1-bit stream (:math:`f_{\text{pdm}}`) to an intermediate rate (:math:`f_{\text{cic}}`). +2. **Arithmetic Shifter & Headroom Normalizer**: Bit-alignment logic mapping the 26-bit CIC accumulator into + the 22-bit input word of the FIR stage while preventing fixed-point overflow. +3. **Finite Impulse Response (FIR) Multirate Filter**: Precision shaping filter inverting the :math:`\text{sinc}^5` + passband droop of the CIC filter and downsampling to the target audio sample rate (:math:`f_s`). +4. **DC-Offset Compensation (DCCOMP)**: First-order high-pass Infinite Impulse Response (IIR) filter + eliminating transducer DC bias and thermal drift. +5. **Channel Gain Multipliers**: 20-bit scaling registers balancing acoustic sensitivities across all + elements of the microphone array. + +Stage 1: 5th-Order Cascaded Integrator-Comb (CIC) Filter +-------------------------------------------------------- + +The primary decimation stage is implemented as a 5th-order Cascaded Integrator-Comb (CIC) filter. Because +CIC filters require no multiplier units (utilizing only adders, subtractors, and delay registers), they +operate directly at the multi-megahertz PDM clock rate with minimal power dissipation. + +The discrete-time transfer function of an :math:`N`-th order CIC filter with decimation factor :math:`M_{\text{cic}}` is: + +.. math:: + + H_{\text{CIC}}(z) = \left( \frac{1 - z^{-M_{\text{cic}}}}{1 - z^{-1}} \right)^N = \left( \sum_{k=0}^{M_{\text{cic}}-1} z^{-k} \right)^N + +In Intel cAVS and ACE DSP architectures, the filter order is fixed at :math:`N = 5` (5 cascaded integrator stages +followed by 5 cascaded comb stages). The decimation factor :math:`M_{\text{cic}}` is programmable between +:math:`5 \le M_{\text{cic}} \le 31`. + +The continuous-frequency magnitude response of the 5th-order CIC filter normalized to :math:`f_{\text{pdm}}` is: + +.. math:: + + |H_{\text{CIC}}(f)| = \left| \frac{\sin(\pi M_{\text{cic}} f / f_{\text{pdm}})}{\sin(\pi f / f_{\text{pdm}})} \right|^5 + +At zero frequency (:math:`f = 0`), the DC power gain of the filter is: + +.. math:: + + G_{\text{CIC}} = M_{\text{cic}}^N = M_{\text{cic}}^5 + +Because :math:`M_{\text{cic}} \le 31`, the maximum theoretical DC gain is :math:`31^5 = 28,629,151` (:math:`\approx 149.1\text{ dB}`). +The maximum bit growth through the five integration stages is: + +.. math:: + + B_{\text{growth}} = \lceil 5 \log_2(M_{\text{cic}}) \rceil + +For :math:`M_{\text{cic}} = 31`, :math:`B_{\text{growth}} = \lceil 5 \times 4.954 \rceil = 25\text{ bits}`. Including the +1-bit input sign, the internal accumulator requires 26 bits of precision, which matches the hardware width +constant ``DMIC_HW_BITS_CIC = 26``. + +Stage 2: Shifter Arithmetic & Headroom Normalization +---------------------------------------------------- + +The FIR decimation engine expects signed 22-bit inputs (``DMIC_HW_BITS_FIR_INPUT = 22``). To bridge the +26-bit CIC accumulator to the 22-bit FIR input without clipping, an arithmetic right shifter scales the +CIC output. The required word length :math:`B_{\text{needed}}` and the right shift offset are calculated by: + +.. math:: + + B_{\text{needed}} = \lfloor \log_2(M_{\text{cic}}^5) + 1 \rfloor + 1 + +.. math:: + + \text{cic\_shift} = B_{\text{needed}} - \text{DMIC\_HW\_BITS\_FIR\_INPUT} = B_{\text{needed}} - 22 + +The shift value is programmed into the ``CIC_CONFIG`` register (bits 27:24) within the legal hardware range +:math:`-8 \le \text{cic\_shift} \le 4`. Because integer shifting attenuates signals by powers of 2 (:math:`2^{\text{cic\_shift}}`), +the residual fractional gain headroom is transferred to the FIR coefficient scaling multiplier: + +.. math:: + + G_{\text{to\_fir}} = \frac{2^{\text{DMIC\_HW\_BITS\_FIR\_INPUT} - 1}}{M_{\text{cic}}^5 \cdot 2^{-\text{cic\_shift}}} = \frac{2^{21}}{M_{\text{cic}}^5 \cdot 2^{-\text{cic\_shift}}} + +Stage 3: FIR Decimator & Sinc Droop Inversion +--------------------------------------------- + +While the CIC filter suppresses high-frequency quantization noise, it introduces a pronounced :math:`\text{sinc}^5` +gain droop across the audio passband: + +.. math:: + + A_{\text{droop}}(f) \approx \left( \frac{\sin(\pi f / f_{\text{cic}})}{\pi f / f_{\text{cic}}} \right)^5 + +At the edge of the passband (:math:`f = 20\text{ kHz}` at :math:`f_s = 48\text{ kHz}`), this droop attenuates high audio +frequencies by up to :math:`-3.5\text{ dB}` to :math:`-6.0\text{ dB}`, degrading vocal clarity and acoustic accuracy. + +The second decimation stage employs a precision multirate FIR filter that performs two critical tasks: + +1. Downsamples the audio stream by decimation factor :math:`M_{\text{fir}}` (:math:`2 \le M_{\text{fir}} \le 15`). +2. Equalizes the passband by implementing an exact inverse :math:`\text{sinc}^5` frequency characteristic: + +.. math:: + + |H_{\text{FIR}}(f)| \approx \frac{1}{|H_{\text{CIC}}(f)|} \quad \text{for } 0 \le f \le f_{\text{pass}} + +The FIR filters are configured with the following characteristics: + +* **Passband Ripple**: :math:`\le \pm 0.1\text{ dB}` across :math:`0\text{ Hz}` to :math:`0.4375 \times f_s` (e.g. :math:`0 - 21\text{ kHz}` at :math:`48\text{ kHz}`). +* **Stopband Attenuation**: :math:`\ge 90\text{ dB}` to :math:`95\text{ dB}` beginning at :math:`0.5100 \times f_s`, preventing alias reflection. +* **Coefficient Storage**: Up to 250 filter taps (``DMIC_HW_FIR_LENGTH_MAX = 250``) stored in dedicated SRAM as + 20-bit signed integers (``DMIC_HW_BITS_FIR_COEF = 20``). Symmetric linear-phase filters exploit symmetry + to store only :math:`\lceil N_{\text{taps}} / 2 \rceil` unique coefficients. + +Pipeline Hardware Cycle Constraints +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The FIR engine shares computational MAC units across active channels. For every audio output frame, the +maximum number of FIR taps :math:`N_{\text{taps}}` is bounded by the ratio between the DSP IO clock frequency +(:math:`f_{\text{io}}`) and the output sample rate (:math:`f_s`): + +.. math:: + + N_{\text{taps}} \le \min\left( 250, \, \left\lfloor \frac{f_{\text{io}}}{2 \cdot f_s} \right\rfloor - 5 \right) + +The subtraction of 5 cycles represents the internal pipeline reload overhead (``DMIC_FIR_PIPELINE_OVERHEAD = 5``). +For standard clock configurations: + +* At :math:`f_{\text{io}} = 19.2\text{ MHz}` and :math:`f_s = 48\text{ kHz}`: :math:`N_{\text{taps}} \le \min(250, \, 200 - 5) = 195\text{ taps}`. +* At :math:`f_{\text{io}} = 38.4\text{ MHz}` and :math:`f_s = 48\text{ kHz}`: :math:`N_{\text{taps}} \le \min(250, \, 400 - 5) = 250\text{ taps}` (clamped at hardware maximum). + +Stage 4: DC-Offset Compensation (DCCOMP) +---------------------------------------- + +Digital MEMS microphones frequently exhibit intrinsic DC offsets resulting from diaphragm mechanical bias, +:math:`\Sigma\Delta` integrator leakage, and thermal drift. Uncompensated DC bias restricts downstream dynamic +headroom, induces audible clicks and pops during stream starts, and corrupts time-domain acoustic feature +extractors. + +The hardware incorporates an independent first-order Infinite Impulse Response (IIR) DC-blocking high-pass +filter on each channel: + +.. math:: + + y[n] = x[n] - x[n-1] + \alpha \cdot y[n-1] + +The feedback pole :math:`\alpha = 1 - 2^{-k}` determines the high-pass cutoff frequency :math:`f_c \approx 2^{-k} \cdot f_s / (2\pi)`. +The hardware provides 8 selectable time constants (``DCCOMP_TC0`` to ``DCCOMP_TC7``): + +.. list-table:: DCCOMP Hardware Time Constants & Cutoff Frequencies + :widths: 15 20 25 40 + :header-rows: 1 + + * - Time Constant + - Bit Shift (:math:`k`) + - Cutoff Frequency @ 48 kHz + - Settling Time (:math:`\tau`) + * - **TC0** + - 5 + - :math:`238.7\text{ Hz}` + - :math:`0.67\text{ ms}` (Fastest settling) + * - **TC1** + - 6 + - :math:`119.4\text{ Hz}` + - :math:`1.33\text{ ms}` + * - **TC2** + - 7 + - :math:`59.7\text{ Hz}` + - :math:`2.67\text{ ms}` + * - **TC3** + - 8 + - :math:`29.8\text{ Hz}` + - :math:`5.33\text{ ms}` + * - **TC4** + - 9 + - :math:`14.9\text{ Hz}` + - :math:`10.67\text{ ms}` + * - **TC5** + - 10 + - :math:`7.5\text{ Hz}` + - :math:`21.33\text{ ms}` (Production default) + * - **TC6** + - 11 + - :math:`3.7\text{ Hz}` + - :math:`42.67\text{ ms}` + * - **TC7** + - 12 + - :math:`1.9\text{ Hz}` + - :math:`85.33\text{ ms}` (Infrasonic audio) + +Stage 5: Channel Output Gain Trimming +------------------------------------- + +Acoustic enclosures, cosmetic mesh grilles, and manufacturing tolerances introduce sensitivity variations +between microphone capsules. To present a balanced multichannel stream to downstream beamformers, the hardware +provides a dedicated 20-bit linear gain multiplier register for each channel: ``OUT_GAIN_LEFT_A``, +``OUT_GAIN_RIGHT_A``, ``OUT_GAIN_LEFT_B``, and ``OUT_GAIN_RIGHT_B``. + +The registers are encoded in unsigned :math:`Q1.19` fixed-point format (where :math:`1.0\text{ (unity gain)} = 2^{19} = 524,288 = \text{0x080000}`). +For a target acoustic trim of :math:`\Delta G\text{ dB}`, the register value is: + +.. math:: + + \text{OUT\_GAIN} = \text{round}\left( 10^{\frac{\Delta G}{20}} \times 2^{19} \right) \quad \text{clamped to } [0, \, \text{0x0FFFFF}] + +Clock Generation & Dual-FIFO Architecture +========================================= + +Primary Clock Division & Duty-Cycle Constraints +------------------------------------------------ + +The DSP clock generation block divides the platform main IO clock (:math:`f_{\text{io}} = 19.2\text{ MHz}` on +older cAVS platforms or :math:`38.4\text{ MHz}` on ACE platforms) to generate the physical ``PDM_CLK``: + +.. math:: + + f_{\text{pdm}} = \frac{f_{\text{io}}}{\text{clkdiv}} + +The 8-bit divider parameter is programmed into the ``MIC_CONTROL`` register as :math:`\text{PDM\_CLKDIV} = \text{clkdiv} - 2`. +The relationship between main IO clock, divider, and resulting PDM frequencies is illustrated in :numref:`fig_dmic_clocking_modes`. + +.. _fig_dmic_clocking_modes: +.. figure:: images/dmic_tuning_clocking_modes.svg + :alt: DMIC Clock Generation and Dual-FIFO Mode Matching Architecture + :width: 100% + :align: center + + DMIC Clock Generation and Dual-FIFO Mode Matching Architecture + +Odd dividers generate asymmetric clock high and low periods, altering the clock duty cycle: + +.. math:: + + C_1 = \left\lfloor \frac{\text{clkdiv}}{2} \right\rfloor, \quad D_{\text{min}} = 100 \times \frac{C_1}{\text{clkdiv}}, \quad D_{\text{max}} = 100 - D_{\text{min}} + +MEMS microphone datasheets enforce strict duty cycle operational limits, typically :math:`40\% \le D \le 60\%`. +If an odd divider produces a duty cycle outside this range (e.g. :math:`\text{clkdiv} = 3 \implies D_{\text{min}} = 33.3\%`), +the :math:`\Sigma\Delta` modulator comparator timing fails, leading to noise floor rise or phase distortion. +Furthermore, in cAVS 1.5 to 2.5 hardware, :math:`\text{clkdiv} \le 4` is strictly prohibited by hardware timing paths. + +Dual-FIFO Multirate Mode Matching +--------------------------------- + +A platform often requires two concurrent capture streams operating at different sample rates: + +1. **FIFO A (Communications / Recording)**: High-fidelity capture at :math:`f_{s,\text{A}} = 48\text{ kHz}`. +2. **FIFO B (Voice Wake / Keyword Detection)**: Ultra-low-power processing at :math:`f_{s,\text{B}} = 16\text{ kHz}`. + +Because both FIFOs receive data from the same physical microphones, they **must share** the exact same PDM clock +frequency (:math:`f_{\text{pdm}}`) and the exact same CIC decimation factor (:math:`M_{\text{cic}}`). The multirate +adaptation is achieved entirely within the FIR decimation stage by selecting different decimation factors +:math:`M_{\text{fir,A}}` and :math:`M_{\text{fir,B}}`: + +.. math:: + + f_{\text{cic}} = \frac{f_{\text{pdm}}}{M_{\text{cic}}} = f_{s,\text{A}} \times M_{\text{fir,A}} = f_{s,\text{B}} \times M_{\text{fir,B}} + +Taking the standard ratio between :math:`48\text{ kHz}` and :math:`16\text{ kHz}` (:math:`3:1`): + +.. math:: + + M_{\text{fir,B}} = 3 \times M_{\text{fir,A}} + +For example, on a platform with :math:`f_{\text{io}} = 38.4\text{ MHz}`: + +* Select :math:`\text{clkdiv} = 16 \implies f_{\text{pdm}} = 38.4\text{ MHz} / 16 = 2.40\text{ MHz}` (Duty cycle = :math:`50.0\%`). +* Select :math:`M_{\text{cic}} = 25 \implies f_{\text{cic}} = 2.40\text{ MHz} / 25 = 96\text{ kHz}`. +* For FIFO A (:math:`48\text{ kHz}`): Select :math:`M_{\text{fir,A}} = 2 \implies 96\text{ kHz} / 2 = 48\text{ kHz}`. Total :math:`\text{OSR} = 25 \times 2 = 50`. +* For FIFO B (:math:`16\text{ kHz}`): Select :math:`M_{\text{fir,B}} = 6 \implies 96\text{ kHz} / 6 = 16\text{ kHz}`. Total :math:`\text{OSR} = 25 \times 6 = 150`. + +Both streams run concurrently from a single physical PDM wire pair without clock conflict. + +Microphone Array Acoustic Calibration & Phase Matching +====================================================== + +Array Geometry & Spatial Directivity +------------------------------------ + +Modern laptops and smart devices combine multiple digital microphones into spatial arrays to run Time-Domain +Filter-and-Sum Beamformers (TDFB) or Real-Time Noise Reduction (RTNR) algorithms. The spatial geometry +and propagation delays are illustrated in :numref:`fig_dmic_array_phase_matching`. + +.. _fig_dmic_array_phase_matching: +.. figure:: images/dmic_tuning_array_phase_matching.svg + :alt: Microphone Array Acoustic Geometry and Inter-Channel Phase Matching + :width: 100% + :align: center + + Microphone Array Acoustic Geometry and Inter-Channel Phase Matching + +For two microphones spaced by distance :math:`d`, an acoustic plane wave arriving at incident angle :math:`\theta` (where +:math:`\theta = 0^\circ` corresponds to broadside on-axis) experiences a physical propagation delay of: + +.. math:: + + \tau = \frac{d \cdot \sin(\theta)}{c} + +Where :math:`c = 343\text{ m/s}` is the speed of sound in air at :math:`20^\circ\text{C}`. The corresponding frequency-dependent +phase shift is: + +.. math:: + + \Delta\phi(f) = 2\pi f \cdot \tau = \frac{2\pi f d \sin(\theta)}{c} + +Impact of Acoustic & Transducer Mismatch +---------------------------------------- + +Spatial beamformers create directional beams and steerable nulls by forming linear combinations of delayed +microphone signals: + +.. math:: + + Y(f) = W_1(f) X_1(f) + W_2(f) X_2(f) + +To place a deep null in the direction of ambient noise (:math:`\theta_{\text{null}}`), the beamformer weights are +designed so that :math:`W_1(f) X_1(f) + W_2(f) X_2(f) = 0`. + +However, real-world hardware introduces two major sources of mismatch: + +1. **Magnitude Imbalance** (:math:`\Delta G`): Component manufacturing tolerances cause :math:`\pm 1.0\text{ dB}` to + :math:`\pm 1.5\text{ dB}` sensitivity variations. Cosmetic acoustic mesh resistance and port hole dust seals + introduce further attenuation differences. +2. **Phase Skew** (:math:`\Delta\phi`): Acoustic port cavities act as acoustic low-pass Helmholtz resonators. + Minor dimensional deviations in adhesive gasket thickness or acoustic port diameter shift the resonant + frequency, introducing up to :math:`10^\circ` to :math:`15^\circ` of inter-channel phase error at :math:`4\text{ kHz}` to :math:`8\text{ kHz}`. + +As shown in :numref:`fig_dmic_array_phase_matching`, a gain mismatch of just :math:`1.0\text{ dB}` degrades spatial null +depth from :math:`> 28\text{ dB}` down to :math:`< 11\text{ dB}`, allowing ambient office noise, keyboard clicks, and echo +to leak directly into the voice stream. + +Acoustic Calibration Laboratory Setup +------------------------------------- + +To eliminate channel imbalance, microphone arrays must undergo acoustic calibration in a controlled metrology +environment, illustrated in :numref:`fig_dmic_sensitivity_calibration_rig`. + +.. _fig_dmic_sensitivity_calibration_rig: +.. figure:: images/dmic_tuning_sensitivity_calibration_rig.svg + :alt: Digital Microphone Acoustic Metrology and Sensitivity Calibration Rig + :width: 100% + :align: center + + Digital Microphone Acoustic Metrology and Sensitivity Calibration Rig + +The calibration test fixture requires: + +1. **Anechoic Test Box / Enclosure**: Sound-isolated acoustic chamber providing :math:`\ge 40\text{ dB}` ambient noise + attenuation and lined with acoustic wedges to eliminate boundary reflections above :math:`200\text{ Hz}`. +2. **Calibrated Reference Sound Source**: Coaxial loudspeaker located at a fixed distance (:math:`d = 0.5\text{ m}` + or :math:`1.0\text{ m}`) on the broadside axis (:math:`\theta = 0^\circ`). +3. **Class 1 Reference Microphone**: Precision measurement microphone (e.g. Brüel & Kjær Type 4190 or GRAS 40AZ) + calibrated using an acoustic calibrator to :math:`94.0\text{ dBSPL} \pm 0.1\text{ dB}` at :math:`1\text{ kHz}`. +4. **Audio Precision APx555 Analyzer**: Precision audio generator driving the sound source and recording the + reference microphone return signal. +5. **Target Device Under Test (DUT)**: Connected via Ethernet or USB to record the uncalibrated multichannel + DMIC PCM stream from SOF. + +Sensitivity Calibration Procedure +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +1. **SPL Normalization**: The Audio Precision analyzer plays a :math:`1\text{ kHz}` sine wave and adjusts generator + output until the reference microphone measures exactly :math:`94.0\text{ dBSPL}` at the DUT position. +2. **Raw Sensitivity Acquisition**: Record a 5-second PCM capture from the DUT at :math:`48\text{ kHz}` across all + channels. Calculate the RMS digital level :math:`V_{\text{rms}, i}` for each channel :math:`i`: + + .. math:: + + S_{\text{meas}, i} = 20 \log_{10}\left( \frac{V_{\text{rms}, i}}{V_{\text{FS}}} \right) \quad [\text{dBFS}] + +3. **Gain Trim Calculation**: Given a target nominal sensitivity :math:`S_{\text{target}}` (typically :math:`-26.0\text{ dBFS}`): + + .. math:: + + \Delta G_i = S_{\text{target}} - S_{\text{meas}, i} \quad [\text{dB}] + + .. math:: + + \text{Trim}_{\text{linear}, i} = 10^{\frac{\Delta G_i}{20}} + +4. **Register Programming**: + + * Hardware :math:`Q1.19` Register: :math:`\text{OUT\_GAIN}_i = \text{round}(\text{Trim}_{\text{linear}, i} \times 524,288)`. + * IPC4 Copier :math:`Q10` Parameter: :math:`\text{gain\_coeffs}[i] = \text{round}(\text{Trim}_{\text{linear}, i} \times 1024)`. + +Post-calibration measurement must verify that channel sensitivity spread is within :math:`\le 0.05\text{ dB}` +across all elements. + +Control Plane ABI & Topology 2 Integration +========================================== + +ACPI NHLT (Non-HD-Audio Link Table) Configuration +------------------------------------------------- + +On Intel platforms, BIOS passes initial hardware decimation settings, clock dividers, and microphone array +geometry to the OS kernel via the ACPI **NHLT** (Non-HD-Audio Link Table). The table contains an endpoint +descriptor for the DMIC gateway, embedding the ``struct dmic_config_blob`` defined in ``src/include/ipc4/dmic.h``: + +.. code-block:: c + + /* Excerpt from src/include/ipc4/dmic.h */ + struct dmic_config_blob { + uint32_t ts_group[4]; /* Time-slot channel mappings */ + union dmic_global_cfg global_cfg;/* Clock-on delay & unmute fade settings */ + uint32_t channel_ctrl_mask : 8; /* Active PDM channels */ + uint32_t clock_source : 8; /* DSP IO clock selection (19.2M / 38.4M) */ + uint32_t rsvd : 16; + struct dmic_channel_cfg channel_cfg[0]; + uint32_t pdm_ctrl_mask; /* Bitmask of active PDM controllers (1..4) */ + struct dmic_pdm_ctrl_cfg pdm_ctrl_cfg[0]; + } __packed __aligned(4); + +The nested ``struct dmic_pdm_ctrl_cfg`` holds the exact register images for each PDM controller: + +* ``cic_control``: Soft reset, MIC A/B polarity, stereo mode. +* ``cic_config``: ``COMB_COUNT`` (:math:`M_{\text{cic}} - 1`) and ``CIC_SHIFT``. +* ``mic_control``: ``PDM_CLKDIV`` (:math:`\text{clkdiv} - 2`) and clock edge selection. +* ``fir_config[2]``: Length, decimation factor, DC offset, and channel gains for FIR A and B. +* ``fir_coeffs[0]``: Array of 20-bit FIR coefficients (or packed 24-bit representations). + +ALSA Topology 2 Declarations +---------------------------- + +In Sound Open Firmware Topology 2, digital microphone DAIs are instantiated using ``Class.Dai."DMIC"`` +defined in ``tools/topology/topology2/include/dais/dmic.conf``. A production topology declaration specifies: + +.. code-block:: text + + # Production Topology 2 DMIC DAI instantiation + Object.Dai.DMIC."0" { + name "dmic01" + dai_index 0 + direction "capture" + driver_version 1 + io_clk 38400000 + sample_rate 48000 + clk_min 1000000 + clk_max 4800000 + duty_min 40 + duty_max 60 + num_pdm_active 2 + fifo_word_length 32 + unmute_ramp_time_ms 200 + + Object.Base.hw_config."0" { + id 0 + } + Object.Base.pdm_config."0" { + ctrl_id 0 + mic_a_enable 1 + mic_b_enable 1 + } + Object.Base.pdm_config."1" { + ctrl_id 1 + mic_a_enable 1 + mic_b_enable 1 + } + } + +Hardware Unmute Logarithmic Gain Ramp +------------------------------------- + +When digital microphones power on and clocking begins, capacitive charge stabilization in the MEMS capsule +generates a low-frequency transient voltage thump. To eliminate audible pops, the SOF DMIC driver applies +an automated logarithmic unmute gain ramp: + +.. math:: + + T_{\text{ramp}} = \text{clamp}\left( \text{round}\left( \text{coef} \times f_s + \text{offset} \right), \, 10\text{ ms}, \, 1000\text{ ms} \right) + +In ``src/include/sof/drivers/dmic.h``: + +* ``LOGRAMP_START_DB`` = :math:`-90\text{ dB}` (starting gain). +* Linear ramp equation: :math:`T_{\text{ramp}} = 200\text{ ms}` at :math:`48\text{ kHz}` and :math:`400\text{ ms}` at :math:`16\text{ kHz}`. +* Hardware unmute triggers: Unmute CIC at :math:`1\text{ ms}` (``DMIC_UNMUTE_CIC = 1``) and unmute FIR at :math:`2\text{ ms}` (``DMIC_UNMUTE_FIR = 2``). + +IPC4 Copier Runtime Gain Control +-------------------------------- + +Under IPC4, runtime acoustic trims can be injected without rebuilding the BIOS NHLT table. The driver +sends a ``DMA_CONTROL`` IPC containing the ``DMIC_SET_GAIN_COEFFICIENTS`` TLV (Type 2): + +.. list-table:: IPC4 DMIC Gain Control TLV Structure + :widths: 20 20 25 35 + :header-rows: 1 + + * - Field + - Byte Offset + - Type / Format + - Description + * - **Type** + - 0x00 + - uint32 (2) + - ``DMIC_SET_GAIN_COEFFICIENTS = 2`` + * - **Length** + - 0x04 + - uint32 (8) + - Payload length in bytes (8 bytes) + * - **gain_coeffs[0]** + - 0x08 + - uint16 (Q10) + - Channel 0 Gain Trim (:math:`1.0 = 1024`) + * - **gain_coeffs[1]** + - 0x0A + - uint16 (Q10) + - Channel 1 Gain Trim (:math:`1.0 = 1024`) + * - **gain_coeffs[2]** + - 0x0C + - uint16 (Q10) + - Channel 2 Gain Trim (:math:`1.0 = 1024`) + * - **gain_coeffs[3]** + - 0x0E + - uint16 (Q10) + - Channel 3 Gain Trim (:math:`1.0 = 1024`) + +Standalone Python Calibration CLI Tool +====================================== + +To streamline decimation parameter calculation, mode matching, and gain trim packaging, SOF provides the +standalone CLI utility ``sof_dmic_tool.py`` located in ``tools/tune/dmic/``. + +Searching Decimation Modes +-------------------------- + +To evaluate all valid single or dual-rate decimation modes for a platform IO clock: + +.. code-block:: bash + + # Search matched modes for 48 kHz (comm) and 16 kHz (voice wake) on a 38.4 MHz platform + python3 tools/tune/dmic/sof_dmic_tool.py modes --ioclk 38.4e6 --rates 48000,16000 + +Example Tool Output: + +.. code-block:: text + + ============================================================================== + SOF DMIC Decimation Mode Search (IO Clock = 38.40 MHz) + ============================================================================== + Dual-FIFO Matched Modes: FIFO A = 48000 Hz, FIFO B = 16000 Hz (Found 3 matches): + Idx clkdiv PDM Clock Duty M_cic M_fir_A M_fir_B CIC Shift + ------------------------------------------------------------------------------ + 0 8 4.80 MHz 50.0% 25 4 12 3 + 1 10 3.84 MHz 50.0% 20 4 12 1 + 2 16 2.40 MHz 50.0% 25 2 6 3 + ============================================================================== + +Calculating Sensitivity Trims & Building Binary Blobs +----------------------------------------------------- + +Given laboratory sensitivity measurements across a 4-channel microphone array: + +.. code-block:: bash + + # Calibrate measured sensitivities to a target of -26.0 dBFS and output IPC4 binary blob + python3 tools/tune/dmic/sof_dmic_tool.py gain-trim \ + --sens -25.2,-26.8,-24.9,-27.1 \ + --target -26.0 \ + --out dmic_gain_calibrated.bin + +Example Tool Output: + +.. code-block:: text + + ============================================================================== + SOF Digital Microphone Acoustic Sensitivity Calibration + Target Sensitivity: -26.00 dBFS at 94 dBSPL (1 kHz) + ============================================================================== + Ch Meas (dBFS) Trim (dB) Linear OUT_GAIN Reg (Q1.19) Copier (Q10) + ------------------------------------------------------------------------------ + 0 -25.20 -0.80 0.9120 0x74BCC (478156 ) 0x03A6 (934) + 1 -26.80 0.80 1.0965 0x8C596 (574870 ) 0x0463 (1123) + 2 -24.90 -1.10 0.8810 0x70C63 (461923 ) 0x0386 (902) + 3 -27.10 1.10 1.1350 0x91481 (595073 ) 0x048A (1162) + ============================================================================== + Successfully generated binary IPC4 gain blob (16 bytes): dmic_gain_calibrated.bin + +Production Calibration Recipes +============================== + +Recipe 1: Dual-Microphone Laptop Bezel (Broadside Array) +-------------------------------------------------------- + +Designed for standard clamshell and convertible laptops with two microphones spaced :math:`60\text{ mm}` apart +in the top display bezel. + +* **Target Use-Case**: High-definition video conferencing (Zoom, Teams) at :math:`48\text{ kHz}` combined with + background voice wake detection at :math:`16\text{ kHz}`. +* **Clock Architecture**: :math:`f_{\text{io}} = 38.4\text{ MHz}`, :math:`\text{clkdiv} = 16 \implies f_{\text{pdm}} = 2.40\text{ MHz}` (Duty cycle: :math:`50.0\%`). +* **Filter Configuration**: + * :math:`M_{\text{cic}} = 25 \implies f_{\text{cic}} = 96\text{ kHz}`, :math:`\text{cic\_shift} = 3`. + * FIFO A (:math:`48\text{ kHz}`): :math:`M_{\text{fir,A}} = 2` (Filter: ``pdm_decim_int32_02``, 63 taps). + * FIFO B (:math:`16\text{ kHz}`): :math:`M_{\text{fir,B}} = 6` (Filter: ``pdm_decim_int32_06``, 127 taps). +* **DCCOMP**: Mode ``TC5`` (:math:`f_c = 7.5\text{ Hz}`). +* **Unmute Ramp**: :math:`200\text{ ms}` logarithmic ramp. + +Recipe 2: Quad-Microphone Conference Tabletop Array (Circular) +-------------------------------------------------------------- + +Designed for executive conference systems and smart hubs with 4 microphones arranged in a :math:`100\text{ mm}` +diameter circular geometry for :math:`360^\circ` spatial speaker tracking. + +* **Target Use-Case**: 4-channel studio-quality capture with high acoustic overload ceiling (:math:`130\text{ dBSPL}`). +* **Clock Architecture**: :math:`f_{\text{io}} = 38.4\text{ MHz}`, :math:`\text{clkdiv} = 8 \implies f_{\text{pdm}} = 4.80\text{ MHz}` (High performance mode). +* **Filter Configuration**: + * :math:`M_{\text{cic}} = 25 \implies f_{\text{cic}} = 192\text{ kHz}`, :math:`\text{cic\_shift} = 3`. + * FIFO A (:math:`48\text{ kHz}`): :math:`M_{\text{fir,A}} = 4` (Filter: ``pdm_decim_int32_04``, 143 taps, Stopband: :math:`> 95\text{ dB}`). +* **DCCOMP**: Mode ``TC6`` (:math:`f_c = 3.7\text{ Hz}`) for extended low-frequency vocal response. +* **Sensitivity Alignment**: Calibrated to :math:`-26.0\text{ dBFS} \pm 0.05\text{ dB}` across all 4 channels. + +Recipe 3: Ultra-Low-Power Edge Wake-on-Voice +-------------------------------------------- + +Designed for battery-constrained standby modes where the DSP monitors for keyword activation while drawing +sub-milliwatt power. + +* **Target Use-Case**: Single or dual-mic keyword listening (:math:`16\text{ kHz}`). +* **Clock Architecture**: :math:`f_{\text{io}} = 19.2\text{ MHz}`, :math:`\text{clkdiv} = 25 \implies f_{\text{pdm}} = 768\text{ kHz}` (Ultra-low-power mode, :math:`D = 48.0\%`). +* **Filter Configuration**: + * :math:`M_{\text{cic}} = 16 \implies f_{\text{cic}} = 48\text{ kHz}`, :math:`\text{cic\_shift} = 0`. + * FIFO B (:math:`16\text{ kHz}`): :math:`M_{\text{fir}} = 3` (Filter: ``pdm_decim_int32_03``, 45 taps). +* **Power Dissipation**: :math:`< 1.2\text{ mW}` total digital subsystem power. + +End-to-End Tuning Toolchain Workflow +==================================== + +The complete end-to-end DMIC engineering workflow is illustrated in :numref:`fig_dmic_tuning_toolchain_workflow`. + +.. _fig_dmic_tuning_toolchain_workflow: +.. figure:: images/dmic_tuning_toolchain_workflow.svg + :alt: End-to-End DMIC Tuning and Calibration Toolchain Workflow + :width: 100% + :align: center + + End-to-End DMIC Tuning and Calibration Toolchain Workflow + +The workflow encompasses 5 coordinated stages: + +1. **Hardware Specification**: Reviewing microphone datasheet limits (PDM clock min/max, duty cycle tolerances, + sensitivity, AOP) and physical acoustic port enclosure geometry. +2. **Filter Tuning & Mode Selection**: Running ``sof_dmic_tool.py`` or Octave ``dmic_init.m`` to select valid + integer decimation tuples and generate droop-compensating FIR filter taps. +3. **Acoustic Calibration**: Measuring the DUT array in an anechoic box with an Audio Precision APx555, + deriving channel sensitivity deltas, and calculating :math:`Q1.19` and :math:`Q10` gain trim coefficients. +4. **Blob Packaging & Compilation**: Populating ACPI NHLT descriptors and ALSA Topology 2 configuration files, + then compiling target binary artifacts (``.tplg`` and ``nhlt-*.bin``). +5. **Target Deployment & Sign-Off**: Deploying binaries to the DUT, verifying live streams via ``arecord`` and + ``sof-ctl``, and confirming that THD+N, frequency flatness, and beamformer directivity satisfy requirements. + +Interactive Live Injection & Diagnostics Matrix +=============================================== + +Runtime Gain Verification via sof-ctl +------------------------------------- + +To inspect or inject digital microphone gain trims on a live DUT over SSH: + +.. code-block:: bash + + # Step 1: Query active mixer controls on the DMIC capture card + ssh root@ "amixer -c 0 scontrols | grep -i dmic" + + # Step 2: Set capture volume via ALSA mixer (in decibels) + ssh root@ "amixer -c 0 sset 'DMIC01 Capture Volume' 20dB" + + # Step 3: Inject binary gain calibration blob into active IPC4 copier component + # Widget ID 12 corresponds to the DMIC ingress copier + scp dmic_gain_calibrated.bin root@:/tmp/dmic_gain.bin + ssh root@ "sof-ctl -D hw:0 -w 12 -s /tmp/dmic_gain.bin" + + # Step 4: Record a 10-second multi-channel test capture to verify audio integrity + ssh root@ "arecord -D hw:0,1 -f S32_LE -c 4 -r 48000 -d 10 /tmp/dmic_test.wav" + +Diagnostic Troubleshooting Matrix +--------------------------------- + +.. list-table:: Digital Microphone Troubleshooting & Diagnostics + :widths: 20 25 25 30 + :header-rows: 1 + + * - Symptom + - Root Cause + - Diagnostic Command + - Remediation Action + * - **Audible Thump/Click on Stream Start** + - Capsule DC bias during clock power-up ramp + - Inspect kernel dmesg: ``dmesg | grep dmic`` + - Increase ``unmute_ramp_time_ms`` in topology (e.g. from 50ms to 200ms); set ``DCCOMP`` to mode ``TC4`` or ``TC5``. + * - **Digital Clipping / Hard Saturation at Moderate SPL Levels** + - CIC shifter underflow or gain multiplier overflow + - Analyze recorded WAV: peak at :math:`0\text{ dBFS}` with flat tops + - Re-evaluate ``cic_shift`` using ``sof_dmic_tool.py``. Ensure :math:`\text{cic\_shift} \ge B_{\text{needed}} - 22`. + * - **Severe Noise Floor Rise / Modulator Hash** + - Non-compliant clock duty cycle (:math:`< 40\%`) from odd ``clkdiv`` + - Measure ``PDM_CLK`` on Saleae logic analyzer or oscilloscope + - Avoid odd dividers that yield duty cycles outside :math:`[40\%, 60\%]`; select higher :math:`f_{\text{io}}` clock. + * - **Degraded Beamformer Null Depth (< 15 dB)** + - Channel gain spread :math:`> 0.5\text{ dB}` or acoustic port leakage + - Run APx555 sensitivity sweep or compare RMS power across recorded channels + - Re-run acoustic calibration in anechoic box; inject precise gain trims via ``DMIC_SET_GAIN_COEFFICIENTS`` TLV. + * - **180° Inverted Channel Polarity** + - Inverted clock edge selection in hardware + - Inspect waveform polarity on dual-channel impulse stimulus + - Toggle ``mic_swap`` or invert ``CIC_CONTROL_MIC_A/B_POLARITY`` bit in driver or topology. + * - **FIFO Buffer Overrun / DSP Panic** + - FIR tap count exceeds allowed MAC cycles per frame interval + - Check SOF trace log: ``sof-logger -t`` shows DMA underrun errors + - Reduce FIR tap count (:math:`N_{\text{taps}} \le \lfloor f_{\text{io}} / (2 f_s) \rfloor - 5`) or increase platform IO clock speed. + +Related Documentation +===================== + +* :ref:`time-domain-fixed-beamformer`: Time-Domain Fixed Beamformer array geometry and spatial filter design. +* :ref:`rtnr`: Real-Time Noise Reduction (RTNR) firmware processing guide. +* :ref:`kpb_wov`: Key Phrase Buffer (KPB) and Wake-on-Voice architecture. +* :ref:`runtime_tuning_sof_ctl`: Unified runtime parameter injection, ABI serialization, and ``sof-ctl`` guide. +* :ref:`drc_tuning`: Dynamic Range Compression & Multiband DRC tuning guide. +* :ref:`crossover_tuning`: Crossover Filter Design & Multi-Driver Speaker Tuning guide. +* :ref:`sound_dose_tuning`: Sound Dose Evaluator & Hearing Health Calibration guide. diff --git a/developer_guides/tuning/images/dmic_tuning_array_phase_matching.svg b/developer_guides/tuning/images/dmic_tuning_array_phase_matching.svg new file mode 100644 index 00000000..083b5cfe --- /dev/null +++ b/developer_guides/tuning/images/dmic_tuning_array_phase_matching.svg @@ -0,0 +1,166 @@ + + + + + + + + + + + + + + + + + + + + + Microphone Array Acoustic Geometry & Inter-Channel Phase Matching + + + Wavefront Propagation Delay, Channel Imbalance, and Beamformer Directivity Degradation + + + + + + + + ACOUSTIC WAVE PROPAGATION & ARRAY MODEL + + + + + + + + + Planar Acoustic Wavefront (Speed c = 343 m/s) + + + + θ + + + + + + + MIC 1 + Reference Channel + + + + MIC 2 + Delayed Channel + + + + + + Spacing d = 50 - 70 mm + + + + + Δx = d * sin(θ) + + + + + + Theoretical Propagation Delay & Phase: + + + + τ = (d * sin(θ)) / c, Δφ(f) = 2π * f * τ + + + + • At d = 60 mm, Broadside (θ = 0°): τ = 0 μs, Δφ = 0° + + + • At d = 60 mm, Endfire (θ = 90°): τ = 175 μs (8.4 samples @ 48 kHz) + + + Acoustic Port Realities: + + Manufacturing tolerances cause ±1.0 dB sensitivity spread and + + + mesh acoustic resistance induces up to 10° - 15° phase skew! + + + + + + + + + + BEAMFORMER SENSITIVITY & CALIBRATION GAIN + + + + + + Spatial Directivity Attenuation Comparison (Cardioid Beam) + + + + + + + + + + + Calibrated Array + Null Depth: > 28 dB + + + + + + + + + + + + 1 dB / 10° Mismatch + Null Depth: Only 11 dB! + + + + + + + SOF Dual-Stage Acoustic Calibration: + + 1. Broadband Sensitivity Normalization (Gain Trim): + + Trims individual channel gains via OUT_GAIN_LEFT / RIGHT registers + + + Gain_Trim_i = 10^((Sens_target - Sens_measured_i) / 20) + + + 2. Frequency-Dependent Phase Alignment (FIR EQ): + + TDFB / RTNR preprocessing FIR filter compensates mesh acoustic delay: + + + H_comp_i(f) = |H_ref(f)| / |H_i(f)| * exp(-j * (∠H_i(f) - ∠H_ref(f))) + + + Result: Directivity Index restored to theoretical maximum (+6.0 dB) + + + + diff --git a/developer_guides/tuning/images/dmic_tuning_clocking_modes.svg b/developer_guides/tuning/images/dmic_tuning_clocking_modes.svg new file mode 100644 index 00000000..261f3dec --- /dev/null +++ b/developer_guides/tuning/images/dmic_tuning_clocking_modes.svg @@ -0,0 +1,168 @@ + + + + + + + + + + + + + + + + + + + + + DMIC Clock Generation & Dual-FIFO Mode Matching Architecture + + + Shared Physical PDM Clock Distribution, Asymmetric FIR Decimation, and Duty-Cycle Tolerances + + + + + + + + PHYSICAL CLOCK GENERATION & BUS TOPOLOGY + + + + + + DSP IO Clock Source (f_io) + System Oscillator: 19.2 MHz (cAVS) or 38.4 MHz (ACE / PTL) + + + f_pdm = f_io / clkdiv, PDM_CLKDIV register = clkdiv - 2 + + + + + + + Clock Duty Cycle & Jitter Constraints + Even Divider (e.g. clkdiv=16): Perfect 50.0% Duty Cycle + Odd Divider (e.g. clkdiv=13): High phase = 6 cycles, Low = 7 cycles + + + D_min = 100 * floor(clkdiv/2) / clkdiv = 46.15% (OK if >= 40%) + + + MEMS Transducer Specification Limit: Typically 40% <= D <= 60% + + + Warning: clkdiv <= 4 is hardware prohibited in cAVS 1.5 - 2.5 + + + + + + + Stereo PDM Microphone Bus Sharing + + + + PDM_CLK (Shared Clock Line) + + + PDM_DATA (Shared Dual-Edge Data Line) + + + + + Mic A (Left) + SELECT = GND / VDD + Drives on Rising Edge + Tri-states on Falling + + + + + + Mic B (Right) + SELECT = Opposite + Drives on Falling Edge + Tri-states on Rising + + + + + + + + + + DUAL-FIFO CONCURRENT SAMPLING & MATCHED MODES + + + + + + The Multi-Rate Challenge: + + Both capture streams MUST share one physical PDM clock and CIC filter, + + + yet output 48 kHz (telephony/recording) and 16 kHz (keyword detect) concurrently! + + + Solution: f_pdm / M_cic = common base rate (e.g. 96 kHz) + + + + + + + + Shared Hardware CIC + M_cic = 25 (e.g. 2.4 MHz / 25 = 96 kHz) + + + + + + + + + + + + + FIFO A: Normal Audio + M_fir_a = 2 + Target Fs = 48 kHz + 96 kHz / 2 = 48 kHz + Filter: pdm_decim_02 + High Performance DMA + + + + + + + FIFO B: Voice Detect (WOV) + M_fir_b = 6 + Target Fs = 16 kHz + 96 kHz / 6 = 16 kHz + Filter: pdm_decim_06 + Ultra-Low Power KPB + + + + + + + Common Production Matched Modes (f_io = 38.4 MHz): + • Mode 1: clkdiv=16 (2.4 MHz), Mcic=25 (96k), Mfir_a=2 (48k), Mfir_b=6 (16k) + • Mode 2: clkdiv=12 (3.2 MHz), Mcic=20 (160k), Mfir_a=4 (40k), Mfir_b=10 (16k) + • Mode 3: clkdiv=20 (1.92 MHz), Mcic=20 (96k), Mfir_a=2 (48k), Mfir_b=6 (16k) + Selected automatically by SOF dmic_init mode-matching algorithm + + + diff --git a/developer_guides/tuning/images/dmic_tuning_pdm_decimation_pipeline.svg b/developer_guides/tuning/images/dmic_tuning_pdm_decimation_pipeline.svg new file mode 100644 index 00000000..e2c3ec2c --- /dev/null +++ b/developer_guides/tuning/images/dmic_tuning_pdm_decimation_pipeline.svg @@ -0,0 +1,230 @@ + + + + + + + + + + + + + + + + + + + + + + + + + Digital Microphone (DMIC) Hardware Decimation Architecture + + + 1-Bit PDM Ingress, 5th-Order CIC Comb Filtering, FIR Droop Compensation, and DC/Gain Trimming + + + + + + + + HARDWARE SIGNAL PROCESSING CHAIN: PDM STREAM TO LINEAR PCM OUTPUT + + + + + + + MEMS PDM Ingress + + + MIC + + 1-Bit PDM + f_pdm: 0.768 - 4.8 MHz + Dual-Edge DDR Mic + Rising: Mic A (L) + Falling: Mic B (R) + + + + + + + + + + + Stage 1: CIC Decimator + + 5th-Order Comb + H_cic(z) = [ (1-z^-M)/(1-z^-1) ]^5 + Decim Factor: M_cic (5 to 31) + DC Gain: G = M_cic^5 + Internal Accum: 26 Bits + CIC_CONTROL / CONFIG + COMB_COUNT = M_cic - 1 + + + + + + + + + + + Bit Alignment + + CIC Shift Engine + cic_shift: -8 to +4 + Aligns 26b CIC accumulator + to 22b FIR Input Path + b_needed = floor(log2(G)+1)+1 + shift = b_needed - 22 + Headroom Protection + + + + + + + + + + + Stage 2: FIR Decimator + + Multirate FIR Filter + Decim Factor: M_fir (2 to 15) + Taps: Up to 250 (Symm/RAM) + Inverts Sinc^5 Passband Droop + Passband Ripple: < 0.1 dB + Stopband Atten: > 90 dB + PDM_COEFFICIENT_A / B + + + + + + + + + + + Stage 3: DC & Gain Trim + + Acoustic Calibration + DCCOMP: 1st-Order HPF + y[n] = x[n]-x[n-1] + a*y[n-1] + Time Constants: TC0 - TC7 + Channel Gain Trim: 20-Bit + OUT_GAIN_LEFT / RIGHT + Linear 24b/32b PCM Output + + + + + + + + + + + Decimation Mathematics & Clock Relations + + + + + 1. Total Oversampling Ratio (OSR): + + OSR = M_cic * M_fir = f_pdm / f_s >= 50 (40 for >48kHz) + + 2. PDM Sampling Clock Generation: + + f_pdm = f_io / clkdiv, clkdiv in [clkdiv_min, clkdiv_max] + + 3. FIR Hardware Maximum Tap Constraint: + + N_taps <= min( 250, floor(f_io / (2 * f_s)) - 5 ) + + + • At f_io = 19.2 MHz and f_s = 48 kHz: Max FIR taps = min(250, 195) = 195 + + + • At f_io = 38.4 MHz and f_s = 48 kHz: Max FIR taps = min(250, 395) = 250 + + + + + + + + + + Intel DSP DMIC Register Map & Quantization + + + + + + + Register + Bitfield / Precision + Function / Range + + + + MIC_CONTROL + Bits 15:8 (8-bit) + PDM_CLKDIV = clkdiv - 2 + + + + CIC_CONFIG + Bits 15:8 (8-bit) + COMB_COUNT = M_cic - 1 + + + + CIC_CONFIG + Bits 27:24 (4-bit) + CIC_SHIFT (-8 to +4) + + + + FIR_CONFIG_A/B + Bits 20:16 (5-bit) + FIR_DECIM = M_fir - 1 + + + + FIR_CONFIG_A/B + Bits 7:0 (8-bit) + FIR_LENGTH = N_taps - 1 + + + + OUT_GAIN_LEFT/R + Bits 19:0 (20-bit) + Channel Gain Multiplier + + + + DC_OFFSET_L/R + Bits 21:0 (22-bit) + Hardware DC Correction + + + + FIR Coeff RAM: Up to 250 x 20-bit (or packed 24-bit) tap coefficients + + + + + diff --git a/developer_guides/tuning/images/dmic_tuning_sensitivity_calibration_rig.svg b/developer_guides/tuning/images/dmic_tuning_sensitivity_calibration_rig.svg new file mode 100644 index 00000000..10de18f6 --- /dev/null +++ b/developer_guides/tuning/images/dmic_tuning_sensitivity_calibration_rig.svg @@ -0,0 +1,152 @@ + + + + + + + + + + + + + + + + + + + + + Digital Microphone Acoustic Metrology & Sensitivity Calibration Rig + + + Anechoic Enclosure, Class 1 Reference Transducer, Audio Precision APx555, and Gain Offset Derivation + + + + + + + + ANECHOIC TEST BOX / ACOUSTIC CHAMBER + + + + + + + + + + + + + + + + Reference Sound Source + 94 dBSPL @ 1 kHz (1.0 Pa) + + + + + + + + + + d = 0.5 m + + + + + + Reference Mic + B&K 4190 / GRAS 40AZ + Class 1 Calibrated + 50 mV/Pa (±0.1 dB) + Sound Level Calibrator + + + + + + + + + Target DUT + Intel Laptop / DUT Array + 2-ch / 4-ch PDM Microphones + SOF Firmware Ingress + Acoustic Port Ingress + + + + + Measurement Interface Cabling + BNC Coaxial Analog Cable: Reference Mic → Audio Precision Ch 1 + USB-C / Ethernet / SSH: Target DUT PCM Capture Stream → APx555 + + + + + + + + + ANALYSIS SYSTEM & SENSITIVITY CALIBRATION + + + + + + + Audio Precision APx555 / Metrology Host + + Stimulus Generator: 1 kHz Sine @ 94 dBSPL + Log Sweep (20Hz - 20kHz) + Ch 1 (Analog): Measures True SPL via Calibrated Reference Microphone + Ch 2-5 (Digital): Records Multi-Channel DMIC PCM Stream from SOF + FFT Window: 64k Rife-Vincent, Averaging: 16x + + + + + + Sensitivity Calibration Formulae: + + 1. Microphone Sensitivity (dBFS at 94 dBSPL): + + + S_i = 20 * log10( V_rms_i / V_FS ) [dBFS] + + + 2. Gain Trim Calculation to Target (-26.0 dBFS): + + + ΔG_i [dB] = S_target - S_i, Trim_i = 10^(ΔG_i / 20) + + + 3. Fixed-Point Register Programming (Q1.19 format): + + + OUT_GAIN = round( Trim_i * 2^19 ) & 0x000FFFFF + + + In IPC4 copier: gain_coeffs[i] encoded in Q10 format (round(Trim_i * 1024)) + + + + + + + Typical Production Tolerances & Trims: + • Raw Mic 0: -25.2 dBFS → Trim = -0.8 dB → OUT_GAIN = 478,550 (0x074D56) + • Raw Mic 1: -26.9 dBFS → Trim = +0.9 dB → OUT_GAIN = 581,510 (0x08E086) + • Post-Trim Delta: < 0.05 dB across all array elements + + Result: 100% Phase Coherence & Optimal Beamformer Null Depth + + + + diff --git a/developer_guides/tuning/images/dmic_tuning_toolchain_workflow.svg b/developer_guides/tuning/images/dmic_tuning_toolchain_workflow.svg new file mode 100644 index 00000000..582f0d5e --- /dev/null +++ b/developer_guides/tuning/images/dmic_tuning_toolchain_workflow.svg @@ -0,0 +1,225 @@ + + + + + + + + + + + + + + + + + + + + + End-to-End DMIC Tuning & Calibration Toolchain Workflow + + + From Transducer Metrology & Decimation Synthesis to Topology2 / NHLT Blobs and Live Runtime Injection + + + + + + + + + STAGE 1: HARDWARE + + MEMS Specification + • Clock range (0.768 - 4.8M) + • Duty cycle limits (40-60%) + • Nom. Sensitivity (-26 dBFS) + • AOP (120 - 135 dBSPL) + • SNR (64 - 72 dBA) + + + + Acoustic Enclosure + • Port hole diameter & depth + • Mesh acoustic resistance + • Gasket seal integrity + • Inter-element spacing d + + + + Target Use-Cases + • Dual-mic Laptop (48k/16k) + • Quad-mic Conference (48k) + • Edge WOV (<1mW 16k) + + + Platform IO Clock + 19.2M / 38.4M + + + + + + + + + + + STAGE 2: FILTER TUNE + + Decimation Synthesis + • Run sof_dmic_tool.py + • Search valid (clkdiv, M_cic) + • Verify OSR >= 50 + • Duty cycle verification + • Match dual-FIFO modes + + + + FIR Filter Selection + • Invert Sinc^5 CIC droop + • Ripple: < 0.1 dB + • Stopband: > 90 dB + • Tap check: <= 250 taps + + + + Shift & Headroom + • Calculate cic_shift + • Normalize to 22-bit + • FIR coefficient scaling + + + sof_dmic_tool.py + Calculates Register Set + + + + + + + + + + + STAGE 3: CALIBRATION + + Anechoic Metrology + • APx555 sound generator + • 94 dBSPL @ 1kHz acoustic + • Class 1 ref mic return + • Capture DUT PCM array + + + + Channel Gain Trim + • S_i = 20*log10(V_rms/V_FS) + • Trim_i = S_target - S_i + • OUT_GAIN_LEFT/R (Q1.19) + • Balance to < 0.05 dB delta + + + + DC & Phase Match + • DCCOMP HPF selection + • Time constant (TC0-TC7) + • Inter-mic phase delta check + • TDFB FIR alignment + + + Channel Trims + Bit-Exact Multipliers + + + + + + + + + + + STAGE 4: BLOB BUILD + + ACPI NHLT Tables + • Endpoint descriptor + • Array geometry (x, y, z) + • Mic types (cardioid/omni) + • nhlt-*-dmic-*.bin + + + + Topology 2 Config + • dais/dmic.conf + • dmic-default.conf + • PDM_MIC_A/B_ENABLE + • Unmute ramp ms + + + + Compilation + • alsatplg -c sof-*.conf + • Builds .tplg binary + • Verifies token bindings + + + alsatplg / nhlt + Generates Target Binaries + + + + + + + + + + + STAGE 5: TARGET RUN + + Target DUT Deploy + • /lib/firmware/intel/sof/ + • Deploy .tplg & NHLT + • Reload driver / reboot + + + + Runtime Injection + • sof-ctl -s (IPC4 copier) + • DMIC_SET_GAIN_COEFFS + • On-the-fly sensitivity trim + + + + Acoustic Sign-Off + • THD+N < 0.5% @ 94 dBSPL + • Flat Freq Response ±1dB + • Phase coherence < 3° + • Beamformer DI >= 5.5 dB + • Zero click/pop on unmute + + + Live Verification + arecord & APx555 + + + + + + + + Production Deployment & Continuous Integration Verification Pipeline + + + 1. Run sof_dmic_tool.py --fs 48000,16000 --ioclk 38.4e6 to generate bit-exact decimation settings and FIR taps. + + + 2. Measure DUT array in anechoic box, derive gain trim delta, update OUT_GAIN_LEFT/RIGHT in topology or inject via sof-ctl. + + + 3. Validate beamformer directionality, noise suppression depth, and wake-on-voice trigger accuracy under real acoustic conditions. + + + From 1700304c93cc5eb50d54b1ab669fd895b40ec1e1 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 17:24:39 +0100 Subject: [PATCH 37/64] doc: developer_guides: add level multiplier & aria agc dynamic range tuning guide - Author comprehensive Level Multiplier & Aria AGC Dynamic Range Control Tuning Guide establishing coordinated dual-stage dynamic range management architecture. - Document Level Multiplier fixed-point Q9.23 precision scaling (-138.47 dB to +48.17 dB), elementary quantization step, and multi-format shift math across S16, S24, and S32. - Detail zero-overhead fast-path memory copy bypass at unity gain (0 dB). - Formulate Aria AGC 4 operating boost modes (0 dB to +18 dB) and dynamic headroom thresholds. - Provide mathematical proof of Aria strict 0 dBFS hard clamp guaranteeing zero digital clipping. - Specify invariant 1 ms circular lookahead buffer, 10-state minimum envelope, and anti-zipper linear ramp. - Create 5 native SVG vector diagrams covering signal chains, bit allocations, transfer functions, lookahead buffers, and calibration workflows. - Update data/modules.yaml SSOT and developer_guides/index.rst navigation. Signed-off-by: Liam Girdwood --- data/modules.yaml | 2 + developer_guides/index.rst | 6 +- ...aria_dynamic_regimes_transfer_function.svg | 215 +++++ .../images/aria_lookahead_timing_ramp.svg | 249 ++++++ .../level_multiplier_aria_signal_chain.svg | 194 +++++ .../level_multiplier_aria_tuning_workflow.svg | 218 ++++++ .../level_multiplier_q9_23_dynamic_range.svg | 147 ++++ .../tuning/level_multiplier_aria_tuning.rst | 739 ++++++++++++++++++ 8 files changed, 1768 insertions(+), 2 deletions(-) create mode 100644 developer_guides/tuning/images/aria_dynamic_regimes_transfer_function.svg create mode 100644 developer_guides/tuning/images/aria_lookahead_timing_ramp.svg create mode 100644 developer_guides/tuning/images/level_multiplier_aria_signal_chain.svg create mode 100644 developer_guides/tuning/images/level_multiplier_aria_tuning_workflow.svg create mode 100644 developer_guides/tuning/images/level_multiplier_q9_23_dynamic_range.svg create mode 100644 developer_guides/tuning/level_multiplier_aria_tuning.rst diff --git a/data/modules.yaml b/data/modules.yaml index de4baf87..6c90e5df 100644 --- a/data/modules.yaml +++ b/data/modules.yaml @@ -113,6 +113,7 @@ modules: - "High-precision Q9.23 fixed-point multiplier (-138.47 dB to +48.17 dB)" - "Zero-overhead fast-path bypass when configured for unity gain (0 dB)" - "Runtime IPC4 calibration and LLEXT dynamic module packaging" + tuning_guide: "developer_guides/tuning/level_multiplier_aria_tuning" - id: up_down_mixer name: "Up/Down Mixer" @@ -174,6 +175,7 @@ modules: - "Target pre-amplification boost (0, 6, 12, 18 dB)" - "Instantaneous regressive ducking to prevent 0 dBFS clipping" - "1ms lookahead circular buffer and per-sample linear interpolation" + tuning_guide: "developer_guides/tuning/level_multiplier_aria_tuning" - id: drc name: "Dynamic Range Compressor (DRC)" diff --git a/developer_guides/index.rst b/developer_guides/index.rst index 00a3ad58..c50df86d 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -60,8 +60,8 @@ Audio Processing Modules & Algorithms * :ref:`kpb_wov` (High-level architecture; also see driver guide :ref:`keyword_detect`) * :ref:`tone` (High-level architecture; also see upstream `Tone README `_) * :ref:`up_down_mixer` (High-level architecture; also see upstream `Up/Down Mixer README `_) -* :ref:`aria` (High-level architecture; also see upstream `Aria README `_) -* :ref:`level_multiplier` (High-level architecture; also see upstream `Level Multiplier README `_) +* :ref:`aria` (High-level architecture; also see upstream `Aria README `_ & tuning guide :ref:`level_multiplier_aria_tuning`) +* :ref:`level_multiplier` (High-level architecture; also see upstream `Level Multiplier README `_ & tuning guide :ref:`level_multiplier_aria_tuning`) * :ref:`phase_vocoder` (High-level architecture; also see upstream `Phase Vocoder source tree `_) * :ref:`stft_process` (High-level architecture; also see upstream `STFT Process README `_) * :ref:`media_codecs` (High-level architecture; also see upstream `Cadence Codec module adapter `_ & `Codec README `_) @@ -126,6 +126,7 @@ Dynamics & Transducer Protection Tuning ======================================= * :ref:`drc_tuning` (Single-band DRC and Multiband DRC compression curves, adaptive ballistics, and speaker protection) +* :ref:`level_multiplier_aria_tuning` (Level Multiplier Q9.23 precision scaling, zero-overhead fast-path bypass, Aria AGC target pre-amplification boost, 1 ms lookahead circular buffering, and regressive anti-clipping protection) * :ref:`smart_amp_tuning` (Smart Amplifier Dynamic Speaker Management, I/V sense feedback calibration, Thiele-Small modeling, thermal and excursion protection) * :ref:`sound_dose_tuning` (Sound Dose Evaluator, IEC 61672-1 Class 1 A-weighting, EN 50332 / IEC 62368-1 compliance, HATS acoustic sensitivity calibration, and closed-loop exposure regulation) @@ -144,6 +145,7 @@ Acoustic, Transducer & Array Tuning tuning/runtime_tuning_sof_ctl tuning/drc_tuning + tuning/level_multiplier_aria_tuning tuning/smart_amp_tuning tuning/sound_dose_tuning tuning/crossover_tuning diff --git a/developer_guides/tuning/images/aria_dynamic_regimes_transfer_function.svg b/developer_guides/tuning/images/aria_dynamic_regimes_transfer_function.svg new file mode 100644 index 00000000..1381beb5 --- /dev/null +++ b/developer_guides/tuning/images/aria_dynamic_regimes_transfer_function.svg @@ -0,0 +1,215 @@ + + + + + + + + + + + + + + + + + + + + + + + + + Aria AGC Transfer Functions & Regressive Headroom Dynamics + + + Target Boost (0/6/12/18 dB), Headroom Knee Thresholds & Mathematical 0 dBFS Clamping Proof + + + + + + + + + + + + + DIGITAL OVERFLOW / CLIPPING ZONE (> 0 dBFS) + + 0 dBFS (A_FS) + + + + + + + + + + + + + + + + + + + + + + + -36 + -30 + -24 + -18 + -12 + -6 + 0 + + -36 + -30 + -24 + -18 + -12 + -6 + 0 + + + + Peak Input Amplitude (dBFS) + + + Output Amplitude (dBFS) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + att = 3 (+18 dB) + att = 2 (+12 dB) + att = 1 (+6 dB) + att = 0 (0 dB / Bypass) + + + + + Strict 0 dBFS Clamping + + + + + + + + + Aria Headroom Thresholds (A_thresh) + + + Mode + Boost + Headroom + A_thresh (Hex) + Ratio + + + + 0 + 0 dB + 0.00 dBFS + 0x007FFFFF + 1.0× + + + 1 + +6 dB + -6.02 dBFS + 0x003FFFFF + 2.0× + + + 2 + +12 dB + -12.04 dBFS + 0x001FFFFF + 4.0× + + + 3 + +18 dB + -18.06 dBFS + 0x000FFFFF + 8.0× + + + Formula: A_thresh = A_FS >> att = (2^23 - 1) >> att + + + + + + + Mathematical Clamping Invariance + + When peak input max_data > A_thresh: + + + + g = floor[ (A_FS × 2^31) / (max_data × 2^att) ] + + + Output sample multiplication with shift (31 - att): + + + + y[n] = (x[n] × g) >> (31 - att) = x[n] × (A_FS / max_data) + + + + ∴ |y_peak| = max_data × (A_FS / max_data) ≡ A_FS (0 dBFS) + + + + + + + + Operating Summary: + + • Signals below A_thresh receive constant linear pre-amplification boost (e.g. +12 dB) to maximize speech recognition sensitivity. + + + • Sudden shouts or acoustic bursts dynamically compress gain so the peak is locked exactly at 0 dBFS, entirely avoiding harsh digital distortion. + + + diff --git a/developer_guides/tuning/images/aria_lookahead_timing_ramp.svg b/developer_guides/tuning/images/aria_lookahead_timing_ramp.svg new file mode 100644 index 00000000..223128f8 --- /dev/null +++ b/developer_guides/tuning/images/aria_lookahead_timing_ramp.svg @@ -0,0 +1,249 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Aria 1 ms Lookahead Circular Buffer & Linear Ramp Engine + + + Timeline Phasing (t + 1ms vs t), 10-State Minimum Envelope Search & Sample-by-Sample Anti-Zipper Interpolation + + + + + + + + Timeline Phasing Across 1 ms Execution Window + + + + + + + Future Stream: t + 1 ms + + + Source DMA Buffer + + + + + aria_algo_calc_gain() + + + Scans chunk peak: max_data + + + Calculates gain before peak hits output + + + + + + + + + + + 1 ms Circular Ring Buffer + + + cd->data_addr .. cd->data_end + + + + + cir_buf_copy() & cir_buf_wrap() + + + Holds 48 samples/ch (at 48 kHz) + + + Strict 1.000 ms invariant delay + + + + + + + + + + + Delayed Stream: t (Now) + + + Sink Egress Buffer + + + + + cd->aria_get_data() + + + Multiplies delayed audio by ramped gain + + + Zipper-free protected output + + + + + + + + + + 10-State Sliding Window Minimum-Envelope Search + + + + + Sliding Gain History Array: cd->gains[0..9] + + + + + + + S0 + + S1 + + S2 + + S3 + + S4 + + S5 + + S6 + + S7 + + + S8 + + + S9 + + + + + + Minimum Envelope Algorithm: + + + gain_begin = min(gains[gain_state + 2 .. gain_state + 9]) + + + gain_end = min(gains[gain_state + 3 .. gain_state + 10]) + + + + + + Lookahead Attack Envelope Benefit: + + + • Inspecting 10 future/sliding states identifies incoming loud peaks. + + + • Gain begins ducking prior to peak arrival, smoothing the attenuation. + + + • Post-transient release recovers smoothly across frames without pumping. + + + + + + + + + Per-Sample Linear Interpolation Ramp Engine + + + + + Slope & Step Derivation: + + step = (gain_end - gain_begin) / frames + + + gain[n+1] = gain[n] + step (evaluated per audio frame) + + + + + + + + Naive Step Change (No Ramp) + + + + + + Discontinuity: Zipper Noise & Clicks + + + + + + + + Aria Linear Interpolation Ramp + + + + + C0 Continuity: Artifact-Free + + + + + + + Tensilica HiFi5 Hardware Circular Addressing: + + + AE_SETCBEGIN0 / AE_SETCEND0 → Dedicated HW Registers + + + AE_L32X2_XC & AE_S32X2_XC1 execute auto-wrap in 0 cycles! + + + + + + + + + + + + + diff --git a/developer_guides/tuning/images/level_multiplier_aria_signal_chain.svg b/developer_guides/tuning/images/level_multiplier_aria_signal_chain.svg new file mode 100644 index 00000000..6735739f --- /dev/null +++ b/developer_guides/tuning/images/level_multiplier_aria_signal_chain.svg @@ -0,0 +1,194 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Dual-Stage Dynamic Range Architecture: Level Multiplier & Aria AGC + + + Static Q9.23 Sensor Alignment (0 ms Latency) + Dynamic Lookahead Regressive Pre-Amplification (1 ms Invariant Latency) + + + + + + + Audio Ingress + + + + DMIC / SoundWire + DAI Copier Gateway + S24_4LE / S32_LE + + + Raw Capture Bus + Multi-Channel Ring + fs = 16k / 48k + + + Acoustic Specs + • Sens: -26 dBFS + • Spread: ±1.5 dB + • Gasket leakage + • Port mesh resist + Requires Trim + + + + + + + + + + Stage 1: Level Multiplier (0 ms) + + + + Gain == 0 dB? + 0x00800000 + + + + FAST-PATH + + + source_to_sink_copy() + Zero-Math Memory Copy + + + + Gain != 0dB + + + Q9.23 Scaling Core + • Range: -138.47 dB to +48.17 dB + • Shift: 23 bits (S16, S24, S32) + • Saturation: Symmetric 24/32-bit + y[n] = q_multsr_sat(x[n], gain, 23) + + + + IPC4 Control Handler + Payload: 4-byte Q9.23 integer + UUID: 56:74:39:30:61:46:44:46... + + + + + + + + + + Stage 2: Aria Dynamic AGC (1 ms) + + + + Lookahead Peak Detection (t + 1ms) + max_data = max(|x[k, ch]|) in future chunk + Target Boost: 0, +6, +12, +18 dB (att = 0..3) + + + + Regressive Gain Evaluator + If max_data > A_thresh: G_reg = A_FS / max_data + 10-State Sliding Window Minimum Search + + + + 1 ms Circular Delay Buffer (t) + Stores 1 ms history in cd->data_addr + Invariant Latency (active & bypass modes) + + + + Per-Sample Linear Ramp & Clamp + step = (gain_end - gain_begin) / frames + Peak Clamped Strictly <= 0 dBFS (No Clipping!) + + + + Aria Runtime Control + Param ID: ARIA_SET_ATTENUATION (1) + UUID: 6d:16:f7:99:2c:37:ef:43... + + + + + + + + + + Sink Egress + + + TDFB + Beamformer + Spatial Array Filter + + + RTNR + Noise Suppressor + Stationary Cleaning + + + ASR / Telephony + Speech Engine + Optimal WER Level + + + Total Latency + 1.000 ms + (0 ms LvMult + + 1 ms Aria Ring) + Deterministic + + + + + + Architectural Division of Responsibility: + + • Level Multiplier: Static precision calibration across channels (-138.47 dB to +48.17 dB in Q9.23), zero latency, fast-path bypass at 0 dB. + + + • Aria AGC: Dynamic target pre-amplification boost (0, 6, 12, 18 dB), instant regressive ducking, 1 ms lookahead, zero flat-top clipping. + + + + + + + + + + + + + diff --git a/developer_guides/tuning/images/level_multiplier_aria_tuning_workflow.svg b/developer_guides/tuning/images/level_multiplier_aria_tuning_workflow.svg new file mode 100644 index 00000000..000d20d3 --- /dev/null +++ b/developer_guides/tuning/images/level_multiplier_aria_tuning_workflow.svg @@ -0,0 +1,218 @@ + + + + + + + + + + + + + + + + + + + + + End-to-End Level Multiplier & Aria AGC Tuning Workflow + + + From Acoustic Metrology to Q9.23 Sensitivity Trims, Dynamic Headroom Sizing, Topology 2 & DUT Verification + + + + + + + + + STAGE 1: METROLOGY + + + Acoustic Testing + • Anechoic chamber + • 94 dBSPL @ 1 kHz + • Class 1 reference mic + • APx555 analyzer + + + Raw Capture + Ch 0: -24.5 dBFS + Ch 1: -26.0 dBFS + Spread: 1.5 dB + + + Dynamic Headroom + Max SPL: 120 dBSPL + AOP margin: 26 dB + ASR Target: -20 dBFS + + + Output: S_meas + Per-channel sens vector + + + + + + + + + + + STAGE 2: LEVEL MULT + + + Sensitivity Trim + • Target: -26.0 dBFS + • ΔG = S_t - S_m + • Linear Q9.23 scaling + • Shift = 23 bits + + + Fast-Path Check + If ΔG == 0.0 dB: + Gain = 0x00800000 + Bypass active loops + + + Python Tool + sof_level_multiplier_ + aria_tool.py + calibrate --target -26 + + + Output: Q9.23 Gains + Per-channel 32-bit words + + + + + + + + + + + STAGE 3: ARIA AGC + + + Mode Selection + • Mode 0: 0 dB (Bypass) + • Mode 1: +6 dB (Close) + • Mode 2: +12 dB (Mid) + • Mode 3: +18 dB (Far) + + + 1 ms Lookahead + Circular buffer delay + 10-state min search + Linear ramp interpolation + + + Simulation Run + sof_level_multiplier_ + aria_tool.py aria-sim + --att 2 --peak -6 + + + Output: Att Mode + att = 0, 1, 2, or 3 + + + + + + + + + + + STAGE 4: PACKAGING + + + IPC4 Packaging + • Level Mult blob (4B) + • Aria blob (4B att) + • Base module config + • ABI header binding + + + Topology 2 + level_multiplier.conf + aria.conf widget + mixout-aria-gain pipeline + + + alsatplg Build + alsatplg -c sof-... + -o sof-aria.tplg + Binary compiler build + + + Output: .tplg & .bin + Target topology binary + + + + + + + + + + + STAGE 5: VERIFICATION + + + Target DUT + • Spider / Aphid / ARL + • Deploy .tplg via scp + • Driver reload + • Confirm UUIDs in dmesg + + + Acoustic Sign-Off + Linear boost: ±0.1 dB + Peak clamp: 0.00 dBFS + Flat factor: 0.00 (clean) + + + Live Control + amixer cset ... + sof-ctl -D hw:0 -s + Runtime injection + + + Passed Sign-Off + Optimal ASR Accuracy + + + + + + + Production Deployment & Calibration Command Pipeline: + + + 1. python3 tools/tune/level_multiplier_aria/sof_level_multiplier_aria_tool.py calibrate --sens -24.5 -26.0 -25.2 -27.1 --target -26.0 + + + 2. python3 tools/tune/level_multiplier_aria/sof_level_multiplier_aria_tool.py build-blob --gain-db 1.5 --out lvmult_ch0.bin + + + 3. ssh root@<dut> 'sof-ctl -D hw:0 -n "level_multiplier.1.1.extctl" -s /tmp/lvmult_ch0.bin && amixer -c 0 cset name="aria.1.1.extctl" 2' + + + + + + + + + + diff --git a/developer_guides/tuning/images/level_multiplier_q9_23_dynamic_range.svg b/developer_guides/tuning/images/level_multiplier_q9_23_dynamic_range.svg new file mode 100644 index 00000000..546be73e --- /dev/null +++ b/developer_guides/tuning/images/level_multiplier_q9_23_dynamic_range.svg @@ -0,0 +1,147 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + Fixed-Point Q9.23 Number System & Dynamic Range + + + 32-Bit Word Allocation, 186.64 dB Dynamic Span, Fast-Path Unity Point & Universal 23-Bit Right-Shift + + + + + + 32-Bit Signed Q9.23 Word Memory Organization + + + + + b31 + Sign (s) + + + + b30 . . . . . . . . b23 + 8 Integer Bits (Magnitude: 0 .. 255) + + + + b22 . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . b0 + 23 Fractional Bits (Quantization Step: Δ = 2^-23 ≈ 1.19209 × 10^-7) + + Linear Equation: Gain = (-1)^s × [ (b30..b23) + (b22..b0) / 2^23 ] + + + + + + Full Dynamic Range & Critical Operating Setpoints + + + + + + + + -∞ dB + 0x00000000 + Complete Mute + + + + -138.47 dB + 0x00000001 + 1 LSB (2^-23) + + + + -20.0 dB + 0x000CCCCD + 0.1000× + + + + + 0.00 dB (Unity) + 0x00800000 + FAST-PATH BYPASS + + + + +20.0 dB + 0x05000000 + 10.000× + + + + +48.17 dB + 0x7FFFFFFF + Max Boost (256.0×) + + + + + + Universal Shift Mathematics (Q_SHIFT_BITS Macro) + + + + S16_LE + Shift = 15 + 23 - 15 = 23 bits + q_multsr_sat_32x32_16(*x, gain, 23) → Clamps to [-32768, 32767] + + + + S24_4LE + Shift = 23 + 23 - 23 = 23 bits + q_multsr_sat_32x32_24(sign_extend_s24(*x), gain, 23) → Clamps to [-2^23, 2^23 - 1] + + + + S32_LE + Shift = 31 + 23 - 31 = 23 bits + q_multsr_sat_32x32(*x, gain, 23) → Full 64-bit product saturated to [-2^31, 2^31 - 1] + + + + + + + Zero-Overhead Fast-Path Bypass + + Condition: + cd->gain == 0x00800000 + + Execution Bypass: + • Bypasses multiplication loops + • Bypasses bit-shifting & saturation + source_to_sink_copy() + Wide 64/128-bit block memory copy + + + 0.000 ms Latency | Minimal CPC + + diff --git a/developer_guides/tuning/level_multiplier_aria_tuning.rst b/developer_guides/tuning/level_multiplier_aria_tuning.rst new file mode 100644 index 00000000..72a4db68 --- /dev/null +++ b/developer_guides/tuning/level_multiplier_aria_tuning.rst @@ -0,0 +1,739 @@ +.. _level_multiplier_aria_tuning: + +Level Multiplier & Aria AGC Dynamic Range Control Tuning Guide +############################################################## + +In modern digital audio signal processing architectures, dynamic range management and signal level calibration are critical requirements across capture and playback pipelines. Voice capture front-ends—such as far-field microphone arrays, automated speech recognition (ASR) engines, and VoIP conferencing pipelines—must operate across extreme acoustic sound pressure level (SPL) ranges: from faint whispers at :math:`35\text{ dBSPL}` to intense shouting, clapping, or acoustic shocks exceeding :math:`115\text{ dBSPL}`. + +To accommodate this dynamic span without introducing digital clipping or compromising signal-to-noise ratio (SNR), Sound Open Firmware (SOF) provides a coordinated, dual-stage gain management architecture: + +1. **Static Precision Level Calibration**: The **Level Multiplier** subsystem applies an ultra-low-latency, high-precision fixed-point **Q9.23** linear scaling amplifier (:math:`-138.47\text{ dB}` to :math:`+48.17\text{ dB}`). Introducing identically :math:`0\text{ ms}` of algorithmic delay, the Level Multiplier eliminates transducer sensitivity manufacturing spread across multi-microphone arrays and provides an automated **zero-overhead fast-path memory bypass** when configured for unity gain (:math:`0\text{ dB}`). +2. **Dynamic Regressive Pre-Amplification & Peak Limiting**: The **Aria** (**Automatic Regressive Input Amplifier**) subsystem applies a selectable target pre-amplification boost (:math:`0\text{ dB}`, :math:`+6\text{ dB}`, :math:`+12\text{ dB}`, or :math:`+18\text{ dB}`) to lift conversational speech into the optimal operational dynamic range of downstream feature extractors. When loud transients or shouts enter the pipeline, Aria automatically and *regressively* ducks the gain below target, enforcing a strict :math:`0\text{ dBFS}` peak clamp without flat-top clipping. To eliminate zipper noise, Aria introduces an invariant :math:`1\text{ ms}` circular lookahead buffer and smooth per-sample linear interpolation. + +This guide provides the authoritative engineering specification for calibrating the Level Multiplier, tuning Aria AGC dynamic headroom regimes, deploying the Python tuning toolchain, configuring ALSA Topology 2, and verifying acoustic performance on target platforms. + +.. contents:: Table of Contents + :local: + :depth: 3 + +Architectural Foundations & Dual-Stage Gain Paradigm +===================================================== + +The dynamic range of a 24-bit linear PCM audio stream is theoretically :math:`144.49\text{ dB}` (:math:`20 \log_{10}(2^{24})`). However, real-world acoustic front-ends face substantial physical constraints: + +* **Microphone Sensitivity Spread**: Commercial MEMS digital microphones exhibit nominal sensitivity tolerances of :math:`\pm 1.0\text{ dB}` to :math:`\pm 2.0\text{ dB}` at :math:`94\text{ dBSPL} / 1\text{ kHz}`. When mounted inside device enclosures, acoustic mesh acoustic impedance variations and gasket compression irregularities widen channel sensitivity disparities up to :math:`\pm 3.0\text{ dB}`. +* **ASR Dynamic Headroom Constraints**: Deep learning Automatic Speech Recognition (ASR) acoustic models achieve minimum Word Error Rates (WER) when conversational speech input resides consistently between :math:`-22\text{ dBFS}` and :math:`-12\text{ dBFS}` RMS. Unamplified far-field speech (e.g. at 3 meters distance) often registers at :math:`-45\text{ dBFS}` to :math:`-35\text{ dBFS}`, requiring up to :math:`+18\text{ dB}` of digital boost. +* **Acoustic Shock & Digital Saturation**: Applying a static :math:`+18\text{ dB}` boost to capture pipelines causes severe digital clipping whenever a user speaks close to the microphone or claps, corrupting downstream beamforming (TDFB) and noise suppression (RTNR) algorithms. + +.. _fig_level_multiplier_aria_signal_chain: + +.. figure:: images/level_multiplier_aria_signal_chain.svg + :alt: Dual-Stage Dynamic Range Architecture: Level Multiplier and Aria AGC Signal Chain + :align: center + :width: 100% + + Dual-Stage Dynamic Range Architecture: Static Q9.23 Transducer Leveling (0 ms Delay) paired with Dynamic Lookahead Regressive Pre-Amplification (1 ms Invariant Latency) + +Component Comparison & Trade-Offs +--------------------------------- + +To clarify component boundaries within SOF signal graphs, :numref:`tab_dynamic_range_comp` compares the Level Multiplier and Aria against related level control components: + +.. _tab_dynamic_range_comp: + +.. list-table:: Architectural Comparison: Level Multiplier vs Aria vs Volume vs DRC + :widths: 18 22 20 20 20 + :header-rows: 1 + + * - Parameter + - Level Multiplier + - Aria AGC + - Volume Control + - DRC (Compressor) + * - **Numeric Format** + - Linear Q9.23 fixed-point + - Mode index :math:`\text{att} \in \{0,1,2,3\}` + - Linear Q1.31 / Log dB + - Multi-segment knee curves + * - **Gain Range** + - :math:`-138.47\text{ dB}` to :math:`+48.17\text{ dB}` + - :math:`0\text{ dB}`, :math:`+6\text{ dB}`, :math:`+12\text{ dB}`, :math:`+18\text{ dB}` + - :math:`-\infty\text{ dB}` to :math:`0\text{ dB}` + - Threshold, ratio, makeup gain + * - **Algorithmic Latency** + - **0.000 ms** (Instantaneous) + - **1.000 ms** (Lookahead circular ring) + - **0.000 ms** (Instantaneous) + - :math:`0\text{ ms}` to :math:`5\text{ ms}` (Lookahead) + * - **Smoothing Profile** + - Direct scalar application + - Continuous per-sample linear ramp + - Multi-ms linear ramp + - Attack/release envelope filters + * - **Fast-Path Bypass** + - Automated memory copy at :math:`0\text{ dB}` + - Invariant ring routing at :math:`0\text{ dB}` + - Scalar bypass at :math:`0\text{ dB}` + - None + * - **Primary Operating Role** + - Transducer sensitivity trim & inter-stage matching + - Dynamic speech lift & peak anti-clipping clamp + - User volume slider control + - Speaker transducer excursion & thermal protection + +Level Multiplier: Fixed-Point Q9.23 Precision Scaling +===================================================== + +The Level Multiplier component (:file:`src/audio/level_multiplier`) implements deterministic, ultra-low-latency linear amplification and attenuation across all active stream channels. + +Q9.23 Fixed-Point Number System +------------------------------- + +Linear gain is parameterized as a 32-bit signed integer using the **Q9.23** numeric format defined in :file:`level_multiplier.h`: + +.. code-block:: c + + #define LEVEL_MULTIPLIER_QXY_X 9 + #define LEVEL_MULTIPLIER_QXY_Y 23 + #define LEVEL_MULTIPLIER_GAIN_ONE (1 << LEVEL_MULTIPLIER_QXY_Y) // 0x00800000 = 8388608 + +The 32-bit word allocates bits as follows: + +* **Bit 31 (Sign Bit)** (:math:`s`): Supports non-inverting (:math:`s=0`) and phase-inverting (:math:`s=1`) gain factors. +* **Bits 30..23 (8 Integer Bits)** (:math:`I`): Represents integer magnitudes from :math:`0` to :math:`255` (:math:`2^8 - 1`). +* **Bits 22..0 (23 Fractional Bits)** (:math:`F`): Provides fractional resolution with an elementary quantization step of: + +.. math:: + + \Delta = 2^{-23} \approx 1.1920928955 \times 10^{-7} + +The linear gain factor represented by a Q9.23 word is: + +.. math:: + + \text{Gain}_{\text{linear}} = (-1)^s \cdot \left( I + \frac{F}{2^{23}} \right) = \frac{\text{raw\_gain\_word}}{2^{23}} + +Gain Range Extremes & Unity Definition +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The Q9.23 representation spans a total dynamic range of :math:`186.64\text{ dB}`: + +1. **Maximum Positive Boost**: + Represented by the maximum positive signed 32-bit integer :math:`\text{0x7FFFFFFF}` (:math:`2^{31} - 1`): + + .. math:: + + \text{Gain}_{\text{max}} = 256.0 - 2^{-23} \approx 255.99999988 \implies G_{\text{max}} = 20 \log_{10}(256) \approx +48.1648\text{ dB} + +2. **Unity Gain (0 dB Point)**: + Represented when the integer component equals :math:`1` and fractional bits are zero: + + .. math:: + + \text{LEVEL\_MULTIPLIER\_GAIN\_ONE} = 1 \cdot 2^{23} = 8,388,608 = \text{0x00800000} \implies 20 \log_{10}(1.0) = 0.000\text{ dB} + +3. **Minimum Non-Zero Resolution**: + Represented by a single least-significant bit (LSB) :math:`\text{0x00000001}`: + + .. math:: + + \text{Gain}_{\text{min}} = 2^{-23} \implies G_{\text{min}} = 20 \log_{10}(2^{-23}) \approx -138.4739\text{ dB} + +4. **Digital Silence / Complete Mute**: + Setting :math:`\text{gain} = \text{0x00000000}` mutes the audio stream (:math:`-\infty\text{ dB}`). + +.. _fig_level_multiplier_q9_23_dynamic_range: + +.. figure:: images/level_multiplier_q9_23_dynamic_range.svg + :alt: Fixed-Point Q9.23 Number System and Level Multiplier Dynamic Range + :align: center + :width: 100% + + Fixed-Point Q9.23 Number System: Bit Allocation, Dynamic Range Extremes, and Fast-Path Unity Setpoint + +Decibel to Q9.23 Conversion Matrix +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To compute the 32-bit Q9.23 integer value for a target decibel adjustment :math:`G_{\text{dB}}`: + +.. math:: + + \text{gain}_{\text{Q9.23}} = \left\lfloor 10^{\frac{G_{\text{dB}}}{20}} \cdot 2^{23} + 0.5 \right\rfloor + +.. _tab_q9_23_conversion: + +.. list-table:: Standard Decibel to Q9.23 Conversion & Quantization Precision Matrix + :widths: 18 22 22 20 18 + :header-rows: 1 + + * - Target Gain (:math:`G_{\text{dB}}`) + - Linear Factor + - Decimal Integer + - Hexadecimal Value + - Error (:math:`\text{dB}`) + * - **+40.0 dB** + - :math:`100.000000\times` + - 838,860,800 + - ``0x32000000`` + - :math:`< 10^{-6}` + * - **+30.0 dB** + - :math:`31.622777\times` + - 265,331,865 + - ``0x0FD0A499`` + - :math:`-0.000001` + * - **+20.0 dB** + - :math:`10.000000\times` + - 83,886,080 + - ``0x05000000`` + - :math:`< 10^{-6}` + * - **+12.0 dB** + - :math:`3.981072\times` + - 33,395,684 + - ``0x01FD0AE4`` + - :math:`-0.000001` + * - **+6.0 dB** + - :math:`1.995262\times` + - 16,737,557 + - ``0x00FF6515`` + - :math:`-0.000002` + * - **+3.5 dB** + - :math:`1.496236\times` + - 12,551,334 + - ``0x00BF84A6`` + - :math:`< 10^{-6}` + * - **0.0 dB (Unity)** + - :math:`1.000000\times` + - 8,388,608 + - ``0x00800000`` + - :math:`0.000000` + * - **-6.0 dB** + - :math:`0.501187\times` + - 4,204,263 + - ``0x00402767`` + - :math:`-0.000001` + * - **-12.0 dB** + - :math:`0.251189\times` + - 2,107,123 + - ``0x00202773`` + - :math:`-0.000001` + * - **-20.0 dB** + - :math:`0.100000\times` + - 838,861 + - ``0x000CCCCD`` + - :math:`+0.000002` + * - **-30.0 dB** + - :math:`0.031623\times` + - 265,271 + - ``0x00040C37`` + - :math:`-0.000012` + * - **-40.0 dB** + - :math:`0.010000\times` + - 83,886 + - ``0x000147AE`` + - :math:`-0.000008` + +Multi-Format Processing Kernels & Shift Mathematics +--------------------------------------------------- + +The Level Multiplier provides optimized inner processing loops for three standard PCM container formats: :c:macro:`SOF_IPC_FRAME_S16_LE`, :c:macro:`SOF_IPC_FRAME_S24_4LE`, and :c:macro:`SOF_IPC_FRAME_S32_LE`. + +In all three kernels, the product of an audio sample and the Q9.23 multiplier is right-shifted by a constant calculated using the SOF fixed-point shift macro: + +.. math:: + + Q\_\text{SHIFT\_BITS}(X, Y, Z) = X + Y - Z + +Where :math:`X` is input fractional bits, :math:`Y = 23` is Q9.23 fractional bits, and :math:`Z` is output fractional bits: + +.. math:: + + \text{Shift}_{S16} = 15 + 23 - 15 = 23 + +.. math:: + + \text{Shift}_{S24} = 23 + 23 - 23 = 23 + +.. math:: + + \text{Shift}_{S32} = 31 + 23 - 31 = 23 + +Because the right-shift is universally **23 bits**, the :math:`2^{23}` unity scaling factor cancels identically across all container formats: + +.. code-block:: c + + // S16_LE Inner Processing Loop + *y = q_multsr_sat_32x32_16(*x, gain, 23); // Clamped to [-32768, 32767] + + // S24_4LE Inner Processing Loop + *y = q_multsr_sat_32x32_24(sign_extend_s24(*x), gain, 23); // Clamped to [-8388608, 8388607] + + // S32_LE Inner Processing Loop + *y = q_multsr_sat_32x32(*x, gain, 23); // Clamped to [-2147483648, 2147483647] + +Zero-Overhead Fast-Path Memory Copy Bypass +------------------------------------------ + +To minimize processor cycle consumption during standard passthrough, :c:func:`level_multiplier_process` inspects the active gain variable on every processing chunk before dispatching arithmetic routines: + +.. code-block:: c + + if (cd->gain != LEVEL_MULTIPLIER_GAIN_ONE) + /* Process audio data with requested Q9.23 multiplier */ + return cd->level_multiplier_func(mod, source, sink, frames); + + /* Gain is exactly 0 dB (0x00800000): bypass arithmetic loops entirely */ + source_to_sink_copy(source, sink, true, frames * cd->frame_bytes); + return 0; + +When unity gain is active: + +* **Zero Arithmetic Operations**: No multiplication, shifting, sign-extension, or saturation routines are executed. +* **Maximum Memory Throughput**: :c:func:`source_to_sink_copy` leverages 64-bit or 128-bit wide word block transfers. +* **Power Conservation**: Active DSP core cycles and dynamic power consumption drop to bare memory bus copy minimums. + +Aria AGC: Dynamic Regimes & Lookahead Limiter +============================================= + +The Aria subsystem (:file:`src/audio/aria`) is an intelligent dynamic range pre-amplifier and peak limiter designed to boost low-amplitude signals while protecting audio pipelines from digital saturation. + +Operating Modes & Dynamic Headroom Thresholds +--------------------------------------------- + +Aria provides four pre-amplification boost modes configured via the unsigned integer parameter :math:`\text{att} \in \{0, 1, 2, 3\}`: + +.. math:: + + G_{\text{target}} = 2^{\text{att}} \implies G_{\text{target,dB}} = 20 \log_{10}(2^{\text{att}}) + +To prevent output samples from exceeding positive full scale (:math:`A_{\text{FS}} = 2^{23} - 1 = \text{0x007FFFFF}` in 24-bit representation), the linear input headroom threshold is: + +.. math:: + + A_{\text{thresh}} = \frac{A_{\text{FS}}}{2^{\text{att}}} = \text{0x007FFFFF} \gg \text{att} + +.. _tab_aria_modes: + +.. list-table:: Aria Operating Modes, Target Boosts, Headroom Thresholds & Dynamic Ducking Ratios + :widths: 12 18 22 24 24 + :header-rows: 1 + + * - Mode (:math:`\text{att}`) + - Linear Factor + - Target Boost (:math:`\text{dB}`) + - Threshold (:math:`A_{\text{thresh}}`) + - Headroom Limit (:math:`\text{dBFS}`) + * - **0** + - :math:`1.0\times` + - :math:`0.00\text{ dB}` (Bypass) + - ``0x007FFFFF`` (8,388,607) + - :math:`0.00\text{ dBFS}` + * - **1** + - :math:`2.0\times` + - :math:`+6.02\text{ dB}` + - ``0x003FFFFF`` (4,194,303) + - :math:`-6.02\text{ dBFS}` + * - **2** + - :math:`4.0\times` + - :math:`+12.04\text{ dB}` + - ``0x001FFFFF`` (2,097,151) + - :math:`-12.04\text{ dBFS}` + * - **3** + - :math:`8.0\times` + - :math:`+18.06\text{ dB}` + - ``0x000FFFFF`` (1,048,575) + - :math:`-18.06\text{ dBFS}` + +Mathematical Proof of Strict 0 dBFS Clamping +-------------------------------------------- + +For every processing chunk (e.g. 48 frames spanning :math:`1\text{ ms}` at 48 kHz), Aria detects the peak absolute amplitude across all channels: + +.. math:: + + \text{max\_data} = \max_{k \in \text{chunk}, ch} |x[k, ch]| + +The processing distinguishes between two regimes: + +1. **Linear Pre-Amplification Regime** (:math:`\text{max\_data} \le A_{\text{thresh}}`): + The signal resides safely within available headroom. The normalized gain word is set to maximum fractional scale: + + .. math:: + + g = 2^{31} - 1 = \text{0x7FFFFFFF} + + Output samples are scaled by the dynamic shift :math:`\text{shift} = 31 - \text{att}`: + + .. math:: + + y[n] = \frac{x[n] \cdot g}{2^{31 - \text{att}}} \approx x[n] \cdot 2^{\text{att}} + +2. **Regressive Ducking Limiter Regime** (:math:`\text{max\_data} > A_{\text{thresh}}`): + Applying the target boost would push the output past :math:`0\text{ dBFS}`. Aria computes a regressive gain word via 64-bit integer division: + + .. math:: + + \text{gain} = \left\lfloor \frac{A_{\text{FS}} \cdot 2^{32}}{\text{max\_data}} \right\rfloor \implies g = \text{gain} \gg (\text{att} + 1) = \left\lfloor \frac{A_{\text{FS}} \cdot 2^{31}}{\text{max\_data} \cdot 2^{\text{att}}} \right\rfloor + + Applying this gain with shift :math:`31 - \text{att}`: + + .. math:: + + y[n] = \frac{x[n] \cdot \left( \frac{A_{\text{FS}} \cdot 2^{31}}{\text{max\_data} \cdot 2^{\text{att}}} \right)}{2^{31 - \text{att}}} = \frac{x[n] \cdot A_{\text{FS}} \cdot 2^{31}}{\text{max\_data} \cdot 2^{31}} = x[n] \cdot \frac{A_{\text{FS}}}{\text{max\_data}} + + Evaluating this equation for the peak sample in the chunk (:math:`|x_{\text{peak}}| = \text{max\_data}`): + + .. math:: + + |y_{\text{peak}}| = \text{max\_data} \cdot \frac{A_{\text{FS}}}{\text{max\_data}} \equiv A_{\text{FS}} = 2^{23} - 1 = \text{0x007FFFFF} \equiv 0.00\text{ dBFS} + +The peak output is locked strictly at :math:`0.00\text{ dBFS}`. Digital overflow and flat-top clipping distortion are mathematically impossible. + +.. _fig_aria_dynamic_regimes_transfer_function: + +.. figure:: images/aria_dynamic_regimes_transfer_function.svg + :alt: Aria AGC Transfer Functions and Regressive Headroom Dynamics + :align: center + :width: 100% + + Aria AGC Transfer Functions: Target Linear Boost (0/6/12/18 dB), Knee Thresholds, and Strict 0 dBFS Clamping + +1 ms Lookahead Ring Buffer & Invariant Latency +-------------------------------------------------------------- + +To prevent transient overshoot distortion, Aria evaluates future audio peaks before they reach the output multiplier using an internal circular delay buffer: + +.. math:: + + \text{buff\_size} = \text{ALIGN\_UP}(\text{chan\_cnt} \cdot \text{smpl\_group\_cnt}, 2) + +where :math:`\text{smpl\_group\_cnt}` represents the number of samples per channel in :math:`1\text{ ms}` (e.g. 48 samples at 48 kHz). + +The component executes a 4-step phased pipeline: + +1. **Step 1 (Future Inspection)** (:math:`t + 1\text{ ms}`): :c:func:`aria_algo_calc_gain` inspects incoming samples in the source buffer, calculates :math:`\text{max\_data}`, and records the required gain into the history table. +2. **Step 2 (Delayed Processing)** (:math:`t`): :c:func:`aria_algo_get_data` reads past audio from the circular buffer (which entered :math:`1\text{ ms}` prior), multiplies samples by the linearly interpolated gain, and emits protected audio to the sink buffer. +3. **Step 3 (Ring Ingestion)**: :c:func:`cir_buf_copy` copies the future incoming audio from the source into the circular buffer at ``cd->data_ptr``. +4. **Step 4 (Pointer Wrap)**: The circular buffer pointer wraps using :c:func:`cir_buf_wrap`. + +.. note:: + When :math:`\text{att} = 0` (Bypass), Aria routes audio through the circular delay buffer without arithmetic scaling. The pipeline latency is **identically 1.000 ms in all modes**, ensuring that downstream multichannel beamforming phases remain perfectly invariant when toggling modes. + +10-State Minimum-Envelope Search & Anti-Zipper Linear Ramp +-------------------------------------------------------------------------- + +To eliminate audible zipper noise and clicks during rapid gain changes, Aria employs a 10-state sliding window and continuous per-sample linear interpolation: + +1. **Minimum-Envelope Search**: Aria searches across a multi-state window of past and lookahead gains: + + .. code-block:: c + + int32_t gain_begin = cd->gains[sof_aria_index_tab[gain_state + 2]]; + int32_t gain_end = cd->gains[sof_aria_index_tab[gain_state + 3]]; + + for (i = 1; i < ARIA_MAX_GAIN_STATES - 1; i++) { + if (cd->gains[sof_aria_index_tab[gain_state + 2 + i]] < gain_begin) + gain_begin = cd->gains[sof_aria_index_tab[gain_state + 2 + i]]; + if (cd->gains[sof_aria_index_tab[gain_state + 3 + i]] < gain_end) + gain_end = cd->gains[sof_aria_index_tab[gain_state + 3 + i]]; + } + + By selecting the *minimum* gain across the window, Aria pulls gain downward *before* an acoustic transient arrives at the output. +2. **Per-Sample Linear Stepping**: Across the :math:`N` frames of the chunk, the per-sample step increment is: + + .. math:: + + \text{step} = \frac{\text{gain\_end} - \text{gain\_begin}}{\text{frames}} + + For each sample :math:`n`, the active gain updates continuously: :math:`\text{gain}_{n+1} = \text{gain}_n + \text{step}`, ensuring :math:`C^0` continuity and eliminating spectral splatter. + +.. _fig_aria_lookahead_timing_ramp: + +.. figure:: images/aria_lookahead_timing_ramp.svg + :alt: Aria 1 ms Lookahead Circular Buffer and Linear Ramp Engine + :align: center + :width: 100% + + Aria 1 ms Lookahead Circular Buffer: Timeline Phasing, 10-State Minimum Envelope Search, and Anti-Zipper Linear Ramp + +Synergistic Front-End Design & Dynamic Headroom Budgeting +========================================================= + +When designing high-performance voice capture front-ends, Level Multiplier and Aria operate in series: + +.. code-block:: text + + [DMIC Gateway] ──> [DC Block] ──> [Level Multiplier] ──> [Aria AGC] ──> [TDFB] ──> [RTNR] ──> [ASR Engine] + +Dynamic Headroom Budget Allocation +---------------------------------- + +1. **Raw Sensor Ingress**: + A digital microphone with nominal sensitivity :math:`-26.0\text{ dBFS}` at :math:`94\text{ dBSPL}` produces conversational speech (65 dBSPL at 0.5m) at approximately :math:`-55.0\text{ dBFS}` RMS. +2. **Stage 1 (Level Multiplier Sensitivity Calibration)**: + Individual capsule variations (e.g. Channel 0 at :math:`-24.5\text{ dBFS}`, Channel 1 at :math:`-26.0\text{ dBFS}`) are aligned by Level Multiplier trims (:math:`-1.5\text{ dB}` and :math:`0.0\text{ dB}`). Channel 1 automatically engages the fast-path memory copy bypass. All channels enter downstream processing aligned within :math:`\pm 0.05\text{ dB}`, preserving beamformer (TDFB) spatial directivity. +3. **Stage 2 (Aria AGC Target Speech Lift)**: + Configured for Mode 2 (:math:`\text{att} = 2`, :math:`+12.04\text{ dB}` boost), Aria lifts conversational speech from :math:`-55\text{ dBFS}` to :math:`-43\text{ dBFS}` RMS, positioning speech features closer to the ASR optimum. +4. **Stage 3 (Acoustic Shock Clamping)**: + If the speaker shouts at :math:`110\text{ dBSPL}` (+16 dB transient surge), the input peak reaches :math:`-10\text{ dBFS}` (exceeding :math:`A_{\text{thresh}} = -12.04\text{ dBFS}`). Aria automatically ducks gain by :math:`10.0\text{ dB}`, locking the peak output strictly at :math:`0.00\text{ dBFS}` without clipping. + +Control Plane & ABI Architecture +================================ + +Both components conform to the Intel IPC4 architecture and support runtime parameter injection without tearing down active pipelines. + +Level Multiplier IPC4 ABI +------------------------- + +The Level Multiplier configuration handler (:file:`level_multiplier-ipc4.c`) receives 4-byte payloads: + +* **Fragment Size**: Strictly 4 bytes (`sizeof(int32_t)`). +* **Payload Format**: 32-bit signed integer in little-endian representing the Q9.23 linear multiplier. +* **Component UUID**: ``30397456-4661-4644-97e5-39a9e5ab1778`` (Topology GUID: ``56:74:39:30:61:46:44:46:97:e5:39:a9:e5:ab:17:78``). + +.. code-block:: text + + # ALSA Topology 2 Widget Declaration (tools/topology/topology2/include/components/level_multiplier.conf) + Class.Widget."level_multiplier" { + uuid "56:74:39:30:61:46:44:46:97:e5:39:a9:e5:ab:17:78" + type "effect" + no_pm "true" + num_input_pins 1 + num_output_pins 1 + } + +Aria IPC4 ABI +------------- + +Aria defines module initialization and runtime controls in :file:`aria.h`: + +* **Initialization Configuration**: + + .. code-block:: c + + struct ipc4_aria_module_cfg { + struct ipc4_base_module_cfg base_cfg; + uint32_t attenuation; // att in {0, 1, 2, 3} + } __packed __aligned(8); + +* **Runtime Parameter ID**: :c:macro:`ARIA_SET_ATTENUATION` (1). +* **Payload**: 32-bit unsigned integer representing target attenuation mode :math:`\text{att}`. +* **Component UUID**: ``6d:16:f7:99:2c:37:ef:43:81:f6:22:00:7a:a1:5f:03``. + +Standalone Python Calibration Toolchain Runbook +=============================================== + +SOF provides the standalone Python calibration utility :file:`sof_level_multiplier_aria_tool.py` located at :file:`tools/tune/level_multiplier_aria/`. + +.. _fig_level_multiplier_aria_tuning_workflow: + +.. figure:: images/level_multiplier_aria_tuning_workflow.svg + :alt: End-to-End Level Multiplier and Aria AGC Tuning Workflow + :align: center + :width: 100% + + End-to-End Tuning Workflow: Transducer Metrology, Q9.23 Gain Trim, Dynamic Headroom Sizing, and DUT Verification + +Subcommand 1: Q9.23 Gain Conversion (`calc-gain`) +------------------------------------------------- + +Convert decibels to Q9.23 integer, verify quantization error, and check fast-path bypass eligibility: + +.. code-block:: bash + + # Convert +3.5 dB sensitivity trim to Q9.23 integer + python3 tools/tune/level_multiplier_aria/sof_level_multiplier_aria_tool.py calc-gain --db 3.5 + +Example Tool Output: + +.. code-block:: text + + ====================================================================== + SOF Level Multiplier Q9.23 Gain Converter + ====================================================================== + Desired Gain: +3.5000 dB + Linear Multiplier: 1.496236x + Q9.23 Integer: 12551334 (0x00BF84A6) + Quantization Error: -0.000000 dB + Fast-Path Bypass (0dB): NO (Active SIMD scaling) + ---------------------------------------------------------------------- + Format Processing Shifts (Q_SHIFT_BITS): + • S16_LE: 15 + 23 - 15 = 23 bits (Shift = LEVEL_MULTIPLIER_S16_SHIFT) + • S24_4LE: 23 + 23 - 23 = 23 bits (Shift = LEVEL_MULTIPLIER_S24_SHIFT) + • S32_LE: 31 + 23 - 31 = 23 bits (Shift = LEVEL_MULTIPLIER_S32_SHIFT) + ====================================================================== + +Subcommand 2: Aria Dynamic Simulation (`aria-sim`) +-------------------------------------------------- + +Simulate Aria dynamic ducking and verify that output peaks clamp strictly to :math:`0.00\text{ dBFS}`: + +.. code-block:: bash + + # Simulate Mode 2 (+12 dB) with an input peak of -6.0 dBFS + python3 tools/tune/level_multiplier_aria/sof_level_multiplier_aria_tool.py aria-sim --att 2 --peak-dbfs -6.0 + + # Run a full dynamic sweep from -42 dBFS to 0 dBFS + python3 tools/tune/level_multiplier_aria/sof_level_multiplier_aria_tool.py aria-sim --att 2 --sweep + +Example Tool Output (Sweep): + +.. code-block:: text + + ============================================================================== + SOF Aria AGC Dynamic Range & Regressive Limiter Simulator + ============================================================================== + Aria Attenuation Mode: 2 (Target Boost: +12.04 dB) + Headroom Threshold: -12.04 dBFS (A_thresh = 0x001FFFFF / 2097151) + Algorithmic Lookahead: 1.000 ms (Circular buffer delay) + ------------------------------------------------------------------------------ + Input (dBFS) Peak Amplitude Regime Output (dBFS) Ducking (dB) + ------------------------------------------------------------------------------ + -42.0 dBFS 66633 Linear Pre-Amplification -29.96 dBFS +0.00 dB + -36.0 dBFS 132950 Linear Pre-Amplification -23.96 dBFS +0.00 dB + -30.0 dBFS 265271 Linear Pre-Amplification -17.96 dBFS +0.00 dB + -24.0 dBFS 529285 Linear Pre-Amplification -11.96 dBFS +0.00 dB + -18.0 dBFS 1056063 Linear Pre-Amplification -5.96 dBFS +0.00 dB + -15.0 dBFS 1491729 Linear Pre-Amplification -2.96 dBFS +0.00 dB + -12.0 dBFS 2107123 Regressive Ducking Limiter -0.00 dBFS +12.00 dB + -9.0 dBFS 2976390 Regressive Ducking Limiter -0.00 dBFS +9.00 dB + -6.0 dBFS 4204263 Regressive Ducking Limiter -0.00 dBFS +6.00 dB + -3.0 dBFS 5938679 Regressive Ducking Limiter -0.00 dBFS +3.00 dB + +0.0 dBFS 8388607 Regressive Ducking Limiter +0.00 dBFS +0.00 dB + ============================================================================== + Verification: Output peak amplitude is strictly clamped <= 0.00 dBFS across all inputs. + +Subcommand 3: Multi-Channel Array Calibration (`calibrate`) +----------------------------------------------------------- + +Calibrate a 4-channel microphone array to :math:`-26.0\text{ dBFS}` reference sensitivity and export binary configuration blobs: + +.. code-block:: bash + + python3 tools/tune/level_multiplier_aria/sof_level_multiplier_aria_tool.py calibrate \ + --sens -24.5 -26.0 -25.2 -27.1 \ + --target-sens -26.0 \ + --target-headroom -20.0 \ + --out-dir /tmp/dmic_calib_blobs + +Example Tool Output: + +.. code-block:: text + + ================================================================================ + SOF Multi-Microphone Array Sensitivity Calibration & Dynamic Sizing + ================================================================================ + Number of Channels: 4 + Target Sensitivity: -26.00 dBFS (at 94 dBSPL / 1 kHz reference) + ASR Headroom Target: -20.00 dBFS nominal speech + -------------------------------------------------------------------------------- + Ch Meas (dBFS) Trim (dB) Q9.23 Int Hex Fast-Path + -------------------------------------------------------------------------------- + 0 -24.50 -1.50 7058134 0x006BB2D6 NO + 1 -26.00 +0.00 8388608 0x00800000 YES (0 dB) + 2 -25.20 -0.80 7650501 0x0074BCC5 NO + 3 -27.10 +1.10 9521161 0x00914809 NO + -------------------------------------------------------------------------------- + Aria Pre-Amplification Recommendation: + • Recommended Mode: att = 2 (+12.04 dB Boost) + • Headroom Margin: Transients duck regressively above -12.04 dBFS + • Anti-Clipping: Strict 0 dBFS hard clamp guarantees zero ASR front-end saturation + ================================================================================ + +Production Calibration Recipes +============================== + +The following recipes represent validated configurations across target hardware deployments: + +.. _tab_level_aria_recipes: + +.. list-table:: Production Calibration Presets: Level Multiplier & Aria Configurations + :widths: 22 24 24 30 + :header-rows: 1 + + * - Deployment Target + - Level Multiplier Setting + - Aria Mode Setting + - Acoustic Characteristics + * - **Recipe 1: Far-Field Smart Speaker** + - Per-channel trim (:math:`\pm 2.5\text{ dB}` to align array) + - Mode 2 (:math:`\text{att} = 2`, :math:`+12.04\text{ dB}`) + - Boosts distant speech (3m) into optimal ASR window; clamps shouting shocks. + * - **Recipe 2: Laptop Video Conferencing** + - Unity gain (:math:`0.0\text{ dB}`, Fast-Path engaged) + - Mode 1 (:math:`\text{att} = 1`, :math:`+6.02\text{ dB}`) + - Minimal power overhead; gentle pre-amplification for close-talk voice calls. + * - **Recipe 3: Industrial High-SPL Environment** + - Headroom padding (:math:`-6.0\text{ dB}`) + - Mode 0 (:math:`\text{att} = 0`, Bypass with active limiter) + - Preserves :math:`6\text{ dB}` extra digital headroom; prevents factory noise clipping. + +Interactive Live Injection & Diagnostics Matrix +=============================================== + +Runtime Gain Verification via sof-ctl +------------------------------------- + +To query and dynamically adjust Level Multiplier and Aria parameters over SSH on a target DUT: + +.. code-block:: bash + + # Step 1: Query mixer controls on active sound card + ssh root@ "amixer -c 0 scontrols | grep -E 'level_multiplier|aria'" + + # Step 2: Set Aria attenuation mode to Mode 2 (+12 dB boost) via ALSA mixer + ssh root@ 'amixer -c 0 cset name="aria.1.1.extctl" 2' + + # Step 3: Inject Q9.23 gain blob (+3.5 dB, 0x00BF84A6) into Level Multiplier + # Write binary 4-byte word: \xa6\x84\xbf\x00 + python3 tools/tune/level_multiplier_aria/sof_level_multiplier_aria_tool.py build-blob \ + --comp level_multiplier --gain-db 3.5 --out /tmp/lvmult_gain.bin + scp /tmp/lvmult_gain.bin root@:/tmp/lvmult_gain.bin + ssh root@ "sof-ctl -D hw:0 -n 'level_multiplier.1.1.extctl' -s /tmp/lvmult_gain.bin" + + # Step 4: Record audio test stream and verify peak levels + ssh root@ "arecord -D hw:0,0 -f S24_4LE -c 2 -r 48000 -d 5 /tmp/test_capture.wav" + ssh root@ "sox /tmp/test_capture.wav -n stats" + +Diagnostic Troubleshooting Matrix +--------------------------------- + +.. _tab_level_aria_troubleshooting: + +.. list-table:: Diagnostic Troubleshooting Matrix: Level Multiplier & Aria + :widths: 22 24 24 30 + :header-rows: 1 + + * - Symptom + - Root Cause + - Diagnostic Procedure + - Remediation Action + * - **Digital Saturation / Harsh Flat-Top Clipping** + - Aria target boost set too high without regressive limiting, or Level Multiplier set past :math:`+30\text{ dB}` without headroom. + - Inspect recorded WAV with :command:`sox stats`; check for ``Flat factor > 0.00``. + - Enable Aria in capture chain; ensure :math:`\text{att} \ge 1` so regressive limiter clamps peaks to :math:`0.00\text{ dBFS}`. + * - **Audible Zipper Noise / Discontinuities** + - Modifying Level Multiplier Q9.23 gain at high frequency during active audio without ramp smoothing. + - Inspect audio spectrogram for broadband spectral vertical lines. + - Use Level Multiplier strictly for static initialization/calibration; use Aria or Volume for runtime dynamic ramps. + * - **Degraded Beamformer (TDFB) Directivity** + - Gain mismatch across microphone array channels exceeding :math:`\pm 0.5\text{ dB}`. + - Run acoustic sweep with APx555; check inter-channel RMS levels. + - Re-run :command:`sof_level_multiplier_aria_tool.py calibrate` to equalize channel sensitivities to target. + * - **Excessive DSP Cycle Footprint (High CPC)** + - Level Multiplier running active multiplication loops on unity gain stream (:math:`0\text{ dB}`). + - Inspect DSP CPC using :command:`dut-monitor --telemetry`. + - Ensure gain is set exactly to :c:macro:`LEVEL_MULTIPLIER_GAIN_ONE` (``0x00800000``) to trigger fast-path bypass. + * - **Inter-Channel Phase Cancellation** + - Negative Q9.23 gain word accidentally inverting channel phase by :math:`180^\circ`. + - Inspect cross-channel correlation in recorded stereo/quad WAV. + - Verify sign bit :math:`b_{31} = 0`; ensure Q9.23 integer is strictly positive unless phase inversion is intended. + +Related Documentation +===================== + +* :ref:`dmic_tuning`: Digital Microphone Acoustic Calibration & Decimation Tuning Guide. +* :ref:`drc_tuning`: Dynamic Range Compression & Multiband DRC Tuning Guide. +* :ref:`smart_amp_tuning`: Smart Amplifier (DSM) & Transducer Protection Calibration Guide. +* :ref:`sound_dose_tuning`: Sound Dose / Hearing Health Calibration & Acoustic Protection Guide. +* :ref:`runtime_tuning_sof_ctl`: Unified Runtime Tuning, Control Blobs & Parameter Injection Guide. +* :ref:`time-domain-fixed-beamformer`: Time Domain Fixed Beamformer (TDFB) Architecture & Array Tuning. From d14fa7c8d933541796efcd0f4b8934cb72602d26 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 17:33:06 +0100 Subject: [PATCH 38/64] doc: developer_guides: add mfcc & audio feature extraction tuning guide Add comprehensive Mel-Frequency Cepstral Coefficients (MFCC) & Audio Feature Extraction Tuning Guide in developer_guides/tuning/mfcc_tuning.rst. Key technical content: - Psychoacoustic Mel scale frequency warping derivations and Slaney area normalization - Sparse packed triangular filterbank vector storage (>95% SRAM reduction) - Dual operating regimes: Whisper-compatible 80-bin Mel log spectrogram mode and classical 13-coefficient MFCC mode with 16-bit DCT-II and sinusoidal liftering - Integrated A-weighted Voice Activity Detection (VAD) and Discontinuous Transmission (DTX) silence suppression protocol - Complete 116-byte packed struct sof_mfcc_config ABI breakdown table - Production tuning recipes for Keyword Spotting (TFLM), Whisper ASR, and Wake-on-Voice - Interactive live injection runbook and diagnostic troubleshooting matrix - 5 native vector visual diagrams (signal pipeline, filterbank response, spectrogram feature maps, VAD/DTX energy dynamics, and tuning workflow) - Integrated into data/modules.yaml SSOT and developer_guides/index.rst Signed-off-by: Liam Girdwood --- data/modules.yaml | 13 +- developer_guides/index.rst | 6 + ...mfcc_mel_filterbank_frequency_response.svg | 195 +++++ .../mfcc_signal_processing_pipeline.svg | 247 ++++++ .../images/mfcc_spectrogram_feature_map.svg | 237 ++++++ .../tuning/images/mfcc_tuning_workflow.svg | 200 +++++ .../images/mfcc_vad_dtx_timing_energy.svg | 166 ++++ developer_guides/tuning/mfcc_tuning.rst | 784 ++++++++++++++++++ 8 files changed, 1843 insertions(+), 5 deletions(-) create mode 100644 developer_guides/tuning/images/mfcc_mel_filterbank_frequency_response.svg create mode 100644 developer_guides/tuning/images/mfcc_signal_processing_pipeline.svg create mode 100644 developer_guides/tuning/images/mfcc_spectrogram_feature_map.svg create mode 100644 developer_guides/tuning/images/mfcc_tuning_workflow.svg create mode 100644 developer_guides/tuning/images/mfcc_vad_dtx_timing_energy.svg create mode 100644 developer_guides/tuning/mfcc_tuning.rst diff --git a/data/modules.yaml b/data/modules.yaml index 6c90e5df..a64344b7 100644 --- a/data/modules.yaml +++ b/data/modules.yaml @@ -391,12 +391,15 @@ modules: source: "SOF" category: "Voice & Telephony" status: "Upstream" - description: "Speech feature extraction engine computing triangular Mel filterbank energies and DCT-II cepstra." - simd: ["HiFi 3", "Scalar C"] + description: "Speech feature extraction engine computing triangular Mel filterbank energies, Slaney normalization, and DCT-II cepstra." + tuning_guide: "developer_guides/tuning/mfcc_tuning" + simd: ["HiFi 3", "HiFi 4", "Scalar C"] key_features: - - "Configurable triangular Mel filterbanks (100 Hz to 8 kHz)" - - "Discrete Cosine Transform (DCT-II) with cepstral liftering" - - "Direct integration with wake word spotting and ASR frontends" + - "Configurable triangular Mel filterbanks (20 Hz to 8 kHz) with Slaney area normalization" + - "Dual-mode operation: 80-bin Mel spectrogram (Whisper ASR) or 13-cepstra MFCC (TFLM microWakeWord)" + - "Discrete Cosine Transform (DCT-II) with sinusoidal cepstral liftering" + - "Embedded Voice Activity Detection (VAD) and Discontinuous Transmission (DTX) silence suppression" + - "Sparse packed triangular filterbank vector storage with >95% SRAM memory reduction" - id: mic_privacy_manager name: "Microphone Privacy Manager" diff --git a/developer_guides/index.rst b/developer_guides/index.rst index c50df86d..c83c3428 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -140,6 +140,11 @@ Acoustic, Transducer & Array Tuning * :ref:`sample_rate_conversion` (Polyphase FIR filter design and multi-stage resampling) * :ref:`demux` (Multi-channel routing matrix configuration) +Machine Learning & Speech Feature Extraction +============================================ + +* :ref:`mfcc_tuning` (Mel-Frequency Cepstral Coefficients (MFCC), triangular Mel filterbank design, Slaney normalization, Whisper-compatible Mel spectrogram scaling, Voice Activity Detection (VAD), and TensorFlow Lite Micro (TFLM) keyword spotting co-design) + .. toctree:: :maxdepth: 1 @@ -150,6 +155,7 @@ Acoustic, Transducer & Array Tuning tuning/sound_dose_tuning tuning/crossover_tuning tuning/dmic_tuning + tuning/mfcc_tuning algorithms/eq/equalizers_tuning algorithms/tdfb/time_domain_fixed_beamformer algorithms/src/sample_rate_conversion diff --git a/developer_guides/tuning/images/mfcc_mel_filterbank_frequency_response.svg b/developer_guides/tuning/images/mfcc_mel_filterbank_frequency_response.svg new file mode 100644 index 00000000..14c34557 --- /dev/null +++ b/developer_guides/tuning/images/mfcc_mel_filterbank_frequency_response.svg @@ -0,0 +1,195 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + Triangular Mel Filterbank & Slaney Normalization Architecture + Non-Linear Frequency Warping, Critical Band Triangular Spacing, Slaney Energy Area Balance, and Sparse Packed Storage + + + + + + PSYCHOACOUSTIC FREQUENCY WARPING: MEL SCALE VS LINEAR HZ + m = 2595 · log₁₀(1 + f / 700) = 1127 · ln(1 + f / 700) + + + + + + + + + + + 0 Hz + + + 1000 Hz + + + 2000 Hz + + + 4000 Hz + + + 8000 Hz (Nyquist) + + + + 0 Mel + + + 1000 Mel + + + 2840 Mel + + + + Linear Domain (f < 1 kHz) + Logarithmic Compression Domain (f > 1 kHz) + + + + + + + + + + + + TRIANGULAR MEL FILTERBANK: SLANEY AREA NORMALIZATION VS CONSTANT HEIGHT + + + + + Slaney Area-Normalized (Height = 2 / Δf) + + Unnormalized (Height = 1.0) + + + + + + + + + + + 20 Hz (f_low) + + + 1000 Hz + + + 2000 Hz + + + 4000 Hz + + + 8000 Hz (f_high) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Narrow Bands: Dense formants (F0, F1) + Wide Bands: Energy normalized to prevent HF blowout + + + + + + + SOF EMBEDDED DSP OPTIMIZATION: SPARSE TRIANGULAR FILTER PACKING + + + + + Traditional Dense Matrix Storage (Wasteful) + Size = (N_fft/2 + 1) × M_mel = 257 × 80 = 20,560 words (41.1 KB) + 94.2% of matrix entries are zero! Wastes critical DSP SRAM. + + + + + 95% RAM Reduction ➔ + + + + + + SOF Packed Vector Storage (struct psy_mel_filterbank) + [next_triangle_idx, start_fft_bin, length, weight₀..weightₖ] + Total footprint: ~1,850 int16_t words (3.7 KB). Zero-overhead inner loop. + + + diff --git a/developer_guides/tuning/images/mfcc_signal_processing_pipeline.svg b/developer_guides/tuning/images/mfcc_signal_processing_pipeline.svg new file mode 100644 index 00000000..ea52237f --- /dev/null +++ b/developer_guides/tuning/images/mfcc_signal_processing_pipeline.svg @@ -0,0 +1,247 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MFCC Audio Feature Extraction Pipeline Architecture + End-to-End SOF Fixed-Point DSP Engine: Pre-Emphasis, Overlap Framing, FFT, Mel Filterbank, Dual-Mode Mel/DCT, and Embedded VAD/DTX + + + + + + + + + AUDIO INGRESS + PCM Audio In + S16 / S24 / S32 + Mono / Selected Ch + fs = 16000 Hz + + + + + + + + + + STAGE 1: PRE-EMPHASIS + 1st-Order Highpass + y[n] = x[n] - α·x[n-1] + α = 0.97 (Q1.15 = 31785) + +6 dB/oct Spectral Tilt + + + + + + + + + + STAGE 2: FRAMING & WIN + Sliding Window + Frame: 25 ms (400 smp) + Shift: 10 ms (160 smp) + Hamming / Hann / Povey + + + + + + + + + + STAGE 3: FFT & POWER + 512-point Real FFT + P[k] = Re[k]² + Im[k]² + 257 unique freq bins + Δf = 16000/512 = 31.25 Hz + + + + + + + + + + STAGE 4: MEL FILTERBANK + Triangular Filterbank + M = 23 (MFCC) or 80 (Mel) + 20 Hz to 8000 Hz (Nyquist) + Slaney Area Normalized + + + + + + + + + + STAGE 5: LOG COMPRESSION + Perceptual Energy + E_log = ln(E_m + p_min) + p_min = 1e-10 (Floor) + Fixed-Point Q9.23 Format + + + + + + + + + + + + + Path A: Mel-Only Mode (num_ceps = 0, e.g. 80 bins) + + + + Path B: MFCC Cepstral Mode (num_ceps = 13..40) + + + + + + MODE A: WHISPER-COMPATIBLE MEL SPECTROGRAM (80 Bins) + + + 1. Dynamic Peak Tracking: + mmax decay with coeff mmax_coef (Q1.15) + + 2. Headroom Clamp: + clamp(E_mel, mmax - top_db, mmax) + + 3. Scale & Offset: + Out = (E_mel + mel_offset) × mel_scale + + 4. Whisper Setpoint: + mel_offset = 4.0, mel_scale = 0.25 (Q4.12) + + + Direct Input to OpenAI Whisper & End-to-End Neural Networks + + + + + + + + MODE B: CEPSTRAL COEFFICIENT ENGINE (13 Coefficients) + + + 1. Q-Format Conversion: + Truncate Q9.23 to Q9.7 16-bit integers + + 2. 16-bit DCT-II Matrix: + c_n = Σ_m E_m · cos(π n (m + 0.5) / M) + + 3. Cepstral Liftering: + w_n = 1 + (L/2)·sin(π·n / L), L = 22.0 + + 4. Variance Equalization: + Balances high/low order cepstral noise + + + Input Tensor to TensorFlow Lite Micro & microWakeWord + + + + + + + + + INTEGRATED ENERGY SUBSYSTEM: VOICE ACTIVITY DETECTION (VAD) & DISCONTINUOUS TRANSMISSION (DTX) + + + + + + 1. A-Weighted Noise Floor Tracking + • IEC 61672-1:2013 A-weighting interpolated + • Weights w_i in Q1.15, peak normalized at 2.5 kHz + Floor[i] = instant drop, slow α rise + • α_fast = 0.020 (init 1s), α_slow = 0.003 (Q1.15 = 98) + Adaptive background acoustic tracking + + + + + + 2. Speech Detection & Hangover + • Signal Energy: E_sig = Σ w_i · Mel[i] (Q9.23) + • Noise Energy: E_noise = Σ w_i · Floor[i] (Q9.23) + ΔE = E_sig - E_noise > 0.30 (Q9.23 = 2516582) + • Hangover = 20 hops (200 ms) speech bridge + Eliminates speech clipping & chatter + + + + + + 3. DTX Protocol & struct mfcc_data_header + struct { magic=0x6d666363, frame_num, + energy, noise_energy, vad_flag }; + • Prepend 24-byte header before features + • DTX silences frames when vad=0 (trailing: 20) + > 80% Host Bus & Memory Bandwidth Savings + + + diff --git a/developer_guides/tuning/images/mfcc_spectrogram_feature_map.svg b/developer_guides/tuning/images/mfcc_spectrogram_feature_map.svg new file mode 100644 index 00000000..4ecc9351 --- /dev/null +++ b/developer_guides/tuning/images/mfcc_spectrogram_feature_map.svg @@ -0,0 +1,237 @@ + + + + + + + + + + + + + + + + + + + + + + + Feature Representation Evolution: Waveform to 2D ML Tensor + Dimensionality Reduction from 16 kHz Time Samples to Log Mel Spectrogram (80 Bins) and DCT-II MFCC Cepstra (13 Coefficients) + + + + + + + + 1. RAW AUDIO (TIME) + + + + + + + + Utterance: "Hey SOF" (1.0 s) + + + + + Sample Rate: + 16,000 smp/s + + Total Samples: + 16,000 (32 KB) + + Information: + Phase + Amplitude + + Redundancy: + Extremely High + + + Contains raw acoustic pressure; + too sparse for compact CNNs. + + + + + + + + 2. LINEAR STFT SPECTRUM + + + + + + + + + + + + + + + + + Sparse High-Freq Tail (> 4 kHz) + + + + + FFT Size: + 512 points + + Output Bins: + 257 freq bins + + Frame Hop: + 10 ms (100 fps) + + Data Volume: + 25.7 k-values/s + + + Equal 31.25 Hz resolution; + wastes bins on high frequencies. + + + + + + + + 3. LOG MEL SPECTROGRAM + + + + + + + + + + + + + + Cochlear Critical Bands (F0..F3) + + + + + Mel Bins: + 80 bands (Slaney) + + Dynamic Span: + 80 dB (top_db clamp) + + Data Volume: + 8.0 k-values/s + + Target Model: + Whisper / Conformer + + + Standard ASR Feature Representation; + captures human ear perception. + + + + + + + + 4. MFCC TENSOR (13 Coeffs) + + + + + + + + c0: Energy + + + + c1: Tilt + + + + c2-c5: Formants + + + + c6-c12: Detail + + + + + Cepstra Count: + 13 (DCT-II) + + Lifter Parameter: + L = 22.0 (Sinusoidal) + + Data Volume: + 1.3 k-values/s (92% drop) + + Target Model: + TFLM microWakeWord + + + Orthogonal & Decorrelated; + Ultra-low CPU & RAM footprint. + + + + + + + DOWNSTREAM NEURAL NETWORK INGESTION MATRIX: CHOOSING MEL VS MFCC + + + + + FEATURE FORMAT + DIMENSIONS + DATA RATE (10ms HOP) + RECOMMENDED ML BACKEND + DSP ADVANTAGES + + + + Mel Spectrogram + 80 Bins (Q9.23) + 8,000 values/sec + OpenAI Whisper, Conformer ASR + High acoustic fidelity; speech synthesis/ASR + + + + MFCC Cepstra + 13 Coeffs (Q9.7) + 1,300 values/sec + TFLM microWakeWord, KWS CNN + Decorrelated; minimal SRAM and compute load + + + + DTX-Silenced + Variable Rate + < 300 values/sec (idle) + Edge Wake-on-Voice / Hostless DSP + > 80% bus sleep duty cycle in quiet rooms + + + diff --git a/developer_guides/tuning/images/mfcc_tuning_workflow.svg b/developer_guides/tuning/images/mfcc_tuning_workflow.svg new file mode 100644 index 00000000..636cdaba --- /dev/null +++ b/developer_guides/tuning/images/mfcc_tuning_workflow.svg @@ -0,0 +1,200 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + End-to-End MFCC & ML Front-End Tuning Workflow + 5-Stage Engineering Methodology: Model Sizing, Bit-Exact Python Simulation, Topology 2 Blob Packaging, and On-Device Verification + + + + + + + + STAGE 1: MODEL SIZING + + + Target Objective + • Select Model Category: + KWS / ASR / Audio Event + + Feature Mode + • Mel-Only (80 Bins) for Whisper + • MFCC (13 Coeffs) for KWS + + Temporal Bounds + • Frame: 25 ms (400 smp) + • Hop: 10 ms (160 smp) + • Sample Rate: 16000 Hz + + Output Interface + • Standard PCM (zero padded) + • Compressed (variable size) + + + Acoustic Specs Locked + Input window & hop rate + match neural net architecture + + + + + + + + + + + STAGE 2: PYTHON SIM + + + Offline Toolchain + sof_mfcc_tool.py + • Bit-exact fixed-point math + + Filterbank Design + • Triangular centers on Mel + • Slaney area normalization + • Bandpass: 20 Hz to 8 kHz + + Dataset Training + • Extract training features + • Train TFLM / Whisper + • Prevent float-fix mismatch + + + Bit-Exact Parity + Model trains on exactly what + SOF firmware DSP emits + + + + + + + + + + + STAGE 3: VAD & DTX + + + A-Weighting Table + • IEC 61672-1:2013 curve + • Peaks at 2.5 kHz (vowels) + + Threshold Calibration + • Energy threshold (0.30 Q9.23) + • Noise rise α_slow = 0.003 + • Hangover = 20 hops (200 ms) + + DTX Duty Cycling + • Trailing hops: 20 + • Silence ping interval: 500 + • Enable mixer notifications + + + VAD Sensitivity Sized + Zero clipping of soft whispers; + >85% quiet bandwidth reduction + + + + + + + + + + + STAGE 4: BLOB & TPLG2 + + + Blob Generation + sof_mfcc_tool.py build-blob + • 116-byte packed struct + • SOF4 IPC4 envelope (144 B) + + Topology 2 Config + • UUID: 73:a7:10:db:a4:... + • Export default.conf / mel80 + • Include in alsatplg build + + Compile Pipeline + • Build tplg2 binary + • Deploy to /lib/firmware/ + + + Topology Signed-Off + Ready for deployment into + target DUT audio topology + + + + + + + + + + + STAGE 5: VERIFY + + + Target DUT Run + • Reload driver + • arecord MFCC sink + + Live Telemetry + • Parse data header + magic = 0x6d666363 + • Monitor VAD ctl + + Acoustic Sign-Off + • Test FAR & FRR + • Noise robustness + • Benchmark CPC + + + Production Pass + Low FAR / FRR + Verified on Hardware + + + + + + + AUTOMATED TOOLCHAIN COMMAND QUICK-REFERENCE + + + # Design 80-bin Mel filterbank for Whisper: python3 sof_mfcc_tool.py design --sample-rate 16000 --num-mel 80 --norm slaney --out mel80.json + # Generate IPC4 Topology 2 config blob: python3 sof_mfcc_tool.py build-blob --mel-only --num-mel 80 --enable-dtx --out mel80_compress.conf + # Extract bit-exact features from test audio: python3 sof_mfcc_tool.py extract --in speech.wav --num-ceps 13 --enable-vad --out features.npy + + + diff --git a/developer_guides/tuning/images/mfcc_vad_dtx_timing_energy.svg b/developer_guides/tuning/images/mfcc_vad_dtx_timing_energy.svg new file mode 100644 index 00000000..7a886995 --- /dev/null +++ b/developer_guides/tuning/images/mfcc_vad_dtx_timing_energy.svg @@ -0,0 +1,166 @@ + + + + + + + + + + + + + + + + + + + + + + + Voice Activity Detection (VAD) & DTX Silence Suppression Timeline + A-Weighted Noise Floor Tracking, Asymmetric Convergence, Hangover Smoothing, and Discontinuous Transmission Bandwidth Savings + + + + + + + PANEL 1: A-WEIGHTED SPEECH SIGNAL ENERGY (E_sig) VS ADAPTIVE NOISE FLOOR (E_noise) + + + + + Signal Energy E_sig (Q9.23) + + Noise Floor E_noise + + + + + + + + + + + INIT CONVERGENCE + 100 frames (α_fast = 0.020) + + + QUIET BACKGROUND + + + ACTIVE SPOKEN UTTERANCE ("HEY SOF") + + + HANGOVER (200 ms) + + + SILENCE INTERVAL + + + + + + + + + + ΔE >> 0.30 + + + + + + + PANEL 2: ENERGY DELTA ΔE, THRESHOLD (0.30 Q9.23), AND BINARY VAD DECISION WITH HANGOVER + + + + + + + + Threshold = 0.30 (2516582) + + + + + + + + VAD=0 (Silence) + VAD=1 (Speech + Hangover) + + + + + + 20 Hops (200 ms) Hold + + + + + + + PANEL 3: DTX PACKET FLOW & BUS BANDWIDTH COMPARISON + + + + Standard Continuous Transmission (DTX Disabled): + 100 frames/sec sent continuously. 100% bus awake duty cycle. + + + + Continuous Stream: Constant DSP DMA, Host Interrupts, and Memory Bus Thrashing + + + + + + SOF DTX-Enabled Stream (dtx_trailing_silence_hops = 20, dtx_silence_hops_interval = 500): + > 85% Transmission Reduction in Quiet Environments + + + + + [SILENCE SUPPRESSED: 0 BYTES TRANSMITTED] + + + + ACTIVE SPEECH FRAMES + + + + TRAILING SILENCE + + + + [BUS ASLEEP] + + + + PING + + + + diff --git a/developer_guides/tuning/mfcc_tuning.rst b/developer_guides/tuning/mfcc_tuning.rst new file mode 100644 index 00000000..43afc6e8 --- /dev/null +++ b/developer_guides/tuning/mfcc_tuning.rst @@ -0,0 +1,784 @@ +.. _mfcc_tuning: + +Mel-Frequency Cepstral Coefficients (MFCC) & Audio Feature Extraction Tuning Guide +################################################################################## + +In modern embedded audio architectures, digital signal processors (DSPs) increasingly serve as real-time sensory front-ends for machine learning (ML) and deep neural network (DNN) inference. Edge processing applications—including on-device Keyword Spotting (KWS), Automated Speech Recognition (ASR), Voice Activity Detection (VAD), and Acoustic Event Detection (AED)—require compact, perceptually relevant acoustic feature representations. Feeding raw 16-bit or 24-bit linear PCM audio directly into embedded neural networks wastes limited DSP memory bandwidth, inflates multiply-accumulate (MAC) cycle footprints, and overwhelms small microcontroller SRAM allocations. + +Sound Open Firmware (SOF) addresses these constraints through its native **MFCC & Audio Feature Extraction** component (:file:`src/audio/mfcc`), which implements a deterministic, highly optimized fixed-point psychoacoustic feature generation engine: + +1. **Psychoacoustic Frequency Warping & Slaney Normalization**: Transforms linear acoustic frequencies into the logarithmic **Mel scale**, mimicking human cochlear critical band resolution. The engine computes triangular bandpass filterbanks with **Slaney area normalization**, equalizing energy accumulation across varying filter bandwidths while employing sparse packed vector storage to reduce filterbank SRAM memory consumption by over :math:`95\%`. +2. **Dual Operating Regimes (Mel-Only vs MFCC Cepstral)**: + + * **Whisper-Compatible Mel Spectrogram Mode** (:math:`\text{num\_ceps} = 0`): Emits 80-bin Mel log spectrogram frames directly to large neural speech recognition models (e.g. OpenAI Whisper, Conformer), supporting dynamic peak tracking (:math:`m_{\text{max}}`), :math:`\text{top\_db}` dynamic headroom clamping, and post-scaling/offset calibration. + * **Classical MFCC Cepstral Mode** (:math:`\text{num\_ceps} \in [10, 40]`): Applies a 16-bit Discrete Cosine Transform (DCT-II) and sinusoidal cepstral liftering to decorrelate filterbank energies into orthogonal cepstral coefficients, creating compact 2D tensor inputs for lightweight inference runtimes such as **TensorFlow Lite for Microcontrollers (TFLM)** and **microWakeWord**. +3. **Integrated Voice Activity Detection (VAD) & Discontinuous Transmission (DTX)**: Tracks an adaptive, A-weighted background acoustic noise floor across Mel bins using asymmetric fast/slow convergence, compares speech-frequency energy against a calibrated threshold, provides hangover smoothing, and silences transmission during inactive periods—slashing host bus wake-ups and memory bus power dissipation by more than :math:`80\%`. + +This guide presents the engineering foundation, mathematical derivations, fixed-point Q-format specifications, VAD/DTX tuning procedures, Python toolchain workflows, ALSA Topology 2 configuration, and diagnostic protocols for the SOF MFCC subsystem. + +.. contents:: Table of Contents + :local: + :depth: 3 + +Architectural Foundations & Signal Processing Pipeline +====================================================== + +Speech sounds are produced by acoustic excitation (glottal vocal cord pulses or turbulent noise) resonating through the human vocal tract cavities (pharynx, oral, and nasal cavities). In the frequency domain, vocal tract resonances manifest as prominent spectral peaks termed **formants** (:math:`F_1, F_2, F_3`), whose relative frequencies and temporal transitions uniquely identify phonemes and spoken words. + +Human auditory perception of frequency is approximately linear below :math:`1\text{ kHz}` and logarithmic above :math:`1\text{ kHz}`. The Mel scale models this non-linear cochlear frequency mapping. By warping FFT spectral power onto triangular Mel filterbanks, the feature extractor compresses high-frequency redundancy while preserving dense formant resolution in the critical speech intelligibility spectrum. + +.. _fig_mfcc_signal_processing_pipeline: + +.. figure:: images/mfcc_signal_processing_pipeline.svg + :alt: MFCC Audio Feature Extraction Pipeline Architecture + :align: center + :width: 100% + + SOF MFCC Fixed-Point Audio Feature Extraction Signal Processing Pipeline: Pre-Emphasis, Overlap Framing, FFT, Mel Filterbank, Dual-Mode Mel/DCT Paths, and Embedded VAD/DTX Engine + +The SOF MFCC processing pipeline executes through seven sequential stages: + +Stage 1: Pre-Emphasis High-Pass Filtering +----------------------------------------- + +Human speech naturally exhibits a spectral tilt of approximately :math:`-6\text{ dB/octave}` above :math:`1\text{ kHz}` due to glottal volume velocity pulse shaping and lips radiation impedance. Consequently, higher-order formants (:math:`F_2, F_3, F_4`) exhibit significantly lower energy than the fundamental voice pitch (:math:`F_0`) and first formant (:math:`F_1`). + +To balance the dynamic range across all spectral bins and prevent high-frequency formants from being masked by numerical quantization floor noise, the input copy routine (:c:func:`mfcc_source_copy_s16`, :c:func:`mfcc_source_copy_s24`, :c:func:`mfcc_source_copy_s32`) applies a first-order finite impulse response (FIR) high-pass pre-emphasis filter: + +.. math:: + + H_{\text{pre}}(z) = 1 - \alpha z^{-1} + +In the time domain: + +.. math:: + + y[n] = x[n] - \alpha \cdot x[n-1] + +where :math:`\alpha` is the pre-emphasis coefficient represented as a signed 16-bit fixed-point integer in **Q1.15** format (:c:member:`sof_mfcc_config.preemphasis_coefficient`). + +* For speech recognition, :math:`\alpha` is typically configured between :math:`0.95` and :math:`0.97`: + +.. math:: + + \text{preemphasis\_coefficient} = \text{round}(0.97 \times 2^{15}) = 31785 \quad (\texttt{0x7C29}) + +* Setting :math:`\alpha = 0` completely disables the pre-emphasis filter without computational penalty. + +Stage 2: Overlap Framing & Tapering Windows +------------------------------------------- + +Speech signals are non-stationary over long intervals but quasi-stationary over short acoustic durations (:math:`10\text{ ms}` to :math:`35\text{ ms}`). The input audio stream is segmented into overlapping temporal frames using an internal circular buffer: + +* **Frame Length** (:math:`T_{\text{frame}}`): Typically :math:`25\text{ ms}` (:math:`400\text{ samples}` at :math:`16\text{ kHz}`). +* **Frame Shift / Hop Size** (:math:`T_{\text{hop}}`): Typically :math:`10\text{ ms}` (:math:`160\text{ samples}` at :math:`16\text{ kHz}`), producing :math:`100\text{ feature frames/sec}`. + +To eliminate Gibbs phenomenon and spectral leakage caused by rectangular truncation, a tapering window :math:`w[n]` is applied to the frame before Fourier transformation: + +.. math:: + + x_w[n] = x[n] \cdot w[n], \quad 0 \le n < N_{\text{frame}} + +SOF provides five selectable window functions via :c:enum:`sof_mfcc_fft_window_type`: + +1. **Hamming Window** (Default, :c:macro:`MFCC_HAMMING_WINDOW`): + + .. math:: + + w[n] = 0.54 - 0.46 \cos\left( \frac{2\pi n}{N - 1} \right) + + Suppresses the first side-lobe to :math:`-43\text{ dB}`, offering the optimal trade-off between main-lobe width and spectral leakage for ASR. + +2. **Hann Window** (:c:macro:`MFCC_HANN_WINDOW`): + + .. math:: + + w[n] = 0.5 \left( 1 - \cos\left( \frac{2\pi n}{N - 1} \right) \right) + + Side-lobes decay at :math:`-18\text{ dB/octave}`, minimizing far-off spectral contamination. Recommended for OpenAI Whisper feature extraction. + +3. **Blackman Window** (:c:macro:`MFCC_BLACKMAN_WINDOW`): + + .. math:: + + w[n] = a_0 - 0.5 \cos\left( \frac{2\pi n}{N - 1} \right) + (0.5 - a_0) \cos\left( \frac{4\pi n}{N - 1} \right) + + Parameter :math:`a_0` is configured via :c:member:`sof_mfcc_config.blackman_coef` in Q1.15 (typically :math:`0.42`). First side-lobe attenuation exceeds :math:`-58\text{ dB}`. + +4. **Povey Window** (:c:macro:`MFCC_POVEY_WINDOW`): + + .. math:: + + w[n] = \left( 0.5 - 0.5 \cos\left( \frac{2\pi n}{N - 1} \right) \right)^{0.85} + + Standard window function used by the Kaldi speech recognition toolkit. + +5. **Rectangular Window** (:c:macro:`MFCC_RECTANGULAR_WINDOW`): Uniform weighting (:math:`w[n] = 1`). + +Stage 3: Real-to-Complex Fast Fourier Transform (FFT) +----------------------------------------------------- + +The windowed frame is zero-padded up to the next power-of-two FFT size :math:`N_{\text{fft}}` (typically :math:`512` points for :math:`400\text{ samples}`) according to :c:member:`sof_mfcc_config.pad`: + +* :c:macro:`MFCC_PAD_END`: Audio samples occupy indices :math:`0 \dots N_{\text{frame}}-1`; zeros pad the tail :math:`N_{\text{frame}} \dots N_{\text{fft}}-1`. +* :c:macro:`MFCC_PAD_CENTER`: Zeros pad equally on the left and right, centering the audio impulse response. +* :c:macro:`MFCC_PAD_START`: Zeros pad the beginning. + +The discrete Fourier transform converts the real-valued signal :math:`x_w[n]` into a complex spectrum: + +.. math:: + + X[k] = \sum_{n=0}^{N_{\text{fft}}-1} x_w[n] \cdot e^{-j \frac{2\pi k n}{N_{\text{fft}}}}, \quad 0 \le k < N_{\text{fft}} + +The elementary frequency resolution between adjacent FFT bins is: + +.. math:: + + \Delta f = \frac{f_s}{N_{\text{fft}}} = \frac{16000\text{ Hz}}{512} = 31.25\text{ Hz} + +Due to Hermitian symmetry for real inputs (:math:`X[N-k] = X^*[k]`), only the first :math:`K = \frac{N_{\text{fft}}}{2} + 1 = 257` unique positive frequency bins are retained. + +The power spectral density :math:`P[k]` is computed as: + +.. math:: + + P[k] = |X[k]|^2 = X_{\text{re}}^2[k] + X_{\text{im}}^2[k], \quad 0 \le k \le \frac{N_{\text{fft}}}{2} + +To compensate for internal FFT bit-shifts and scaling factors, SOF calculates a scale shift offset: + +.. math:: + + \text{mel\_scale\_shift} = \text{input\_shift} - \text{fft\_plan}\to\text{len} + +where :math:`\text{fft\_plan}\to\text{len} = \log_2(512) = 9` for a 512-point FFT. + +Stage 4: Triangular Mel Filterbank & Slaney Normalization +--------------------------------------------------------- + +The Mel frequency scale is defined psychoacoustically by: + +.. math:: + + m = 2595 \cdot \log_{10}\left( 1 + \frac{f}{700} \right) = 1127 \cdot \ln\left( 1 + \frac{f}{700} \right) + +The inverse transformation from Mel to linear frequency is: + +.. math:: + + f = 700 \cdot \left( 10^{\frac{m}{2595}} - 1 \right) = 700 \cdot \left( e^{\frac{m}{1127}} - 1 \right) + +.. _fig_mfcc_mel_filterbank_frequency_response: + +.. figure:: images/mfcc_mel_filterbank_frequency_response.svg + :alt: Triangular Mel Filterbank and Slaney Normalization Frequency Response + :align: center + :width: 100% + + Triangular Mel Filterbank Spacing, Slaney Area Normalization, and SOF Packed Vector Sparse Storage Optimization + +A filterbank of :math:`M` triangular filters (:math:`M = 23` for standard MFCC, :math:`M = 80` for Whisper) is constructed between lower cutoff :math:`f_{\text{low}}` (e.g. :math:`20\text{ Hz}`) and upper cutoff :math:`f_{\text{high}}` (e.g. :math:`8000\text{ Hz}`): + +1. **Center Frequency Spacing**: Convert :math:`f_{\text{low}}` and :math:`f_{\text{high}}` to Mel values :math:`m_{\text{low}}` and :math:`m_{\text{high}}`. +2. Generate :math:`M + 2` linearly spaced points in the Mel domain: + + .. math:: + + m_i = m_{\text{low}} + i \cdot \frac{m_{\text{high}} - m_{\text{low}}}{M + 1}, \quad i \in [0, M+1] + +3. Map each Mel point :math:`m_i` back to linear Hz (:math:`f_i`) and then to discrete FFT bin indices :math:`k_i`: + + .. math:: + + k_i = \left\lfloor \frac{N_{\text{fft}} \cdot f_i}{f_s} + 0.5 \right\rfloor + +4. The triangular weighting function for filter :math:`m \in [1, M]` across bin :math:`k` is: + + .. math:: + + H_m[k] = \begin{cases} + 0 & k < k_{m-1} \\ + \frac{k - k_{m-1}}{k_m - k_{m-1}} & k_{m-1} \le k \le k_m \\ + \frac{k_{m+1} - k}{k_{m+1} - k_m} & k_m \le k \le k_{m+1} \\ + 0 & k > k_{m+1} + \end{cases} + +Slaney Area Normalization +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Without normalization, high-frequency triangular filters—which span wide bandwidths in linear Hertz—integrate over vastly more FFT bins than low-frequency filters, artificially inflating high-frequency energies. + +When Slaney normalization is enabled (:c:macro:`MFCC_MEL_NORM_SLANEY`), each triangular filter is scaled by its bandwidth: + +.. math:: + + H_{m,\text{slaney}}[k] = H_m[k] \cdot \frac{2}{f_{m+1} - f_{m-1}} + +This equalizes the total filter area to unity across all frequencies, ensuring that flat white noise produces uniform spectral energy across the entire Mel filterbank. + +SOF Sparse Packed Triangular Vector Storage +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In a standard matrix implementation, storing a 257-bin by 80-filter matrix requires: + +.. math:: + + N_{\text{dense}} = 257 \times 80 = 20,560 \text{ words} \quad (41.1\text{ KB}) + +Because triangular filters are strictly local, :math:`96.0\%` of dense matrix elements are zeroes. SOF stores the filterbank in a compressed sequential vector (:c:struct:`psy_mel_filterbank`). For each triangle :math:`m`, the packed vector contains: + +* **Word 0**: Offset index to next triangle. +* **Word 1**: Starting FFT bin index :math:`k_{m-1}`. +* **Word 2**: Length of non-zero triangle segment (:math:`k_{m+1} - k_{m-1} + 1`). +* **Words 3..N**: Non-zero fractional weights :math:`H_m[k]` stored in **Q1.15** format. + +This packed structure reduces total filterbank memory consumption from :math:`41.1\text{ KB}` to **1.6 KB**, allowing the entire table to reside permanently in high-speed L1 DSP SRAM cache. + +Stage 5: Logarithmic Energy Compression +--------------------------------------- + +The energy in Mel band :math:`m` is computed by multiplying the power spectrum by the triangular filter weights: + +.. math:: + + E_m = \sum_{k=k_{m-1}}^{k_{m+1}} P[k] \cdot H_m[k] + +Human auditory loudness perception is logarithmic rather than linear. The raw energy is compressed using a logarithmic scale selected via :c:enum:`sof_mfcc_mel_log_type`: + +* **Natural Log** (:c:macro:`MEL_LOG_IS_LOG`): :math:`\ln(E_m + p_{\text{min}})`. +* **Base-10 Log** (:c:macro:`MEL_LOG_IS_LOG10`): :math:`\log_{10}(E_m + p_{\text{min}})`. Standard for OpenAI Whisper. +* **Decibels** (:c:macro:`MEL_LOG_IS_DB`): :math:`10 \cdot \log_{10}(E_m + p_{\text{min}})`. Standard for Librosa. + +The parameter :math:`p_{\text{min}}` (:c:member:`sof_mfcc_config.pmin`) establishes a numerical energy floor (e.g. :math:`10^{-10}` in Q1.31), preventing arithmetic underflow or :math:`\log(0)` singularities during digital silence. The output is formatted as a 32-bit signed integer in **Q9.23** precision. + +.. _fig_mfcc_spectrogram_feature_map: + +.. figure:: images/mfcc_spectrogram_feature_map.svg + :alt: Feature Representation Evolution: Raw Waveform to 2D Machine Learning Tensor + :align: center + :width: 100% + + Feature Representation Evolution: Raw 16 kHz Time Samples to Linear Spectrogram, Log Mel Spectrogram (80 Bins), and DCT-II MFCC Cepstra (13 Coefficients) + +Dual Operating Regimes: Mel Spectrogram vs MFCC +=============================================== + +Depending on the downstream machine learning architecture, the SOF MFCC component operates in one of two distinct functional modes governed by :c:member:`sof_mfcc_config.num_ceps`. + +Mode A: Mel Spectrogram Engine (OpenAI Whisper & Modern ASR) +------------------------------------------------------------ + +When :c:member:`sof_mfcc_config.num_ceps` is set to :math:`0` (:c:member:`mfcc_state.mel_only` is true), the component bypasses the DCT stage and directly outputs the 80-bin Mel log spectrum. This mode is specifically tailored for deep neural networks such as OpenAI Whisper, Conformer, and RNN-T engines. + +In this mode, SOF executes three post-processing steps: + +1. **Dynamic Peak Tracking** (:math:`m_{\text{max}}`): + When :c:member:`sof_mfcc_config.dynamic_mmax` is enabled, the firmware tracks the maximum Mel energy peak across all bands: + + .. math:: + + \text{peak} = \max_{j=0 \dots M-1} \text{mel\_log}[j] + + If :math:`\text{peak} > m_{\text{max}}`, :math:`m_{\text{max}}` jumps to the peak immediately. If :math:`\text{peak} \le m_{\text{max}}`, :math:`m_{\text{max}}` decays exponentially according to :c:member:`sof_mfcc_config.mmax_coef`: + + .. math:: + + m_{\text{max}}[t] = m_{\text{max}}[t-1] + \text{mmax\_coef} \cdot (\text{peak} - m_{\text{max}}[t-1]) + +2. **Top-dB Dynamic Headroom Clamping**: + Values lower than :math:`m_{\text{max}} - \text{top\_db}` are clamped: + + .. math:: + + \text{clamp\_val} = m_{\text{max}} - \text{top\_db} + + .. math:: + + E_{\text{clamped}}[j] = \max(\text{mel\_log}[j], \text{clamp\_val}) + + For decibels (:c:macro:`MEL_LOG_IS_DB`), :c:member:`sof_mfcc_config.top_db` is typically set to :math:`80.0\text{ dB}`. For base-10 log (:c:macro:`MEL_LOG_IS_LOG10`), :c:member:`sof_mfcc_config.top_db` is set to :math:`8.0`. + +3. **Whisper Scale and Offset Normalization**: + To match the exact normalization expected by Whisper acoustic encoders, the clamped values are scaled and offset: + + .. math:: + + E_{\text{whisper}}[j] = (E_{\text{clamped}}[j] + \text{mel\_offset}) \cdot \text{mel\_scale} + + * :c:member:`sof_mfcc_config.mel_offset`: Set to :math:`4.0` in **Q8.7** format (:math:`512`). + * :c:member:`sof_mfcc_config.mel_scale`: Set to :math:`0.25` in **Q4.12** format (:math:`1024`). + +Mode B: MFCC Cepstral Engine (TFLM & microWakeWord) +--------------------------------------------------- + +When :c:member:`sof_mfcc_config.num_ceps` is greater than :math:`0` (typically :math:`13`), the component applies the Discrete Cosine Transform (DCT-II) and cepstral liftering: + +1. **Fixed-Point Conversion**: Truncates 32-bit Q9.23 Mel values into signed 16-bit **Q9.7** integers. +2. **Discrete Cosine Transform (DCT-II)**: + Mel band energies are highly correlated due to overlapping triangular filters. The DCT-II acts as an orthogonal linear transform, concentrating the dominant spectral envelope information into the lowest cepstral coefficients while discarding high-frequency ripple: + + .. math:: + + c_n = \sum_{m=0}^{M-1} E_m \cdot \cos\left( \frac{\pi n (m + 0.5)}{M} \right), \quad 0 \le n < N_{\text{ceps}} + + * :math:`c_0`: Represents the total frame acoustic energy / perceived loudness. + * :math:`c_1`: Captures the overall spectral tilt (balance between low and high frequencies). + * :math:`c_2 \dots c_5`: Encodes the broad vocal tract formant positions (:math:`F_1, F_2, F_3`), representing phoneme identities. + * :math:`c_6 \dots c_{12}`: Captures fine spectral details and speaker-specific characteristics. + +3. **Sinusoidal Cepstral Liftering**: + Higher-order cepstral coefficients naturally exhibit smaller numerical variances than low-order coefficients, making neural network gradient optimization difficult. SOF applies a sinusoidal cepstral lifter (:c:member:`sof_mfcc_config.cepstral_lifter`, typically :math:`L = 22.0` in **Q7.9** format): + + .. math:: + + w_n = 1 + \frac{L}{2} \cdot \sin\left( \frac{\pi n}{L} \right), \quad 0 \le n < N_{\text{ceps}} + + .. math:: + + c_{n,\text{lifted}} = c_n \cdot w_n + + This normalizes coefficient variance, improving Word Error Rate (WER) and False Rejection Rate (FRR) in microWakeWord models. + +Integrated VAD & Discontinuous Transmission (DTX) +================================================= + +Continuously transmitting audio feature tensors across memory buses to host processors dissipates significant dynamic power, even when the user is silent. The SOF MFCC module embeds a real-time Voice Activity Detector (VAD) and Discontinuous Transmission (DTX) silence suppression engine directly into the DSP feature extraction loop. + +.. _fig_mfcc_vad_dtx_timing_energy: + +.. figure:: images/mfcc_vad_dtx_timing_energy.svg + :alt: Voice Activity Detection and DTX Silence Suppression Timeline + :align: center + :width: 100% + + Embedded Voice Activity Detection (VAD) and Discontinuous Transmission (DTX) Energy Dynamics, Noise Floor Tracking, and Transmission Savings + +A-Weighted Speech Energy Formulation +------------------------------------ + +The VAD constructs an A-weighting spectral filter by linearly interpolating the IEC 61672-1:2013 standard curve across the center frequency of each Mel bin (:c:func:`mfcc_vad_build_weights`): + +* Peak sensitivity occurs at :math:`2500\text{ Hz}` (:math:`w_{\text{peak}} = 32767` in Q1.15), matching the human ear's resonant ear canal response. +* Low frequencies (:math:`< 100\text{ Hz}`) and ultra-high frequencies (:math:`> 10\text{ kHz}`) are attenuated, preventing HVAC rumble and mechanical chassis vibrations from falsely triggering the VAD. +* Weights are normalized so that :math:`\sum_{i=0}^{M-1} w_i = 1.0` in Q1.15. + +Asymmetric Adaptive Noise Floor Tracking +---------------------------------------- + +The noise floor :math:`N_i` is tracked independently for each Mel bin: + +1. **Initialization Phase**: During the first :math:`N_{\text{init}} = 100\text{ frames}` (:math:`1.0\text{ s}`), a fast rise coefficient :math:`\alpha_{\text{fast}} = 0.020` is applied to rapidly converge to ambient background room acoustics. +2. **Operational Phase**: After initialization, a slow rise coefficient :math:`\alpha_{\text{slow}} = 0.003` (:c:macro:`MFCC_VAD_NOISE_RISE_ALPHA`, Q1.15 = :math:`98`) is applied: + + .. math:: + + N_i[t] = \begin{cases} + E_i[t] & \text{if } E_i[t] < N_i[t-1] \quad \text{(Instant Follow-Down)} \\ + N_i[t-1] + \alpha_{\text{slow}} \cdot (E_i[t] - N_i[t-1]) & \text{if } E_i[t] \ge N_i[t-1] \quad \text{(Slow Rise)} + \end{cases} + +This asymmetric tracking guarantees that background noise floors adapt during quiet intervals but do not rise during prolonged speech utterances. + +Energy Delta & Hangover Smoothing +--------------------------------- + +The total speech-weighted signal energy and noise energy are computed in 64-bit precision and scaled to Q9.23: + +.. math:: + + E_{\text{sig}} = \sum_{i=0}^{M-1} w_i \cdot E_i[t], \quad E_{\text{noise}} = \sum_{i=0}^{M-1} w_i \cdot N_i[t] + +The energy delta is: + +.. math:: + + \Delta E = E_{\text{sig}} - E_{\text{noise}} + +Speech is declared when :math:`\Delta E` exceeds the energy threshold (:c:macro:`MFCC_VAD_ENERGY_THRESHOLD` = :math:`0.30 \times 2^{23} = 2,516,582`): + +.. math:: + + \text{Speech Detected} \iff \Delta E > E_{\text{thresh}} + +To prevent phoneme dropout during quiet consonant terminations, plosives, and brief pauses between words, a hangover counter (:c:macro:`MFCC_VAD_HANGOVER_FRAMES` = :math:`20\text{ frames} = 200\text{ ms}`) holds the VAD in the active state after the signal drops below threshold. + +Discontinuous Transmission (DTX) Protocol +----------------------------------------- + +When DTX is enabled (:c:member:`sof_mfcc_config.enable_dtx`), the component optimizes memory and DMA transmission: + +1. **Active Speech**: Frames are continuously written to the output sink buffer. +2. **Trailing Silence**: Upon speech termination, exactly :c:member:`sof_mfcc_config.dtx_trailing_silence_hops` (typically :math:`20`) are transmitted to ensure downstream wake-word models capture acoustic decay. +3. **Silence Suppression**: Subsequent silent frames are completely suppressed. Zero bytes are written to the sink buffer. +4. **Periodic Keepalive Ping**: To prevent downstream pipelines from reporting buffer underruns, a silence frame is emitted every :c:member:`sof_mfcc_config.dtx_silence_hops_interval` hops (e.g. :math:`500\text{ hops} = 5.0\text{ s}`). + +Output Frame Header: struct mfcc_data_header +-------------------------------------------- + +Every output frame emitted by the MFCC component begins with a 24-byte metadata header (:c:struct:`mfcc_data_header`): + +.. code-block:: c + + struct mfcc_data_header { + uint32_t magic; /**< Magic word MFCC_MAGIC (0x6d666363, 'mfcc') */ + uint32_t frame_number; /**< Incrementing hop index starting from 0 */ + int32_t reserved; /**< Set to 0 */ + int32_t energy; /**< Weighted signal energy in Q9.23 */ + int32_t noise_energy; /**< Weighted noise floor energy in Q9.23 */ + int32_t vad_flag; /**< VAD decision: 1 = speech, 0 = silence */ + }; + +Downstream neural network runtimes inspect :c:member:`mfcc_data_header.vad_flag` to bypass inference computation when :math:`\text{vad\_flag} = 0`. + +Control Plane ABI & Topology 2 Configuration +============================================ + +The MFCC module registers with the SOF processing module framework using the following parameters: + +* **Component UUID**: ``73:a7:10:db:a4:1a:ea:4c:a2:1f:2d:57:a5:c9:82:eb`` +* **Component Type**: ``effect`` (:c:macro:`SOF_COMP_EFFECT`) +* **Switch Control Index**: ``MFCC_CTRL_INDEX_VAD`` (:math:`0`) for host VAD event notification. + +Configuration Structure: struct sof_mfcc_config +----------------------------------------------- + +The component is initialized via an IPC configuration blob containing the 116-byte packed structure :c:struct:`sof_mfcc_config` (:file:`include/user/mfcc.h`): + +.. list-table:: SOF MFCC Configuration Structure (struct sof_mfcc_config, 116 Bytes) + :widths: 18 14 18 50 + :header-rows: 1 + + * - Field Name + - Data Type + - Format / Range + - Functional Description + * - ``size`` + - ``uint32_t`` + - 116 Bytes + - Total size of the configuration structure in bytes. + * - ``mel_offset`` + - ``int16_t`` + - Q8.7 (:math:`0` or :math:`4.0`) + - Post-scaling offset for Mel spectrogram mode (use 4.0 for Whisper). + * - ``mel_scale`` + - ``int16_t`` + - Q4.12 (:math:`1.0` or :math:`0.25`) + - Post-scaling gain for Mel spectrogram mode (use 0.25 for Whisper). + * - ``mmax_init`` + - ``int16_t`` + - Q8.7 (:math:`0`) + - Initial peak Mel value for headroom clamping. + * - ``mmax_coef`` + - ``int16_t`` + - Q1.15 + - Exponential decay coefficient for dynamic :math:`m_{\text{max}}` tracking. + * - ``dtx_trailing`` + - ``uint16_t`` + - :math:`0 \dots 100` hops + - Number of trailing silence hops to transmit after speech ends (default: 20). + * - ``dtx_interval`` + - ``uint16_t`` + - :math:`0 \dots 1000` hops + - Periodic keepalive hop interval during continuous silence (default: 500). + * - ``sample_freq`` + - ``int32_t`` + - :math:`8000 \dots 64000\text{ Hz}` + - Sampling frequency in Hertz (default: 16000). + * - ``pmin`` + - ``int32_t`` + - Q1.31 (:math:`10^{-10}`) + - Linear power floor to prevent logarithmic underflow during silence. + * - ``mel_log`` + - ``enum`` + - :math:`0=\text{log}, 1=\log_{10}, 2=\text{dB}` + - Mathematical scale for logarithmic energy compression. + * - ``norm`` + - ``enum`` + - :math:`0=\text{none}, 1=\text{slaney}` + - Triangular filterbank area normalization mode. + * - ``pad`` + - ``enum`` + - :math:`0=\text{end}, 1=\text{center}, 2=\text{start}` + - Zero-padding alignment within the FFT input buffer. + * - ``window`` + - ``enum`` + - :math:`0 \dots 4` + - Tapering window: Rectangular, Blackman, Hamming, Hann, or Povey. + * - ``dct`` + - ``enum`` + - :math:`1=\text{DCT\_II}` + - Discrete Cosine Transform algorithm (must be DCT-II). + * - ``blackman_coef`` + - ``int16_t`` + - Q1.15 (:math:`0.42`) + - Parameter :math:`a_0` when Blackman window is selected. + * - ``cepstral_lifter`` + - ``int16_t`` + - Q7.9 (:math:`22.0`) + - Sinusoidal lifter parameter :math:`L` for variance equalization. + * - ``channel`` + - ``int16_t`` + - :math:`-1` (mono), :math:`0 \dots 7` + - Audio stream channel index to extract for feature processing. + * - ``frame_length`` + - ``int16_t`` + - Samples (:math:`400`) + - Frame analysis window length (:math:`25\text{ ms}` at :math:`16\text{ kHz}`). + * - ``frame_shift`` + - ``int16_t`` + - Samples (:math:`160`) + - Frame advance step size (:math:`10\text{ ms}` at :math:`16\text{ kHz}`). + * - ``high_freq`` + - ``int16_t`` + - Hertz (:math:`8000`) + - High cutoff frequency for Mel filterbank (0 for Nyquist). + * - ``low_freq`` + - ``int16_t`` + - Hertz (:math:`20`) + - Low cutoff frequency for Mel filterbank. + * - ``num_ceps`` + - ``int16_t`` + - :math:`0` (Mel-only), :math:`1 \dots 40` + - Number of cepstral coefficients to emit. + * - ``num_mel_bins`` + - ``int16_t`` + - :math:`10 \dots 128` + - Number of internal Mel filterbank bands (23 for KWS, 80 for Whisper). + * - ``preemphasis`` + - ``int16_t`` + - Q1.15 (:math:`0.97`) + - High-pass pre-emphasis filter coefficient (0 to disable). + * - ``top_db`` + - ``int16_t`` + - Q8.7 (:math:`80.0\text{ dB}` or :math:`8.0`) + - Dynamic range clamp span below peak :math:`m_{\text{max}}`. + * - ``dynamic_mmax`` + - ``bool`` + - :math:`0` or :math:`1` + - Enables dynamic peak tracking for Mel headroom clamping. + * - ``enable_vad`` + - ``bool`` + - :math:`0` or :math:`1` + - Enables embedded Mel-energy Voice Activity Detection. + * - ``enable_dtx`` + - ``bool`` + - :math:`0` or :math:`1` + - Enables discontinuous transmission silence frame suppression. + * - ``update_controls`` + - ``bool`` + - :math:`0` or :math:`1` + - Dispatches IPC switch control notification to host on VAD state change. + * - ``compress_output`` + - ``bool`` + - :math:`0` or :math:`1` + - Enables variable-size compressed PCM output without zero padding. + +Topology 2 Configuration Template +--------------------------------- + +In ALSA Topology 2, the MFCC widget is defined using :file:`tools/topology/topology2/include/components/mfcc.conf` and packaged with its binary configuration block: + +.. code-block:: text + + # Topology 2 MFCC Component Definition + Object.Widget.mfcc."1" { + index 1 + instance 1 + num_input_pins 1 + num_output_pins 1 + num_input_audio_formats 1 + num_output_audio_formats 1 + + # Include compiled binary configuration blob (144 bytes SOF4) + + } + +Standalone Python Calibration Toolchain Runbook +=============================================== + +SOF provides the standalone Python calibration utility :file:`sof_mfcc_tool.py` located at :file:`tools/tune/mfcc/`: + +.. _fig_mfcc_tuning_workflow: + +.. figure:: images/mfcc_tuning_workflow.svg + :alt: End-to-End MFCC and Machine Learning Front-End Tuning Workflow + :align: center + :width: 100% + + End-to-End 5-Stage MFCC and ML Front-End Tuning Methodology: Model Sizing, Bit-Exact Python Simulation, Topology 2 Blob Packaging, and On-Device Verification + +Subcommand 1: Filterbank Design (design) +---------------------------------------- + +To calculate Mel filterbank center frequencies, verify Slaney area normalization, and inspect DSP SRAM memory savings: + +.. code-block:: bash + + $ python3 tools/tune/mfcc/sof_mfcc_tool.py design \ + --sample-rate 16000 \ + --fft-size 512 \ + --num-mel 80 \ + --norm slaney + + ================================================================================ + SOF Mel Filterbank Design Summary + ================================================================================ + Sample Frequency: 16000 Hz + FFT Size: 512 points (Δf = 31.25 Hz) + Mel Bins: 80 + Frequency Span: 20.0 Hz to 8000.0 Hz + Normalization: SLANEY + Dense Matrix Footprint: 20560 int16 words (40.2 KB) + Sparse Packed Storage: 825 int16 words (1.6 KB) + DSP SRAM RAM Reduction: 96.0% + -------------------------------------------------------------------------------- + Sample Filter Center Frequencies: + Bin 0: Center = 42.5 Hz | Span = [ 1.. 3] ( 3 taps) + Bin 10: Center = 309.9 Hz | Span = [ 9.. 11] ( 3 taps) + Bin 20: Center = 673.7 Hz | Span = [ 20.. 23] ( 4 taps) + Bin 30: Center = 1168.5 Hz | Span = [ 36.. 39] ( 4 taps) + Bin 40: Center = 1841.6 Hz | Span = [ 56.. 61] ( 6 taps) + Bin 50: Center = 2757.1 Hz | Span = [ 85.. 92] ( 8 taps) + Bin 60: Center = 4002.3 Hz | Span = [124..133] (10 taps) + Bin 70: Center = 5696.1 Hz | Span = [176..189] (14 taps) + ================================================================================ + +Subcommand 2: Binary Blob & Topology 2 Export (build-blob) +---------------------------------------------------------- + +To generate an IPC4 configuration blob and ALSA Topology 2 `.conf` include file for a Whisper-compatible 80-bin Mel engine with DTX: + +.. code-block:: bash + + $ python3 tools/tune/mfcc/sof_mfcc_tool.py build-blob \ + --mel-only \ + --num-mel 80 \ + --mel-offset 4.0 \ + --mel-scale 0.25 \ + --top-db 8.0 \ + --dynamic-mmax \ + --enable-vad \ + --enable-dtx \ + --dtx-trailing 20 \ + --dtx-interval 500 \ + --out tools/topology/topology2/include/components/mfcc/mel80_compress_dtx.conf + +Subcommand 3: VAD & DTX Energy Simulation (vad-sim) +--------------------------------------------------- + +To evaluate VAD threshold sensitivity and simulate memory bus transmission savings on an audio utterance: + +.. code-block:: bash + + $ python3 tools/tune/mfcc/sof_mfcc_tool.py vad-sim \ + --sample-rate 16000 \ + --duration 5.0 \ + --speech-duration 1.5 \ + --num-mel 80 + + ================================================================================ + SOF VAD & DTX Discontinuous Transmission Simulation + ================================================================================ + Audio Duration: 5.00 s (497 hops) + Active Speech Frames: 172 hops (1.72 s) + Silence Frames: 325 hops (3.25 s) + DTX-Suppressed Frames: 305 hops + Memory & Bus Bandwidth: 61.4% REDUCTION + ================================================================================ + +Production Tuning Recipes +========================= + +The following configurations represent validated production profiles across speech recognition and edge wake-word deployments: + +.. _tab_mfcc_production_recipes: + +.. list-table:: Production Tuning Presets: MFCC & Mel Feature Extraction Configurations + :widths: 22 24 24 30 + :header-rows: 1 + + * - Deployment Target + - Feature Extraction Mode + - VAD & DTX Parameters + - Downstream ML Architecture + * - **Recipe 1: Edge Keyword Spotting** + - 13 MFCCs, 23 Mel bins, Hamming, :math:`\alpha = 0.97`, :math:`L = 22.0` + - VAD enabled, DTX enabled (:math:`N_{\text{trailing}} = 20`) + - TensorFlow Lite for Microcontrollers (TFLM) microWakeWord CNN. + * - **Recipe 2: OpenAI Whisper ASR** + - 80 Mel bins, Mel-only (:math:`\text{num\_ceps} = 0`), Hann, Slaney norm, Offset 4.0, Scale 0.25 + - VAD enabled, DTX enabled (:math:`N_{\text{interval}} = 500`) + - Whisper Tiny/Base/Small Transformer acoustic encoder. + * - **Recipe 3: Ultra-Low-Power Wake-on-Voice** + - 10 MFCCs, 16 Mel bins, Rectangular, :math:`\alpha = 0` + - Aggressive DTX (:math:`N_{\text{trailing}} = 5`, :math:`E_{\text{thresh}} = 0.45`) + - Hostless sub-milliwatt DSP keyword detection running in D0ix listening state. + +Interactive Live Injection & Diagnostics Matrix +=============================================== + +Runtime Verification via arecord & sof-ctl +------------------------------------------ + +To verify MFCC streaming, capture feature tensors, and monitor VAD switch events on a target DUT: + +.. code-block:: bash + + # Step 1: Monitor VAD switch events from the active sound card + ssh root@ "amixer -c 0 sget 'mfcc.1.1.switch'" + + # Step 2: Stream MFCC frames directly to file + ssh root@ "arecord -D hw:0,1 -f S16_LE -c 1 -r 16000 -d 5 /tmp/mfcc_capture.raw" + + # Step 3: Inspect the 24-byte struct mfcc_data_header from the captured stream + ssh root@ "hexdump -C -n 24 /tmp/mfcc_capture.raw" + + # Expected Output: + # 00000000 63 63 66 6d 00 00 00 00 00 00 00 00 2a 3b 10 00 |ccfm........*;..| + # 00000010 12 18 04 00 01 00 00 00 |........| + # Note: 63 63 66 6d represents ASCII 'mfcc' (0x6d666363) in little-endian. + # vad_flag = 0x00000001 (Speech active) + +Diagnostic Troubleshooting Matrix +--------------------------------- + +.. _tab_mfcc_troubleshooting_matrix: + +.. list-table:: Diagnostic Troubleshooting Matrix: MFCC & Audio Feature Extraction + :widths: 22 25 25 28 + :header-rows: 1 + + * - Symptom + - Root Cause + - Diagnostic Procedure + - Remediation Action + * - **Clipped Speech Onsets & Phoneme Drops** + - VAD energy threshold set too high, or hangover counter too short to bridge pauses. + - Inspect :c:member:`mfcc_data_header.vad_flag` during soft whispers or plosives. + - Lower :c:member:`sof_mfcc_config.top_db` threshold or increase :c:macro:`MFCC_VAD_HANGOVER_FRAMES` to :math:`25` hops. + * - **Out-of-Band Noise Aliasing** + - Upper Mel cutoff :math:`f_{\text{high}}` set beyond the Nyquist frequency (:math:`f_s / 2`). + - Review filterbank design table in :command:`sof_mfcc_tool.py design`. + - Set :c:member:`sof_mfcc_config.high_freq` strictly to :math:`0` or :math:`\le f_s / 2`. + * - **Whisper Transcription Garbage / Hallucinations** + - Mel spectrogram scaling or offset mismatched with model training expectations. + - Compare exported Mel frame values against Librosa reference vectors. + - Ensure :c:member:`sof_mfcc_config.mel_offset` is :math:`4.0` (Q8.7 = 512) and :c:member:`sof_mfcc_config.mel_scale` is :math:`0.25` (Q4.12 = 1024). + * - **High DSP Cycle Footprint (CPC)** + - Dense matrix multiplication invoked instead of sparse packed triangular indexing. + - Inspect compiler flags for SIMD vector dot products in :file:`mfcc_hifi4.c`. + - Verify Kconfig selects ``MATH_16BIT_MEL_FILTERBANK`` and HiFi SIMD optimization routines. + * - **Downstream Buffer Underrun during Silence** + - DTX periodic keepalive interval disabled (:math:`\text{dtx\_silence\_hops\_interval} = 0`). + - Check kernel :command:`dmesg` for pipeline XRUNs during silence. + - Configure :c:member:`sof_mfcc_config.dtx_silence_hops_interval` to :math:`500` hops (:math:`5.0\text{ s}`) to transmit periodic keepalives. + +Related Documentation +===================== + +* :ref:`dmic_tuning`: Digital Microphone Acoustic Calibration & Decimation Tuning Guide. +* :ref:`level_multiplier_aria_tuning`: Level Multiplier & Aria AGC Dynamic Range Control Tuning Guide. +* :ref:`drc_tuning`: Dynamic Range Compression & Multiband DRC Tuning Guide. +* :ref:`smart_amp_tuning`: Smart Amplifier (DSM) & Transducer Protection Calibration Guide. +* :ref:`sound_dose_tuning`: Sound Dose / Hearing Health Calibration & Acoustic Protection Guide. +* :ref:`runtime_tuning_sof_ctl`: Unified Runtime Tuning, Control Blobs & Parameter Injection Guide. +* :ref:`time-domain-fixed-beamformer`: Time Domain Fixed Beamformer (TDFB) Architecture & Array Tuning. From a870ed8a7f5f7f9130b05b0793f1af6a6b87dd58 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 17:54:16 +0100 Subject: [PATCH 39/64] doc: developer_guides: eliminate duplicate links in firmware and tuning sections - Move core infrastructure and audio processing module toctree entries directly under their respective subsections with :hidden: so that they remain indexed by Sphinx and visible in sidebar navigation without rendering redundant unannotated duplicate link lists in the page body. - Retain only packaging, dynamic loadable modules (LLEXT), and hostless subsystems under 'Firmware Packaging, Dynamic Modules & Subsystems'. - Add :hidden: to the tuning toctree to remove the duplicate link list under the tuning section. Signed-off-by: Liam Girdwood --- developer_guides/index.rst | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/developer_guides/index.rst b/developer_guides/index.rst index c83c3428..e97ddea8 100644 --- a/developer_guides/index.rst +++ b/developer_guides/index.rst @@ -39,6 +39,16 @@ Core Infrastructure & Pipeline * :ref:`ipc_infrastructure` (High-level architecture; also see upstream `IPC README `_) * :ref:`fw_init_boot` (High-level architecture; also see upstream `init README `_) +.. toctree:: + :hidden: + + firmware/pipeline_architecture + firmware/module_framework + firmware/scheduler_architecture + firmware/audio_buffer_management + firmware/ipc_infrastructure + firmware/fw_init_boot + Audio Processing Modules & Algorithms ------------------------------------- @@ -66,20 +76,9 @@ Audio Processing Modules & Algorithms * :ref:`stft_process` (High-level architecture; also see upstream `STFT Process README `_) * :ref:`media_codecs` (High-level architecture; also see upstream `Cadence Codec module adapter `_ & `Codec README `_) -Firmware Architecture, Packaging & Core Subsystems -================================================== - -Core pipeline architecture, firmware image packaging, cryptographic signing, loadable modules, and standalone hostless embedded firmware: - .. toctree:: - :maxdepth: 1 + :hidden: - firmware/pipeline_architecture - firmware/module_framework - firmware/scheduler_architecture - firmware/audio_buffer_management - firmware/ipc_infrastructure - firmware/fw_init_boot firmware/volume_module firmware/mixin_mixout firmware/src_asrc @@ -103,6 +102,15 @@ Core pipeline architecture, firmware image packaging, cryptographic signing, loa firmware/phase_vocoder firmware/stft_process firmware/media_codecs + +Firmware Packaging, Dynamic Modules & Subsystems +================================================ + +Firmware image packaging, cryptographic signing, loadable modules, and standalone hostless embedded firmware: + +.. toctree:: + :maxdepth: 1 + rimage/index.rst firmware/llext_modules firmware/hostless_firmware @@ -146,7 +154,7 @@ Machine Learning & Speech Feature Extraction * :ref:`mfcc_tuning` (Mel-Frequency Cepstral Coefficients (MFCC), triangular Mel filterbank design, Slaney normalization, Whisper-compatible Mel spectrogram scaling, Voice Activity Detection (VAD), and TensorFlow Lite Micro (TFLM) keyword spotting co-design) .. toctree:: - :maxdepth: 1 + :hidden: tuning/runtime_tuning_sof_ctl tuning/drc_tuning From 925e744d418e366580333a667e0df9e18e3a96f5 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 19 Sep 2026 18:20:08 +0100 Subject: [PATCH 40/64] doc: rimage: comprehensive architecture, binary layout, and signing modernization Modernize and significantly expand the Rimage developer documentation to establish an authoritative, deeply technical reference for firmware packaging, cryptographic signing, and hardware boot handshakes: - Author comprehensive developer_guides/rimage/index.rst covering: * Dual implementations: production C99 toolchain (tools/rimage/) and the upstream Rust port (thesofproject/rimage). * Byte-level binary image layout (.ri) from offset 0x0000 to EOF, covering XMan, CSE (), CSS (), Signed Pkg metadata, Firmware Descriptor (), and Module Table (). * Platform memory matrix comparing cAVS (1.5, 1.8, 2.5) and ACE (1.5, 2.0, 3.0) addressing, cached/uncached SRAM aliases, IMR staging windows, L2 cache bypass, and alignment rules. * Dual cryptographic signing workflows (native RSA PKCS#1 v1.5 vs Intel MEU/HSM) and Silicon Root-of-Trust (RoT) eFuse authentication. * Zephyr LLEXT dynamic module packaging (-l). * Declarative TOML configuration (platform-*.toml and *.toml.h). * Complete CLI reference, CMake/west integration, and troubleshooting matrix with hexdump and readelf diagnostic recipes. - Rewrite developer_guides/rimage/extended_manifest.rst into an in-depth architectural specification of the Extended Manifest (XMan), documenting header invariants, all element types (0 through 6), alignment constraints (EXT_MAN_ALIGN = 16), Linux kernel host driver handshake flow (loader.c), and step-by-step element extension guide. - Replace legacy PlantUML .pu diagrams with 5 publication-quality native vector SVG diagrams: * rimage_pipeline_architecture.svg (5-stage packaging pipeline) * rimage_binary_layout.svg (byte-level binary layout) * rimage_extended_manifest_handshake.svg (XMan element taxonomy & handshake) * rimage_crypto_signing_flow.svg (signing workflows & hardware RoT) * rimage_platform_memory_matrix.svg (cAVS vs ACE memory & virtual addressing) Signed-off-by: Liam Girdwood --- developer_guides/rimage/extended_manifest.rst | 345 ++++++++++- .../rimage/images/ext_man_build_flow.pu | 33 -- .../rimage/images/image_build_flow.pu | 70 --- .../rimage/images/rimage_binary_layout.svg | 257 ++++++++ .../images/rimage_crypto_signing_flow.svg | 270 +++++++++ .../rimage_extended_manifest_handshake.svg | 221 +++++++ .../images/rimage_pipeline_architecture.svg | 295 ++++++++++ .../images/rimage_platform_memory_matrix.svg | 218 +++++++ developer_guides/rimage/index.rst | 547 ++++++++++++++++-- 9 files changed, 2069 insertions(+), 187 deletions(-) delete mode 100644 developer_guides/rimage/images/ext_man_build_flow.pu delete mode 100644 developer_guides/rimage/images/image_build_flow.pu create mode 100644 developer_guides/rimage/images/rimage_binary_layout.svg create mode 100644 developer_guides/rimage/images/rimage_crypto_signing_flow.svg create mode 100644 developer_guides/rimage/images/rimage_extended_manifest_handshake.svg create mode 100644 developer_guides/rimage/images/rimage_pipeline_architecture.svg create mode 100644 developer_guides/rimage/images/rimage_platform_memory_matrix.svg diff --git a/developer_guides/rimage/extended_manifest.rst b/developer_guides/rimage/extended_manifest.rst index 3cddcd94..6bfcf98d 100644 --- a/developer_guides/rimage/extended_manifest.rst +++ b/developer_guides/rimage/extended_manifest.rst @@ -1,32 +1,333 @@ .. _extended_manifest: -Extended Manifest -################# +Extended Manifest Architecture & Host Handshake +############################################### -The extended manifest is a place to store build-time known firmware metadata -such as the firmware version or a used compiler description. Given that -information is read on the host side before firmware startup, this is -especially important for ABI compatibility checks. -This part of the output binary is located as a first structure in the binary -file and it is skipped in the DSP loading routine; so, the attached -information does not affect DSP memory. +The **Extended Manifest** (abbreviated as **XMan**) is an extensible, un-signed metadata container embedded at the very beginning (offset ``0x0000``) of compiled Sound Open Firmware binary images (``.ri`` files). It conveys vital compile-time hardware, toolchain, and ABI configuration details to the host operating system kernel (Linux ``snd-sof``) **before** the audio DSP is initialized or booted. +Because the host driver parses the extended manifest prior to initiating DSP reset and DMA transfer, the host dynamically verifies ABI compatibility, configures memory windows for IPC mailboxes and debug dumps, and logs firmware version information without requiring the DSP core to be powered on. Crucially, the extended manifest is stripped or bypassed by the host DMA loader, ensuring **zero memory footprint** inside the DSP SRAM. -Build flow -========== +.. figure:: images/rimage_extended_manifest_handshake.svg + :alt: SOF Extended Manifest Architecture and Host Driver Handshake + :align: center + :width: 100% -.. uml:: images/ext_man_build_flow.pu - :caption: Extended manifest generation + Extended Manifest architecture, element taxonomy, and Linux kernel host driver handshake flow. +Extended Manifest Binary Header +******************************* -Add a new element -================= +The extended manifest begins with a fixed, backwards-compatible header defined in ``include/sound/sof/ext_manifest.h``: -To add a new element to the extended manifest, do the following: +.. code-block:: c -#. Add a new element definition in the ``ext_manifest.h`` file located in - the firmware and driver repository. -#. Add a new element declaration in the ``ext_manifest.c`` file located in - the firmware repository. -#. Add a new element handling routine in the driver repository: - ``sound/soc/sof/loader.c:snd_sof_fw_ext_man_parse()`` \ No newline at end of file + /* Magic number in ASCII: 'XMan' (0x6e614d58 in little-endian) */ + #define SOF_EXT_MAN_MAGIC_NUMBER 0x6e614d58 + + /* Version encoding: MMmmmppp (Major: bits 31-24, Minor: bits 23-12, Patch: bits 11-0) */ + #define SOF_EXT_MAN_BUILD_VERSION(MAJOR, MINOR, PATCH) ( \ + ((uint32_t)(MAJOR) << 24) | \ + ((uint32_t)(MINOR) << 12) | \ + (uint32_t)(PATCH)) + + #define SOF_EXT_MAN_VERSION SOF_EXT_MAN_BUILD_VERSION(1, 0, 0) + + /* Structural alignment required for every extended manifest element */ + #define EXT_MAN_ALIGN 16 + + struct sof_ext_man_header { + uint32_t magic; /* Identification magic: EXT_MAN_MAGIC_NUMBER */ + uint32_t full_size; /* Full size of ext_man in bytes (header + all elements + padding) */ + uint32_t header_size; /* Size of this header in bytes (enables future header growth) */ + uint32_t header_version; /* Header version: SOF_EXT_MAN_VERSION */ + + /* Immediately followed by contiguous sequence of struct sof_ext_man_elem_header elements */ + } __packed; + +Header Invariant Rules +====================== + +1. **Magic Identification**: The host driver inspects the first 4 bytes of any requested firmware file. If ``magic != 0x6e614d58``, the driver deduces that no extended manifest is present and treats offset ``0x0000`` as the hardware payload. +2. **Forward & Backward Compatibility**: + - If a newer firmware binary expands ``sof_ext_man_header`` with additional fields, older drivers read only up to ``header_size`` bytes and start element parsing at ``iptr = fw->data + head->header_size``. + - Version consistency is checked using the major version mask: ``SOF_EXT_MAN_VERSION_INCOMPATIBLE(host_ver, cli_ver)`` checks ``(host_ver & 0xFF000000) != (cli_ver & 0xFF000000)``. +3. **Payload Offset Isolation**: The ``full_size`` field indicates the exact byte count of the extended manifest region. Once parsing succeeds, the Linux driver configures ``sdev->basefw.payload_offset = head->full_size``, directing the hardware DMA engine to skip this un-signed metadata entirely. + +Element Taxonomy & Structures +***************************** + +Following the main header, the extended manifest contains an arbitrary sequence of self-describing metadata elements. Each element begins with a generic element header: + +.. code-block:: c + + struct sof_ext_man_elem_header { + uint32_t type; /* Element type enum: SOF_EXT_MAN_ELEM_* */ + uint32_t size; /* Total size in bytes, including this 8-byte header and payload */ + + /* Immediately followed by type-dependent payload struct */ + } __packed; + +Standard Element Types +====================== + +The Linux kernel and SOF firmware define standard element types in ``enum sof_ext_man_elem_type``: + +.. list-table:: Standard Extended Manifest Element Types + :widths: 10 32 58 + :header-rows: 1 + + * - Type ID + - Enumeration Constant + - Description & Payload Structure + * - ``0`` + - ``SOF_EXT_MAN_ELEM_FW_VERSION`` + - **Firmware Version Metadata**: Contains ``struct sof_ipc_fw_version`` (major, minor, micro, build, tag, commit hash) and ABI flags. + * - ``1`` + - ``SOF_EXT_MAN_ELEM_WINDOW`` + - **Memory Windows**: Contains ``struct sof_ipc_window``, defining host-accessible memory windows for IPC mailboxes, debug/panic dumps, and trace buffers. + * - ``2`` + - ``SOF_EXT_MAN_ELEM_CC_VERSION`` + - **Compiler & Toolchain Version**: Contains ``struct sof_ipc_cc_version`` describing the C compiler, LLVM/Clang version, and build flags used to compile the binary. + * - ``3`` + - ``SOF_EXT_MAN_ELEM_PROBE_INFO`` + - **Trace Probe Configuration**: Describes dynamic trace probe instrumentation points and DMA stream allocation. + * - ``4`` + - ``SOF_EXT_MAN_ELEM_DBG_ABI`` + - **Debug ABI Information**: Contains ``struct ext_man_dbg_abi`` defining the ABI version used by debugfs and telemetry interfaces. + * - ``5`` + - ``SOF_EXT_MAN_ELEM_CONFIG_DATA`` + - **Hardware Configuration Tokens**: Hardware configuration parameters, clock frequencies, and power management defaults. + * - ``6`` + - ``SOF_EXT_MAN_ELEM_PLATFORM_CONFIG_DATA`` + - **Platform Configuration Data**: Platform-specific board overrides parsed by ``snd_sof_dsp_parse_platform_ext_manifest()``. + +IPC4 Extended Manifest 4 ($AE1) +=============================== + +For modern Intel platforms running the IPC4 protocol (cAVS 2.5 and ACE 1.5/2.0/3.0), SOF also supports **Extended Manifest 4**, identified by the ASCII magic ``$AE1`` (``0x31454124``): + +.. code-block:: c + + #define SOF_EXT_MAN4_MAGIC_NUMBER 0x31454124 + + struct sof_ext_manifest4_hdr { + uint32_t id; /* Magic: 0x31454124 ($AE1) */ + uint32_t len; /* Length of extension manifest */ + uint16_t version_major; /* Header version major */ + uint16_t version_minor; /* Header version minor */ + uint32_t num_module_entries; /* Count of module entries described */ + } __packed; + +Extended Manifest 4 provides detailed component descriptors, module UUIDs, scheduling capabilities (sample period multipliers), pin direction caps, and supported audio sample rates directly to the IPC4 pipeline manager in the kernel. + +Build-Time Generation in Rimage +******************************* + +Rimage extracts extended manifest elements directly from the compiled firmware ELF executable during image packaging: + +1. **Metadata Section Declaration**: + Firmware source files declare metadata structures in C and place them in the dedicated ``.fw_metadata`` ELF section: + + .. code-block:: c + + /* Example: Firmware version metadata element in SOF source */ + static const struct { + struct sof_ext_man_elem_header elem_header; + struct sof_ipc_fw_version version; + uint32_t flags; + } fw_ver_elem __section(".fw_metadata") __aligned(EXT_MAN_ALIGN) = { + .elem_header = { + .type = SOF_EXT_MAN_ELEM_FW_VERSION, + .size = sizeof(fw_ver_elem), + }, + .version = { + .major = SOF_MAJOR, + .minor = SOF_MINOR, + .micro = SOF_MICRO, + .build = SOF_BUILD, + .tag = SOF_TAG, + }, + .flags = 0, + }; + +2. **Linker Placement**: + The platform linker script collects all symbols marked with ``__section(".fw_metadata")`` into a contiguous, non-allocatable ELF section named ``.fw_metadata``. +3. **Extraction & Validation**: + Rimage inspects the primary ELF executable using ``elf_section_header_get_by_name()``. It calls ``ext_man_validate()`` to verify two critical integrity constraints: + - Every element header specifies a non-zero ``size`` that is an exact multiple of ``EXT_MAN_ALIGN`` (16 bytes). + - The cumulative sum of all element sizes matches the total size of the ``.fw_metadata`` section exactly, ensuring no trailing garbage or unaligned offsets exist. +4. **Binary Emission**: + Rimage writes the ``struct ext_man_header`` at byte 0 of the destination ``.ri`` file, followed by the contents of ``.fw_metadata``. In addition, if requested, Rimage writes an un-signed standalone file named ``.xman`` for host validation tooling. + +Linux Kernel Host Driver Handshake +********************************** + +When the Linux kernel sound driver loads the firmware file via the standard firmware subsystem, the following handshake occurs in ``sound/soc/sof/loader.c`` and ``sound/soc/sof/ipc3-loader.c``: + +.. code-block:: text + + [ User / Udev ] + | + v + request_firmware(&sdev->basefw.fw, fw_filename, sdev->dev) + | + v + sdev->ipc->ops->fw_loader->parse_ext_manifest(sdev) + | + +---> ipc3_fw_ext_man_size(sdev, fw) + | | + | +---> Check head->magic == 0x6e614d58 ("XMan") + | +---> Return head->full_size + | + +---> Loop over elements: iptr = fw->data + head->header_size + | | + | +---> elem_hdr->type == SOF_EXT_MAN_ELEM_FW_VERSION + | | Parse version, commit hash, display in dmesg + | | + | +---> elem_hdr->type == SOF_EXT_MAN_ELEM_WINDOW + | | Map PCI BAR memory windows for IPC & trace + | | + | +---> elem_hdr->type == SOF_EXT_MAN_ELEM_CC_VERSION + | | Store compiler version in debugfs + | | + | +---> default: + | Safely skip unknown elements via elem_hdr->size + | + v + sdev->basefw.payload_offset = ext_man_size + | + v + DSP DMA Loader transfers payload starting at sdev->basefw.payload_offset + (Extended Manifest is NOT copied to DSP SRAM) + +Graceful Fallback & Extensibility +================================= + +The kernel parser loop guarantees robust forward compatibility: + +.. code-block:: c + + while (remaining > sizeof(*elem_hdr)) { + elem_hdr = (struct sof_ext_man_elem_header *)iptr; + + if (elem_hdr->size < sizeof(*elem_hdr) || elem_hdr->size > remaining) { + dev_err(sdev->dev, "invalid sof_ext_man header size, type %d size %#x\n", + elem_hdr->type, elem_hdr->size); + return -EINVAL; + } + + switch (elem_hdr->type) { + case SOF_EXT_MAN_ELEM_FW_VERSION: + ret = ipc3_fw_ext_man_get_version(sdev, elem_hdr); + break; + case SOF_EXT_MAN_ELEM_WINDOW: + ret = ipc3_fw_ext_man_get_windows(sdev, elem_hdr); + break; + /* ... additional element types ... */ + default: + dev_info(sdev->dev, "unknown sof_ext_man header type %d size %#x\n", + elem_hdr->type, elem_hdr->size); + break; + } + + /* Advance pointer to next element using declared element size */ + iptr += elem_hdr->size; + remaining -= elem_hdr->size; + } + +If a newer firmware image contains novel element types, older Linux kernels log an informational message (``unknown sof_ext_man header type ...``) and advance to the next element without failing firmware initialization. + +Guide: Adding a New Extended Manifest Element +********************************************* + +To add a new metadata element to Sound Open Firmware, follow this four-step procedure across firmware, rimage, and the kernel: + +Step 1: Declare the Element in Header Files +=========================================== + +In the SOF firmware tree (``src/include/kernel/ext_manifest.h``) and the Linux kernel tree (``include/sound/sof/ext_manifest.h``), define the new element type and payload structure: + +.. code-block:: c + + /* 1. Add new element type to enum */ + enum sof_ext_man_elem_type { + /* ... existing elements ... */ + SOF_EXT_MAN_ELEM_CUSTOM_TELEMETRY = 7, + }; + + /* 2. Define packed structure with mandatory elem_header */ + struct sof_ext_man_custom_telemetry { + struct sof_ext_man_elem_header hdr; + uint32_t sampling_interval_ms; + uint32_t feature_flags; + uint8_t custom_name[32]; + } __packed; + +Step 2: Instantiate Element in Firmware Code +============================================ + +In the relevant firmware platform or subsystem file, declare an instance of the struct and place it in the ``.fw_metadata`` section: + +.. code-block:: c + + #include + + static const struct sof_ext_man_custom_telemetry custom_telem + __section(".fw_metadata") __aligned(EXT_MAN_ALIGN) = { + .hdr = { + .type = SOF_EXT_MAN_ELEM_CUSTOM_TELEMETRY, + .size = sizeof(struct sof_ext_man_custom_telemetry), + }, + .sampling_interval_ms = 100, + .feature_flags = 0x00000003, + .custom_name = "SOF_TELEMETRY_V1", + }; + +.. note:: + Ensure that ``sizeof(struct sof_ext_man_custom_telemetry)`` is an exact multiple of ``EXT_MAN_ALIGN`` (16 bytes). Use explicit padding fields if necessary to prevent ``ext_man_validate()`` from rejecting the binary during build. + +Step 3: Implement Parser Handler in Linux Kernel +================================================ + +In ``sound/soc/sof/ipc3-loader.c`` (or ``ipc4-loader.c``), add a case branch to handle the new element type: + +.. code-block:: c + + static int ipc3_fw_ext_man_get_custom_telemetry(struct snd_sof_dev *sdev, + const struct sof_ext_man_elem_header *hdr) + { + const struct sof_ext_man_custom_telemetry *telem = + (const struct sof_ext_man_custom_telemetry *)hdr; + + dev_info(sdev->dev, "Custom Telemetry: %s, interval %u ms, flags 0x%08x\n", + telem->custom_name, telem->sampling_interval_ms, telem->feature_flags); + + /* Store telemetry configuration in sdev private context */ + sdev->custom_telemetry_interval = telem->sampling_interval_ms; + return 0; + } + + /* In sof_ipc3_fw_parse_ext_man(): */ + case SOF_EXT_MAN_ELEM_CUSTOM_TELEMETRY: + ret = ipc3_fw_ext_man_get_custom_telemetry(sdev, elem_hdr); + break; + +Step 4: Verification and Debugging +================================== + +1. Rebuild the firmware using ``west build`` and inspect the Rimage output: + + .. code-block:: bash + + # Rimage logs verbose parsing of all discovered fw_metadata elements: + Extended manifest found module, type: 0x0007 size: 0x0030 ( 48) offset: 0x0070 + +2. Deploy the signed ``.ri`` binary to the target DUT and reload the kernel driver: + + .. code-block:: bash + + # Verify kernel dmesg output on the DUT: + dmesg | grep -i "sof_ext_man\|telemetry" + # Expected output: + # sof-audio-pci-intel-tgl: found sof_ext_man header type 7 size 0x30 + # sof-audio-pci-intel-tgl: Custom Telemetry: SOF_TELEMETRY_V1, interval 100 ms, flags 0x00000003 \ No newline at end of file diff --git a/developer_guides/rimage/images/ext_man_build_flow.pu b/developer_guides/rimage/images/ext_man_build_flow.pu deleted file mode 100644 index 2ca7f731..00000000 --- a/developer_guides/rimage/images/ext_man_build_flow.pu +++ /dev/null @@ -1,33 +0,0 @@ -@startuml - -title Extended manifest build flow - -class firmware_elf_file << (F, orchid) >> { - suffix: - -- Content -- - +FW metadata section - +other sections -} - -rectangle rimage { - class ext_man_write { - 1. Create .ri.xman file - 2. Find elf file with .fw_metadata section - 3. Read .fw_metadata section - 4. Build ext_man_header - 5. Validate sum of elements size with section size - 6. Save output to .ri.xman file - } -} - -class ext_man_file << (F, orchid) >> { - suffix: .ri.xman - -- Content -- - + ext_man_header - + .fw_metadata section content -} - -firmware_elf_file -down-> ext_man_write -ext_man_write -down-> ext_man_file - -@enduml diff --git a/developer_guides/rimage/images/image_build_flow.pu b/developer_guides/rimage/images/image_build_flow.pu deleted file mode 100644 index 0550e3c9..00000000 --- a/developer_guides/rimage/images/image_build_flow.pu +++ /dev/null @@ -1,70 +0,0 @@ -@startuml - -title image build flow - -class bootloader_elf_file << (F, orchid) >> { - -- note -- - Used for CAVS 1.5 and newer -} -class firmware_elf_file << (F, orchid) >> { -} - -class rimage { - +write_firmware() - +write_firmware_meu() - +ext_man_write() -} - -class adsp_manifest_file << (F, orchid) >> { - suffix: .ri.met -} - -class image_file << (F, orchid) >> { - suffix: .ri - -- Content -- - +CSE manifest - +CSS manifest - +ADSP manifest - +runtime code -} - -class unsigned_image_file << (F, orchid) >> { - suffix: .ri.uns - -- Content -- - +ADSP manifest - +runtime code -} - -class ext_man_file << (F, orchid) >> { - suffix: .ri.xman -} - -class build_step { - + glue binary files() -} -hide build_step circle - -class final_image_file << (F, orchid) >> { - suffix: .ri - -- Content -- - +Extended manifest - +CSE manifest - +CSS manifest - +ADSP manifest - +runtime code -} - -firmware_elf_file -down-> rimage -bootloader_elf_file -down-> rimage -rimage -down-> ext_man_file : with -e flag -ext_man_file -down-> build_step -rimage -down-> adsp_manifest_file -rimage -down-> image_file : without MEU\nwithout -s argument -rimage -down-> unsigned_image_file : with MEU \nwith -s argument -unsigned_image_file -down-> MEU -MEU -down-> image_file -adsp_manifest_file -down-> image_file -image_file -down-> build_step -build_step -down-> final_image_file - -@enduml diff --git a/developer_guides/rimage/images/rimage_binary_layout.svg b/developer_guides/rimage/images/rimage_binary_layout.svg new file mode 100644 index 00000000..b6957e6f --- /dev/null +++ b/developer_guides/rimage/images/rimage_binary_layout.svg @@ -0,0 +1,257 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SOF Binary Firmware Image (.ri) Internal Layout & Header Hierarchy + + + Byte-level anatomical breakdown of Extended Manifest, Converged Security Headers, Module Descriptors, and Page-Aligned Payload + + + + + + + + + + Binary File Layout (.ri) + + + + + + Extended Manifest (XMan) + Magic: 0x6e614d58 ('XMan') + Host Driver Metadata (Unsigned) + + + + + + + Host Driver Skips (offset = full_size) + + + + + + + CSE Directory ($CPD) + Magic: 0x44504324 ($CPD) + Entries: ADSP.man, .met, cavs/ace + + + + + + CSS Signature ($MN2) + RSA-2048/3072 Modulus & Exp + PKCS#1 v1.5 RSA Signature + Vendor: 0x8086 | BCD Date | SVN + + + + + + Signed Pkg Info & Meta Ext + V2.5 / ACE 1.5 Security Ext + + + + + + FW Desc ($AM1) & Modules ($AME) + num_modules = N | preload_pages + UUID, Affinity Mask, Entry Point + SHA-256 / SHA-384 Module Hashes + + + + + + + Page Padding (0x00.. MAN_PAGE_SIZE) + + + + + + + Payload: Executable Segments + Module 0 (Base FW Code/Data) + .text (Executable Code Pages) + .rodata (Constants & Tables) + Module 1..N (LLEXT / Dynamic) + 4KB Page Aligned (MAN_PAGE_SIZE) + + + + + + + + + + + + Header 1: Extended Manifest (struct ext_man_header) — Pre-Boot Host Driver Handshake + + + + + + + struct ext_man_header { + uint32_t magic; /* 0x6e614d58 ('XMan') */ + uint32_t full_size; /* Total size (hdr + elems + pad) */ + uint32_t header_size; /* sizeof(struct ext_man_header) */ + uint32_t header_version; /* EXT_MAN_VERSION (1.0.0) */ + }; + + + + + Extended Manifest Elements (struct ext_man_elem_header, 16B aligned): + + • Type 0 (EXT_MAN_ELEM_FW_VERSION): Major, Minor, Hotfix, Build, Git Tag + • Type 1 (EXT_MAN_ELEM_WINDOW): Memory window descriptor (base, size, flags) + • Type 2 (EXT_MAN_ELEM_CC_VERSION): Compiler description (Clang/XCC version) + • Type 3 (EXT_MAN_ELEM_PROBE_INFO): Probe extraction & trace buffer mapping + • Type 4 (EXT_MAN_ELEM_DBG_ABI): User ABI version compatibility handshake + + + + + + + + + + Header 2: Security Headers — CSE Directory ($CPD) & CSS Authentication ($MN2) + + + + + + + + CSE Partition Directory Header (struct CsePartitionDirHeader_v2_5) + + • header_marker: 0x44504324 ("$CPD") + • nb_entries: 3 (ADSP.man, ADSP.met, ADSP/cavs) + • partition_name: "ADSP" + • checksum: CRC32 of partition headers + Partition Table: + Entry 0: "ADSP.man" (offset=0x5c, len=0x464..0x4b8) + Entry 1: "ADSP.met" (offset=0x4c0) | Entry 2: "ADSP" (offset=0x540) + + + + + CSS Cryptographic Header (struct css_header_v2_5) + + • header_id: "$MN2" | module_vendor: 0x8086 (Intel) + • date: BCD timestamp (yYyy:mm:dd) + • svn: Security Version Number (Anti-Rollback Counter) + • modulus_size: 256B (RSA-2048) or 384B (RSA-3072) + • exponent: 4B (0x00010001 = 65537) + • signature: RSA PKCS#1 v1.5 SHA-256 / SHA-384 + Authenticates entire manifest against fused OEM public key hash + + + + + + + + + + Header 3: Firmware Descriptor ($AM1) & Module Entries ($AME) — Subsystem Metadata + + + + + + + + FW Descriptor (struct sof_man_fw_desc) + + • struct_id: "$AM1" + • preload_page_count: Preload size + • num_module_entries: Total modules + • major/minor/build version + Parsed by DSP Boot ROM to allocate base L2/SRAM heap before executing entry point + + + + + Module Table Array (struct sof_man_module module_entries[num_modules]) + + • struct_id: "$AME" | name[8]: "BASEFW", "VOLUME", "MIXER", etc. + • uuid: 128-bit RFC 4122 unique component identifier + • entry_point: Virtual address executed upon thread initialization + • type: load_type (0=builtin, 1=module, 2=LLEXT, 3=LLEXT_AUX), affinity_mask + • segment[3]: text, data, bss (file_offset, v_base_addr, flags, length in 4KB pages) + • hash[32/48]: Cryptographic SHA-256 or SHA-384 digest of module code segment + + + + + + + + Memory Alignment & Loading Invariance: + + + 1. Extended Manifest (0x6e614d58) is placed at the exact head of the file. Host kernel parses it and jumps to full_size. + + + 2. The signed partition begins with CSE ($CPD). All module text/data segments are strictly aligned to MAN_PAGE_SIZE (4096 bytes) for direct DSP DMA execution. + + + diff --git a/developer_guides/rimage/images/rimage_crypto_signing_flow.svg b/developer_guides/rimage/images/rimage_crypto_signing_flow.svg new file mode 100644 index 00000000..ef4c9445 --- /dev/null +++ b/developer_guides/rimage/images/rimage_crypto_signing_flow.svg @@ -0,0 +1,270 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SOF Cryptographic Signing Architecture & Hardware Root-of-Trust Authentication + + + Dual Signing Modes (Native RSA PKCS#1 v1.5 vs Intel MEU/HSM) and CSME / DSP Boot ROM eFuse Verification + + + + + + + + + + Build-Time Cryptographic Signing: Native Engine vs External Intel MEU / OEM HSM Flow + + + + + + + Flow A: Native Rimage Signing (-k otc_private.pem) + + + Used for Community Builds, Engineering Silicon & Pre-Production Validation + + + + + + + 1. SHA-256 (CAVS) / SHA-384 (ACE) per module .text/.data + + + + + 2. Hash CSS + CSE + ADSP Manifest Payload Block + + + + + 3. RSA-2048/3072 PKCS#1 v1.5 Sign with Private Key + + + Directly generates signed container image (sof-*.ri) + + + + + + + + + Flow B: Production External Signing (-s meu_offset) + + + Used for Commercial Tier-1 OEM Production Laptops & Fused Hardware + + + + + + + 1. rimage outputs unsigned .ri.uns & metadata .ri.met + + + + + 2. Intel MEU Tool / Air-Gapped HSM signs with OEM Key + + + + + 3. rimage -q (Resign) or -y (Verify Signature) + + + Stitches signed headers and validates production cryptographic parameters + + + + + + + + + + Loaded on Target DUT + + + + + + + + + + Hardware Boot ROM / CSME Authentication Pipeline (Silicon Execution) + + + + + + + + + 1 + + eFuse Root Validation + + + + Hardware reads Public Key Modulus from CSS header ($MN2). + + + + H(Modulus) == eFuse_Hash? + + + Match against burned OTP fuses + + + + FAIL ➔ Hard Reset / Lockup + + + PASS ➔ Proceed to Step 2 + + + + + + + + + + + 2 + + CSS RSA Verification + + + + Decrypts RSA signature using authenticated Public Key. + + + + RSA_Verify(Sig, Manifest) + + + SHA-256 / SHA-384 PKCS#1 v1.5 + + + + FAIL ➔ CSS auth error + + + PASS ➔ Proceed to Step 3 + + + + + + + + + + + 3 + + SVN & PV Bit Check + + + + Validates Security Version Number against hardware counter. + + + + CSS.svn >= HW_Min_SVN + + + PV Bit matches silicon state + + + + FAIL ➔ Rollback detected + + + PASS ➔ Proceed to Step 4 + + + + + + + + + + + 4 + + Core Execution + + + + Validates SHA digests in $AME per module before jumping. + + + + Jump to entry_point + + + DSP Core 0 Released + + + + FW BOOT SUCCESS + + + + + + + Production Silicon Policy: If any hardware fuse check, RSA signature decryption, or SVN anti-rollback check fails, the DSP Boot ROM terminates boot execution immediately, keeping the audio subsystem in hard D3 state. + + + diff --git a/developer_guides/rimage/images/rimage_extended_manifest_handshake.svg b/developer_guides/rimage/images/rimage_extended_manifest_handshake.svg new file mode 100644 index 00000000..72831d0a --- /dev/null +++ b/developer_guides/rimage/images/rimage_extended_manifest_handshake.svg @@ -0,0 +1,221 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Extended Manifest (XMan) Data Structure & Host Kernel Handshake + + + Pre-boot metadata extraction in Linux snd-sof driver (snd_sof_fw_ext_man_parse()) and seamless payload handover to DSP + + + + + + + + + + Binary Head: Extended Manifest Structure (.ri file head) + + + + + + + struct ext_man_header (Offset 0x0000) + + + • magic: 0x6e614d58 (ASCII 'XMan' in little-endian) + + + • full_size: Total bytes of Extended Manifest (header + elements + pad) + + + • header_size: sizeof(struct ext_man_header) | header_version: 0x01000000 + + + + + + + Sequential Elements Array (struct ext_man_elem_header, EXT_MAN_ALIGN = 16B): + + + + + + Type 0: EXT_MAN_ELEM_FW_VERSION + + + Major, Minor, Micro, Build Version, ABI Version, Git Commit SHA Tag + + + + + + Type 1: EXT_MAN_ELEM_WINDOW (SOF_IPC_EXT_WINDOW) + + + Host/DSP Shared Memory Windows (Win 0: In/Outbox, Win 1: Debug Trace, Win 2: Stream) + + + + + + Type 2: EXT_MAN_ELEM_CC_VERSION (Compiler Description) + + + Toolchain banner string (e.g. Clang 18.1.0 Xtensa, GCC, or Cadence XCC) + + + + + + Type 3: EXT_MAN_ELEM_PROBE_INFO & Type 4: DBG_ABI + + + Direct probe point extraction registers and User ABI compatibility token + + + + + + Type 5: CONFIG_DATA & Type 6: PLATFORM_CONFIG_DATA + + + Hardware topology configuration hints, IPC capability flags, clock setpoints + + + + + + + + Parsed By + + + + + + + + + Linux Host Driver Handshake (sound/soc/sof/loader.c) + + + + + + + + 1 + + Firmware Loading: request_firmware() + + + Host fetches sof-<platform>.ri from filesystem (/lib/firmware/intel/sof/) + + + + + + 2 + + Magic Verification: snd_sof_fw_ext_man_parse() + + + if (head->magic == EXT_MAN_MAGIC_NUMBER) { /* 0x6e614d58 */ } + + + Validates header length and major ABI version consistency + + + + + + 3 + + Element Dispatch Loop: Iterating elem_size + + + Iterates over elements; unknown element types are gracefully skipped: + + + offset += elem->elem_size; /* Allows forward compatibility */ + + + + + + 4 + + Telemetry & Capability Registration + + + Registers Mailbox base addresses, SRAM windows, debug log buffers + + + Emits kernel log: "Firmware info: version 2.12.0-xxx, build ... Clang 18" + + + + + + 5 + + Payload Handover: Skip full_size to Signed Manifest + + + fw_payload = fw->data + ext_man->full_size; + + + Driver transmits the signed CSE/CSS payload to DSP SRAM. Extended Manifest does not consume DSP memory. + + + + + + + + + Key Architectural Decoupling: + + + The Extended Manifest acts as an un-signed preamble specifically intended for the Linux host operating system. Because target hardware boot ROMs and CSME coprocessors strictly authenticate cryptographic signatures starting at the CSE partition header ($CPD), the driver strips/skips the Extended Manifest prior to initiating DMA boot transfers, ensuring zero impact on internal DSP L1/L2 SRAM memory budgets. + + + diff --git a/developer_guides/rimage/images/rimage_pipeline_architecture.svg b/developer_guides/rimage/images/rimage_pipeline_architecture.svg new file mode 100644 index 00000000..8972ad9c --- /dev/null +++ b/developer_guides/rimage/images/rimage_pipeline_architecture.svg @@ -0,0 +1,295 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SOF Rimage: End-to-End Firmware Image Packaging & Cryptographic Signing Pipeline + + + Transformation from compiled Xtensa/Host ELF binaries into validated, hardware-authenticated .ri container images + + + + + + + + + + + + 1. ELF Ingestion + + + + + + Target ELF Binaries + • zephyr.elf (Base FW) + • bootloader.elf + + + Section Extraction + • .text (Executable Code) + • .rodata (Const Data) + • .bss (Zero-init Size) + + + Metadata Section + • .fw_metadata + - FW Version & ABI + - Memory Windows + + + Dynamic Modules + • *.llext (Relocatable) + - Module Flag: -l + - Non-bootloader entry + + + ELF Validator + Relocatable / Absolute + + + + + + + + + + + + + 2. TOML Configuration + + + + + + Platform TOML + • config/tgl.toml + • config/mtl.toml.h + • config/ptl.toml.h + + + Memory Zones + • ROM: Boot Code + • IMR: Isolated RAM + • HP/LP-SRAM Blocks + • Cached/Uncached VMA + + + Manifest Layout + • format = "cse/css" + • version = 2.5 / 3.0 + • image_size bound + + + Module Registry + • UUID bindings + • Affinity & instance max + + + tomlc99 Parser + Strict Bounds Verification + + + + + + + + + + + + + 3. Manifest Synthesis + + + + + + Extended Manifest + • Magic: 0x6e614d58 + • Tag: 'XMan' + • Skipped by Boot ROM + + + Security Headers + • CSE: $CPD (Partitions) + • CSS: $MN2 (Signatures) + • Signed Pkg Info + • ADSP Metadata Ext + + + FW & Module Table + • FW Desc: $AM1 + • Modules: $AME + • Entry point & VMA + • V2.5/3.0 offsets + + + Page Alignment + • 4096-Byte Padding + • MAN_PAGE_SIZE bound + + + Contiguous Buffer + DMA-Ready Image Memory + + + + + + + + + + + + + 4. Crypto & Signing + + + + + + Module Hashing + • SHA-256 (CAVS) + • SHA-384 (ACE) + • Hashed per .text/.data + • Stored in $AME entry + + + RSA Signatures + • PKCS#1 v1.5 padding + • RSA-2048 / RSA-3072 + • Signs CSS payload + • Key: otc_private.pem + + + External MEU / HSM + • Flag: -s meu_offset + • Production OEM Keys + • Emits .uns & .met + • Resign via -q / verify -y + + + Security Attributes + • PV Bit Flag: -p + • Anti-Rollback SVN + + + Crypto Engine + OpenSSL / Native Backend + + + + + + + + + + + + + 5. Image Artifacts + + + + + + Primary Signed Image + • sof-*.ri + 1. [XMan] Ext Manifest + 2. [CSE/CSS] Signatures + 3. [Payload] Modules + + + Unsigned Payload + • sof-*.ri.uns + • Stripped of CSS/RSA + • Input for MEU tool + + + Manifest Metadata + • sof-*.ri.met + • Standalone manifest + • Verification artifact + + + Target Deployment + /lib/firmware/intel/sof/ + • Loaded via snd-sof + • CSME / DSP Boot ROM + • Verified in Hardware + + + + + + + + Key Build Directive: + + + west sign -t rimage -- -k keys/otc_private.pem -c config/tgl.toml -o build/sof-tgl.ri build/zephyr.elf + + + The host kernel driver (snd-sof) parses the initial Extended Manifest for versioning and window geometry, skips past it to the CSE/CSS signature, and transfers the signed payload to DSP SRAM. The hardware Boot ROM authenticates the RSA PKCS#1 v1.5 signature against fused OEM root keys prior to core execution. + + + diff --git a/developer_guides/rimage/images/rimage_platform_memory_matrix.svg b/developer_guides/rimage/images/rimage_platform_memory_matrix.svg new file mode 100644 index 00000000..8de15b92 --- /dev/null +++ b/developer_guides/rimage/images/rimage_platform_memory_matrix.svg @@ -0,0 +1,218 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SOF Platform Memory Architectures & Virtual Addressing Matrix + + + cAVS (1.5, 1.8, 2.5) vs ACE (1.5, 2.0, 3.0): SRAM Aliasing, Cache Bypass Windows, IMR Layout, and Alignment Rules + + + + + + + + + + Intel cAVS Memory Architecture (cAVS 1.5, 1.8, 2.5 / Tiger Lake, Ice Lake) + + + + + + 0xBE000000 + HP-SRAM (Cached Alias) + Code (.text), RoData, Stack, Heap + + + + 0x9E000000 + HP-SRAM (Uncached Alias) + IPC Mailboxes, DMA Ring Descriptors + + + + + + + + 0xBF000000 + LP-SRAM (Cached) + Low-power retention + + + + 0x9F000000 + LP-SRAM (Uncached) + Wakeup buffers + + + + 0xB0000000 + IMR (Isolated Memory Region) in Host DRAM + Pre-allocated carveout for runtime firmware staging, large buffers, and crashdump capture + + + + cAVS Memory Management Parameters: + • Cache Bit Mask: Bit 29 / Bit 30 controls caching (Uncached = base & ~0x20000000) + • Page Alignment: 4096 bytes (4 KB) strictly required by CSE and DMA engines + • Boot Vector: Reset vector mapped at 0xBE000000 (HP-SRAM entry point) + • Host DMA Load: DMA transfers firmware directly into SRAM before DSP core is reset + + + + + + + + + Intel ACE Memory Architecture (ACE 1.5, 2.0, 3.0 / Arrow Lake, Panther Lake) + + + + + + 0xA0000000 + DSP SRAM (Cached Window) + Normal execution, L1/L2 data cached + + + + 0x40000000 + DSP SRAM (Uncached Window) + L2 Cache Bypass: IPC, Trace, DMA + + + + + + + + Unified L2 Cache / Memory Subsystem + Coherent multi-core DSP cluster (Core 0-3); hardware snooping across DSP cores + + + + 0xA104A000 / 0xA1000000 + IMR Boot & Dynamic Module Staging + CSME stages initial payload; LLEXT relocatable modules loaded on demand + + + + ACE Memory Management Parameters: + • Uncached Remapping: 0x40000000 aliases physical SRAM without cache allocation + • Dynamic Modules: Zephyr LLEXT relocatable segments mapped dynamically + • Multi-Core Snooping: HW MESI/L2 cache coherency between primary and secondary cores + • CSME Preload: CSME verifies manifest & copies signed image from Host DRAM into IMR/SRAM + + + + + + + + + Rimage ELF Section Mapping, Page Alignment, and Cache Flush Handshake + + + + + + + 1. ELF Section Segmentation + • .text, .literal: SRAM Cached, Executable + • .rodata: SRAM Cached, Read-Only + • .data, .bss: SRAM Cached, Read-Write + • .shmem, .mailbox: SRAM Uncached Alias + • .llext.*: Position-independent ELF sections + Extracted and validated against TOML limits + + + + + + 2. Alignment & Page Constraints + • MAN_PAGE_SIZE = 4096 (4 KB) + • Segment padding to 4KB page boundaries + • preload_page_count defines boot DMA size + • CSS / CPD Manifests aligned to 16/64 bytes + • Extended Manifest aligned to 16 bytes + Mismatched alignment causes CSME verify fault + + + + + + 3. Cache Coherency Handshake + • Host DMA writes bypass DSP L1/L2 caches + • DSP invalidates I-Cache before executing code + • Firmware performs dcache_writeback + • IPC messaging uses uncached window + • Shared buffers require fence instructions + Prevents stale memory hazard between Host & DSP + + + diff --git a/developer_guides/rimage/index.rst b/developer_guides/rimage/index.rst index 1ac2c423..8a99f896 100644 --- a/developer_guides/rimage/index.rst +++ b/developer_guides/rimage/index.rst @@ -1,96 +1,519 @@ .. _rimage: -Rimage Firmware Image Creation & Signing -######################################## +Rimage Firmware Packaging & Signing Architecture +################################################ -**Rimage** is the official DSP firmware image packaging and cryptographic signing tool for Sound Open Firmware (SOF). Implemented in modern Rust (`thesofproject/rimage `_), rimage transforms compiled ELF executables into validated, hardware-loadable binary images (``.ri`` files) for Intel, NXP, AMD, and other DSP architectures. +**Rimage** is the official firmware image creation, packaging, and cryptographic signing toolchain for Sound Open Firmware (SOF). It translates compiled Executable and Linkable Format (ELF) object files produced by the Xtensa/LLVM compiler into validated, secure, hardware-bootable binary images (``.ri`` files). -Rimage parses declarative target platform configuration files (TOML), verifies memory section alignments, packages dynamic modules and static manifests, and applies cryptographic digital signatures required by hardware boot ROMs. +Rimage is responsible for enforcing platform memory boundaries, generating Converged Security Engine (CSE) partition manifests, computing cryptographic digest hashes, inserting the un-signed Extended Manifest for host kernel initialization, bundling relocatable dynamic modules (Zephyr LLEXT), and applying digital signatures required by hardware boot ROMs and Silicon Root-of-Trust (RoT) engines. -Key Features & Capabilities -*************************** +.. figure:: images/rimage_pipeline_architecture.svg + :alt: SOF Rimage End-to-End Packaging and Signing Pipeline + :align: center + :width: 100% -* **Rust Architecture**: High-performance, memory-safe signing and image generation engine. -* **Declarative TOML Configuration**: Platform memory layouts, modules, and hardware parameters are defined in clean, human-readable TOML files under `config/`. -* **Hardware Manifest Generation**: - * **Extended Manifest**: Describes firmware versioning, compiler toolchains, and ABI metadata for the Linux host driver (`snd-sof`). - * **CSE Manifest**: Converged Security Engine descriptors for modern Intel platforms. - * **CSS Manifest**: Common Security Signature for Intel CAVS / ACE security coprocessors. - * **ADSP Manifest**: Audio DSP hardware memory segment descriptors. -* **Cryptographic Signing**: Supports RSA PKCS#1 v1.5 with SHA-256 and SHA-384, utilizing OpenSSL or pure Rust crypto backends. -* **IPC4 Multi-Module Packaging**: Bundles base firmware images with loadable library modules (LLEXT). + End-to-end Rimage 5-stage packaging pipeline: ELF extraction, TOML hardware mapping, manifest synthesis, cryptographic signing, and binary assembly. .. toctree:: :maxdepth: 1 extended_manifest -TOML Platform Configuration -*************************** +Architecture & Dual Implementations +*********************************** -Rimage relies on platform-specific TOML files to describe hardware memory mappings and signing requirements. For example, a target configuration defines memory segments, cache settings, and manifest types: +Rimage exists in two implementations within the Sound Open Firmware ecosystem: + +1. **Production C99 Toolchain** (``tools/rimage/``): + The authoritative packaging tool integrated directly into the SOF build tree and invoked by Zephyr's CMake/west build system. Implemented in high-performance C99, it utilizes `tomlc99` for TOML parsing and OpenSSL (`libcrypto`) for cryptographic hashing and RSA PKCS#1 v1.5 digital signing. This tool is built on-the-fly or executed from the host system during firmware compilation. +2. **Upstream Rust Implementation** (``thesofproject/rimage``): + A modular, memory-safe reimplementation in Rust designed to provide clean library crates for ELF parsing, manifest serialization, and cryptographic verification, supporting modern continuous integration pipelines. + +Both implementations adhere to the identical binary layout specifications, TOML schemas, cryptographic structures, and hardware alignment constraints described in this guide. + +Binary Image Layout & Header Hierarchy +************************************** + +A fully packaged Sound Open Firmware binary file (``.ri``) consists of a multi-tiered header stack followed by page-aligned executable and data payloads. The image is structured so that each layer can be parsed and validated by the relevant hardware or software entity: the Linux host driver, the Intel Converged Security Engine (CSME), the DSP Boot ROM, and finally the SOF Base Firmware runtime. + +.. figure:: images/rimage_binary_layout.svg + :alt: SOF Binary Image Layout and Header Hierarchy + :align: center + :width: 100% + + Comprehensive byte-level binary layout of an SOF ``.ri`` firmware image from offset ``0x0000`` to end-of-file. + +Binary Structure Anatomy +======================== + +The following table summarizes the structural hierarchy of a modern SOF image: + +.. list-table:: SOF Binary Image (``.ri``) Section Hierarchy + :widths: 14 18 20 48 + :header-rows: 1 + + * - Offset + - Structure / Magic + - Target Consumer + - Function & Key Fields + * - ``0x0000`` + - ``sof_ext_man_header`` (``0x6e614d58`` / ``XMan``) + - Linux Host Driver (``snd-sof``) + - **Extended Manifest**: Un-signed compile-time metadata. Conveys firmware version, compiler version, IPC memory windows, and debug ABI. Skipped during DSP DMA transfer. + * - ``+full_size`` + - ``cse_header`` (``0x44504324`` / ``$CPD``) + - Intel CSME / Hardware Boot ROM + - **CSE Partition Directory**: Declares image partitions: ``ADSP.man`` (manifest), ``ADSP.met`` (metadata), and ``ADSP`` (executable payload). + * - Variable + - ``css_header`` (``0x324e4d24`` / ``$MN2``) + - CSME / Boot ROM Cryptographic Engine + - **Common Security Signature**: Contains vendor ID (``0x8086``), BCD date, security version number (SVN), RSA modulus (2048/3072-bit), exponent (``0x10001``), and PKCS#1 v1.5 digital signature. + * - Variable + - Signed Package Info & Metadata + - Boot ROM Verifier + - Security extension metadata and ADSP partition integrity hashes. + * - Page-aligned + - ``adsp_fw_desc`` (``0x314d4124`` / ``$AM1``) + - DSP Boot ROM / Base Firmware Loader + - **Firmware Descriptor**: Specifies preload page count, entry point address, and the count of executable modules. + * - Variable + - ``adsp_module_entry`` (``0x454d4124`` / ``$AME``) + - DSP Module Loader / IPC4 Dispatcher + - **Module Descriptor Table**: Array of module definitions with UUIDs, entry points, memory segment lists (text/data/bss), and cryptographic hashes. + * - ``4096``-aligned + - Executable Payload Segments + - Audio DSP Core Execution + - High-Performance SRAM and IMR code/data pages loaded via DMA bursts. + +Alignment Constraints +===================== + +Strict alignment rules are enforced by Rimage to guarantee hardware compatibility: + +- **Page Alignment** (``MAN_PAGE_SIZE = 4096``): Executable segments and binary partitions must be aligned to 4 KB page boundaries to allow direct DMA streaming into DSP SRAM without partial-page DMA faults. +- **Extended Manifest Alignment** (``EXT_MAN_ALIGN = 16``): All extended manifest element structures must be aligned to 16 bytes. +- **Cryptographic Manifest Alignment**: The CSS manifest and signature blocks must be aligned to 64 bytes for hardware SHA-256 and SHA-384 hardware acceleration blocks. + +Platform Memory Architectures & Addressing Matrix +************************************************* + +Intel audio DSP architectures are divided into two primary evolutionary families: **cAVS** (Intel Converged Audio, Voice and Speech) and **ACE** (Intel Audio Core Engine). Each architecture defines distinct virtual memory maps, cache bypass windows, and Isolated Memory Region (IMR) interfaces. + +.. figure:: images/rimage_platform_memory_matrix.svg + :alt: SOF Platform Memory Architectures and Virtual Addressing Matrix + :align: center + :width: 100% + + Memory architectural comparison between Intel cAVS and ACE platforms: virtual addressing, cached/uncached aliasing, and IMR windows. + +cAVS Memory Architecture (cAVS 1.5, 1.8, 2.5) +============================================== + +Platforms such as Apollo Lake (cAVS 1.5), Cannon Lake (cAVS 1.8), and Tiger Lake (cAVS 2.5) utilize an Xtensa DSP core memory architecture where caching behavior is controlled by high-order virtual address bits: + +- **Cached HP-SRAM Alias** (``0xBE000000``): High-Performance SRAM accessed via the L1 instruction and data caches. Executable code (``.text``), read-only data (``.rodata``), stack, and heap are mapped here. +- **Uncached HP-SRAM Alias** (``0x9E000000``): Physical SRAM accessed by bypassing the L1 cache. Used for inter-processor communication (IPC) mailboxes, DMA ring buffers, and host-DSP shared telemetry memory. +- **Low-Power SRAM (LP-SRAM)**: Mapped at cached ``0xBF000000`` and uncached ``0x9F000000``. Retained during low-power D3 states for Wake-on-Voice (WOV) buffering. +- **Isolated Memory Region (IMR)**: Mapped at ``0xB0000000`` in host DRAM carveout, used for runtime firmware staging and large audio stream buffering. + +ACE Memory Architecture (ACE 1.5, 2.0, 3.0) +=========================================== + +Starting with Meteor Lake / Arrow Lake (ACE 1.5), Lunar Lake (ACE 2.0), and Panther Lake (ACE 3.0), the memory architecture was redesigned around a unified L2 cache subsystem and dynamic module loading: + +- **Cached SRAM Window** (``0xA0000000``): Unified DSP SRAM mapped with L1/L2 hardware cache snooping. Core 0 reset vector is located at ``0xA0000000``. +- **Uncached / Cache-Bypass Window** (``0x40000000``): Physical SRAM alias that completely bypasses the L2 cache controller. Essential for host DMA buffers, IPC descriptors, and trace logging streams to prevent cache stale hazards. +- **ACE IMR Window** (``0xA104A000`` / ``0xA1000000``): Dedicated DRAM carveout managed by the Intel Converged Security and Management Engine (CSME) for cold-store firmware execution and dynamic loadable library (LLEXT) staging. + +Memory Mapping Comparison +========================= + +.. list-table:: Platform Memory Mapping Comparison Matrix + :widths: 24 26 26 24 + :header-rows: 1 + + * - Parameter / Region + - cAVS 2.5 (Tiger Lake) + - ACE 1.5 (Arrow Lake) + - ACE 3.0 (Panther Lake) + * - **HP-SRAM Cached** + - ``0xBE000000`` + - ``0xA0000000`` + - ``0xA0000000`` + * - **HP-SRAM Uncached** + - ``0x9E000000`` + - ``0x40000000`` + - ``0x40000000`` + * - **IMR Base Address** + - ``0xB0000000`` + - ``0xA1000000`` + - ``0xA104A000`` + * - **Cache Control Mechanism** + - Virtual address bit flipping + - Window remapping (``0x40000000``) + - Window remapping + HW snooping + * - **Hash Algorithm** + - SHA-256 + - SHA-384 + - SHA-384 + * - **Signature Algorithm** + - RSA-2048 / 3072 PKCS#1 v1.5 + - RSA-3072 PKCS#1 v1.5 + - RSA-3072 PKCS#1 v1.5 + +Cryptographic Digital Signing Architecture +****************************************** + +Sound Open Firmware enforces cryptographic authentication to protect the audio DSP subsystem from unauthorized code execution. During system boot, the hardware CSME and DSP Boot ROM verify the digital signature embedded in the CSS manifest before releasing the DSP reset vector. + +.. figure:: images/rimage_crypto_signing_flow.svg + :alt: SOF Cryptographic Signing Architecture and Flow + :align: center + :width: 100% + + Cryptographic signing pipeline comparing native Rimage RSA signing against the external Intel MEU / OEM HSM workflow, followed by Boot ROM eFuse verification. + +Dual Signing Workflows +====================== + +Rimage supports two distinct cryptographic signing flows depending on the target deployment environment: + +Native Rimage Signing Engine (Development & Community) +------------------------------------------------------ + +In development environments and for engineering hardware, Rimage signs the binary directly using an RSA private key supplied via the command line (``-k ``): + +1. **Digest Calculation**: Rimage hashes the manifest header and executable module payloads using SHA-256 (cAVS) or SHA-384 (ACE). +2. **PKCS#1 v1.5 Formatting**: The digest is encapsulated in an ASN.1 ``DigestInfo`` prefix and padded according to RSA PKCS#1 v1.5 standards. +3. **Modular Exponentiation**: The signature is computed using OpenSSL's RSA engine: + + .. math:: + + S = M^d \pmod{n} + +4. **Public Key Embedding**: The RSA public modulus (*n*) and exponent (*e* = ``0x10001``) are written into the CSS manifest header alongside the signature (*S*). + +External Intel MEU & OEM HSM Signing (Production Devices) +--------------------------------------------------------- + +On commercial production platforms, the private signing key resides within a secure Hardware Security Module (HSM) or an Intel Management Engine Utility (MEU) signing pipeline. Rimage accommodates this workflow via the ``-s`` offset parameter: + +1. Rimage builds the complete image structure, calculates all module hashes, formats the CSS header, and leaves a blank signature placeholder of specified size. +2. The partial image is submitted to the OEM HSM or Intel MEU signing server. +3. The HSM signs the CSS manifest and injects the resulting signature and production certificate chain back into the ``.ri`` binary. +4. Rimage verifies the final signature using its verification option: ``rimage -c -y -k ``. + +Hardware Root-of-Trust (RoT) Authentication Flow +================================================ + +When the signed binary is loaded by the host driver: + +1. **CSME Pre-Boot Inspection**: The Intel CSME DMA engine transfers the image from Host DRAM into IMR memory. +2. **eFuse Hash Match**: The CSME reads the public key modulus from the CSS manifest, calculates its cryptographic hash, and compares it against one-time-programmable (OTP) eFuses burned into the SoC silicon: + + .. math:: + + H_{\text{computed}} = \text{SHA-384}(K_{\text{public}}) \stackrel{?}{=} \text{eFuse}_{\text{OEM\_KEY\_HASH}} + +3. **Signature Verification**: If the key hash matches the hardware fuses, the CSME decrypts the signature using the embedded public key and confirms that the decrypted hash matches the calculated image digest. +4. **Boot Vector Release**: If verification succeeds, power is applied to DSP Core 0, the reset line is de-asserted, and execution begins at the entry point specified in the manifest. If verification fails, the DSP remains held in reset and a security error is reported to the host via IPC. + +Dynamic Modular Packaging (Zephyr LLEXT) +**************************************** + +Modern Sound Open Firmware architectures (IPC4 on MTL, ARL, LNL, and PTL) support dynamic loading of audio processing modules as **Linkable Loadable Extensions (LLEXT)**. Instead of linking every audio codec, filter, and algorithm into a monolithic base firmware image, modules are compiled as standalone relocatable ELF objects (e.g. ``volume.llext``, ``eq_iir.llext``, ``copier.llext``) and packaged into the firmware release. + +Rimage Dynamic Module Mode (``-l``) +=================================== + +When invoked with the ``-l`` flag, Rimage alters its packaging logic: + +- **No Bootloader Assumption**: Rimage does not treat the first ELF module as a DSP bootloader; instead, all segments are treated as dynamic library modules. +- **Module Table Generation**: An ``adsp_module_entry`` (``$AME``) descriptor is constructed for each module, containing: + - 128-bit Component UUID (registered in SOF topology files). + - Module entry point and symbol relocations. + - Memory footprint requirements (text, data, bss). + - Core affinity mask (e.g. ``0x1`` for Core 0, ``0x3`` for Cores 0 and 1). +- **Runtime Host Loading**: At runtime, the Linux host driver loads these modular ``.ri`` files into DSP memory on demand using IPC4 ``LARGE_CONFIG_SET`` commands when an audio pipeline containing that module is instantiated. + +Declarative Platform Configuration (TOML) +***************************************** + +Rimage uses declarative TOML (Tom's Obvious Minimal Language) files combined with the C preprocessor to define hardware platform parameters. + +Platform TOML Anatomy (``platform-*.toml``) +=========================================== + +Base platform files (e.g. ``platform-ptl.toml``, ``platform-tgl.toml``) declare static hardware boundaries: .. code-block:: toml - [platform] - name = "tgl" - arch = "xtensa" + # Example: Panther Lake platform definition (platform-ptl.toml) + version = [3, 0] - [manifest] - version = 4 - format = "cse" + [adsp] + name = "ptl" + image_size = "0x2C0000" # 22 memory banks * 128 KB + alias_mask = "0xE0000000" - [[memory.regions]] - name = "iram" - vma = 0xa0000000 - size = 0x80000 - type = "code" + [[adsp.mem_zone]] + type = "ROM" + base = "0x1FF80000" + size = "0x400" - [[memory.regions]] - name = "dram" - vma = 0xa0080000 - size = 0x60000 - type = "data" + [[adsp.mem_zone]] + type = "IMR" + base = "0xA104A000" + size = "0x2000" -Command-Line Usage -****************** + [[adsp.mem_zone]] + type = "SRAM" + base = "0xA00F0000" + size = "0x100000" -While rimage is typically invoked automatically by the Zephyr build system during `west build`, it can also be run standalone: + [[adsp.mem_alias]] + type = "uncached" + base = "0x40000000" -.. code-block:: bash + [[adsp.mem_alias]] + type = "cached" + base = "0xA0000000" - # Generate a signed Tiger Lake (TGL) image using test keys - rimage -k keys/otc_private.pem \ - -c config/tgl.toml \ - -o build/sof-tgl.ri \ - build/sof-tgl.elf + [cse] + partition_name = "ADSP" + + [[cse.entry]] + name = "ADSP.man" + offset = "0x5c" + length = "0x4b8" + + [[cse.entry]] + name = "ADSP.met" + offset = "0x4c0" + length = "0x70" + + [[cse.entry]] + name = "ADSP" + offset = "0x540" + length = "0x0" # Computed automatically by rimage + + [css] + + [signed_pkg] + name = "ADSP" + [[signed_pkg.module]] + name = "ADSP.met" + + [adsp_file] + [[adsp_file.comp]] + base_offset = "0x2000" + + [fw_desc.header] + name = "ADSPFW" + load_offset = "0x40000" + +C Preprocessor TOML Templates (``*.toml.h``) +============================================ + +To synchronize firmware module IDs and Kconfig features with Rimage, platform templates (e.g. ``ptl.toml.h``) leverage the C preprocessor: + +.. code-block:: c + + /* Excerpt from ptl.toml.h */ + #include "platform-ptl.toml" + + [[module.entry]] + name = "BRNGUP" + uuid = UUIDREG_STR_BRNGUP + affinity_mask = "0x1" + instance_count = "1" + domain_types = "0" + load_type = "0" + module_type = "0" + auto_start = "0" + index = __COUNTER__ + + [[module.entry]] + name = "BASEFW" + uuid = UUIDREG_STR_BASEFW + affinity_mask = "3" + instance_count = "1" + domain_types = "0" + load_type = "0" + module_type = "0" + auto_start = "0" + index = __COUNTER__ + + #if defined(CONFIG_COMP_VOLUME) || defined(LLEXT_FORCE_ALL_MODULAR) + #include