diff --git a/.wokeignore b/.wokeignore index 44bd49d3..95b8e7ef 100644 --- a/.wokeignore +++ b/.wokeignore @@ -16,11 +16,9 @@ contribute/process/bug-tracking.rst contribute/process/images/* getting_started/setup/setup_up_2_board.rst developer_guides/setup_special_device/setup_up_2_board.rst -developer_guides/virtualization/virtualization.rst developer_guides/firmware/component-tutorial/tut-ii-topology.rst developer_guides/subsystem_architecture/host/linux_driver/architecture/sof_driver_arch.rst developer_guides/tech/compile_wsl.rst -developer_guides/virtualization/files/q-v6.sh developer_guides/topology/topology.rst developer_guides/fuzzing/testbench_afl_fuzzing.rst diff --git a/architectures/index.rst b/architectures/index.rst index 8c1a36d0..14698d0a 100644 --- a/architectures/index.rst +++ b/architectures/index.rst @@ -605,6 +605,9 @@ At the heart of the firmware is the audio processing pipeline framework: ctl_eq -> comp_eq [style=dashed, color="#d35400", label="IPC Set Data"]; } +.. seealso:: + For an in-depth architectural explanation of how pipelines, modules, Low-Latency (LL) and Data Processing (DP) scheduling domains, circular buffers, lifecycle management, and the runtime state machine operate, see the :ref:`pipeline_architecture` developer guide. + Topology 2 Architecture ======================= diff --git a/data/modules.yaml b/data/modules.yaml index 0b277a58..a64344b7 100644 --- a/data/modules.yaml +++ b/data/modules.yaml @@ -50,6 +50,22 @@ modules: - "Continuous clock drift tracking" - "Decoupled clock domain bridging" + - id: dmic + name: "Digital Microphone (DMIC) Decimation & Array Tuning" + source: "SOF" + category: "Foundational DSP" + status: "Upstream" + tuning_guide: "developer_guides/tuning/dmic_tuning" + description: "Hardware PDM ingress, 5th-order CIC comb decimation, multirate FIR droop compensation, DC-offset compensation, and multichannel array acoustic calibration." + simd: ["Hardware Accelerator", "Scalar C"] + key_features: + - "5th-order Cascaded Integrator-Comb (CIC) filter with up to 31x decimation" + - "Multirate droop-compensating FIR filters with passband ripple < 0.1 dB and stopband > 90 dB" + - "Dual-FIFO mode matching for concurrent 48 kHz communications and 16 kHz wake-on-voice" + - "Acoustic sensitivity calibration and inter-channel gain trimming for beamforming arrays" + - "Automated logarithmic unmute gain ramping eliminating stream start pops" + - "Standalone Python calibration CLI (sof_dmic_tool.py) and ACPI NHLT / Topology 2 integration" + - id: demux name: "Audio Demux" source: "SOF" @@ -86,6 +102,31 @@ modules: - "Mono to stereo/surround replication" - "Channel swap and mute masking" + - id: level_multiplier + name: "Level Multiplier" + source: "SOF" + category: "Basic Routing & Level" + status: "Upstream" + description: "Ultra-low-latency Q9.23 linear scaling amplifier for capture sensitivity calibration and inter-stage matching." + simd: ["HiFi 3", "HiFi 4", "HiFi 5", "Scalar C"] + key_features: + - "High-precision Q9.23 fixed-point multiplier (-138.47 dB to +48.17 dB)" + - "Zero-overhead fast-path bypass when configured for unity gain (0 dB)" + - "Runtime IPC4 calibration and LLEXT dynamic module packaging" + tuning_guide: "developer_guides/tuning/level_multiplier_aria_tuning" + + - id: up_down_mixer + name: "Up/Down Mixer" + source: "SOF" + category: "Basic Routing & Level" + status: "Upstream" + description: "Configurable matrix-based channel upmixer and downmixer with per-coefficient attenuation." + simd: ["HiFi 3", "HiFi 4", "HiFi 5", "Scalar C"] + key_features: + - "Matrix coefficients for stereo, surround 5.1, and 7.1 mapping" + - "Channel energy normalization and clipping prevention" + - "Zero-copy passthrough when channel geometry matches" + - id: tone name: "Tone Generator" source: "SOF" @@ -123,6 +164,19 @@ modules: - "Parametric peak, notch, low/high shelf" - "Low computational latency" + - id: aria + name: "Aria (Automatic Regressive Input Amplifier)" + source: "SOF" + category: "Audio Enhancement" + status: "Upstream" + description: "Dynamic pre-amplifier and lookahead peak limiter with 1ms algorithmic latency." + simd: ["HiFi 3", "HiFi 4", "HiFi 5", "Scalar C"] + key_features: + - "Target pre-amplification boost (0, 6, 12, 18 dB)" + - "Instantaneous regressive ducking to prevent 0 dBFS clipping" + - "1ms lookahead circular buffer and per-sample linear interpolation" + tuning_guide: "developer_guides/tuning/level_multiplier_aria_tuning" + - id: drc name: "Dynamic Range Compressor (DRC)" source: "SOF" @@ -171,17 +225,57 @@ modules: - "Sub-audible rumble attenuation" - "Near-zero phase distortion in audio band" + - id: phase_vocoder + name: "Phase Vocoder" + source: "SOF" + category: "Audio Enhancement" + status: "Upstream" + description: "Frequency-domain time-scale modification (0.5x to 2.0x speed) without pitch alteration." + simd: ["HiFi 3", "Scalar C"] + key_features: + - "Real-time Short-Time Fourier Transform (STFT) analysis & synthesis" + - "Variable speed scaling (0.5x to 2.0x) with exact GCD counter normalization" + - "Interactive phase re-anchoring and mono downmix optimization" + + - id: stft_process + name: "STFT Process" + source: "SOF" + category: "Audio Enhancement" + status: "Upstream" + description: "Modular Short-Time Fourier Transform frequency-domain filtering and synthesis engine." + simd: ["HiFi 3", "Scalar C"] + key_features: + - "Multi-channel 32-bit forward and inverse FFT with COLA windowing" + - "Dual-domain processing: Cartesian complex and polar magnitude/phase" + - "Single contiguous buffer layout and zero-copy polar memory overlay" + - id: smart_amp - name: "Smart Amp Protection" + name: "Smart Amp Protection (DSM)" source: "SOF" category: "Speaker Protection" status: "Upstream" - description: "Speaker protection algorithm monitoring voltage/current feedback to maximize loudness safely." + description: "Closed-loop dynamic speaker management monitoring real-time voltage/current (I/V) feedback to maximize loudness and prevent mechanical and thermal destruction." simd: ["HiFi 3", "HiFi 4", "Scalar C"] key_features: - - "Real-time voice coil temperature estimation" - - "Membrane excursion limiting" - - "Maximizes acoustic output without damage" + - "Real-time voice coil temperature estimation via continuous Re(t) tracking" + - "Nonlinear membrane excursion prediction and adaptive high-pass limiting" + - "Closed-loop hardware I/V sense feedback via SoundWire and I2S/TDM" + - "Two-layer modular architecture supporting Maxim DSM and vendor engines" + - "Live runtime parameter injection and telemetry readback via sof-ctl" + + - id: sound_dose + name: "Sound Dose & Exposure" + source: "SOF" + category: "Speaker Protection" + status: "Upstream" + description: "Auditory health monitoring and cumulative sound exposure limiter complying with IEC 62368-1 Clause 10.6, EN 50332-1/-2/-3, and WHO-ITU H.870." + simd: ["HiFi 3", "Scalar C"] + key_features: + - "IEC 61672-1 Class 1 A-weighting cascaded Direct Form I IIR biquad filtering" + - "Overflow-proof 64-bit real-time energy accumulation and integer base-2 logarithm decibel conversion" + - "Autonomous 1-second asynchronous IPC4 notification dispatch without host polling" + - "Smooth per-frame exponential slew gain limiter (0.05 dB/frame) eliminating clicks and pops" + - "Acoustic laboratory HATS calibration, rolling 7-day CSD tracking, and runtime control via sof-ctl" # --- Voice, Telephony & Speech --- - id: tdfb @@ -292,7 +386,58 @@ modules: - "TFLite Micro runtime execution" - "Low false-reject and false-alarm rates" + - id: mfcc + name: "Mel-Frequency Cepstral Coefficients (MFCC)" + source: "SOF" + category: "Voice & Telephony" + status: "Upstream" + description: "Speech feature extraction engine computing triangular Mel filterbank energies, Slaney normalization, and DCT-II cepstra." + tuning_guide: "developer_guides/tuning/mfcc_tuning" + simd: ["HiFi 3", "HiFi 4", "Scalar C"] + key_features: + - "Configurable triangular Mel filterbanks (20 Hz to 8 kHz) with Slaney area normalization" + - "Dual-mode operation: 80-bin Mel spectrogram (Whisper ASR) or 13-cepstra MFCC (TFLM microWakeWord)" + - "Discrete Cosine Transform (DCT-II) with sinusoidal cepstral liftering" + - "Embedded Voice Activity Detection (VAD) and Discontinuous Transmission (DTX) silence suppression" + - "Sparse packed triangular filterbank vector storage with >95% SRAM memory reduction" + + - id: mic_privacy_manager + name: "Microphone Privacy Manager" + source: "SOF" + category: "Voice & Telephony" + status: "Upstream" + description: "Hardware-enforced microphone capture mute and privacy state management." + simd: ["Scalar C"] + key_features: + - "Zero-sample hardware mute interlock" + - "GPIO privacy LED synchronization" + - "Host-independent privacy state enforcement" + + - id: rtnr + name: "Realtek Neural Noise Reduction (RTNR)" + source: "Realtek" + category: "Voice & Telephony" + status: "Upstream" + description: "Deep neural network noise suppression engine isolating speech from non-stationary background noise." + simd: ["HiFi 4", "Scalar C"] + key_features: + - "Neural network recurrent inference" + - "Non-stationary transient acoustic noise suppression" + - "Dual-microphone directional voice enhancement" + # --- Codecs & Compression --- + - id: media_codecs + name: "Media Codecs (Cadence XA & Compress-Offload)" + source: "SOF / Cadence" + category: "Codecs & Compression" + status: "Upstream" + description: "Hardware-accelerated compressed audio offload decoders and encoders using the Cadence Xtensa Audio (XA) standard." + simd: ["HiFi 3", "HiFi 4", "HiFi 5", "Scalar C"] + key_features: + - "ALSA compress-offload playback (MP3, AAC, Vorbis, PCM passthrough) and capture (MP3 enc)" + - "Standardized Cadence Xtensa Audio (XA) four-class memory tables and state machine" + - "Deep-buffer DMA host wakeup suppression enabling prolonged C10 deep sleep" + - id: aac_dec name: "AAC Decoder" source: "FFmpeg" diff --git a/data/platforms.yaml b/data/platforms.yaml index e8b3796c..ab181887 100644 --- a/data/platforms.yaml +++ b/data/platforms.yaml @@ -22,7 +22,6 @@ platforms: - IPC3 zephyr_target: intel_adsp_cavs25 target_alias: tgl - test_dut: Spider status: Mainline Active - id: tgl_h @@ -204,7 +203,6 @@ platforms: - IPC4 zephyr_target: intel_adsp_ace15_mtlm target_alias: arl-s - test_dut: Dragon Fly status: Mainline Active - id: lnl @@ -245,7 +243,6 @@ platforms: - IPC4 zephyr_target: intel_ace30_ptl target_alias: ptl - test_dut: Aphid status: Mainline Active - id: wcl @@ -618,9 +615,8 @@ platforms: - N/A zephyr_target: esp32p4 target_alias: esp32-p4 - test_dut: "Pallas (Tx) / Ceres (Rx) Loopback Pair" status: Active Integration - notes: "Essential test card for automated I2S/PDM loopback verification across Spider and Aphid" + notes: "Essential test card for automated I2S/PDM loopback verification across target DUTs" - id: esp32_c6 vendor: Espressif diff --git a/data/sof_bin_releases.json b/data/sof_bin_releases.json index 95d46b51..e0f223c7 100644 --- a/data/sof_bin_releases.json +++ b/data/sof_bin_releases.json @@ -1,4 +1,15 @@ [ + { + "tag_name": "v2026.09", + "name": "v2026.09", + "fw_version": "v2.15", + "published_at": "2026-09-17", + "html_url": "https://github.com/thesofproject/sof-bin/releases/tag/v2026.09", + "asset_name": "sof-bin-2026.09.tar.gz", + "asset_url": "https://github.com/thesofproject/sof-bin/releases/download/v2026.09/sof-bin-2026.09.tar.gz", + "asset_size_mb": 16.7, + "prerelease": false + }, { "tag_name": "v2025.12.2", "name": "v2025.12.2", @@ -119,16 +130,5 @@ "asset_url": "https://github.com/thesofproject/sof-bin/releases/download/v2024.06/sof-bin-2024.06.tar.gz", "asset_size_mb": 9.4, "prerelease": false - }, - { - "tag_name": "v2024.03", - "name": "v2024.03", - "fw_version": "v2.9", - "published_at": "2024-04-02", - "html_url": "https://github.com/thesofproject/sof-bin/releases/tag/v2024.03", - "asset_name": "sof-bin-2024.03.tar.gz", - "asset_url": "https://github.com/thesofproject/sof-bin/releases/download/v2024.03/sof-bin-2024.03.tar.gz", - "asset_size_mb": 7.8, - "prerelease": false } ] \ No newline at end of file diff --git a/developer_guides/algorithms/eq/equalizers_tuning.rst b/developer_guides/algorithms/eq/equalizers_tuning.rst index 45335c60..9b237ffd 100644 --- a/developer_guides/algorithms/eq/equalizers_tuning.rst +++ b/developer_guides/algorithms/eq/equalizers_tuning.rst @@ -3,6 +3,13 @@ Equalizers, IIR and FIR ####################### +.. seealso:: + + For a high-level firmware architectural overview of both Finite Impulse Response (FIR) + and Infinite Impulse Response (IIR) equalizers—including transversal filter structures, + Direct Form I biquad cascades, parametric filter topologies, dynamic IPC blob swapping, + and SIMD acceleration—see :ref:`eq_fir_iir`. + .. contents:: :depth: 3 diff --git a/developer_guides/algorithms/src/sample_rate_conversion.rst b/developer_guides/algorithms/src/sample_rate_conversion.rst index 22c3655b..ebd6f539 100644 --- a/developer_guides/algorithms/src/sample_rate_conversion.rst +++ b/developer_guides/algorithms/src/sample_rate_conversion.rst @@ -3,6 +3,13 @@ Sample Rate Conversion ###################### +.. seealso:: + + For a high-level firmware architectural overview of both Synchronous (SRC) and + Asynchronous (ASRC) converters—including multi-stage factorization, continuous + Farrow drift compensation, push vs pull topologies, and SIMD acceleration—see + :ref:`src_asrc`. + Introduction ************ diff --git a/developer_guides/debugability/coredump-reader/images/coredump_architecture.svg b/developer_guides/debugability/coredump-reader/images/coredump_architecture.svg new file mode 100644 index 00000000..0e472a59 --- /dev/null +++ b/developer_guides/debugability/coredump-reader/images/coredump_architecture.svg @@ -0,0 +1,242 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Figure 330: SOF Firmware Crash Diagnostics & Zephyr Coredump Architecture + + + Hardware Fault Vectors, Memory Window Preserves, Kernel State Retention & Symbolic GDB Backtrace + + + + + + + + + + 1. Hardware Exception Vectors & Fatal Fault Trapping (Xtensa / RISC-V DSP) + + + + + + + + Hardware Exception Traps + EXCCAUSE: LoadStoreError, WindowOverflow + Faulting address latched into EXCVADDR register + + + + + + Zephyr Fatal Error Dispatcher + z_xtensa_fatal_error() / fatal.c + Freezes IRQs; preserves register window ar0..ar63, a0..a15 + + + + + + Software Panic & Watchdog Triggers + SOF_ASSERT() | k_panic() + Watchdog expiration triggers isolated emergency stack + + + + + + + + + + + + + + + + 2A. Zephyr Coredump Subsystem & Memory Window Serialization + + + + + + + + Intel ADSP Memory Window Backend + CONFIG_DEBUG_COREDUMP_BACKEND_INTEL_ADSP_MEM_WINDOW=y + Direct write into PCI memory window; zero dynamic heap allocation + Guaranteed operation even under severe memory pool exhaustion + + + + + + Structured Binary Coredump Packet Format + Header: Magic 0x5A45 ('ZE') | Target Arch: Xtensa LX6/LX7 + Payload: [CPU Register Window] + [Thread Stacks] + [Memory Blocks] + Encodes thread control blocks, priority, and faulting instruction pointer + + + + + + ROM Status Register & Hardware Lock + • FW Status Update: Latches FW_STATUS_PANIC into outbox register + • Trace Buffer Retention: Last trace DMA burst preserved in SRAM + • DSP Core Halt: Executes idle loop preventing cascading corruptions + • Hardware Watchdog Strobe: Suppresses watchdog to allow host dump + + + + + + + + + + + 2B. Linux Kernel State Retention & debugfs Extraction + + + + + + + + DSP Power State Retention Policy + CONFIG_SND_SOC_SOF_DEBUG_RETAIN_DSP_CONTEXT=y + options sof_pci_dev sof_pci_debug=1 [Inhibits runtime suspend] + Prevents PCI runtime power management from cutting DSP memory power + + + + + + Kernel debugfs Exception Node + /sys/kernel/debug/sof/exception [Direct binary crash dump] + dmesg log emits decoded registers and exception backtrace + Exposes full hardware exception window directly to userspace tools + + + + + + Automated Host Artifact Capture + • One-Line Dump: cat /sys/.../exception > dsp-coredump.bin + • Remote Capture via SSH: Automated crash capture in CI + • Crash Archival: Timestamped tarball with kernel dmesg and FW ELF + • DUT Recovery: Graceful recovery via driver reload or relay toggle + + + + + + + + + + + + + + + 3. Host Symbolic Analysis & Interactive GDB Post-Mortem Triaging + + + + + + + + Zephyr Coredump GDB Server + coredump_gdbserver.py build/zephyr.elf \ + --gdb-port 1234 dsp-coredump.bin + Emulates remote target presenting frozen DSP state + + + + + + Interactive GDB / LLDB Session + xt-gdb / gdb-multiarch -ex 'target remote :1234' + • bt: Full symbolic backtrace with file and line + • info registers: Inspect PC, AR registers, window state + + + + + + Offline & Legacy Parsers + sof-coredump-reader.py -v -i dsp-coredump + • Generates automated GDB script (.gdb) + • Fully compatible with offline CI regression triage + + + + diff --git a/developer_guides/debugability/coredump-reader/index.rst b/developer_guides/debugability/coredump-reader/index.rst index 1ca6d5d2..9f459c6e 100644 --- a/developer_guides/debugability/coredump-reader/index.rst +++ b/developer_guides/debugability/coredump-reader/index.rst @@ -1,86 +1,252 @@ .. _dbg-coredump-reader: -Coredump-reader -############### +DSP Crash Diagnostics & Zephyr Coredump +####################################### -NOTE: These instructions do not work with SOF running on Zephyr, -please refer to -https://docs.zephyrproject.org/latest/services/debugging/coredump.html +Sound Open Firmware (SOF) incorporates an automated crash preservation and post-mortem analysis framework. Because embedded audio DSPs frequently operate without virtual memory management units (MMUs) or operating system paging, memory safety violations, unaligned memory accesses, or software assertions result in immediate CPU exception traps. -Tool for processing FW stack dumps. In verbose mode it prints the stack leading -to the core dump including DSP registers and function calls. -It outputs unwrapped gdb command function call addresses to human readable -function call format either to a file or stdout. +To prevent critical fault telemetry from being lost upon a crash, modern SOF running on the **Zephyr RTOS** captures processor register state, call frames, and memory segments into hardware memory windows, enabling full symbolic post-mortem backtracing under GDB. -Coredump-reader usage +.. figure:: images/coredump_architecture.svg + :alt: SOF Firmware Crash Diagnostics and Zephyr Coredump Architecture + :align: center + :width: 100% + + Figure 330: SOF Firmware Crash Diagnostics & Zephyr Coredump Architecture + +--- + +Architecture Overview ********************* -Usage sof-coredump-reader.py [-h] [-a ARCH] [-c] [-l COLUMNCOUNT] [-v] (--stdout | -o OUTFILE) [--stdin | -i INFILE] +The crash diagnostics framework spans four coordinated execution tiers: + +1. **Hardware Fault Trapping**: When a fatal fault occurs on the DSP core, the hardware exception vector invokes Zephyr's architecture-specific fatal error handler (``arch/xtensa/core/fatal.c``), freezing interrupts and capturing the CPU register state. +2. **Zero-Allocation In-Memory Dump**: The Intel ADSP Memory Window coredump backend (``coredump_backend_intel_adsp_mem_window.c``) serializes register blocks, thread metadata, and active stack frames directly into a shared PCI memory window without performing any dynamic heap allocations. +3. **Kernel Power Retention**: The Linux ``snd-sof`` driver inhibits runtime power management, preventing the host operating system from powering down DSP SRAM and erasing crash telemetry. The crash image is exposed via ``debugfs``. +4. **Interactive GDB Post-Mortem**: Host tools (``coredump_gdbserver.py`` or ``sof-coredump-reader.py``) parse the binary crash dump and establish a GDB session against the firmware ELF binary, providing full symbolic backtraces and variable inspection. + +--- + +Zephyr Coredump Subsystem Configuration +*************************************** + +SOF enables the native Zephyr coredump framework using the following Kconfig directives in target board configurations: + +.. code-block:: cfg + + # Enable Zephyr Coredump Core + CONFIG_DEBUG_COREDUMP=y + CONFIG_DEBUG_COREDUMP_BACKEND_INTEL_ADSP_MEM_WINDOW=y + CONFIG_DEBUG_COREDUMP_MEMORY_DUMP_MIN=y + + # Capture thread stacks and register windows + CONFIG_DEBUG_COREDUMP_SHELL=n + +Memory Window Backend Mechanics +=============================== + +During a fatal exception, the DSP heap may be corrupted, exhausted, or inaccessible. The ``coredump_backend_intel_adsp_mem_window`` backend operates under strict emergency constraints: + +* **Static Buffering**: Writes directly into the pre-mapped host-accessible DSP memory window (SRAM Window 0/3). +* **Zero Allocation**: Executes without calling ``k_malloc()``, ``malloc()``, or acquiring RTOS synchronization primitives. +* **ROM Status Handshake**: Latches ``FW_STATUS_PANIC`` into the DSP status outbox register, signaling the host kernel that a panic dump is ready for extraction. +* **Halt Loop**: Enters a controlled low-power idle loop to prevent cascading memory corruption or repeated exception loops. + +--- + +Captured Processor Architecture State +************************************* + +On Tensilica Xtensa DSP architectures (e.g. Intel cAVS 2.5 on Tiger Lake, ACE 1.5 on Arrow Lake, ACE 3.0 on Panther Lake), the coredump captures complete architectural state: + +Special Registers +================= + +* **``PC`` (Program Counter)**: Exact instruction address executing at the time of the fault. +* **``PS`` (Processor State)**: CPU privilege level, interrupt mask, and register window pointer. +* **``EXCCAUSE`` (Exception Cause)**: Hardware fault code identifying the failure type. +* **``EXCVADDR`` (Exception Virtual Address)**: Memory address that triggered the violation (for load/store errors). +* **``EPC1`` .. ``EPC7``**: Saved program counters across nested interrupt priority levels. + +Register Window File +==================== + +Xtensa processors employ a windowed register architecture consisting of up to 64 physical registers (``ar0`` .. ``ar63``). At any given moment, the active function operates on a 16-register sliding window (``a0`` .. ``a15``): + +* **``a0``**: Function return address (used to reconstruct caller stack frames). +* **``a1``**: Stack pointer (points to local variables and spilled register frames). +* **``a2`` .. ``a7``**: Incoming function parameters and return values. +* **``a8`` .. ``a15``**: Local variables and temporary registers. + +The coredump backend dumps both the active register window and the spilled register frames on the stack, allowing GDB to reconstruct the full call hierarchy across all active function calls. + +--- + +Kernel State Retention & Crash Extraction +***************************************** + +Preventing Runtime D3 Power-Off +=============================== + +By default, Linux runtime power management (Runtime PM) automatically places idle audio DSPs into low-power D3 suspend, cutting power to DSP SRAM. If a crash occurs and the audio stream halts, Runtime PM would power off the DSP and permanently erase the coredump before the developer can inspect it. + +To preserve the crash telemetry in memory, configure the driver retention policy: --h show this help message and exit --a ARCH determine architecture of dump file; valid archs are: LE64bit, LE32bit --c set output to be colourful --l COLUMNCOUNT set how many colums to group the output in --v increase output verbosity ---stdin input is from stdin --i INFILE path to sys dump bin ---stdout output is to stdout --o OUTFILE output is to FILE +1. **Kernel Configuration**: + Ensure ``CONFIG_SND_SOC_SOF_DEBUG_RETAIN_DSP_CONTEXT=y`` is enabled in the host kernel. +2. **Module Parameter**: + Set ``sof_pci_debug=1`` in ``/etc/modprobe.d/sof.conf``: -sof-coredump-to-gdb.sh shows example usage of sof-coredump-reader.py -We read from dump file into sof-coredump-reader.py, then we pipe its output to xt-gdb, which operates on given elf-file. + .. code-block:: text + + # Prevent DSP power-down on fatal exceptions + options snd_sof_pci sof_pci_debug=1 + +Extracting the Dump File +======================== + +Once an exception occurs, the Linux driver logs the failure in ``dmesg`` and populates the ``debugfs`` exception node: + +.. code-block:: bash + + # Verify crash event in dmesg + sudo dmesg | grep -i "dsp exception" + + # Extract raw coredump binary + sudo cat /sys/kernel/debug/sof/exception > /tmp/dsp-coredump.bin + + # Check dump size + ls -lh /tmp/dsp-coredump.bin + +--- + +Interactive GDB Post-Mortem Debugging Runbook +********************************************* + +Step 1: Launch Zephyr Coredump GDB Server +========================================= + +The Zephyr RTOS provides ``coredump_gdbserver.py``, which reads the binary dump file, maps the frozen DSP register and memory state, and emulates a live GDB remote stub: + +.. code-block:: bash + + # Launch GDB server on localhost:1234 + python3 ~/work/sof-tgl/zephyr/scripts/coredump/coredump_gdbserver.py \ + --gdb-port 1234 \ + build-sof-staging/sof/sof-tgl.elf \ + /tmp/dsp-coredump.bin + +Step 2: Connect Interactive GDB Session +======================================= + +In a second terminal, launch the target-specific cross-debugger (``xt-gdb`` or ``gdb-multiarch``) with the matching firmware ELF binary: .. code-block:: bash - ./sof-coredump-to-gdb.sh sof-apl dump_file + # For Cadence Xtensa toolchain: + xt-gdb build-sof-staging/sof/sof-tgl.elf -ex 'target remote :1234' + + # For Open-Source LLVM / multiarch toolchains: + gdb-multiarch build-sof-staging/sof/sof-tgl.elf -ex 'target remote :1234' + +Step 3: Post-Mortem Triage Commands +=================================== + +Once attached, execute standard GDB inspection commands: + +.. code-block:: text + + (gdb) bt + #0 eq_fir_process (dev=0x9e0a4e78) at src/audio/eq_fir/eq_fir.c:142 + #1 0xbe02fb29 in comp_copy (dev=0x9e0a4e78) at src/audio/component.c:85 + #2 0xbe04e277 in pipeline_task (arg=0x9e0a37d0) at src/audio/pipeline/pipeline.c:320 + #3 0xbe050a28 in z_thread_entry (entry=0xbe04e200, p1=0x9e0a37d0, p2=0, p3=0) + + (gdb) info registers + pc 0xbe051b00 0xbe051b00 + ps 0x60020 393248 + exccause 0xc 12 (LoadStoreError) + excvaddr 0xdeadbeef -559038737 + a0 0xbe02fb29 -1107092695 + a1 0x9e0a4044 -1643495356 + a2 0x9e0a4e78 -1643491720 + + (gdb) frame 0 + (gdb) print *dev + $1 = {state = 2, frames = 48, rate = 48000, channels = 2, ...} + + (gdb) list + 140 for (int i = 0; i < dev->frames; i++) { + 141 /* Attempting to read filter coefficients from unmapped address */ + 142 int32_t coef = cd->fir_coefs[i]; + 143 accum += (sample * coef) >> 15; + +--- -Usage with Linux SOF Driver -*************************** +Legacy & Offline Coredump Reader +******************************** -If a core dump occurs after a DSP error, the Linux SOF driver allows -accessing the dump via debugfs. Consider the following example of capturing -the dump file and processing it with coredump-reader: +For environments without Python GDB server support or when triaging pre-Zephyr dumps, the ``sof-coredump-reader.py`` tool converts binary dumps into GDB script files: .. code-block:: bash - dut> cat /sys/kernel/debug/sof/exception >dsp-coredump - # transfer file to host - host> sof/tools/coredumper/sof-coredump-reader.py -v -l 4 -i dsp-coredump -o dsp-coredump.gdb - host> xt-gdb sof/build_tlg_xcc/sof --command=dsp-coredump.gdb - [cut] - $1 = "Exception location:" - 0xbe02fb29 is in ipc_glb_debug_message (/home/user/sof/src/ipc/handler-ipc3.c:1371). - [cut] - $2 = "backtrace" - #0 0xbe051b00 in literals () - #1 0xbe04e277 in dump_stack (p=3187705884, addr=0x1cc6c29b, offset=3270769662, limit=380, stack_ptr=0x1) at /home/user//sof/src/arch/xtensa/include/arch/lib/cache.h:79 - #2 0xbe04e2f7 in panic_dump (p=233492486, panic_info=0x0, data=0xbe0a4130) at /home/user/sof/src/arch/xtensa/include/arch/debug/panic.h:45 - #3 0xbe02dfd9 in exception () at /home/user/sof/src/arch/xtensa/init.c:115 - #4 0xbe050a28 in _GeneralException () - #5 0xbe02fb29 in ipc_glb_debug_message (header=394016) at /home/user/sof/src/ipc/handler-ipc3.c:1373 - [cut] - (xt-gdb) info all-registers - pc 0xbe051b00 0xbe051b00 - ar0 0x0 0 - ar1 0xbe00a044 -1107255228 - ar2 0x10000 65536 - -Notes: - -- Coredump-reader only works with the xcc toolchain. - -- If the Linux kernel fails to probe, the exception file cannot be read. - -- To prevent runtime suspend from powering off the DSP and erasing - the exception data, perform one of the following steps: - - - Set the ``CONFIG_SND_SOC_SOF_DEBUG_RETAIN_DSP_CONTEXT`` option in the - kernel to ensure DSP is left powered on if a DSP crash occurs. - - - Disable runtime power management (PM) with a module parameter. - For example, for PCI devices:: - options sof_pci_dev sof_pci_debug=1 - -- The DSP core dump information is also printed to kernel dmesg, but - sof-coredump-reader.py cannot parse this core dump format. + # Convert dump to GDB script + python3 tools/coredumper/sof-coredump-reader.py -v -l 4 \ + -i /tmp/dsp-coredump.bin \ + -o /tmp/dsp-coredump.gdb + + # Run xt-gdb with generated script + xt-gdb build-sof-staging/sof/sof-tgl.elf --command=/tmp/dsp-coredump.gdb + +Command-Line Options +==================== + +.. list-table:: sof-coredump-reader.py Flags + :widths: 20 80 + :header-rows: 1 + + * - Option + - Description + * - ``-a `` + - Target architecture format (``LE32bit`` or ``LE64bit``). + * - ``-v`` + - Increase output verbosity, printing raw stack offsets and registers. + * - ``-l `` + - Group memory and stack dump columns for improved terminal readability. + * - ``-i `` + - Path to binary crash dump extracted from ``/sys/kernel/debug/sof/exception``. + * - ``-o `` + - Output path for generated GDB batch command script. + +--- + +Common DSP Exception Causes & Triage Guide +****************************************** + +.. list-table:: Common Xtensa EXCCAUSE Fault Codes & Resolutions + :widths: 15 20 65 + :header-rows: 1 + + * - Cause Code + - Exception Name + - Typical Root Cause & Debugging Action + * - **0** + - ``IllegalInstruction`` + - Execution jumped to an invalid memory location or uninitialized function pointer. Inspect ``a0`` (return address) and stack backtrace to identify corrupt callback structures. + * - **9** + - ``LoadStoreAlignment`` + - An unaligned 32-bit or 64-bit load/store was attempted on an odd address boundary. Ensure audio sample pointers are aligned to 4 or 8 bytes (``ALIGN_UP(ptr, 4)``). + * - **12** + - ``InstructionFetchError`` + - Attempted to execute code from non-executable or powered-off DSP memory bank. Check dynamic power gating of SRAM banks or LLEXT dynamic module memory permissions. + * - **13** + - ``LoadStoreError`` + - Attempted to access non-existent MMIO address or unmapped host DMA window. Inspect ``excvaddr`` in GDB to determine the illegal pointer address. + * - **28** + - ``IntegerDivideByZero`` + - Division by zero in audio rate calculation or period size. Validate sample rate and channel count configurations received via IPC before dividing. + * - **Software Panic** + - ``k_panic() / SOF_ASSERT`` + - Explicit assertion failure triggered by defensive runtime checks (e.g. buffer size overrun). Locate the assertion line from the symbol table and verify parameter constraints. diff --git a/developer_guides/debugability/index.rst b/developer_guides/debugability/index.rst index cb81c3e2..bb1a1ff7 100644 --- a/developer_guides/debugability/index.rst +++ b/developer_guides/debugability/index.rst @@ -1,14 +1,99 @@ .. _api-debugability: +.. _sof_debugability_portal: -Debugability -############ +DSP Telemetry, Logging & Diagnostics Portal +########################################### -.. toctree:: - :maxdepth: 1 +Sound Open Firmware (SOF) provides an asynchronous, zero-overhead diagnostic and telemetry infrastructure designed for hard real-time embedded audio DSP execution. In audio signal processing, processing periods execute on sub-millisecond deadlines (typically 1 ms or 200 µs intervals). Blocking the DSP core on synchronous I/O operations—such as UART serial transmission or blocking host IPC calls—introduces buffer starvation, audible glitches, and fatal pipeline dropouts. - traces/index - coredump-reader/index - probes/index - ri-info/index - perf-counters/index - shell/index +To provide continuous visibility into the firmware runtime without compromising acoustic deadlines, SOF decouples event generation from data transmission through a multi-tier observability stack: + +* **Compile-Time String Metadata Extraction (:ref:`dbg-traces`)**: Format strings and filenames are stripped from the firmware binary by the ``smex`` tool into an external dictionary file (``.ldc``), leaving compact 32-bit entry IDs and packed arguments in firmware text. +* **Autonomous Hardware Trace DMA**: Log entries and performance metrics are written to high-speed internal SRAM circular buffers and transferred to host memory windows by background DMA engines without CPU intervention. +* **Network-Accessible Telemetry Server (:ref:`dbg-probes`)**: High-throughput daemon (``sof_probe_server``) streaming live trace DMA packets over TCP port ``9999`` to remote development clients and the multi-pane ``dut-monitor`` dashboard. +* **Zero-Allocation Fatal Crash Preservation (:ref:`dbg-coredump-reader`)**: Dedicated hardware memory window backends preserve CPU register windows, call stacks, and exception causes upon fatal CPU traps for GDB post-mortem backtrace analysis. +* **Zero-IPC Interactive Terminal (:ref:`dbg-zephyr-shell`)**: Full Zephyr shell access over shared memory windows (``cavstool.py``), remaining fully operational even when the IPC subsystem is unresponsive or deadlocked. + +.. list-table:: SOF Debugability & Telemetry Framework Breakdown + :widths: 20 25 25 30 + :header-rows: 1 + + * - Diagnostic Subsystem + - Target Mechanism + - Host Ingestion Interface + - Primary Use Case & Capabilities + * - **DSP Traces & Telemetry** + - Compile-time ``smex`` extraction, Zephyr logging, internal SRAM ring buffers, background trace DMA. + - Linux kernel debugfs (``/sys/kernel/debug/sof/trace``) & ``sof-logger``. + - Real-time event tracing, state transition verification, microsecond timing benchmarks, module logging. + * - **Crash Diagnostics & Coredump** + - Zephyr coredump subsystem, ADSP memory window backend, CPU exception vector capture. + - Linux kernel debugfs (``/sys/kernel/debug/sof/exception``) & ``coredump_gdbserver.py``. + - Post-mortem root-cause analysis of fatal DSP faults, memory corruption, divide-by-zero, and assert panics. + * - **Audio Data Probes** + - Dynamic ALSA widget buffer injection and extraction tap points across processing DAG. + - ALSA Compress Offload (``crecord``), ``sof-probes -p`` WAV demuxer. + - In-flight audio sample extraction, intermediate waveform validation in Audacity, algorithm tuning. + * - **Network Probe Server** + - C streaming daemon (``sof_probe_server``), 1MB thread-safe ring buffer, TCP port 9999. + - Host Python client (``sof_probe_client.py``) & multi-pane ``dut-monitor`` dashboard. + - Continuous remote log streaming over private lab networks, decoupled from SSH session latency. + * - **Zephyr Interactive Shell** + - Shared SRAM memory window backend (``CONFIG_SHELL_BACKEND_ADSP_MEMORY_WINDOW``). + - Host ``cavstool.py -l -p`` bridge spawning pseudo-terminal (``/dev/pts/X``). + - Interactive runtime inspection of thread states, stack high-water marks, memory heap pools, and D0ix sleep states. + * - **Performance Counters** + - Hardware Tensilica CCOUNT registers, 64-bit platform timers, per-component cycle tracking. + - Periodic trace emission via trace DMA ring buffers decoded by ``sof-logger``. + - Cycle budget accounting, million cycles per second (MCPS) calculations, multi-core workload balancing. + * - **Manifest & Binary Inspection** + - Signed firmware binary manifest structures (``$CPD``, ``$AM1``, ``$AME``, ``$AE1``, ``XMan``). + - Host Python parsing tool ``sof-ri-info`` & ``rimage`` inspector. + - Binary layout validation, load segment addresses, entry points, module UUIDs, and crypto signatures. + +--- + +Diagnostic Decision Tree & Troubleshooting Matrix +************************************************* + +Select the appropriate diagnostic tool based on the observed system behavior: + +.. list-table:: Symptom-Based Diagnostic Tool Selection Matrix + :widths: 25 25 50 + :header-rows: 1 + + * - Observed Symptom + - Recommended Toolchain + - Diagnostic Runbook & Action Plan + * - **Audible Glitch / Dropout** + - :ref:`Audio Probes ` & :ref:`sof-logger ` + - Attach probe points before and after suspect audio components; extract intermediate buffers via ``crecord``; demux with ``sof-probes -p`` to pinpoint where the waveform degrades. + * - **DSP Kernel Panic / Freeze** + - :ref:`Coredump & GDB ` + - Capture ``/sys/kernel/debug/sof/exception``; launch ``coredump_gdbserver.py``; connect GDB to inspect backtrace, faulting instruction pointer (``PC``), and corrupted registers. + * - **Early DSP Boot Failure** + - :ref:`snd-sof-probes ` (Boot Logging) + - Load probe driver with ``logging_boot_enable=1`` to capture pre-buffered firmware initialization logs (up to 4 KB) prior to userspace audio server startup. + * - **High CPU / Execution Overrun** + - :ref:`Performance Counters ` + - Enable ``CONFIG_PERFORMANCE_COUNTERS=y``; analyze peak platform and CPU ticks in ``sof-logger``; calculate component MCPS against 1 ms pipeline budgets. + * - **Stack Overflow / Leak** + - :ref:`Zephyr Shell ` + - Attach terminal via ``cavstool.py -l -p``; execute ``kernel stacks`` and ``kernel threads`` to observe per-thread unused stack margins and dynamic heap allocations. + * - **Firmware Signature / Boot Reject** + - :ref:`sof-ri-info ` + - Run ``sof-ri-info.py -v -i sof-platform.ri`` to verify CSE partition directories, ADSP manifest headers, entry addresses, and cryptographic hashes. + +--- + +.. seealso:: + + For dedicated specifications, architectural deep-dives, and step-by-step developer runbooks for each observability subsystem, refer to the individual guides in the :ref:`telemetry_diagnostics_pillar`: + + * :ref:`dbg-traces`: Compile-time dictionary extraction, lockless trace DMA buffers, and live ``sof-logger`` decoding. + * :ref:`dbg-coredump-reader`: Native Zephyr RTOS coredump, memory window register preservation, and interactive GDB backtrace analysis. + * :ref:`dbg-probes`: Dynamic audio buffer probe points, ALSA Compress Offload (``crecord``), and high-throughput TCP probe server (port 9999). + * :ref:`dbg-zephyr-shell`: Zero-IPC interactive Zephyr memory window shell, ``cavstool.py`` terminal bridge, and thread/stack monitoring. + * :ref:`dbg-perf-counters`: Hardware Tensilica CCOUNT registers, platform timers, and mathematical MCPS calculation formulas. + * :ref:`dbg-ri-info`: Firmware binary manifests, partition directories, module manifests, and cryptographic signature validation. + * :ref:`uuid`: Universal Unique Identifier (UUID) registry, little-endian wire format translation, and IPC4 dynamic module loading. diff --git a/developer_guides/debugability/perf-counters/index.rst b/developer_guides/debugability/perf-counters/index.rst index 4e1ea77b..c6e14114 100644 --- a/developer_guides/debugability/perf-counters/index.rst +++ b/developer_guides/debugability/perf-counters/index.rst @@ -1,63 +1,152 @@ .. _dbg-perf-counters: -Performance Counters -#################### +Performance Counters & MCPS Profiling +##################################### -Firmware can be configured to trace performance counters for each processing -component. Each performance trace entry includes: +In hard real-time embedded audio DSP development, meeting acoustic deadlines requires strict management of CPU cycle budgets. If an audio processing component (such as a parametric equalizer, sample rate converter, or dynamic range compressor) consumes more CPU cycles than permitted by its pipeline scheduling interval, the audio buffer starves, triggering audible clicks, pops, or fatal DMA dropouts. -- component UUID +Sound Open Firmware (SOF) provides a built-in **Performance Counter Subsystem** that measures exact hardware CPU and platform timer consumption for every active audio component on each execution period. -- peak platform and cpu timer ticks consumed during component's copy processing +--- -.. note:: - Performance timestamp macros use both the platform timer and cpu timer in case - the latter is not always running. +Architecture: Hardware Timers & Cycle Accounting +************************************************ + +The performance counter subsystem utilizes low-overhead hardware registers on the DSP core: + +1. **Tensilica CCOUNT Register**: Increments once per processor clock cycle at the full DSP core frequency (e.g. 400 MHz on Tiger Lake, 800 MHz on Arrow Lake and Panther Lake). It provides single-cycle timing resolution for measuring component execution times. +2. **64-Bit Platform Timer**: Operates off an external hardware oscillator (e.g. 19.2 MHz or 24 MHz) that continues running even if the DSP core dynamically alters its frequency or enters low-power clock gating. + +During each execution period, the pipeline scheduler wraps the component's ``comp_copy()`` processing function with timestamp measurement macros: + +.. code-block:: c + + /* Pipeline scheduler component execution */ + uint32_t ccount_start = arch_timer_get_system(); + int err = comp_copy(dev); + uint32_t ccount_end = arch_timer_get_system(); + + uint32_t cycles = ccount_end - ccount_start; + if (cycles > comp->peak_cpu_ticks) { + comp->peak_cpu_ticks = cycles; + } + +--- + +Enabling Performance Counters +***************************** + +Performance counters can be enabled in firmware via Kconfig: + +.. code-block:: cfg + + # Enable component-level cycle profiling + CONFIG_PERFORMANCE_COUNTERS=y + + # Optional: Set periodic reporting frequency (in periods) + CONFIG_PERFORMANCE_COUNTERS_PERIOD=1000 + +When enabled, the firmware periodically logs peak performance metrics for each active audio component over the trace DMA ring buffer: + +.. code-block:: text -Performance Counters usage -************************** + [ 8481257.031250] ( 51.562500) c0 eq_fir 1.2 src/audio/pipeline.c:206 perf comp_copy peak plat 782 cpu 8136 -Currently, you can only enable performance counters statically during FW build in -one of two ways: +Trace Field Breakdown: -- Select **Performance counter** from **Debug** menu using ``make menuconfig``. +* **``c0``**: Processing DSP core ID (Core 0). +* **``eq_fir 1.2``**: Audio component name and pipeline-unique component ID. +* **``plat 782``**: Peak platform timer cycles consumed during the copy cycle. +* **``cpu 8136``**: Peak CPU core clock cycles (CCOUNT) consumed during the copy cycle. -- Add ``CONFIG_PERFORMANCE_COUNTERS=y`` to specific FW config, for - example, tigerlake_defconfig. +--- +Mathematical MCPS Calculation +***************************** -After you enable the performance counters, they are logged periodically for each -active component with the pipeline period frequency. +Million Cycles Per Second (MCPS) is the standard metric used in audio DSP engineering to quantify computational load. -Example -******* +General Formula +=============== -Performance counter trace example: +The MCPS consumed by an audio component is given by: - .. code-block:: bash +.. math:: - [ 8481257.031250] ( 51.562500) c0 demux 1.2 src/audio/pipeline.c:206 perf comp_copy peak plat 782 cpu 8136 + \text{MCPS} = \frac{\text{cpu\_ticks}}{\text{pipeline\_period\_seconds} \times 10^6} -``demux 1.2`` - processing component +Standard 1 ms Pipeline Period +============================= -``plat 782`` - peak platform cycles consumed +For standard 1 ms audio pipelines (:math:`T_{\text{period}} = 10^{-3}\text{ s}`): -``cpu 8136`` - peak CPU cycles consumed +.. math:: -MCPS calculation ----------------- + \text{MCPS} = \frac{\text{cpu\_ticks}}{10^{-3} \times 10^6} = \frac{\text{cpu\_ticks}}{1000} -The equation below illustrates how to calculate component MCPS (million cycles -per second) consumption. +In the trace example above, ``cpu_ticks = 8136``: - .. code-block:: bash +.. math:: - MCPS = cpu_ticks / (pipeline_period[s] * 10^6) - - // for common pipeline_period = 1ms it can be simplified to - MCPS = cpu_ticks / 1000 + \text{MCPS} = \frac{8136}{1000} = 8.136 \text{ MCPS} + +Low-Latency 200 µs Pipeline Period +================================== + +For ultra-low latency pipelines (:math:`T_{\text{period}} = 200\,\mu\text{s} = 2 \times 10^{-4}\text{ s}`): + +.. math:: + + \text{MCPS} = \frac{\text{cpu\_ticks}}{200 \times 10^{-6} \times 10^6} = \frac{\text{cpu\_ticks}}{200} + +If a component consumes ``1400`` CPU ticks in a 200 µs pipeline: + +.. math:: + + \text{MCPS} = \frac{1400}{200} = 7.0 \text{ MCPS} + +--- + +DSP Workload Budgeting & Multi-Core Allocation +********************************************** + +Core Capacity & Headroom Guidelines +=================================== + +Total available MCPS is directly proportional to the DSP core clock frequency: + +.. list-table:: Core Frequency & Available MCPS Budget + :widths: 25 25 25 25 + :header-rows: 1 + + * - Platform + - Core Clock Frequency + - Total Raw MCPS + - Safe Usable Budget (65%) + * - **Tiger Lake (TGL)** + - 400 MHz + - 400 MCPS + - ~260 MCPS + * - **Meteor Lake (MTL)** + - 400 / 600 MHz + - 400 / 600 MCPS + - ~260 / 390 MCPS + * - **Arrow Lake (ARL)** + - 800 MHz + - 800 MCPS + - ~520 MCPS + * - **Panther Lake (PTL)** + - 800 MHz + - 800 MCPS + - ~520 MCPS + +.. note:: + Always maintain at least **30–35% headroom** below total raw capacity. This reserved bandwidth accommodates RTOS context switches, DMA interrupts, IPC deserialization, cache misses, and external bus contention. -In the trace example above, cpu_ticks = 8136, the pipeline_period is 1ms so the -demux consumption equals 8,136 MCPS +Multi-Core Load Balancing +========================= +When an audio processing pipeline exceeds the recommended single-core budget: +1. **Offload Heavy Modules**: Shift compute-heavy algorithms (such as Acoustic Echo Cancellation, Valve Steam Audio 3D binaural spatialization, or Deep Learning RTNR noise suppression) to secondary DSP cores (Core 1, Core 2, or Core 3) via IPC4 module binding. +2. **SIMD Vector Optimization**: Refactor processing loops to utilize Tensilica HiFi Vector Floating-Point Unit (VFPU) SIMD intrinsics (``AE_MULFP32X2``, ``AE_ADDANDSUB``). Vectorized implementations typically reduce component MCPS consumption by 4x to 10x compared to scalar C implementations. diff --git a/developer_guides/debugability/probes/images/probes_data_pipeline.svg b/developer_guides/debugability/probes/images/probes_data_pipeline.svg new file mode 100644 index 00000000..08e8e1a0 --- /dev/null +++ b/developer_guides/debugability/probes/images/probes_data_pipeline.svg @@ -0,0 +1,242 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Figure 331: Audio Buffer Probing & Zephyr Interactive Shell Architecture + + + Intermediate Audio Buffer Injection/Extraction, Compressed Offload Streams & Zero-IPC Memory Window Terminal + + + + + + + + + + 1. Live Audio Pipeline Intermediate Buffer Tap Points & Injection Ports + + + + + + + + Audio Processing Chain Tap Points + [Host DMA] -> (B1) -> [EQ] -> (B2) -> [DRC] -> (B3) + Tap into inter-module circular buffers (B1, B2, B3) + Zero topology reconfiguration needed + + + + + + Dynamic Probe Attachment Interface + echo <buffer_id,purpose,stream_tag> > probe_points + • IPC3: Monolithic buffer counter (e.g. 7,1,0) + • IPC4: Bitfield module_id:instance_id:type:index (2,0,0) + + + + + + Bidirectional Extraction & Injection + • EXTRACTION: Tap and copy audio frames to host + • INJECTION: Inject test vectors directly into buffer + Isolates algorithm faults from upstream pipelines + + + + + + + + + + + + + + + + 2A. Audio Probe Capture & Multi-Channel Extraction Pipeline + + + + + + + + ALSA Compress Offload Capture Stream (crecord) + crecord -c3 -d0 -b8192 -f4 -FS32_LE -R48000 -C4 extract.dat + Transfers packed multi-point probe packets over DMA + Captures up to 16 concurrent probe points in single stream + + + + + + sof-probes Stream Demultiplexer + sof-probes -p extract.dat -> buffer_1.wav, buffer_2.wav + Parses probe packet headers and separates individual buffers + Generates standard multi-channel uncompressed WAV files + + + + + + Acoustic Analysis & Differential Verification + • Audacity Inspection: Compare pre-EQ vs post-EQ audio visually + • MATLAB / Octave: Compute THD+N, SNR, and frequency response + • Glitch Hunting: Identify buffer dropouts, clicks, or wrap errors + • Algorithm Tuning: Calibrate DRC compression thresholds + + + + + + + + + + + 2B. Interactive Zephyr Shell (In-Memory Terminal Bridge) + + + + + + + + Memory Window Shell Backend + CONFIG_SHELL_BACKEND_ADSP_MEMORY_WINDOW=y + Bidirectional ring buffer in DSP shared SRAM window + Zero dependency on IPC; completely immune to IPC deadlocks + + + + + + Host cavstool.py Terminal Bridge + cavstool.py -l -p -> spawns pseudo-terminal /dev/pts/4 + minicom -p /dev/pts/4 connects standard terminal emulator + Maps PCIe BARs; transparently monitors DSP boot & handoff + + + + + + Real-Time DSP Inspection Commands + • kernel uptime: Accurate DSP RTOS tick counter + • kernel threads: Priority, state (running, pending), entry points + • kernel stacks: Per-thread stack high-water mark & overflow check + • sof memory: Dynamic heap allocation & pool fragmentation + + + + + + + + + + + + + + + 3. Power Management Resilience & Transparent D0ix / D3 Suspend Recovery + + + + + + + + Low-Power D0ix / D3 Sleep State + • DSP enters low-power clock gating + • Memory window temporarily inoperable + Terminal appears paused; no kernel crash + + + + + + Transparent Wakeup & Auto-Resume + • Audio playback resumes DSP to D0 active + • cavstool.py detects memory window alive + Interactive shell session resumes without restarting + + + + + + Zero Runtime Performance Impact + • Probes activate only when stream opened + • Shell task runs at lowest background priority + Safe for deployment in development builds + + + + diff --git a/developer_guides/debugability/probes/index.rst b/developer_guides/debugability/probes/index.rst index 98e6dedc..5c30d25f 100644 --- a/developer_guides/debugability/probes/index.rst +++ b/developer_guides/debugability/probes/index.rst @@ -1,265 +1,296 @@ .. _dbg-probes: -Probes -###### +Audio Data Probes & Network Telemetry +##################################### -Typically, pipeline for audio data processing contains several components -separated by data buffers; the probe module is a debug feature that allows -for data extraction from (or injection into) these buffers. It aids in -finding audio issues or bugs in audio components with possible data analysis -from each buffer. +In complex audio DSP processing graphs, a playback or capture pipeline contains multiple sequential processing modules (such as Volume, Equalizers, Dynamic Range Compressors, Sample Rate Converters, and Mixers) connected by intermediate circular audio buffers. When audio distortion, phase cancellation, or audible dropouts occur, inspecting only the final hardware endpoint does not reveal which component in the graph corrupted the audio stream. -Requirements -************ +The SOF **Probe Subsystem** provides a dynamic, non-intrusive tap mechanism that allows developers to: -.. _install-tinycompress: +1. **Extract Intermediate Audio Data**: Tap into any circular audio buffer in the pipeline graph in real time and capture raw audio samples via ALSA Compress Offload. +2. **Inject Test Audio Vectors**: Feed synthetic test signals (chirps, impulse responses, multi-tone bursts) directly into an intermediate component buffer, isolating downstream algorithm behavior. +3. **Stream High-Throughput Firmware Telemetry**: Utilize dedicated probe DMA channels to stream binary logs over the network via the **TCP Probe Server** (port 9999). -- Install `tinycompress `_ (crecord tool) +.. figure:: images/probes_data_pipeline.svg + :alt: Audio Buffer Probes and Zephyr Interactive Shell Architecture + :align: center + :width: 100% -Enabling Probes -*************** + Figure 331: Audio Buffer Probes & Zephyr Interactive Shell Architecture -.. _kernel-side: +--- -Kernel side -=========== +Enabling Probes in Kernel and Firmware +************************************** -- The probes support is enabled by Kconfig on supported platforms as a SOF client - driver, check the kernel config for ``SND_SOC_SOF_DEBUG_PROBES``. - The debugfs also needs to be enabled for the probes to be usable. +Kernel Configuration +==================== - .. code-block:: bash +Probes operate as an ALSA SOF client driver backed by ``debugfs``: - CONFIG_DEBUG_FS=y +1. **Kernel Kconfig Options**: -- The probes client needs to be enabled via the 'enable' module parameter (e.g. ``/etc/modprobe.d/sof.conf``): + .. code-block:: cfg - .. code-block:: bash + CONFIG_SND_SOC_SOF_DEBUG_PROBES=y + CONFIG_DEBUG_FS=y - options snd_sof_probes enable=1 +2. **Module Configuration** (``/etc/modprobe.d/sof.conf``): + Enable the probes client driver upon boot and optionally lock its card slot assignment (e.g. Card 3): - To make sure that the sound card for the probes is consistent between boots, a - card slot can be forced for the module. - For example to use card3, this can be added to the sof.conf file: + .. code-block:: text - .. code-block:: bash + options snd_sof_probes enable=1 + options snd slots=,,,snd_sof_probes - options snd slots=,,,snd_sof_probes +3. **Reload Driver**: - Remove and re-load the driver: - - .. code-block:: bash - - rmmod snd_sof_probes - modprobe snd_sof_probes - - Verify that the card is available (if not, try to reboot): - - .. code-block:: bash - - cat /proc/asound/cards | grep sofprobes - -.. _firmware-side: - -Firmware side -============= + .. code-block:: bash -- The Probe module can be enabled under the 'Probe' menu's 'Probes enabled' prompt (``PROBES``) - To edit the ``kconfig`` use this command: + sudo rmmod snd_sof_probes 2>/dev/null + sudo modprobe snd_sof_probes + cat /proc/asound/cards | grep -i "sofprobes" - .. code-block:: bash +Firmware Configuration +====================== - make menuconfig +In the SOF firmware Zephyr build, enable probe points in Kconfig: - The following options available +.. code-block:: cfg - Required for audio probes: + # Enable runtime audio buffer probes + CONFIG_PROBE=y + CONFIG_PROBE_POINTS_MAX=16 - .. code-block:: bash + # Enable probe logging backend + CONFIG_LOG_BACKEND_SOF_PROBE=y + CONFIG_ZEPHYR_LOG=y - CONFIG_PROBE=y # enable probes - CONFIG_PROBE_POINTS_MAX=16 # max probepoints +No modifications to the ALSA Topology file are required; probe tap points are attached dynamically to running pipelines at runtime. - Required for logging through probes interface: +--- - .. code-block:: bash +Audio Buffer Extraction & Injection Workflow +******************************************** - CONFIG_LOG_BACKEND_SOF_PROBE=y - CONFIG_ZEPHYR_LOG=y +Step 1: Discover Buffer IDs +=========================== - Refer to :ref:`Simple logging case` for quick guide to use probes logging interface. +To attach a probe point, determine the internal buffer identifier of the target module: -- Refer to **Step 3 Build firmware binaries** in :ref:`Build SOF from Scratch ` for reference on how to build SOF FW. +For IPC3 Platforms +------------------ +Buffer IDs follow component instantiation order. Inspect kernel ``dmesg``: -Note that you do not need to modify the audio topology file. +.. code-block:: bash -Data extraction -*************** + dmesg | grep "tplg: widget " + # Output: + # snd_sof: tplg: widget 2 (gain.1.1) is ready [type: 6, pipe: 1, pins: 1 / 1] + # snd_sof: tplg: widget 3 (eq_fir.1.1) is ready [type: 12, pipe: 1, pins: 1 / 1] + # snd_sof: tplg: widget 7 (dai-copier.HDA.playback) is ready [type: 27, pipe: 1, pins: 1 / 0] -Extraction is the most common use case. It allows for data extraction from -the audio component data buffer. It requires starting the compress stream by -starting the crecord tool. Note that one compress stream may contain data -from several extraction probe points which means data parsing is needed at -the last stage of extraction. +For IPC4 Platforms +------------------ +On IPC4 platforms (e.g. Tiger Lake, Meteor Lake, Arrow Lake, Panther Lake), the buffer ID is a structured 32-bit bitfield (``probe_point_id_t``): -#. Start the crecord tool to prepare the extraction stream (read the crecord - readme file): +.. code-block:: c - .. code-block:: bash + typedef union probe_point_id { + uint32_t full_id; + struct { + uint32_t module_id : 16; /**< Target module ID (e.g. 2 for gain) */ + uint32_t instance_id : 8; /**< Module instance counter (e.g. 0) */ + uint32_t type : 2; /**< Probe point type (input vs output queue) */ + uint32_t index : 6; /**< Pin/queue index within target module */ + } fields; + } probe_point_id_t; - crecord -c3 -d0 -b8192 -f4 -FS32_LE -R48000 -C4 /tmp/extract.dat +Step 2: Start Compress Offload Capture (crecord) +================================================ - Usage: +The probe stream is captured through the ALSA Compress Offload interface using the ``crecord`` tool (provided by `tinycompress `_): - .. code-block:: none +.. code-block:: bash - -c : card number; 3 in the above example if a slot is forced - -d : device ID; equals 0 in the above example (probes card only have 1 compressed capture stream). - -b : buffer size. For probes, this is part of the probe - initialization IPC and denotes the extraction stream buffer size on the host side. - -f : fragments is basically number of periods for compress stream. + # Start capture on Card 3, Device 0 with 8KB buffer and 4 periods + crecord -c3 -d0 -b8192 -f4 -FS32_LE -R48000 -C4 /tmp/extract.dat & + CRECORD_PID=$! - The other parameters are "don't-cares" for the driver. +Step 3: Attach Probe Points via debugfs +======================================= - - Use ``aplay`` to start the playback stream. - - Pause the playback stream. (optional) - - Add probe points via the ``debugfs`` "probe_points" entry in ``/sys/kernel/debug/sof`` +Write the probe attachment triplet into ``/sys/kernel/debug/sof/probe_points``: +.. code-block:: bash - For example, to add buffer 7 with a probe point (IPC3): + # Format: echo ,, > /sys/kernel/debug/sof/probe_points - .. code-block:: bash + # IPC3 Example: Extract from buffer 3 (EQ output), purpose=1 (EXTRACTION), stream_tag=0 + echo 3,1,0 | sudo tee /sys/kernel/debug/sof/probe_points - echo 7,1,0 > probe_points + # IPC4 Example: Extract from module 2, instance 0, purpose=0 (EXTRACTION), stream_tag=0 + echo 2,0,0 | sudo tee /sys/kernel/debug/sof/probe_points - Refer to the host side struct sof_probe_point_desc defined in ``sound/soc/sof/probe.h`` - or struct probe_point in ``/src/include/ipc/probe.h`` from sof for the meaning of the triplets: +Step 4: Play Audio & Stop Capture +================================= - .. code-block:: c +.. code-block:: bash - /** - * Description of probe point - */ - struct probe_point { - uint32_t buffer_id; /**< ID of buffer to which probe is attached */ - uint32_t purpose; /**< PROBE_PURPOSE_EXTRACTION or PROBE_PURPOSE_INJECTION */ - uint32_t stream_tag; /**< Stream tag of DMA via which data will be provided for injection. - * For extraction purposes, stream tag is ignored when received, - * but returned actual extraction stream tag via INFO function. - */ - } __attribute__((packed)); + # Start audio playback through the pipeline + aplay -D hw:0 -r 48000 -c 2 -f S16_LE /path/to/test_sweep.wav - In the above example, 7 stands for the ``buffer_id`` which is a monolithic - counter value that follows a component instantiation order. + # When complete, terminate crecord + kill -SIGINT $CRECORD_PID - One way to find out the right instance of ``buffer_id`` is to enable - dev_dbg in ``sound/sound/soc/sof/topology.c`` and search for the widget id - from the following messages: +--- - .. code-block:: c +Multi-Channel Demultiplexing (sof-probes) +***************************************** - dev_dbg(scomp->dev, - "tplg: widget %d (%s) is ready [type: %d, pipe: %d, pins: %d / %d, stream: %s]\n", - swidget->comp_id, w->name, swidget->id, index, - swidget->num_input_pins, swidget->num_output_pins, - strnlen(w->sname, SNDRV_CTL_ELEM_ID_NAME_MAXLEN) > 0 ? w->sname : "none"); +Because a single compress capture stream can aggregate audio data from up to 16 simultaneous probe tap points, the raw ``extract.dat`` file contains interleaved multi-point packet frames. - On a booted system the list can be acquired with +Use the **``sof-probes``** utility with the ``-p`` (parse) flag to demultiplex the stream into separate, uncompressed standard WAV files: - .. code-block:: bash +.. code-block:: bash - dmesg | grep "tplg: widget " - ... - snd_sof:sof_widget_ready: sof-audio-pci-intel-tgl 0000:00:1f.3: tplg: widget 2 (gain.1.1) is ready [type: 6, pipe: 1, pins: 1 / 1, stream: none] - snd_sof:sof_widget_ready: sof-audio-pci-intel-tgl 0000:00:1f.3: tplg: widget 3 (mixin.1.1) is ready [type: 4, pipe: 1, pins: 1 / 3, stream: none] - snd_sof:sof_widget_ready: sof-audio-pci-intel-tgl 0000:00:1f.3: tplg: widget 4 (pipeline.1) is ready [type: 32, pipe: 1, pins: 0 / 0, stream: none] - snd_sof:sof_widget_ready: sof-audio-pci-intel-tgl 0000:00:1f.3: tplg: widget 5 (codec0_in) is ready [type: 0, pipe: 1, pins: 0 / 0, stream: none] - snd_sof:sof_widget_ready: sof-audio-pci-intel-tgl 0000:00:1f.3: tplg: widget 6 (iDisp2 Tx) is ready [type: 7, pipe: 1, pins: 0 / 0, stream: none] - snd_sof:sof_widget_ready: sof-audio-pci-intel-tgl 0000:00:1f.3: tplg: widget 7 (dai-copier.HDA.Analog.playback) is ready [type: 27, pipe: 2, pins: 1 / 0, stream: Analog] - ... + # Demux raw probe extraction file + sof-probes -p /tmp/extract.dat + + # Output generated: + # sof-probes: Parsing file: /tmp/extract.dat + # sof-probes: Creating wave file for buffer id: 3 -> buffer_3.wav + # sof-probes: Creating wave file for buffer id: 7 -> buffer_7.wav + # sof-probes: done + +Inspect the resulting ``buffer_*.wav`` files in audio analysis tools such as **Audacity**, MATLAB, or GNU Octave: + +* Compare pre-processing vs post-processing waveforms side-by-side. +* Calculate Total Harmonic Distortion + Noise (THD+N) and Signal-to-Noise Ratio (SNR). +* Visually identify clipping, unaligned sample phase, or dropped audio frames. + +--- + +High-Throughput Network Probe Server (Port 9999) +************************************************ + +On remote hardware test stations (DUTs), running file-based captures over SSH introduces terminal overhead and network latency. SOF provides the **``sof_probe_server``** C streaming daemon listening on TCP port **9999**: + +.. code-block:: text + + +-------------------------------------------------------------------------+ + | Target DUT (Linux Kernel) | + | | + | [ DSP Audio Pipeline ] ---> [ Trace / Probe DMA ] | + | | | + | v | + | [ /dev/snd/comprC3D0 ] | + | | | + | v | + | [ sof_probe_server (TCP :9999) ] | + | (1MB Thread-Safe Circular Queue) | + +-------------------------------------------------------------------------+ + | + Private Lab Network Stream + v + +-------------------------------------------------------------------------+ + | Host Analysis Workstation | + | | + | [ sof_probe_client.py ] or [ dut-monitor Dashboard ] | + | --display ascii --out /tmp/data.bin (Live Decoded Logs: Section 4) | + +-------------------------------------------------------------------------+ + +C Server Architecture +===================== + +* **ALSA Compress Interface**: Directly opens ``/dev/snd/comprC*D*`` using native ALSA compress ioctls. +* **1 MB Circular Buffer**: Uses a POSIX mutex and condition variable (``pthread_mutex_t`` / ``pthread_cond_t``) circular buffer to completely decouple kernel DMA extraction from network TCP socket writes. +* **Low CPU Overhead**: Consumes negligible CPU cycles (<1%) on host and embedded cores. + +Running Probe Server on DUT +=========================== - For IPC4 system, the above example looks like this (extraction from gain.1.1): +.. code-block:: bash - .. code-block:: bash + # Launch probe server on target DUT + timeout 15 ssh -o ConnectTimeout=5 root@ \ + 'nohup /usr/local/bin/sof_probe_server -c 3 -d 0 -p 9999 -v > /tmp/probe_server.log 2>&1 &' - echo 2,0,0 > probe_points +Remote Data Collection via Python Client +======================================== - The semantics of the buffer_id are quite different on IPC4 system: +On the development workstation, run ``sof_probe_client.py``: - .. code-block:: c +.. code-block:: bash - typedef union probe_point_id { - uint32_t full_id; - struct { - uint32_t module_id : 16; /**< Target module ID */ - uint32_t instance_id : 8; /**< Target module instance ID */ - uint32_t type : 2; /**< Probe point type as specified by ProbeType enumeration */ - uint32_t index : 6; /**< Queue index inside target module */ - } fields; - } __attribute__((packed, aligned(4))) probe_point_id_t; + # Stream and save raw binary probe data + python3 tools/sof-probe-server/sof_probe_client.py \ + --host --port 9999 \ + --output /tmp/dut_probe_data.bin - .. code-block:: c + # Display live ASCII logs decoded from probe stream + python3 tools/sof-probe-server/sof_probe_client.py \ + --host --port 9999 \ + --display ascii - /** - * Description of probe point - */ - struct probe_point { - probe_point_id_t buffer_id; /**< ID of buffer to which probe is attached */ - uint32_t purpose; /**< PROBE_PURPOSE_xxx */ - uint32_t stream_tag; /**< Stream tag of DMA via which data will be provided for injection. - * For extraction purposes, stream tag is ignored when received, - * but returned actual extraction stream tag via INFO function. - */ - } __attribute__((packed, aligned(4))); + # Preview raw packet headers in hexadecimal + python3 tools/sof-probe-server/sof_probe_client.py \ + --host --port 9999 \ + --display hex -2. Unpause the playback stream. (optional) -#. Close the playback stream when done. -#. Close the crecord tool. +Integrated dut-monitor Multi-Pane Dashboard +=========================================== -.. _data-parsing: +The ``dut-monitor`` terminal monitoring suite automatically connects to ``sof_probe_server`` on TCP port 9999. Decoded DSP firmware logs are displayed in real-time in **Section 4**, synchronized with live hardware power consumption telemetry (port 8080) and CPU thermal metrics. -Data parsing -************ +--- -As previously mentioned, one compress stream can contain data from several -extraction probe points which means data parsing is needed at the final -stage of extraction. The following example demonstrates how to extract data. Use ``-p`` for parse. +Early Boot Probe Logging (snd-sof-probes) +***************************************** -Usage and ouput: +The ``snd-sof-probes`` kernel module provides the ``logging_boot_enable`` parameter to capture initialization logs from the very start of firmware execution: .. code-block:: bash - $ ./sof-probes -p /tmp/extract.dat - sof-probes: Parsing file: /tmp/extract.dat - sof-probes: Creating wave file for buffer id: 7 - sof-probes: done - -As a result, ``buffer_7.wav`` is generated in the *tools/build_tools/probes* folder. The wave file can then be examined with your tool of choice -such as ``Audacity``. - -.. _simple-logging-case: + # 1. Enable boot logging in kernel module + timeout 20 ssh -o ConnectTimeout=5 root@ ' + rmmod snd_sof_probes 2>/dev/null + modprobe snd_sof_probes logging_boot_enable=1 + dmesg | grep "logging_boot" + ' -Simple logging case -******************* + # 2. Start probe server and initiate dummy stream to drain pre-buffered logs (up to 4 KB) + timeout 15 ssh -o ConnectTimeout=5 root@ ' + pkill -f sof_probe_server; pkill aplay; sleep 1 + nohup /usr/local/bin/sof_probe_server -c 3 -d 0 -p 9999 -v > /tmp/probe.log 2>&1 & + nohup aplay -D hw:0 -r 48000 -c 2 -f S16_LE /dev/zero > /dev/null 2>&1 & + ' -With the :ref:`crecord` and :ref:`sof-probes` in path, FW built with :ref:`probes logging enabled`, and probes enabled from :ref:`Linux side`, it should be possible to extract the logs with following steps: + # 3. Stream early boot trace to workstation + timeout 60 python3 tools/sof-probe-server/sof_probe_client.py \ + --host --port 9999 --display ascii --out /tmp/boot_trace.bin -#. crecord has to be started first: +Subsequent test sessions can be launched back-to-back without reloading the kernel driver, as the ``fw_probe_active`` state machine tracks extraction state across multiple stream lifetimes. -.. code-block:: bash - - crecord -c3 -d0 -b8192 -f4 -FS32_LE -R48000 -C4 | sof-probes -l +--- -#. then to enable logs through probes sysfw interface use following commands as root, +Intel Arrow Lake mtrace Buffer Extraction +***************************************** - IPC3 system: +On Intel Arrow Lake (ARL-S) platforms running ACE 1.5, firmware logs can also be retrieved directly from the hardware ``mtrace`` buffer: .. code-block:: bash - echo 0,1,0 > /sys/kernel/debug/sof/probe_points + # Start mtrace reader prior to test execution or driver reload + timeout 15 ssh -o ConnectTimeout=5 root@ \ + 'nohup ./mtrace-reader.py > /tmp/fw_mtrace.log 2>&1 &' - IPC4 system: + # Execute test audio pipeline + timeout 30 ssh -o ConnectTimeout=5 root@ \ + 'aplay -D hw:0 -r 48000 -c 2 -f S16_LE /dev/zero -d 5' -.. code-block:: bash + # Retrieve formatted mtrace log + scp root@:/tmp/fw_mtrace.log ./fw_mtrace.log - echo 0,0,0 > /sys/kernel/debug/sof/probe_points + # Terminate reader + timeout 15 ssh -o ConnectTimeout=5 root@ 'pkill -f mtrace-reader' diff --git a/developer_guides/debugability/ri-info/index.rst b/developer_guides/debugability/ri-info/index.rst index 6fbf0414..aec48255 100644 --- a/developer_guides/debugability/ri-info/index.rst +++ b/developer_guides/debugability/ri-info/index.rst @@ -1,42 +1,120 @@ .. _dbg-ri-info: -sof-ri-info -########### - -sof-ri-info is a python3 script that parses manifests that are included in -the sof binary. It prints extracted metadata in readable form. The output is -manifest-type dependent content. The binary file layout displays when -verbose mode is selected. - -Currently, the following can be parsed: - -+-------------------+------------------+ -| name | signature | -+===================+==================+ -| CSE Manifest | `$CPD` | -+-------------------+------------------+ -| CSS Manifest | | -+-------------------+------------------+ -| ADSP Manifest | `$AM1`, `$AME` | -+-------------------+------------------+ -| Extended Manifest | `$AE1`, `XMan` | -+-------------------+------------------+ - -Examples -******** - -.. note:: - Run ``sof_ri_info.py -h`` to see how to switch to the appropriate display - mode. +Firmware Binary Manifest & Security Inspection (sof-ri-info) +############################################################ + +Sound Open Firmware (SOF) binaries deployed onto modern silicon architectures (such as Intel cAVS and ACE DSPs) are packaged and cryptographically signed using the **``rimage``** tool. To verify binary integrity, diagnose secure boot rejections, and inspect module layouts before flashing or deploying firmware, developers use the **``sof_ri_info.py``** inspection utility. + +The utility parses binary manifest structures embedded within ``.ri`` firmware images, printing human-readable metadata, cryptographic signing headers, and memory segment descriptors. + +--- + +Firmware Manifest Architecture +****************************** + +An SOF firmware binary contains layered partition and manifest blocks prepended to the executable machine code: + +.. list-table:: Supported Firmware Manifest Structures + :widths: 20 15 65 + :header-rows: 1 + + * - Manifest Name + - Signature + - Architectural Purpose & Contents + * - **CSE Manifest** + - ``$CPD`` + - Converged Security Engine Partition Directory. Defines partition metadata, entry table offsets, partition name (e.g. ``ADSP``), and image checksum. + * - **CSS Manifest** + - N/A + - Crypto Subsystem Manifest. Stores public key modulus (RSA-2048 / RSA-3072), public exponent (``0x10001``), cryptographic signature, and Platform Firmware Authentication extensions. + * - **ADSP Manifest** + - ``$AM1``, ``$AME`` + - Audio DSP Manifest. Specifies hardware core requirements, entry point instruction addresses (``entry_point``), load segment memory targets, and virtual memory page allocations. + * - **Extended Manifest** + - ``$AE1``, ``XMan`` + - Extended Architecture Manifest. Encodes firmware ABI version (IPC3/IPC4), compiler toolchain flags, and the built-in audio module UUID catalog. + +--- + +Using sof_ri_info.py +******************** + +The ``sof_ri_info.py`` script is located in the ``tools/`` directory of the SOF repository: + +.. code-block:: bash + + python3 tools/sof_ri_info.py [-h] [--headers | -v | --full_bytes] + +Command-Line Modes +================== + +.. list-table:: sof_ri_info.py Display Modes + :widths: 25 75 + :header-rows: 1 + + * - Option + - Output & Description + * - ``--headers`` + - **Headers Only Mode**: Prints high-level partition summary, signing key identity (Community vs Production), date, and extension types. + * - ``-v`` + - **Verbose Mode**: Traverses byte-by-byte offsets, displaying entry lengths, section addresses, and CSE directory indexes. + * - ``--full_bytes`` + - **Full Bytes Mode**: Emits complete hexadecimal dumps of cryptographic moduli, exponent blocks, and signature arrays. + * - ``--no_colors`` + - Suppresses terminal ANSI escape formatting for automated logging and pipe redirection. + +--- + +Manifest Inspection Examples +**************************** + +Headers-Only Mode +================= + +Displays high-level partition metadata and signing authority: .. literalinclude:: output_headers.txt - :caption: Example of "headers only" mode. + :caption: Example of "headers only" mode output. + :language: text :linenos: +Verbose Byte-Offset Mode +======================== + +Displays sequential file offsets and partition entry lengths: + .. literalinclude:: output_verbose.txt - :caption: Example of "verbose" mode. + :caption: Example of "verbose" mode output. + :language: text :linenos: +Full Cryptographic Bytes Mode +============================= + +Dumps complete RSA public key modulus, exponent, and signature arrays: + .. literalinclude:: output_full_bytes.txt - :caption: Example of "full bytes" mode - complete content of relevant binary objects is printed out. + :caption: Example of "full bytes" mode output. + :language: text :linenos: + +--- + +Boot Authentication Troubleshooting +*********************************** + +When deploying firmware to pre-production development boards and target DUTs, secure boot failures typically manifest as a DSP ROM stall (``ROM_STATUS = 0x80000000`` or timeout): + +1. **Verify Community vs Production Key**: + In ``--headers`` output, inspect the Modulus line: + + .. code-block:: text + + Modulus size (dwords) 64 + 85 00 e1 68 aa eb d2 07 ... 5a 96 28 27 19 af 43 b9 (Community key) + + * Development hardware with open fuses accepts the SOF **Community Key** (``otc_community_key.pem``). + * Secure production hardware with burned vendor fuses requires Intel OEM production signing keys. + +2. **Verify Load Segment Boundaries**: + Run ``-v`` mode to ensure entry segment limits match the physical SRAM bank allocations specified in the platform memory configuration. diff --git a/developer_guides/debugability/shell/index.rst b/developer_guides/debugability/shell/index.rst index 4f0b943c..2785681a 100644 --- a/developer_guides/debugability/shell/index.rst +++ b/developer_guides/debugability/shell/index.rst @@ -1,119 +1,246 @@ .. _dbg-zephyr-shell: -Zephyr Shell -############ +Zephyr Interactive Shell +######################## + +Sound Open Firmware (SOF) incorporates the native **Zephyr RTOS Shell** subsystem, providing an interactive, bidirectional command-line terminal directly on the running audio DSP. While production audio firmware operates headlessly without interactive consoles, developer and validation builds can leverage the interactive shell to: + +* Inspect running thread priorities, states, and entry points. +* Audit stack high-water marks across all RTOS threads to detect impending stack overflows. +* Check dynamic heap pool allocations and detect memory fragmentation. +* Dynamically adjust logging verbosity on a per-module basis without recompiling firmware. +* Monitor audio pipeline scheduling states and component parameters in real time. + +--- + +Architecture: Shared Memory Window Backend +****************************************** + +Traditional embedded shells communicate via dedicated UART serial interfaces. On modern multi-core audio DSPs (such as Intel Tiger Lake, Meteor Lake, Arrow Lake, and Panther Lake), hardware UART lines are frequently unavailable, unrouted on production motherboards, or multiplexed with other high-speed peripherals. + +To solve this, SOF utilizes the **Intel ADSP Memory Window Shell Backend** (``CONFIG_SHELL_BACKEND_ADSP_MEMORY_WINDOW=y``): + +.. code-block:: text + + +-------------------------------------------------------------------------+ + | Host Linux Workstation | + | | + | [ minicom / picocom / screen ] | + | | | + | v | + | [ /dev/pts/4 ] (Pseudo-Terminal) | + | | | + | v | + | [ cavstool.py -l -p ] | + | | | + | v (PCIe MMIO BAR Read / Write) | + +-------------------------------------------------------------------------+ + | + PCIe System Bus + | + +-------------------------------------------------------------------------+ + | Audio DSP (cAVS / ACE) | + | | + | [ Shared SRAM Memory Window ] <---> [ Zephyr Shell Engine ] | + | | | + | [ RTOS Threads & Stacks ] | + | [ SOF Pipeline State ] | + +-------------------------------------------------------------------------+ + +Key Architectural Advantages +============================ + +1. **Zero IPC Dependency**: The memory window shell backend operates via direct host PCIe MMIO memory accesses into shared DSP SRAM. It does not send or receive IPC messages. As a result, the shell remains fully responsive even if the firmware IPC subsystem is deadlocked, hung, or uninitialized. +2. **Transparent Low-Power Resilience**: When the DSP transitions into low-power D0ix or D3 suspend states, the shared memory window is temporarily gated. ``cavstool.py`` detects this condition and pauses terminal I/O. When an audio stream resumes and wakes the DSP back to active D0, the shell terminal resumes immediately without session drops. +3. **Deterministic Real-Time Scheduling**: The shell thread executes at the lowest cooperative background priority (priority 14), ensuring that real-time audio pipeline processing threads (priorities -16 to 0) are never preempted or delayed. + +--- + +Enabling Shell Support in Firmware +********************************** -Zephyr provides a shell subystem for interactive debugging and a channel -to run custom test sequences. SOF supports use of the Zephyr shell. +Firmware builds have the shell disabled by default to minimize memory footprint and power consumption. Enable shell support using build overlays: -The Zephyr shell is documented at: -https://docs.zephyrproject.org/latest/services/shell/index.html +.. code-block:: bash -Requirements -************ + # Build Tiger Lake firmware with Zephyr shell enabled + ./sof/scripts/xtensa-build-zephyr.py tgl -o app/shell_overlay.conf -- SOF target platform must have Zephyr support. + # Build Panther Lake (PTL) firmware with shell enabled + ./sof/scripts/xtensa-build-zephyr.py ptl -o app/shell_overlay.conf -- At least one shell backend (DSP memory window, serial port, RTT, ...) compatible - with target platform. +The ``shell_overlay.conf`` configuration enables the following Kconfig options: -Build SOF with Shell Support -**************************** +.. code-block:: cfg -Shell is typically disabled by default and the firmware needs to be -rebuilt. For common SOF targets, a build overlay is provided in SOF -upstream to easily enable shell suppot in build. + CONFIG_SHELL=y + CONFIG_SHELL_BACKEND_ADSP_MEMORY_WINDOW=y + CONFIG_SHELL_STACK_SIZE=2048 + CONFIG_SHELL_CMD_BUFF_SIZE=256 + CONFIG_THREAD_NAME=y + CONFIG_THREAD_STACK_INFO=y + CONFIG_INIT_STACKS=y - .. code-block:: bash +--- - # example build for Intel Tiger Lake platform - build-sh> sof/scripts/xtensa-build-zephyr.py tgl -o app/shell_overlay.conf +Connecting with cavstool.py +*************************** -Using Shell with Intel cavstool.py -********************************** +The ``cavstool.py`` host utility communicates with the DSP memory window over the PCIe bus and spawns a virtual pseudo-terminal (PTY): + +Step 1: Launch cavstool Bridge +============================== + +Run ``cavstool.py`` on the target machine (or DUT) with ``-l`` (listen) and ``-p`` (pseudo-terminal): + +.. code-block:: bash + + sudo ./cavstool.py -l -p + +Output: + +.. code-block:: text + + INFO:cavs-fw:Existing driver "snd_sof_pci_intel_tgl" found + INFO:cavs-fw:Mapped PCI bar 0 of length 16384 bytes. + INFO:cavs-fw:Selected output stream 15 (GCAP = 0xffffffff) + INFO:cavs-fw:Mapped PCI bar 4 of length 1048576 bytes. + INFO:cavs-fw:Detected cAVS 2.5 hardware + INFO:cavs-fw:Waiting for firmware handoff, ROM_STATUS = 0x5 + INFO:cavs-fw:FW alive, ROM_STATUS = 0x5 + INFO:cavs-fw:shell PTY at: /dev/pts/4 + +Step 2: Attach Terminal Emulator +================================ + +In another terminal, attach to the allocated pseudo-terminal (e.g. ``/dev/pts/4``) using ``minicom``, ``picocom``, or ``screen``: + +.. code-block:: bash + + # Connect using minicom + sudo minicom -p /dev/pts/4 + + # Or connect using picocom + sudo picocom /dev/pts/4 + +Press ``Enter`` to reveal the interactive Zephyr shell prompt: + +.. code-block:: text + + ~$ + +--- + +Command Reference & Diagnostics Runbook +**************************************** + +Kernel & System Information +=========================== + +.. code-block:: text + + ~$ kernel uptime + Uptime: 45210 ms + + ~$ kernel version + Zephyr version 3.7.0 + +Thread State & Scheduling Analysis +================================== + +Inspect all active RTOS threads, priorities, and execution states: + +.. code-block:: text + + ~$ kernel threads + Scheduler: 1 since last call + Threads: + *0x9e0a4e78 ll_thread0 + options: 0x0, priority: -16 timeout: 0 + state: running, entry: 0xbe02e060 + stack size 8192, unused 6752, usage 1440 / 8192 (17 %) + + 0x9e0a37d0 edf_workq + options: 0x0, priority: -14 timeout: 0 + state: pending, entry: 0xbe0189a0 + stack size 8192, unused 6304, usage 1888 / 8192 (23 %) + + 0x9e0a3c48 sysworkq + options: 0x0, priority: -1 timeout: 0 + state: pending, entry: 0xbe019200 + stack size 1024, unused 728, usage 296 / 1024 (28 %) + + 0x9e0a3180 shell_adsp_memory_window + options: 0x0, priority: 14 timeout: 0 + state: running, entry: 0xbe01969c + stack size 2048, unused 760, usage 1288 / 2048 (62 %) + +Stack High-Water Mark & Overflow Auditing +========================================= + +Execute ``kernel stacks`` to audit stack headroom across all audio processing threads: + +.. code-block:: text + + ~$ kernel stacks + 0x9e0a4e78 ll_thread0 (real size 8192): unused 6752 usage 1440 / 8192 (17 %) + 0x9e0a37d0 edf_workq (real size 8192): unused 6304 usage 1888 / 8192 (23 %) + 0x9e0a3c48 sysworkq (real size 1024): unused 728 usage 296 / 1024 (28 %) + 0x9e0a3180 shell_adsp_memory_window (real size 2048): unused 760 usage 1288 / 2048 (62 %) + 0x9e0a3080 logging (real size 4096): unused 3488 usage 608 / 4096 (14 %) + 0x9e0a38b0 idle 00 (real size 1024): unused 824 usage 200 / 1024 (19 %) + 0xbe09df80 IRQ 00 (real size 2048): unused 1712 usage 336 / 2048 (16 %) + +.. note:: + If any thread exhibits usage exceeding **85–90%**, increase its stack allocation in Kconfig or the component configuration to avoid intermittent stack corruption exceptions. + +SOF Pipeline & Component Diagnostics +==================================== + +Inspect active audio pipelines, components, and buffer queues: + +.. code-block:: text + + ~$ sof pipeline list + Pipeline 1: Core 0, Priority 0, State: RUNNING, Period: 1000 us + [0] host-copier (ID: 1, Active) + [1] volume (ID: 2, Active) + [2] eq_iir (ID: 3, Active) + [3] dai-copier (ID: 4, Active) + + ~$ sof mem status + Heap System Pool: + Total: 524288 bytes + Allocated: 184320 bytes (35 %) + Free: 339968 bytes (65 %) + Largest Free Block: 294912 bytes + +Dynamic Logging Configuration +============================= + +Adjust logging levels on a live DSP without stopping the audio stream: + +.. code-block:: text + + # Check active log levels + ~$ log status + eq_fir: 3 (INF) + volume: 3 (INF) + ipc: 3 (INF) + + # Enable verbose debug logging on eq_fir module + ~$ log enable 4 eq_fir + + # Suppress logging on volume module + ~$ log enable 1 volume + +--- + +Handling DSP Low-Power States (D0ix / D3) +***************************************** + +When audio playback or capture stops, the Linux kernel driver allows the audio DSP to transition into low-power states (D0ix clock gating or D3 power gating) to conserve energy: -This section covers use with SOF targets compatible with -CONFIG_SHELL_BACKEND_ADSP_MEMORY_WINDOW backend (for example Audio DSPs -on Intel Tiger Lake and Meteor Lake). - -Running the tool with "-p" to create a pseudo terminal for the shell: - - .. code-block:: bash - - dut-sh> sudo ./cavstool.py -l -p - INFO:cavs-fw:Existing driver "snd_sof_pci_intel_tgl" found - INFO:cavs-fw:Mapped PCI bar 0 of length 16384 bytes. - INFO:cavs-fw:Selected output stream 15 (GCAP = 0xffffffff) - INFO:cavs-fw:Mapped PCI bar 4 of length 1048576 bytes. - INFO:cavs-fw:Detected cAVS 1.8+ hardware - INFO:cavs-fw:Waiting forever for firmware handoff, ROM_STATUS = 0xffffffff - INFO:cavs-fw:FW alive, ROM_STATUS = 0x5 - INFO:cavs-fw:shell PTY at: /dev/pts/4 - -The Zephyr shell is now available at pseudo terminal /dev/pts/4 (see log above) -and can be attached with any terminal program: - - .. code-block:: bash - - dut-sh> sudo minicom -p /dev/pts/4 - Welcome to minicom 2.8 - - OPTIONS: I18n - Port /dev/modem - - Press CTRL-A Z for help on special keys - - ~$ kernel uptime - Uptime: 31600 ms - ~$ kernel stacks - 0x9e0a4e78 ll_thread0 (real size 8192): unused 6752 usage 1440 / 8192 (17 %) - 0x9e0a34b8 (real size 4096): unused 4008 usage 88 / 4096 ( 2 %) - 0x9e0a3400 (real size 4096): unused 4008 usage 88 / 4096 ( 2 %) - 0x9e0a3348 (real size 4096): unused 4008 usage 88 / 4096 ( 2 %) - 0x9e0a3290 (real size 4096): unused 4008 usage 88 / 4096 ( 2 %) - 0x9e0a37d0 edf_workq (real size 8192): unused 6304 usage 1888 / 8192 (23 %) - 0x9e0a3c48 sysworkq (real size 1024): unused 728 usage 296 / 1024 (28 %) - 0x9e0a3180 shell_adsp_memory_window (real size 2048): unused 760 usage 1288 / 2048 (62 %) - 0x9e0a3080 logging (real size 4096): unused 3488 usage 608 / 4096 (14 %) - 0x9e0a38b0 idle 00 (real size 1024): unused 824 usage 200 / 1024 (19 %) - 0xbe09df80 IRQ 00 (real size 2048): unused 1712 usage 336 / 2048 (16 %) - 0xbe09e780 IRQ 01 (real size 2048): unused 0 usage 2048 / 2048 (100 %) - 0xbe09ef80 IRQ 02 (real size 2048): unused 0 usage 2048 / 2048 (100 %) - 0xbe09f780 IRQ 03 (real size 2048): unused 0 usage 2048 / 2048 (100 %) - ~$ kernel threads - Scheduler: 1 since last call - Threads: - 0x9e0a4e78 ll_thread0 - options: 0x0, priority: -16 timeout: 0 - state: pending, entry: 0xbe02e060 - stack size 8192, unused 6752, usage 1440 / 8192 (17 %) - - 0x9e0a34b8 - options: 0x0, priority: -16 timeout: 0 - state: prestart, entry: 0xbe0154cc - stack size 4096, unused 4008, usage 88 / 4096 (2 %) - - [cut] - *0x9e0a3180 shell_adsp_memory_window - options: 0x0, priority: 14 timeout: 0 - state: , entry: 0xbe01969c - stack size 2048, unused 760, usage 1288 / 2048 (62 %) - - 0x9e0a3080 logging - options: 0x0, priority: 14 timeout: 0 - state: pending, entry: 0xbe016710 - stack size 4096, unused 3488, usage 608 / 4096 (14 %) - - 0x9e0a38b0 idle 00 - options: 0x1, priority: 15 timeout: 0 - state: , entry: 0xbe054298 - stack size 1024, unused 824, usage 200 / 1024 (19 %) - ~$ - -The memory window backend does not rely on IPC, so the shell is not -dependent on the IPC version implementation. The cavstool.py is also -implemented to handle cases where the DSP is suspended to lower power -state and the memory window is not accessible to host. When the DSP -is in such state, the shell terminal will appear inactive, but it will -resume immediately after DSP resumes to active state, without need -to rerun the cavstool.py script. +1. **Terminal Pausing**: When the DSP enters D0ix or D3, the hardware memory window becomes inaccessible. In your terminal, keystrokes will not echo and command output will pause. +2. **Transparent Auto-Resume**: As soon as an application initiates an audio stream (or a test script starts ``aplay`` / ``arecord``), the DSP powers back up to active D0. ``cavstool.py`` detects the valid ROM status, re-establishes the memory window pointers, and the terminal resumes immediately without dropping the shell session. +3. **Zero Restart Needed**: Developers do not need to kill ``cavstool.py`` or restart ``minicom`` across multiple playback sessions. diff --git a/developer_guides/debugability/traces/images/dsp_telemetry_architecture.svg b/developer_guides/debugability/traces/images/dsp_telemetry_architecture.svg new file mode 100644 index 00000000..2fb8a1d7 --- /dev/null +++ b/developer_guides/debugability/traces/images/dsp_telemetry_architecture.svg @@ -0,0 +1,242 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Figure 329: Sound Open Firmware (SOF) DSP Telemetry, Logging & Trace Streaming Architecture + + + Asynchronous Zero-Overhead Logging: Build-Time smex Extraction, Trace DMA Ring Buffers & TCP Probe Server (Port 9999) + + + + + + + + + + 1. Build-Time String Extraction & Metadata Catalog Pipeline (smex) + + + + + + + + Zephyr Logging Macro Calls + LOG_MODULE_REGISTER(eq_fir, ...) + Format strings stored in .static_log_entries section + + + + + + smex Metadata Extraction Tool + smex -l sof-tgl.ldc -e build/zephyr.elf + Extracts catalog (.ldc); embeds 32-bit entry IDs in FW + + + + + + Zero-Overhead Firmware Image + sof-tgl.ri / sof-mtl.ri + No text strings in DSP memory; ~10-20 cycles per log + + + + + + + + + + + + + + + + 2A. Audio DSP Core & Hardware Trace DMA Subsystem + + + + + + + + Internal DSP SRAM Trace Ring Buffer + trace_buf [8 KB / 16 KB lockless circular buffer] + Packet: [timestamp:64b | log_entry_id:32b | params:4x32b] + Lockless atomic writes from audio pipeline processing threads + + + + + + Autonomous Hardware Trace DMA Controller + dma_trace_enable() -> background burst transfers + Transfers trace chunks to host memory window without CPU polling + Audio pipelines execute uninterrupted with strict sub-ms latency + + + + + + DSP Memory Window & IPC Handshake + • Host SRAM Window 3: Accessible via host PCIe MMIO BAR + • Trace Position Notification: SOF_IPC_TRACE_DMA_POSITION + • Watermark Thresholds: Triggers DMA burst before buffer wrap + • D0ix Retention: Flushes pending traces before entering low-power sleep + + + + + + + + + + + 2B. Linux Driver Ingestion & sof-logger Decoding + + + + + + + + Linux Kernel Driver (sound/soc/sof) + /sys/kernel/debug/sof/trace [Continuous debugfs stream] + snd-sof-probes logging_boot_enable=1 [Early boot logging] + Manages host DMA ring buffer and wakeups on trace events + + + + + + sof-logger Binary Trace Decoder + sof-logger -t -d /path/to/sof-tgl.ldc -l /sys/.../trace + Resolves entry ID -> source file, line, function, printf format + Live streaming (-t), file dump (-i/-o), timestamp sync + + + + + + Trace Output & Production Diagnostics + • Microsecond Timestamps: Hardware DSP clock synchronization + • Log Level Filtering: --level <1..4> (ERR, WRN, INF, DBG) + • Module Categorization: Per-component prefix (ipc, pipe, dai) + • Automated Test Capture: Integrated with Twister, CTest, ktest + + + + + + + + + + + + + + + 3. Remote Network Probe Server (Port 9999) & Live Telemetry Ecosystem + + + + + + + + TCP Probe Server (sof_probe_server) + sof_probe_server -c 3 -d 0 -p 9999 -v + • ALSA compress capture (/dev/snd/comprC*D*) + • 1MB circular buffer decouples extraction from network + + + + + + Remote Python Client (sof_probe_client.py) + python3 sof_probe_client.py --host <dut> -p 9999 \ + --display ascii --out /tmp/probe_live.bin + High-throughput TCP stream over private lab LAN + + + + + + Integrated dut-monitor Dashboard + Live multi-pane telemetry console + • Auto-connects to port 9999 (Section 4 logs) + • Correlates FW logs with live power (port 8080) + + + + diff --git a/developer_guides/debugability/traces/index.rst b/developer_guides/debugability/traces/index.rst index 1df8bf6b..207b035f 100644 --- a/developer_guides/debugability/traces/index.rst +++ b/developer_guides/debugability/traces/index.rst @@ -1,48 +1,303 @@ .. _dbg-traces: -Traces -###### +DSP Telemetry, Logging & Traces +############################### -A FW developer may log important events by adding ``trace_event(...)`` entries -to the source code. The data is collected in the internal buffer and -transmitted periodically to the host through the DMA. +Sound Open Firmware (SOF) features a high-performance, asynchronous logging and telemetry infrastructure designed specifically for hard real-time embedded audio DSPs. Because audio signal processing operates on strict sub-millisecond scheduling deadlines (e.g. 1 ms or 200 µs periods), DSP firmware cannot block on slow UART serial writes or synchronous host communications. Instead, SOF combines compile-time string dictionary extraction (**smex**), hardware DMA circular buffers, native Zephyr RTOS structured logging, and network-accessible telemetry servers. -Building & Processing Traces +.. figure:: images/dsp_telemetry_architecture.svg + :alt: Sound Open Firmware DSP Telemetry, Logging and Trace Architecture + :align: center + :width: 100% + + Figure 329: Sound Open Firmware (SOF) DSP Telemetry, Logging & Trace Streaming Architecture + +--- + +Architecture Overview +********************* + +The SOF logging infrastructure is split into three decoupled operational stages: + +1. **Build-Time Dictionary Extraction**: C source strings and format specifications are stripped from the target executable and saved into an external Log Dictionary Catalog (``.ldc``), embedding only 32-bit metadata IDs into the firmware binary. +2. **Runtime Execution & Autonomous DMA**: The DSP core writes fixed-size binary trace packets into an internal SRAM circular ring buffer. A dedicated background hardware DMA channel transfers trace chunks to a shared host memory window without stalling audio pipeline processing loops. +3. **Host-Side Ingestion & Real-Time Decoding**: The host Linux kernel exposes binary trace buffers via ``debugfs``, and user-space utilities (``sof-logger``, ``sof_probe_server``) decode entry IDs against the ``.ldc`` dictionary in real time. + +--- + +Zephyr Structured Logging Integration +************************************* + +Modern SOF firmware natively integrates with the Zephyr RTOS logging subsystem (`zephyr/logging/log.h`). Each firmware module registers its logging domain and default verbosity level: + +.. code-block:: c + + #include + #include + + /* Register module with Kconfig-defined default log level */ + LOG_MODULE_REGISTER(eq_fir, CONFIG_SOF_LOG_LEVEL); + + int eq_fir_process(struct comp_dev *dev) + { + LOG_DBG("eq_fir_process: dev %p, frame count %u", dev, dev->frames); + + if (dev->state != COMP_STATE_ACTIVE) { + LOG_WRN("eq_fir: processing called while state=%u not active", dev->state); + return -EINVAL; + } + + /* Processing inner loop executes without logging overhead */ + return 0; + } + +Standard Logging Levels +======================= + +SOF utilizes four standard log levels mapped directly to Zephyr severity ratings: + +.. list-table:: SOF Logging Macro Severity & Guidelines + :widths: 15 15 70 + :header-rows: 1 + + * - Logging Macro + - Numeric Level + - Recommended Production & Debug Usage + * - ``LOG_ERR(...)`` + - Level 1 + - Critical runtime failures, unrecoverable hardware errors, invalid IPC state transitions, memory allocations faults. Always enabled in production. + * - ``LOG_WRN(...)`` + - Level 2 + - Recoverable boundary conditions, parameter sanitization clamps, non-fatal buffer underrun/overrun warnings. + * - ``LOG_INF(...)`` + - Level 3 + - Milestone events: component instantiation, pipeline binding, audio stream start/stop, clock frequency changes, power state transitions (D0 $\leftrightarrow$ D0ix). + * - ``LOG_DBG(...)`` + - Level 4 + - Verbose per-buffer execution traces, coefficient updates, DMA pointer offsets. Disabled in release builds to save CPU cycles and DMA bandwidth. + +--- + +Compile-Time String Extraction with smex +**************************************** + +To minimize the DSP memory footprint and avoid transmitting bulky ASCII text across memory buses, SOF employs the **smex** (String Metadata Extractor) build tool: + +1. **Linker Placement**: String literals, filenames, line numbers, and printf format arguments in ``LOG_*`` invocations are placed in a dedicated read-only section (``.static_log_entries``) of the ELF binary. +2. **Metadata Harvesting**: During the firmware build, ``smex`` parses the ``.static_log_entries`` section of ``zephyr.elf``: + + .. code-block:: bash + + smex -l build/sof-tgl.ldc -e build/zephyr/zephyr.elf + +3. **Dictionary Catalog (``.ldc``)**: ``smex`` extracts format strings and argument typing into the ``.ldc`` catalog file. In the binary firmware image (``.ri`` / ``.bin``), the compiler and linker emit compact 32-bit integer entry IDs. +4. **Footprint Reduction**: This reduces firmware binary size by 40–70% and reduces DSP trace logging execution to approximately 10–20 clock cycles per event. + +--- + +Runtime DSP Trace DMA Engine **************************** -During the compilation and linking, string literals and log entry metadata is -linked to the special *debug* sections that are extracted later to a special -*dictionary* file. This part is not loaded to the DSP and does not occupy the -DSP memory keeping both the memory footprint and the trace DMA payload small. +At runtime, logging operations must never interrupt audio pipelines executing on strict DMA-driven period boundaries: + +.. code-block:: text + + +-------------------------------------------------------------------------+ + | DSP Internal SRAM (trace_buf) | + | | + | [ Audio Thread ] ---> Lockless Atomic Write -> [ Packet 0 | Packet 1 ] | + +-------------------------------------------------------------------------+ + | + Autonomous Trace DMA Transfer + v + +-------------------------------------------------------------------------+ + | Host Shared Memory (SRAM Window 3) | + | | + | [ DMA Position IPC ] -> Host Driver Interrupt -> [ Linux debugfs trace]| + +-------------------------------------------------------------------------+ + +Trace Packet Structure +====================== + +Each binary trace entry emitted into the internal SRAM circular buffer contains a packed binary header: + +.. code-block:: c + + struct sof_log_entry { + uint64_t timestamp; /**< Hardware DSP timer tick count */ + uint32_t log_entry_id; /**< 32-bit metadata ID resolved via .ldc catalog */ + uint32_t params[4]; /**< Up to 4 runtime 32-bit format arguments */ + } __attribute__((packed)); + +Autonomous Background Transfer +============================== + +* **Lockless Ring Buffer**: The internal trace buffer (typically 8 KB or 16 KB) operates locklessly. Audio processing threads append trace packets using atomic pointer operations without acquiring mutexes or disabling interrupts. +* **Trace DMA Controller**: A background hardware DMA channel transfers accumulated trace chunks to host shared memory (SRAM Window 3 on Intel cAVS/ACE architectures). +* **Watermark Triggering**: When the buffer reaches its configured watermark threshold or a periodic timer fires, the DMA burst executes autonomously without DSP CPU polling. +* **Trace Position IPC**: The DSP notifies the host kernel of newly available trace data by posting an asynchronous ``SOF_IPC_TRACE_DMA_POSITION`` message containing the write pointer offset. + +--- + +Host-Side Ingestion & Decoding +****************************** + +Linux Kernel debugfs Trace Node +=============================== + +On Linux hosts with the mainline SOF driver loaded, the raw binary trace buffer is exposed via ``debugfs``: + +.. code-block:: text + + /sys/kernel/debug/sof/trace + +Reading this file yields the continuous binary stream emitted by the DSP Trace DMA engine. + +Early Boot Logging (snd-sof-probes) +=================================== + +To capture early firmware initialization messages prior to userspace audio server startup, the ``snd-sof-probes`` client driver provides the ``logging_boot_enable`` parameter: + +.. code-block:: bash + + # Reload probe driver with boot logging enabled + sudo rmmod snd_sof_probes 2>/dev/null + sudo modprobe snd_sof_probes logging_boot_enable=1 + + # Verify boot logging initialization in kernel dmesg + dmesg | grep "logging_boot" + +When enabled, the driver automatically allocates extraction DMA channels during probe registration and drains pre-buffered firmware initialization logs (up to 4 KB) before ALSA audio streams open. + +--- + +Using sof-logger +**************** + +The ``sof-logger`` host utility reads the binary trace stream, resolves metadata entry IDs using the ``.ldc`` catalog, and prints formatted messages with microsecond-accurate timestamps: + +Live Continuous Streaming +========================= + +.. code-block:: bash + + # Stream and decode live traces directly from debugfs + sof-logger -t -l /lib/firmware/intel/sof-ipc4/tgl/community/sof-tgl.ldc + +Offline Binary Trace Decoding +============================= + +If a binary trace dump was captured during an automated test run or hardware crash: + +.. code-block:: bash + + # Dump raw trace buffer to file + cat /sys/kernel/debug/sof/trace > /tmp/fw_trace.bin + + # Decode offline trace file + sof-logger -d /path/to/sof-tgl.ldc -i /tmp/fw_trace.bin -o /tmp/decoded_trace.txt + +Command-Line Options +==================== + +.. list-table:: sof-logger Common Flags + :widths: 20 80 + :header-rows: 1 + + * - Flag + - Description & Usage + * - ``-t`` + - Enable continuous real-time streaming mode (follows stream until interrupted). + * - ``-l `` + - Specify the Log Dictionary Catalog file matching the target firmware build. + * - ``-i `` + - Read binary trace data from a saved file instead of the default debugfs node. + * - ``-o `` + - Write human-readable decoded trace output to the specified file. + * - ``-p`` + - Strip ANSI color formatting codes for clean file logging. + * - ``--level <1..4>`` + - Filter messages below the specified severity level (1=ERR, 2=WRN, 3=INF, 4=DBG). + +--- + +Network Probe Server Streaming (Port 9999) +****************************************** + +On remote development and automated validation setups (DUTs), running ``sof-logger`` over SSH introduces significant network latency and terminal process overhead. SOF provides a high-throughput C streaming daemon—**``sof_probe_server``**—listening on TCP port **9999**: + +.. code-block:: text + + +--------------------------+ +--------------------------+ + | Target DUT | | Host Analysis Workstation| + | | | | + | [ DSP Trace DMA ] | | | + | | | | | + | v | | | + | [/dev/snd/comprC3D0] | | | + | | | | | + | v | | | + | [sof_probe_server :9999] | --- TCP/LAN --> | [sof_probe_client.py] | + | (1MB Ring Buffer) | (Port 9999) | or | + | | | [dut-monitor Dashboard] | + +--------------------------+ +--------------------------+ + +Running the Probe Server on Target DUT +====================================== + +.. code-block:: bash + + # Launch probe server on target board in background + timeout 15 ssh -o ConnectTimeout=5 root@ \ + 'nohup /usr/local/bin/sof_probe_server -c 3 -d 0 -p 9999 -v > /tmp/probe_server.log 2>&1 &' + +Streaming via Host Python Client +================================ + +On the host workstation, stream and preview logs over the network: + +.. code-block:: bash + + # Connect to DUT probe server and stream live ASCII log output + python3 tools/sof-probe-server/sof_probe_client.py \ + --host --port 9999 --display ascii --out /tmp/dut_trace.bin + +Integrated dut-monitor Dashboard +================================ + +The ``dut-monitor`` terminal dashboard automatically connects to ``sof_probe_server`` on TCP port 9999 and renders live decoded DSP logs in **Section 4**, alongside synchronized power telemetry (port 8080) and CPU metrics. + +--- -.. graphviz:: images/build-traces.dot - :caption: Traces - build process +Troubleshooting & Diagnostics +***************************** -Once the binary trace data is received by the host driver, it is accessible to -the trace decoder (logger) through the files located in the -_/sys/kernel/debug/sof/..._. The logger requires the *dictionary* file to -decode the trace data and "printf" them using format specified in the source -files. +Trace Buffer Wraparound & Missing Entries +========================================= -.. graphviz:: images/process-traces.dot - :caption: Traces - running & processing +* **Symptom**: ``sof-logger`` prints ``[DROPPED X ENTRIES]`` or non-sequential timestamps. +* **Root Cause**: Host reader cannot consume trace DMA packets quickly enough during bursts of ``LOG_DBG`` calls, overflowing the internal SRAM buffer. +* **Resolution**: + 1. Filter out high-frequency debug logs by raising ``CONFIG_SOF_LOG_LEVEL`` to ``CONFIG_LOG_DEFAULT_LEVEL=3`` (INFO). + 2. Increase internal trace buffer size in Kconfig: ``CONFIG_SOF_TRACE_BUF_SIZE=16384``. + 3. Stream via ``sof_probe_server`` using its 1 MB host-side circular queue rather than reading directly through debugfs over SSH. -Enabling Traces -*************** +Dictionary Mismatch (Unresolved IDs) +==================================== -When the traces are enabled by the driver, it stores the FW version information -received along with the *FW Ready* IPC message at the beginning of the local -trace files. It enables simple compatibility check between the trace data and -the *dictionary* file performed by the logger. +* **Symptom**: ``sof-logger`` outputs ````. +* **Root Cause**: The ``.ldc`` dictionary supplied to ``sof-logger`` does not match the exact binary running on the DSP. +* **Resolution**: Ensure the ``.ldc`` file corresponds to the identical Git commit and build configuration: -Note that the trace data may be collected on some machine and sent along with -the dictionary file to another person for investigation. It is important to be -able to verify the consistency of both by having the build version attached to -them. + .. code-block:: bash -.. uml:: images/trace-enable-flow.pu + sof-logger -l build-sof-staging/sof/sof-tgl.ldc -t -Adding Traces -************* +Zero Data from debugfs Node +=========================== -Refer to the *src/include/sof/trace.h*. +* **Symptom**: ``cat /sys/kernel/debug/sof/trace`` returns 0 bytes. +* **Root Cause**: Trace DMA is not enabled or the DSP core is suspended in D0ix sleep. +* **Resolution**: + 1. Start an audio playback stream to bring the DSP into active D0 state: ``aplay -D hw:0 -r 48000 -c 2 -f S16_LE /dev/zero &``. + 2. Verify kernel probe module: ``modprobe snd-sof-probes logging_boot_enable=1``. diff --git a/developer_guides/firmware/aria.rst b/developer_guides/firmware/aria.rst new file mode 100644 index 00000000..fe6f414a --- /dev/null +++ b/developer_guides/firmware/aria.rst @@ -0,0 +1,988 @@ +.. _aria: + +Aria (Automatic Regressive Input Amplifier) Architecture +======================================================== + +The **Aria** (**Automatic Regressive Input Amplifier**) subsystem in Sound Open Firmware (SOF) is a specialized, intelligent dynamic range pre-amplifier and lookahead peak limiter. Designed primarily for capture pipelines (such as microphone front-ends and far-field speech recognition) and sensitive playback chains, Aria applies a selectable target pre-amplification boost (:math:`0\text{ dB}`, :math:`+6\text{ dB}`, :math:`+12\text{ dB}`, or :math:`+18\text{ dB}`) to incoming audio signals. When high-amplitude signals or abrupt transient bursts enter the pipeline, Aria automatically and *regressively* ducks the gain below the target, ensuring that peak signal amplitudes never exceed :math:`0\text{ dBFS}` (:math:`A_{FS} = \text{0x007fffff}` in 24-bit container format) without introducing clipping or digital saturation. + +To perform artifact-free gain modulation, Aria integrates an internal circular delay buffer introducing exactly :math:`1\text{ ms}` of lookahead algorithmic latency. This lookahead window allows the gain calculation engine to inspect future audio peaks before they reach the output, computing an optimal attenuation curve that is applied via sample-by-sample linear interpolation, totally eliminating zipper noise and transient overshoot. + +.. contents:: Table of Contents + :local: + :depth: 3 + +------------------------------------------------------------------------------- + +Architectural Overview & Functional Role +---------------------------------------- + +In modern digital signal processing pipelines, capture front-ends must accommodate a wide dynamic range of acoustic inputs—from faint whispers in distant microphone arrays to loud shouts or unexpected acoustic shocks. Conventional static gain stages and traditional automatic gain controls present fundamental trade-offs: + +- **Static Linear Gain Stages**: + Applying a fixed pre-amplification gain (e.g. :math:`+12\text{ dB}`) boosts quiet signals into the optimal operating range of downstream automatic speech recognition (ASR) engines, but inevitably causes harsh digital clipping whenever loud acoustic transients enter the analog-to-digital converter (ADC). +- **Dynamic Range Compressors (DRC)**: + Standard wideband or multiband compressors can manage high amplitudes, but rely on complex envelope followers (attack/release filters) and non-linear logarithmic curve mappings. When an unexpected transient occurs, feedback compressors cannot react instantaneously without significant lookahead buffers, leading to either initial transient clipping or prolonged gain pumping. +- **Automatic Gain Control (AGC)**: + AGC systems operate on long time horizons (typically 100 to 500 ms). While effective for slow vocal level drift, they are too sluggish to protect against sudden peak clipping. + +The Aria component resolves this challenge by operating as an **Automatic Regressive Input Amplifier**: + +1. **Target Linear Pre-amplification**: + Under nominal conditions where the signal resides safely within available headroom, Aria acts as a fixed linear pre-amplifier, applying the configured target gain of :math:`0\text{ dB}`, :math:`+6\text{ dB}`, :math:`+12\text{ dB}`, or :math:`+18\text{ dB}`. +2. **Instantaneous Regressive Back-off**: + When the peak amplitude of an incoming block exceeds the headroom threshold, the amplification factor automatically regresses (attenuates) in exact proportion to the peak overshoot: + + .. math:: + + G_{regressive} = \frac{A_{FS}}{\text{Peak Amplitude}} + + This guarantees that the peak output amplitude is locked at :math:`A_{FS}`, completely preventing digital overflow. +3. **Deterministic 1 ms Lookahead Latency**: + By buffering :math:`1\text{ ms}` of audio in an internal circular buffer, the peak detection engine evaluates incoming frames in advance. Gain transitions are smoothly interpolated across the entire frame window, eliminating step discontinuities. + +.. _figure_216: + +.. graphviz:: + :align: center + :caption: SOF Aria Subsystem Architecture: Lookahead Buffer, Dynamic Regressive Amplifier & Linear Ramp Engine + + digraph aria_architecture { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_input { + label = "Egress Audio Stream"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + source [label="Audio Source Stream\n(SOF_IPC_FRAME_S24_4LE)\nFrames at t + 1ms", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + subgraph cluster_aria { + label = "Aria Processing Module (UUID: 6d:16:f7:99...)"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + peak_detect [label="Peak Amplitude Detector\n(aria_algo_calc_gain)\nDetect max_data in chunk", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + gain_calc [label="Regressive Gain Evaluator\nIf max > Thresh: g = A_FS / max\nElse: g = 2^att", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + + state_tab [label="10-State Gain History\n(sof_aria_index_tab)\nMinimum Envelope Filter", fillcolor="#1e293b", fontcolor="#94a3b8", color="#475569"]; + + circ_buf [label="1 ms Lookahead Circular Buffer\n(cd->data_addr)\nBuffered Audio at t", fillcolor="#334155", fontcolor="#f8fafc", color="#64748b"]; + + ramp_engine [label="Linear Interpolation Ramp\nstep = (gain_end - gain_begin) / N\nPer-sample gain += step", fillcolor="#0d9488", fontcolor="#ffffff", color="#2dd4bf"]; + mult_sat [label="Multiply & Scale Unit\n(q_multsr_sat_32x32_24)\nout = (in * g) >> (31 - att)", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + subgraph cluster_output { + label = "Ingress Audio Stream"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + sink [label="Protected Sink Stream\n(SOF_IPC_FRAME_S24_4LE)\nPeak Clamped <= 0 dBFS", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + source -> peak_detect [label="Future audio\n(t + 1ms)"]; + source -> circ_buf [label="Write to ring\n(1ms delay)"]; + + peak_detect -> gain_calc [label="max_data"]; + gain_calc -> state_tab [label="Record state\n(gains[gain_idx])"]; + + state_tab -> ramp_engine [label="gain_begin\ngain_end"]; + circ_buf -> mult_sat [label="Delayed audio\n(t)"]; + ramp_engine -> mult_sat [label="Interpolated\ngain[n]"]; + mult_sat -> sink [label="Output frames"]; + } + +Comparison with Other SOF Modules +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To clarify when Aria should be instantiated in an audio graph rather than alternative processing blocks, the following table summarizes functional boundaries across related SOF components: + +.. list-table:: Architectural Comparison: Aria vs Volume vs DRC vs Smart Amp + :widths: 20 25 25 30 + :header-rows: 1 + + * - Subsystem + - Primary Operating Mode + - Dynamic Reaction Speed + - Typical Deployment Target + * - **Aria** + - Target gain (:math:`0/6/12/18\text{ dB}`) with instant regressive back-off + - Instantaneous lookahead (:math:`1\text{ ms}` pre-transient interpolation) + - Microphone capture front-ends and sensitive playback endpoints + * - **Volume** + - User-controlled linear/logarithmic gain slider (:math:`-\infty` to :math:`0\text{ dB}`) + - User-paced smooth ramp (typically :math:`16\text{ ms}` to :math:`500\text{ ms}`) + - Main and per-stream loudness controls + * - **DRC** + - Multi-segment compression knee with ratio, threshold, and makeup gain + - Envelope-follower driven attack (:math:`1\text{ ms}` to :math:`20\text{ ms}`) + and release (:math:`50\text{ ms}` to :math:`1000\text{ ms}`) + - Speaker overload protection and studio post-processing compression + * - **Smart Amp** + - Physical electro-mechanical-thermal speaker excursion modeling + - Fast non-linear displacement tracking with slow thermal decay + - Micro-speaker protection in mobile and thin laptops + +------------------------------------------------------------------------------- + +Mathematical Foundations & Regressive Dynamic Headroom +------------------------------------------------------ + +Aria operates strictly on 24-bit audio packaged inside 32-bit containers (:c:macro:`SOF_IPC_FRAME_S24_4LE`). In this encoding, sample values occupy the 24 least significant bits, sign-extended to 32 bits: + +.. math:: + + -8,388,608 \le x[n] \le +8,388,607 \quad (-2^{23} \le x[n] \le 2^{23} - 1) + +The positive full-scale maximum amplitude is denoted as: + +.. math:: + + A_{FS} = 2^{23} - 1 = \text{0x007FFFFF} = 8,388,607 + +Target Gain Parameterization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The target pre-amplification boost is configured via the unsigned integer parameter :math:`\text{att} \in \{0, 1, 2, 3\}`: + +.. list-table:: Aria Attenuation Parameter to Target Boost Mapping + :widths: 15 20 25 40 + :header-rows: 1 + + * - Parameter :math:`\text{att}` + - Linear Multiplier :math:`2^{\text{att}}` + - Decibel Boost :math:`G_{target}` + - Permissible Input Headroom :math:`A_{thresh}` + * - **0** + - :math:`1.0\times` (:math:`2^0`) + - :math:`0.00\text{ dB}` (Bypass) + - :math:`A_{FS} = \text{0x007FFFFF} = 8,388,607` (:math:`0.00\text{ dBFS}`) + * - **1** + - :math:`2.0\times` (:math:`2^1`) + - :math:`+6.02\text{ dB}` + - :math:`A_{FS} / 2 = \text{0x003FFFFF} = 4,194,303` (:math:`-6.02\text{ dBFS}`) + * - **2** + - :math:`4.0\times` (:math:`2^2`) + - :math:`+12.04\text{ dB}` + - :math:`A_{FS} / 4 = \text{0x001FFFFF} = 2,097,151` (:math:`-12.04\text{ dBFS}`) + * - **3** + - :math:`8.0\times` (:math:`2^3`) + - :math:`+18.06\text{ dB}` + - :math:`A_{FS} / 8 = \text{0x000FFFFF} = 1,048,575` (:math:`-18.06\text{ dBFS}`) + +Headroom Threshold & Regressive Gain Derivation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To prevent any sample from exceeding :math:`A_{FS}` when amplified by :math:`2^{\text{att}}`, the linear input threshold is: + +.. math:: + + A_{thresh} = \frac{A_{FS}}{2^{\text{att}}} = \text{0x007FFFFF} \gg \text{att} + +For every processing chunk (e.g. 48 frames at 48 kHz, spanning :math:`1\text{ ms}`), the algorithm detects the peak absolute amplitude across all channels: + +.. math:: + + \text{max\_data} = \max_{k \in \text{chunk}, ch} |x[k, ch]| + +The mathematical gain computation distinguishes between two regimes: + +1. **Unclipped Linear Regime** (:math:`\text{max\_data} \le A_{thresh}`): + The signal fits completely within available headroom. The raw 64-bit gain word is set to: + + .. math:: + + \text{gain} = 2^{\text{att} + 32} - 1 + + When normalized into a 32-bit state variable, it yields full fractional scale: + + .. math:: + + g = \text{gain} \gg (\text{att} + 1) = 2^{31} - 1 = \text{0x7FFFFFFF} + +2. **Regressive Compression Regime** (:math:`\text{max\_data} > A_{thresh}`): + Applying the target boost would push the output past :math:`A_{FS}`. The raw gain word is dynamically calculated via 64-bit integer division: + + .. math:: + + \text{gain} = \left\lfloor \frac{A_{FS} \cdot 2^{32}}{\text{max\_data}} \right\rfloor = \left\lfloor \frac{\text{0x007FFFFF} \cdot 2^{32}}{\text{max\_data}} \right\rfloor + + The normalized gain state is then scaled: + + .. math:: + + g = \text{gain} \gg (\text{att} + 1) = \left\lfloor \frac{\text{0x007FFFFF} \cdot 2^{31}}{\text{max\_data} \cdot 2^{\text{att}}} \right\rfloor + +Dynamic Shift Output Scaling +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +During output sample synthesis, the sample multiplication applies a dynamic right-shift determined by: + +.. math:: + + \text{shift} = 31 - \text{att} + +The output sample :math:`y[n, ch]` is generated by multiplying the input sample by the normalized gain and right-shifting: + +.. math:: + + y[n, ch] = \frac{x[n, ch] \cdot g}{2^{\text{shift}}} = \frac{x[n, ch] \cdot g}{2^{31 - \text{att}}} + +Mathematical Proof of Anti-Clipping Clamping +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Evaluating this equation in the regressive regime where :math:`\text{max\_data} > A_{thresh}`: + +.. math:: + + y[n, ch] = \frac{x[n, ch] \cdot \left(\frac{A_{FS} \cdot 2^{31}}{\text{max\_data} \cdot 2^{\text{att}}}\right)}{2^{31 - \text{att}}} + = \frac{x[n, ch] \cdot A_{FS} \cdot 2^{31}}{\text{max\_data} \cdot 2^{\text{att}} \cdot 2^{31 - \text{att}}} + = x[n, ch] \cdot \frac{A_{FS}}{\text{max\_data}} + +For the peak sample in the chunk (:math:`|x[n, ch]| = \text{max\_data}`): + +.. math:: + + |y_{peak}| = \text{max\_data} \cdot \frac{A_{FS}}{\text{max\_data}} = A_{FS} = \text{0x007FFFFF} + +The peak output is clamped exactly to :math:`0\text{ dBFS}`, guaranteeing that no digital overflow occurs regardless of the input burst magnitude. + +.. _figure_217: + +.. graphviz:: + :align: center + :caption: Mathematical Dynamics: Target Gain Boost (0/6/12/18 dB), Headroom Thresholds & Regressive Ducking Curve + + digraph aria_math_curves { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_regimes { + label = "Aria Input Dynamic Regimes & Transfer Function"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + node_low [label="Low-Level Signal Regime\n(x <= A_thresh)\nGain = 2^att (Target Boost)\nOutput = x * 2^att", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + node_thresh [label="Headroom Threshold Point\nx = A_FS >> att\n(Output reaches exactly A_FS)", fillcolor="#d97706", fontcolor="#ffffff", color="#fbbf24", shape="diamond"]; + node_high [label="High-Level Transient Regime\n(x > A_thresh)\nGain = A_FS / x (Regressive Ducking)\nPeak Output Clamped to A_FS (0 dBFS)", fillcolor="#dc2626", fontcolor="#ffffff", color="#f87171"]; + } + + subgraph cluster_thresholds { + label = "Headroom Thresholds Across Attenuation Modes"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + t0 [label="att = 0 (0 dB Boost)\nA_thresh = 0x007FFFFF\nFull Scale Headroom", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + t1 [label="att = 1 (+6 dB Boost)\nA_thresh = 0x003FFFFF\n-6.02 dBFS Headroom", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + t2 [label="att = 2 (+12 dB Boost)\nA_thresh = 0x001FFFFF\n-12.04 dBFS Headroom", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + t3 [label="att = 3 (+18 dB Boost)\nA_thresh = 0x000FFFFF\n-18.06 dBFS Headroom", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + node_low -> node_thresh [label="Signal rises"]; + node_thresh -> node_high [label="Exceeds headroom"]; + + node_thresh -> t0 [style="dotted", label="Mode 0"]; + node_thresh -> t1 [style="dotted", label="Mode 1"]; + node_thresh -> t2 [style="dotted", label="Mode 2"]; + node_thresh -> t3 [style="dotted", label="Mode 3"]; + } + +------------------------------------------------------------------------------- + +1 ms Lookahead Circular Buffer & Latency Phasing +------------------------------------------------ + +A fundamental problem in conventional peak limiters is that gain reduction is triggered *after* or *at* the arrival of a peak, causing either initial overshoot clipping or unnatural transient distortion. Aria completely eliminates this issue by introducing a **1 ms lookahead window** realized through an internal circular delay buffer. + +Buffer Sizing & Memory Layout +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The circular buffer is allocated during component initialization (:c:func:`aria_init`) to hold exactly :math:`1\text{ ms}` of audio across all channels: + +.. math:: + + \text{buff\_size} = \text{ALIGN\_UP}(\text{chan\_cnt} \cdot \text{smpl\_group\_cnt}, 2) + +where: + +- :math:`\text{chan\_cnt}` is the number of audio channels (e.g. 2 for stereo, 4 for quad mic array). +- :math:`\text{smpl\_group\_cnt}` is the number of samples per channel in :math:`1\text{ ms}` (e.g. 48 samples at 48 kHz). +- The buffer is aligned to 8-byte boundaries (2 samples of 32-bit audio) to satisfy SIMD vector memory alignment requirements. + +An offset variable is tracked: + +.. math:: + + \text{offset} = (\text{chan\_cnt} \cdot \text{smpl\_group\_cnt}) \& 1 + +ensuring that the circular buffer read and write pointers maintain invariant alignment throughout runtime execution. + +Phased Execution Cycle (The 4-Step Pipeline) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In every processing tick of :c:func:`aria_process_data`, Aria executes four consecutive operations: + +1. **Step 1: Lookahead Peak Inspection** (:math:`t + 1\text{ ms}`): + The function :c:func:`aria_algo_calc_gain` inspects the future incoming frames in ``source``. It scans all channels, calculates the peak absolute value :math:`\text{max\_data}`, evaluates whether regressive compression is required, and stores the computed gain into the gain history table at: + + .. math:: + + \text{gain\_idx} = \text{sof\_aria\_index\_tab}[\text{cd->gain\_state} + 1] + +2. **Step 2: Delayed Audio Retrieval & Gain Application** (:math:`t`): + The function ``cd->aria_get_data`` reads the *past* audio stored in the circular buffer at ``cd->data_ptr`` (which entered the buffer :math:`1\text{ ms}` prior). It linearly interpolates the gain across the block and writes the protected, amplified audio to ``sink``. +3. **Step 3: History Buffer Ingestion**: + The function :c:func:`cir_buf_copy` transfers the future incoming audio from ``source`` into the circular buffer at ``cd->data_ptr``, storing it as history for processing in the subsequent millisecond. +4. **Step 4: Circular Pointer Wrap**: + The circular pointer is advanced by the chunk sample size and wrapped using :c:func:`cir_buf_wrap`: + + .. math:: + + \text{cd->data\_ptr} = \text{cir\_buf\_wrap}(\text{cd->data\_ptr} + \text{sample\_size}, \text{cd->data\_addr}, \text{cd->data\_end}) + +Bypass Invariance +~~~~~~~~~~~~~~~~~ + +When :math:`\text{att} == 0`, Aria operates in bypass mode. Rather than short-circuiting the buffer, :c:func:`aria_process_data` routes audio through the circular delay buffer without applying gain multipliers: + +.. code-block:: c + + if (cd->att) { + aria_algo_calc_gain(cd, sof_aria_index_tab[cd->gain_state + 1], source, frames); + cd->aria_get_data(mod, sink, frames); + } else { + cir_buf_copy(cd->data_ptr, cd->data_addr, cd->data_end, + sink->w_ptr, sink->addr, sink->end_addr, + data_size); + } + +This design ensures that the pipeline latency is **strictly invariant at 1 ms**, preventing downstream phase misalignments or timestamp discontinuities when switching attenuation modes on the fly. + +.. _figure_218: + +.. graphviz:: + :align: center + :caption: Lookahead Buffer Timing & 1 ms Lookahead Latency Phasing (Future Peak Detection vs Delayed Stream Application) + + digraph aria_timing_phasing { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_timeline { + label = "Timeline Phasing Across 1 ms Execution Window"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + t_future [label="Time t + 1 ms (Future Input)\nIncoming stream in source DMA ring\nEvaluated by aria_algo_calc_gain()", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + t_present [label="Circular Delay Ring Buffer\nStores 1 ms history in cd->data_addr\nDecouples peak detection from scaling", fillcolor="#334155", fontcolor="#f8fafc", color="#64748b"]; + t_past [label="Time t (Delayed Audio Output)\nRead from cd->data_ptr into sink\nScaled by interpolated gain[n]", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + subgraph cluster_steps { + label = "Phased Execution Sequence in aria_process_data()"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + s1 [label="1. Peak Detection: Calculate required gain for future frame (t+1ms)", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + s2 [label="2. Scaling & Egress: Multiply delayed audio (t) by ramped gain -> sink", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + s3 [label="3. Ring Update: Copy future audio (t+1ms) from source -> circular buffer", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + s4 [label="4. Ring Wrap: Advance cd->data_ptr with cir_buf_wrap()", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + t_future -> s1 [label="Inspects"]; + s1 -> s2 [label="Advances state"]; + t_present -> s2 [label="Reads delayed audio"]; + s2 -> t_past [label="Writes to sink"]; + t_future -> s3 [label="Transfers"]; + s3 -> t_present [label="Populates ring"]; + s3 -> s4 [label="Completes copy"]; + } + +------------------------------------------------------------------------------- + +Multi-State Gain Follower & Per-Sample Linear Interpolation +----------------------------------------------------------- + +Abrupt gain changes between consecutive processing chunks produce audible discontinuities known as *zipper noise* and generate high-frequency distortion harmonics. To ensure acoustic transparency, Aria utilizes a **10-state sliding gain tracking table** and continuous **per-sample linear interpolation**. + +Sliding Gain History Table +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Aria maintains 10 historical gain values in the array: + +.. code-block:: c + + int32_t gains[ARIA_MAX_GAIN_STATES]; // ARIA_MAX_GAIN_STATES = 10 + +To eliminate expensive runtime modulo arithmetic (:math:`\% 10`), indexing is performed via a pre-computed lookup table: + +.. code-block:: c + + const int32_t sof_aria_index_tab[] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3 + }; + +Lookahead Minimum-Envelope Search +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When determining the starting gain (:math:`\text{gain\_begin}`) and ending gain (:math:`\text{gain\_end}`) for the current :math:`1\text{ ms}` chunk, Aria searches across a multi-state window for the *minimum* gain value: + +.. code-block:: c + + int32_t gain_state_add_2 = cd->gain_state + 2; + int32_t gain_state_add_3 = cd->gain_state + 3; + int32_t gain_begin = cd->gains[sof_aria_index_tab[gain_state_add_2]]; + int32_t gain_end = cd->gains[sof_aria_index_tab[gain_state_add_3]]; + + for (i = 1; i < ARIA_MAX_GAIN_STATES - 1; i++) { + if (cd->gains[sof_aria_index_tab[gain_state_add_2 + i]] < gain_begin) + gain_begin = cd->gains[sof_aria_index_tab[gain_state_add_2 + i]]; + if (cd->gains[sof_aria_index_tab[gain_state_add_3 + i]] < gain_end) + gain_end = cd->gains[sof_aria_index_tab[gain_state_add_3 + i]]; + } + +By tracking the minimum gain across states, Aria establishes a **lookahead attack envelope**: + +- If an impending peak requires severe gain reduction, :math:`\text{gain\_begin}` and :math:`\text{gain\_end}` are pulled downward *before* the peak reaches the output. +- The gain ramps down smoothly toward the required attenuation, so that the signal is already safely compressed when the peak transient hits the output multiplier. +- Conversely, when transitioning out of a transient into quiet audio, the gain recovers smoothly across subsequent blocks without abrupt pumping. + +Continuous Per-Sample Linear Interpolation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Once :math:`\text{gain\_begin}` and :math:`\text{gain\_end}` are determined, Aria computes the per-sample ramp increment: + +.. math:: + + \text{step} = \frac{\text{gain\_end} - \text{gain\_begin}}{\text{frames}} + +The gain accumulator starts at :math:`\text{gain} = \text{gain\_begin}`. For every sample group, the current gain is applied and then updated: + +.. math:: + + \text{gain}_{n+1} = \text{gain}_n + \text{step} + +This ensures :math:`C^0` continuity across the entire audio stream, completely eliminating zipper noise. + +.. _figure_219: + +.. graphviz:: + :align: center + :caption: Multi-State Gain Follower & Linear Interpolation Ramp (Minimum-Envelope Search & Per-Sample Stepping) + + digraph aria_gain_smoothing { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_states { + label = "10-State Circular Gain Table (cd->gains[0..9])"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + s_hist [label="Historical Gain States\ngains[0] .. gains[7]\nPast chunk gains", fillcolor="#1e293b", fontcolor="#94a3b8", color="#475569"]; + s_curr [label="Current Active Gain\ngains[gain_state]\nActive frame chunk", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + s_next [label="Future Lookahead Gain\ngains[gain_state + 1]\nComputed for t + 1ms", fillcolor="#d97706", fontcolor="#ffffff", color="#fbbf24"]; + } + + subgraph cluster_min_search { + label = "Minimum Envelope Search"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + min_eval [label="Envelope Minimum Evaluation\nSearch across ARIA_MAX_GAIN_STATES - 1\ngain_begin = min(gains[...])\ngain_end = min(gains[...])", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + } + + subgraph cluster_ramp { + label = "Per-Sample Linear Stepping"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b22"; + + calc_step [label="Slope Calculation\nstep = (gain_end - gain_begin) / frames", fillcolor="#0d9488", fontcolor="#ffffff", color="#2dd4bf"]; + sample_loop [label="Per-Sample Execution Loop\ny[n] = (x[n] * gain) >> (31 - att)\ngain += step", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + s_hist -> min_eval; + s_curr -> min_eval; + s_next -> min_eval; + + min_eval -> calc_step [label="gain_begin\ngain_end"]; + calc_step -> sample_loop [label="step"]; + } + +------------------------------------------------------------------------------- + +Tensilica HiFi SIMD Acceleration & Hardware Circular Buffers +------------------------------------------------------------ + +The computational throughput of the Aria component is heavily optimized using Cadence Tensilica HiFi SIMD instruction sets, delivering distinct implementations across **Generic Scalar C**, **HiFi3 / HiFi4**, and **HiFi5**. + +Generic Scalar Implementation (:file:`aria_generic.c`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The scalar C fallback performs signed 24-bit sign-extension and fixed-point fractional multiplication: + +.. code-block:: c + + in_sample = sign_extend_s24(*in++); + out[ch] = q_multsr_sat_32x32_24(in_sample, gain, shift); + +While fully functional and portable across any processor architecture (including RISC-V and ARM), the scalar loops require branching for circular wrapping and sample-by-sample clamping. + +Tensilica HiFi3 / HiFi4 Acceleration (:file:`aria_hifi3.c`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The HiFi3/4 kernel introduces vectorized peak detection, odd/even channel specialization, and symmetric rounding: + +1. **Vector Absolute Maximum in a Single Instruction**: + In :c:func:`aria_algo_calc_gain`, the future sample stream is scanned using 64-bit vector alignment loads (:c:macro:`AE_LA64_PP`) and the :c:macro:`AE_MAXABS32S` instruction, which simultaneously computes the absolute value and compares it against the running maximum across dual 32-bit SIMD lanes in a single cycle: + + .. code-block:: c + + AE_LA32X2_IP(in_sample, inu, in); + max_data = AE_MAXABS32S(max_data, AE_SLAI32(in_sample, 8)); + +2. **Channel Specialization (Odd vs Even Channels)**: + To maximize vector register utilization, :c:func:`aria_algo_get_data_func` dynamically binds either :c:func:`aria_algo_get_data_odd_channel` or :c:func:`aria_algo_get_data_even_channel`: + - **Even Channels (Stereo, Quad, 8ch)**: Samples are processed in pairs (:math:`\text{ch} += 2`). Dual 32-bit vector registers :c:macro:`AE_LA32X2_IP` feed high and low 32x32 multipliers: + + .. code-block:: c + + out1 = AE_MUL32_HH(in_sample, gain); + out1 = AE_SRAA64(out1, shift_bits); + out2 = AE_MUL32_LL(in_sample, gain); + out2 = AE_SRAA64(out2, shift_bits); + + - **Odd Channels (Mono, 3ch, 5ch)**: Samples are processed individually with single-lane instructions (:c:macro:`AE_L32_XP` and :c:macro:`AE_S32_L_XP`). +3. **Symmetric Rounding and Saturation**: + Intermediate products are rounded from 48-bit fixed-point back to 24-bit signed representation using :c:macro:`AE_ROUND24X2F48SSYM`, guaranteeing bit-exact symmetry and preventing negative DC bias accumulation. + +Tensilica HiFi5 Hardware Circular Addressing (:file:`aria_hifi5.c`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On Intel platforms equipped with Tensilica HiFi5 cores (such as Panther Lake and Lunar Lake), Aria achieves maximal memory throughput by leveraging dedicated **Hardware Circular Addressing Registers**: + +1. **Hardware Circular Buffer Setup**: + HiFi5 features dedicated circular addressing pointer registers :c:macro:`AE_SETCBEGIN0`, :c:macro:`AE_SETCEND0` for the input delay buffer, and :c:macro:`AE_SETCBEGIN1`, :c:macro:`AE_SETCEND1` for the sink buffer: + + .. code-block:: c + + set_circular_buf0(cd->data_addr, cd->data_end); + set_circular_buf1(audio_stream_get_addr(sink), audio_stream_get_end_addr(sink)); + +2. **Zero-Overhead Automatic Address Wrapping**: + When loading and storing samples, the specialized circular instructions :c:macro:`AE_L32X2_XC` and :c:macro:`AE_S32X2_XC1` automatically wrap the memory pointer back to the buffer start address when the end boundary is reached: + + .. code-block:: c + + AE_L32X2_XC(in_sample, in, inc); + ... + AE_S32X2_XC1(out_sample, out, inc); + + This completely eliminates runtime boundary checking, pointer masking, and branch instructions inside the inner DSP audio loop. +3. **128-Bit SIMD Vector Pipelines**: + In the peak detection stage, HiFi5 utilizes 128-bit vector loads (:c:macro:`AE_LA128_PP` and :c:macro:`AE_LA32X2X2_IP`), processing 4 32-bit audio samples simultaneously per instruction cycle. + +.. _figure_220: + +.. graphviz:: + :align: center + :caption: Tensilica HiFi3/HiFi4 vs HiFi5 SIMD Acceleration (Dual-Channel Multipliers vs Hardware Circular Buffering AE_SETCBEGIN) + + digraph aria_simd_comparison { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_hifi3 { + label = "Tensilica HiFi3 / HiFi4 SIMD Pipeline"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + h3_load [label="Dual Load: AE_LA32X2_IP\nLoads 2 x 32-bit samples (64-bit)", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + h3_mult [label="Dual Multiply: AE_MUL32_HH / LL\nMultiplies high and low lanes by gain", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + h3_round [label="Symmetric Round: AE_ROUND24X2F48SSYM\nRounds 48-bit product to 24-bit", fillcolor="#0d9488", fontcolor="#ffffff", color="#2dd4bf"]; + h3_wrap [label="Software Buffer Wrap: cir_buf_wrap()\nConditional pointer evaluation", fillcolor="#334155", fontcolor="#94a3b8", color="#475569"]; + } + + subgraph cluster_hifi5 { + label = "Tensilica HiFi5 Advanced Hardware Pipeline"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b22"; + + h5_setup [label="Hardware Ring Setup:\nAE_SETCBEGIN0/1 & AE_SETCEND0/1\nConfigures DSP hardware registers", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + h5_load [label="128-Bit Load: AE_LA128_PP / AE_LA32X2X2_IP\nLoads 4 x 32-bit samples per cycle", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + h5_auto [label="Hardware Auto-Wrap Load/Store:\nAE_L32X2_XC & AE_S32X2_XC1\nZero-cycle hardware address wrap", fillcolor="#10b981", fontcolor="#ffffff", color="#6ee7b7"]; + } + + h3_load -> h3_mult -> h3_round -> h3_wrap; + h5_setup -> h5_load -> h5_auto; + } + +------------------------------------------------------------------------------- + +IPC4 Modular Interface, LLEXT Packaging & Topology 2 Graph +---------------------------------------------------------- + +Aria is fully compliant with the Intel IPC4 firmware architecture and supports both static compilation into the core firmware binary and modular dynamic loading via **Zephyr Loadable Linkable Extensions (LLEXT)**. + +IPC4 Configuration Structures +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The module configuration structure is defined in :file:`aria.h`: + +.. code-block:: c + + struct ipc4_aria_module_cfg { + struct ipc4_base_module_cfg base_cfg; + uint32_t attenuation; + } __packed __aligned(8); + +- ``base_cfg``: Standard IPC4 base module configuration specifying input/output buffer sizes, audio stream format (:math:`\text{depth} = 32`, :math:`\text{valid\_depth} = 24`), and channel count. +- ``attenuation``: The target attenuation/boost mode (:math:`\text{att} \in \{0, 1, 2, 3\}`). If the host provides a value greater than :c:macro:`ARIA_MAX_ATT` (3), the firmware clamps it to 3 and emits a warning trace. + +Runtime Control Parameter +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Aria supports dynamic runtime adjustment of target attenuation without tearing down the audio pipeline via IPC4 large configuration messages: + +- **Parameter ID**: :c:macro:`ARIA_SET_ATTENUATION` (1). +- **Payload**: 32-bit unsigned integer representing the new attenuation setting (``cd->att``). +- When received in :c:func:`aria_set_config`, the firmware immediately updates ``cd->att`` and recomputes the baseline gain states via :c:func:`aria_set_gains`. + +Modular LLEXT Packaging +~~~~~~~~~~~~~~~~~~~~~~~ + +When built as a loadable module (``CONFIG_COMP_ARIA = "m"``), Aria is compiled into an independent ELF shared object (:file:`aria.llext`) and exported with a signed module manifest: + +.. code-block:: c + + static const struct sof_man_module_manifest mod_manifest __section(".module") __used = + SOF_LLEXT_MODULE_MANIFEST("ARIA", &aria_interface, 1, SOF_REG_UUID(aria), 8); + +- **Module Name**: ``"ARIA"`` +- **Interface Structure**: ``aria_interface`` +- **Module Version**: ``1`` +- **Component UUID**: ``6d:16:f7:99:2c:37:ef:43:81:f6:22:00:7a:a1:5f:03`` +- **Stack Size**: 8 KB + +Platform Performance Profiles (:file:`aria.toml`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The configuration file :file:`src/audio/aria/aria.toml` specifies processing constraints and Cycles Per Chunk (CPC) metrics across various operational frame sizes: + +.. list-table:: Aria Performance & Resource Allocation across Chunk Sizes + :widths: 20 20 25 35 + :header-rows: 1 + + * - Chunk Frames + - Cycles Per Chunk (CPC) + - Input Buffer Size (IBS) + - Output Buffer Size (OBS) + * - **16 frames** + - 1,063,000 CPS + - 16 samples + - 21 samples + * - **32 frames** + - 2,680,000 CPS + - 32 samples + - 42 samples + * - **64 frames** + - 3,591,000 CPS + - 64 samples + - 85 samples + * - **96 frames** + - 4,477,000 CPS + - 96 samples + - 128 samples + * - **192 frames** + - 7,195,000 CPS + - 192 samples + - 192 samples + +ALSA Topology 2 Integration +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In ALSA Topology 2, Aria is declared as an audio effect widget in :file:`tools/topology/topology2/include/components/aria.conf`: + +.. code-block:: text + + Class.Widget."aria" { + DefineAttribute."index" {} + + + DefineAttribute."cpc" { + token_ref "comp.word" + } + DefineAttribute."is_pages" { + token_ref "comp.word" + } + + Object.Control.bytes."1" { + !access [ tlv_read tlv_callback ] + Object.Base.extops.1 { + name "extctl" + get 258 + put 0 + } + max 4096 + } + + uuid "6d:16:f7:99:2c:37:ef:43:81:f6:22:00:7a:a1:5f:03" + type "effect" + no_pm "true" + cpc 5000 + is_pages 1 + num_input_pins 1 + num_output_pins 1 + } + +Aria is integrated into audio playback and capture pipelines, such as :file:`topology2/include/pipelines/cavs/mixout-aria-gain-mixin-playback.conf`: + +.. code-block:: text + + Object.Base { + route.1 { + source mixout.$index.1 + sink aria.$index.1 + } + route.2 { + source aria.$index.1 + sink gain.$index.1 + } + route.3 { + source gain.$index.1 + sink mixin.$index.1 + } + } + +Tuning Blobs via Octave/MATLAB (:file:`sof_aria_blobs.m`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To generate pre-compiled binary configuration blobs for ALSA topology generation, SOF provides the Octave script :file:`src/audio/aria/tune/sof_aria_blobs.m`. It constructs ABI-compliant configuration containers for: + +- :file:`passthrough.conf`: Sets :math:`\text{att} = 0` (Bypass, 0 dB). +- :file:`param_1.conf`: Sets :math:`\text{att} = 1` (+6 dB). +- :file:`param_2.conf`: Sets :math:`\text{att} = 2` (+12 dB). +- :file:`param_3.conf`: Sets :math:`\text{att} = 3` (+18 dB). + +.. _figure_221: + +.. graphviz:: + :align: center + :caption: IPC4 Configuration Architecture & LLEXT Modular Packaging + + digraph aria_ipc4_llext { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_host { + label = "Host Driver / User-Space ALSA Plane"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + blob_script [label="MATLAB / Octave Generator\n(sof_aria_blobs.m)\nExports param_1.conf..param_3.conf", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + alsa_tplg [label="ALSA Topology 2 Compiler\n(alsatplg)\nCompiles aria.conf widget & routes", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + host_ctl [label="Runtime Mixer Control\n(amixer / ctl)\nSends ARIA_SET_ATTENUATION", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + } + + subgraph cluster_dsp { + label = "SOF Audio DSP Firmware Engine"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + ipc4_handler [label="IPC4 Message Dispatcher\nParses Large Config / Init Data", fillcolor="#0d9488", fontcolor="#ffffff", color="#2dd4bf"]; + llext_loader [label="Zephyr LLEXT Dynamic Linker\nLoads aria.llext via ELF manifest", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + aria_core [label="Aria Processing Core\nUpdates cd->att & recomputes cd->gains[]", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + blob_script -> alsa_tplg [label="Tuning Blobs"]; + alsa_tplg -> ipc4_handler [label="Pipeline Creation"]; + host_ctl -> ipc4_handler [label="Runtime Attenuation"]; + + ipc4_handler -> llext_loader [label="Bind Module"]; + llext_loader -> aria_core [label="Instantiate"]; + ipc4_handler -> aria_core [label="Update Attenuation"]; + } + +.. _figure_222: + +.. graphviz:: + :align: center + :caption: End-to-End Audio Graph & Topology 2 Integration (mixout-aria-gain-mixin Playback Pipeline) + + digraph aria_playback_pipeline { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_ingress { + label = "Host Playback Ingress"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + mixout [label="Mixout Widget\n(mixout.1)\nAudio Stream Egress", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + subgraph cluster_aria_pipe { + label = "Aria Dynamic Protection Pipeline (Pipeline 1)"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + aria_w [label="Aria Widget\n(aria.1.1)\nTarget Boost + Lookahead Limiter\nUUID: 6d:16:f7:99...", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + gain_w [label="Gain Widget\n(gain.1.1)\n32-bit Linear Scaler", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + mixin_w [label="Mixin Widget\n(mixin.1)\nBus Fan-In Node", fillcolor="#0d9488", fontcolor="#ffffff", color="#2dd4bf"]; + } + + subgraph cluster_egress { + label = "Physical Audio Egress"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b22"; + + dai [label="DAI Copier Gateway\n(I2S / SoundWire Link)\nOutput to Codec / Amp", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + mixout -> aria_w [label="Route 1 (S24_4LE)"]; + aria_w -> gain_w [label="Route 2 (Protected)"]; + gain_w -> mixin_w [label="Route 3 (Leveled)"]; + mixin_w -> dai [label="Playback Egress"]; + } + +------------------------------------------------------------------------------- + +Factory Bringup, Acoustic Quality & Verification Runbook +-------------------------------------------------------- + +This section outlines an end-to-end engineering verification procedure to validate Aria functionality, dynamic boost accuracy, regressive anti-clipping clamping, and latency invariance on physical DUTs (such as Panther Lake, Meteor Lake, or Tiger Lake). + +1. Topology Compilation & Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Verify that the target topology includes the Aria widget and compiles cleanly: + +.. code-block:: bash + + # Step 1: Export tuning blobs using GNU Octave + cd tools/tune/aria + octave --no-gui sof_aria_blobs.m + + # Step 2: Compile ALSA Topology 2 binary + cd ../../topology/topology2 + alsatplg -c development/sof-mtl-sdw-benchmark-aria24-simplejack.conf \ + -o sof-mtl-sdw-benchmark-aria24-simplejack.tplg + + # Step 3: Deploy topology to target DUT + scp sof-mtl-sdw-benchmark-aria24-simplejack.tplg root@:/lib/firmware/intel/sof-ipc4/ + +2. Driver Reload & DSP Initialization Check +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Reload the kernel sound driver and inspect :command:`dmesg` to verify module instantiation: + +.. code-block:: bash + + # Reload SOF audio driver on DUT + ssh root@ 'modprobe -r snd_sof_pci_intel_mtl && modprobe snd_sof_pci_intel_mtl' + + # Check dmesg for Aria registration and UUID confirmation + ssh root@ 'dmesg | grep -i aria' + +Expected output: + +.. code-block:: text + + sof-audio-pci-intel-mtl: module ARIA [6d16f799-2c37-43ef-81f6-22007aa15f03] loaded + sof-audio-pci-intel-mtl: aria.1.1: created with attenuation = 1 (target +6 dB) + +3. Dynamic Range & Linear Pre-amplification Verification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Verify that low-amplitude audio receives the exact target boost across all attenuation modes: + +.. code-block:: bash + + # Generate 1 kHz test tone at -30 dBFS (well below all headroom thresholds) + sox -n -r 48000 -c 2 -b 24 test_tone_minus30dBFS.wav synth 5 sine 1000 vol -30dB + + # Play test tone through Aria playback pipeline + ssh root@ 'aplay -D hw:0,0 test_tone_minus30dBFS.wav' + + # Check output level across attenuation modes via amixer + # Mode 0 (att = 0, 0 dB): Output level must equal -30.0 dBFS + # Mode 1 (att = 1, +6 dB): Output level must equal -24.0 dBFS (+/- 0.1 dB) + # Mode 2 (att = 2, +12 dB): Output level must equal -18.0 dBFS (+/- 0.1 dB) + # Mode 3 (att = 3, +18 dB): Output level must equal -12.0 dBFS (+/- 0.1 dB) + +4. Transient Shock & Anti-Clipping Clamping Verification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Verify that full-scale signals and loud acoustic transients are strictly clamped to :math:`0\text{ dBFS}` without digital wrap-around: + +.. code-block:: bash + + # Generate high-amplitude burst signal at -3 dBFS + sox -n -r 48000 -c 2 -b 24 test_burst.wav synth 3 sine 1000 vol -3dB + + # Set Aria to maximum boost mode (att = 3, target +18 dB) + ssh root@ 'amixer -c 0 cset name="aria.1.1.extctl" 3' + + # In a naive amplifier, -3 dBFS + 18 dB = +15 dBFS (massive digital clipping) + # In Aria, output peak must clamp strictly to 0.00 dBFS (0x007FFFFF) + ssh root@ 'aplay -D hw:0,0 test_burst.wav' + + # Record capture loopback and verify maximum peak using sox + sox recorded_output.wav -n stats + +Expected verification statistics: + +.. code-block:: text + + Pk lev dB 0.00 + Max amp 0.999999 + Min amp -0.999999 + Zero crossings 6000 + Flat factor 0.00 <-- Verifies zero flat-top clipping distortion! + +5. 1 ms Lookahead Latency Invariance Verification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Measure group delay through the pipeline with Aria in bypass (:math:`\text{att} = 0`) versus active (:math:`\text{att} = 2`): + +.. code-block:: bash + + # Measure impulse response latency with cross-correlation + python3 -c ' + import numpy as np, scipy.io.wavfile as wf + rate, ref = wf.read("impulse_ref.wav") + rate, cap = wf.read("impulse_cap.wav") + corr = np.correlate(cap[:,0], ref[:,0], mode="full") + delay_ms = (np.argmax(corr) - len(ref) + 1) / rate * 1000.0 + print(f"Measured Algorithmic Delay: {delay_ms:.3f} ms") + ' + +The measured delay delta between bypass and active mode must be **identically 0.000 ms**, confirming that the circular delay buffer maintains constant pipeline latency. diff --git a/developer_guides/firmware/audio_buffer_management.rst b/developer_guides/firmware/audio_buffer_management.rst new file mode 100644 index 00000000..0daa12d3 --- /dev/null +++ b/developer_guides/firmware/audio_buffer_management.rst @@ -0,0 +1,626 @@ +.. _audio_buffer_management: + +Audio Buffer Management +####################### + +The **Audio Buffer Management** subsystem in Sound Open Firmware (SOF) provides the foundational memory and data-transport infrastructure that connects audio processing components into streaming pipelines. By abstracting raw memory allocation, circular pointer math, multi-core cache coherency, and format alignment, the buffer subsystem enables real-time audio streams to flow deterministically across heterogeneous DSP memory architectures. + +This guide provides a high-level conceptual overview of circular ring buffers, lockless single-producer single-consumer (SPSC) mechanics, memory tiers, cache synchronization, sample interleaving, and automated self-healing recovery without focusing on low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +1. Audio Buffer Architecture Overview +************************************* + +Why Real-Time Audio Requires Specialized Buffer Management +=========================================================== + +Unlike general-purpose computing where data buffers can be resized or queued dynamically, embedded audio processing operates under uncompromising real-time constraints: + +1. **Jitter Absorption**: Audio hardware Direct Memory Access (DMA) controllers demand a constant, uninterrupted stream of samples. Buffers absorb transient execution jitter caused by high-priority interrupts, host operating system scheduling delays, or variable algorithmic execution times. +2. **Clock Domain & Period Decoupling**: Components in an audio pipeline often execute at different chunk sizes or period rates (for example, a 1 ms low-latency I/O component feeding a 10 ms acoustic echo canceler). Buffers decouple these mismatched consumption and production rhythms. +3. **Multi-Core Isolation**: In multi-core DSPs, audio buffers act as the shared memory conduits connecting tasks running on different physical cores without requiring coarse-grained cross-core spinlocks. +4. **Hardware DMA Alignment**: Audio interfaces (I2S, SoundWire, HDA) transfer samples in burst transactions that mandate strict memory alignment (e.g., 64-byte or 128-byte boundaries) to achieve maximum memory bus throughput. + +High-Level Architecture +======================= + +.. graphviz:: + :caption: High-Level Audio Buffer Architecture: Decoupling Producers and Consumers + :align: center + + digraph audio_buffer_arch { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_prod { + label = "Upstream Component (Producer)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1b4f72"; + + prod_comp [label="Producer Module\n(Host Copier / Volume / EQ)", fillcolor="#aed6f1"]; + sink_api [label="Sink API\n(sink_get_buffer / commit)", fillcolor="#aed6f1", style="filled,bold"]; + prod_comp -> sink_api [label="Renders\nSamples"]; + } + + subgraph cluster_buffer { + label = "Circular Ring Buffer Container"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1e8449"; + + buf_mem [label="Audio Sample Storage\n(Allocated in SRAM / DRAM)", fillcolor="#a9dfbf", shape=cylinder]; + buf_meta [label="Atomic State Variables\n_write_offset (Producer)\n_read_offset (Consumer)", fillcolor="#a9dfbf"]; + buf_mem -> buf_meta [style=invis]; + } + + subgraph cluster_cons { + label = "Downstream Component (Consumer)"; + style = "filled,rounded"; + color = "#8e44ad"; + fillcolor = "#f4ecf7"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#512e5f"; + + src_api [label="Source API\n(source_get_data / release)", fillcolor="#d7bde2", style="filled,bold"]; + cons_comp [label="Consumer Module\n(Mixer / AEC / DAI Copier)", fillcolor="#d7bde2"]; + src_api -> cons_comp [label="Consumes\nSamples"]; + } + + sink_api -> buf_mem [label="Writes Audio Data\n& Advances _write_offset", color="#2980b9", penwidth=1.5]; + buf_mem -> src_api [label="Reads Audio Data\n& Advances _read_offset", color="#8e44ad", penwidth=1.5]; + } + +The Buffer Abstraction Evolution +================================ + +Sound Open Firmware has evolved its buffer implementation across architectural generations: + +* **Legacy Component Buffers (``comp_buffer``)**: Used in Pipeline 1.0, where buffers were tightly coupled to component devices via linked lists (``source_list`` and ``sink_list``) and relied on direct pointer arithmetic and shared structures. +* **Modern Ring Buffers (``ring_buffer``)**: Introduced in Pipeline 2.0, providing completely asynchronous, lockless Single-Producer Single-Consumer (SPSC) circular queues with independent read and write offsets, explicit cache coherency management, and pluggable Source/Sink APIs. + +--- + +2. Circular Ring Buffers & Lockless SPSC Mechanics +************************************************** + +The foundation of SOF audio streaming is the **Lockless Circular (Ring) Buffer**. In high-performance audio DSPs, acquiring mutexes or spinlocks during audio frame processing introduces unacceptable jitter and risks inter-core priority inversions. SOF solves this by using a Single-Producer Single-Consumer (SPSC) lockless design. + +The Lockless Architecture +========================= + +A ring buffer connects exactly one data producer to exactly one data consumer. Thread-safety and multi-core safety are achieved through two simple architectural principles: + +1. **Only Two Shared State Variables**: + * ``_write_offset``: Represents the cumulative position where the producer writes new samples. It is modified **exclusively** by the producer. + * ``_read_offset``: Represents the cumulative position where the consumer reads samples. It is modified **exclusively** by the consumer. +2. **Atomic 32-Bit Operations**: On modern DSP architectures (Tensilica Xtensa, ARM Cortex-M, RISC-V), 32-bit aligned memory writes and reads are atomic instructions. Because neither component writes to the other component's offset variable, no locks or critical sections are required. + +Resolving the "Buffer Full vs. Buffer Empty" Ambiguity +====================================================== + +In classical circular buffers with an index spanning from ``0`` to ``buffer_size - 1``, when ``write_offset == read_offset``, the system cannot distinguish between a **completely empty** buffer and a **completely full** buffer without maintaining a secondary counter. + +SOF employs an elegant mathematical solution: + +.. graphviz:: + :caption: Circular Ring Buffer Traversal: Resolving Full vs Empty using Double-Size Virtual Offsets + :align: center + + digraph ring_buffer_math { + rankdir=TB; + nodesep=0.4; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_virtual { + label = "Virtual Offset Range (0 to 2 * buffer_size)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1b4f72"; + + v_empty [label="Empty Condition\n_write_offset == _read_offset\n(Available Data = 0)", fillcolor="#d4e6f1"]; + v_data [label="Partially Filled\nAvailable Data = (_write_offset - _read_offset) % (2 * buffer_size)\nFree Space = buffer_size - Available Data", fillcolor="#aed6f1"]; + v_full [label="Full Condition\n_write_offset == _read_offset + buffer_size\n(Free Space = 0)", fillcolor="#d4e6f1"]; + v_empty -> v_data -> v_full [style=invis]; + } + + subgraph cluster_physical { + label = "Physical DSP Memory Buffer (0 to buffer_size - 1)"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1e8449"; + + phys_map [label="Physical Memory Address Calculation\nPhysical Offset = Offset % buffer_size\nMemory Pointer = data_buffer_start + Physical Offset", fillcolor="#a9dfbf", style="filled,bold"]; + } + + subgraph cluster_circular { + label = "Circular Ring Traversal"; + style = "filled,rounded"; + color = "#d35400"; + fillcolor = "#fef5e7"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#a04000"; + + cell0 [label="Cell 0\n[Start]", fillcolor="#fad7a0"]; + cell1 [label="Cell 1", fillcolor="#fad7a0"]; + cell2 [label="Cell 2", fillcolor="#fad7a0"]; + cell3 [label="Cell 3\n[End]", fillcolor="#fad7a0"]; + + cell0 -> cell1 -> cell2 -> cell3; + cell3 -> cell0 [label="Wrap Around", color="#d35400", style=dashed]; + } + + v_data -> phys_map [label="Modulo Mapping"]; + phys_map -> cell0 [label="Accesses Array"]; + } + +1. **Double-Size Virtual Range**: Both ``_write_offset`` and ``_read_offset`` are allowed to increment continuously from ``0`` up to ``2 * buffer_size``. +2. **Deterministic State Detection**: + + * When ``_write_offset == _read_offset``, the buffer is **strictly empty**. + * When ``_write_offset == _read_offset + buffer_size``, the buffer is **strictly full**. + +3. **Physical Addressing**: When reading or writing sample bytes in physical memory, the address is calculated using the modulo operator: + +.. math:: + + \text{Physical Offset} = \text{Offset} \pmod{\text{buffer\_size}} + +This mathematical formulation completely eliminates ambiguous states, avoids secondary count variables, and guarantees glitch-free concurrency across cores. + +--- + +3. Buffer Sizing, Chunk Ratios & Asynchronous Decoupling +******************************************************** + +The Minimum Sizing Criterion +============================ + +Audio streams connect processing blocks that consume and produce data in different chunk sizes. To guarantee that neither component blocks or starves, SOF enforces a mathematical sizing guideline: + +.. math:: + + \text{Buffer Size} \ge 2 \times \max(\text{IBS}, \text{OBS}) + +* **IBS (Input Buffer Size)**: The maximum audio chunk size (in bytes or frames) consumed by the downstream component during each execution step. +* **OBS (Output Buffer Size)**: The maximum audio chunk size (in bytes or frames) produced by the upstream component during each execution step. + +Why Twice the Maximum Chunk Size? +================================= + +Consider an asynchronous scenario where the producer writes 3 frames and the consumer reads 5 frames: + +.. graphviz:: + :caption: Asynchronous Buffer Occupancy Over Time (Unequal IBS and OBS Ratios) + :align: center + + digraph buffer_occupancy { + rankdir=LR; + nodesep=0.2; + ranksep=0.3; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=8, margin="0.1,0.05"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + c0 [label="Cycle 0\nBuffer: 0 frames\nProducer starts", fillcolor="#eaeded"]; + c3 [label="Cycle 3\nProduce 3 frames\nBuffer: 3 frames", fillcolor="#d4e6f1"]; + c6 [label="Cycle 6\nProduce 3 frames\nBuffer: 6 frames\n(Consumer ready)", fillcolor="#aed6f1"]; + c7 [label="Cycle 7\nConsume 5 frames\nBuffer: 1 frame", fillcolor="#d7bde2"]; + c9 [label="Cycle 9\nProduce 3 frames\nBuffer: 4 frames", fillcolor="#aed6f1"]; + c12 [label="Cycle 12\nProduce 3 + Consume 5\nBuffer Peak: 7 frames", fillcolor="#f5b7b1", style="filled,bold"]; + c15 [label="Cycle 15\nProduce 3 + Consume 5\nBuffer: 0 frames", fillcolor="#abebc6"]; + + c0 -> c3 -> c6 -> c7 -> c9 -> c12 -> c15; + } + +Even when average input and output throughput are identical, scheduling latency and thread preemption mean that producer and consumer execution intervals will drift. Allocating at least ``2 * max(IBS, OBS)`` ensures that the producer always has sufficient free space to write its chunk, and the consumer always has sufficient buffered samples to satisfy its read request. + +Topology 2.0 Buffer Declaration +=============================== + +In ALSA Topology 2.0 configuration files (such as ``tools/topology/topology2/include/components/buffer.conf``), buffers are instantiated with explicit period multiples and capability flags: + +.. list-table:: + :widths: 25 25 50 + :header-rows: 1 + + * - Parameter + - Typical Values + - Architectural Purpose + * - **periods** + - ``2``, ``4``, ``8`` + - Number of audio periods buffered (e.g., 2 periods for low-latency, 4–8 for host DMA). + * - **caps** + - ``host``, ``dai``, ``comp``, ``pass`` + - Declares memory placement constraints (e.g., L2 HP-SRAM vs. DMA-accessible memory). + * - **size** + - Automatically computed + - Computed dynamically as ``period_bytes * periods``. + +--- + +4. DSP Memory Tiers & Cache Coherency +************************************* + +Modern audio DSPs (such as Intel cAVS and ACE architectures) feature heterogeneous memory hierarchies with differing access latencies, power profiles, and caching behaviors. + +The DSP Memory Hierarchy +======================== + +.. graphviz:: + :caption: DSP Memory Tiers: Access Latency vs Storage Capacity + :align: center + + digraph memory_tiers { + rankdir=TB; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_l1 { + label = "Tier 1: Core-Local Scratchpad (L1 TCM)"; + style = "filled,rounded"; + color = "#c0392b"; + fillcolor = "#f9ebea"; + fontname = "Verdana-Bold"; + fontsize = 9; + + t_l1 [label="L1 Tightly-Coupled Memory (TCM)\nSingle-cycle latency, private to individual DSP core.\nUsed for module stack, scratch registers, and FIR coefficient delay lines.", fillcolor="#f5b7b1"]; + } + + subgraph cluster_l2 { + label = "Tier 2: High-Performance System SRAM (L2 HP-SRAM)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 9; + + t_l2 [label="L2 High-Performance SRAM (HP-SRAM)\nMulti-banked shared SRAM accessible by all DSP cores and DMA controllers.\nPrimary storage for active ring buffers, module state, and IPC mailboxes.", fillcolor="#aed6f1"]; + } + + subgraph cluster_lp { + label = "Tier 3: Low-Power System SRAM (LP-SRAM)"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 9; + + t_lp [label="Low-Power SRAM (LP-SRAM)\nRetains memory during DSP low-power wait states (D0ix).\nHosts wake-on-voice (WoV) buffers and low-power streaming queues.", fillcolor="#a9dfbf"]; + } + + subgraph cluster_host { + label = "Tier 4: Host Memory (Host DRAM)"; + style = "filled,rounded"; + color = "#7f8c8d"; + fillcolor = "#f2f4f4"; + fontname = "Verdana-Bold"; + fontsize = 9; + + t_dram [label="Host System DRAM (PCIe / Shared DMA Windows)\nGigabyte-scale capacity with high access latency.\nHosts circular ALSA ring buffers managed via Host DMA gateways.", fillcolor="#d5dbdb"]; + } + + t_l1 -> t_l2 [label="Cache Miss / Spilling", style=dashed]; + t_l2 -> t_lp [label="Power Tier Migration", style=dashed]; + t_l2 -> t_dram [label="Host DMA Transfers", color="#2980b9", penwidth=1.5]; + } + +Local Mode vs. Shared Mode +========================== + +The SOF buffer management subsystem automatically configures buffers into one of two operational modes: + +1. **Local Mode (Intra-Core)**: + + * Used when both the producer and consumer components execute on the **same DSP core**. + * The ring buffer structure and audio sample payload reside in local cached SRAM. + * **Zero Cache Overhead**: The CPU core reads and writes directly from L1 cache without issuing cache invalidations or flushes. + +2. **Shared Mode (Cross-Core)**: + + * Used when the producer executes on Core 0 and the consumer executes on Core 1 (or between DSP cores and hardware DMA controllers). + * Because each DSP core maintains its own local L1 data cache, hardware memory lines can quickly become desynchronized. + * SOF enforces cache coherency through explicit kernel primitives: + +.. graphviz:: + :caption: Cross-Core Shared Buffer Synchronization and Cache Coherency + :align: center + + digraph cache_coherency { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_c0 { + label = "DSP Core 0 (Producer Core)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1b4f72"; + + c0_write [label="1. Render Samples\n(Write audio to L1 Cache)", fillcolor="#aed6f1"]; + c0_wb [label="2. Write-Back Cache\ndcache_writeback_region()\n(Flushes dirty lines to SRAM)", fillcolor="#aed6f1", style="filled,bold"]; + c0_write -> c0_wb; + } + + subgraph cluster_sram { + label = "Shared L2 HP-SRAM"; + style = "filled,rounded"; + color = "#d35400"; + fillcolor = "#fef5e7"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#a04000"; + + shared_mem [label="Physical Shared Ring Buffer\n(Audio Samples + Modulo Offsets)", fillcolor="#fad7a0", shape=cylinder]; + } + + subgraph cluster_c1 { + label = "DSP Core 1 (Consumer Core)"; + style = "filled,rounded"; + color = "#8e44ad"; + fillcolor = "#f4ecf7"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#512e5f"; + + c1_inv [label="3. Invalidate Cache\ndcache_invalidate_region()\n(Discards stale L1 lines)", fillcolor="#d7bde2", style="filled,bold"]; + c1_read [label="4. Consume Samples\n(Fetches fresh data from SRAM)", fillcolor="#d7bde2"]; + c1_inv -> c1_read; + } + + c0_wb -> shared_mem [label="Flush Dirty Lines", color="#2980b9", penwidth=1.5]; + shared_mem -> c1_inv [label="Read Updated Memory", color="#8e44ad", penwidth=1.5]; + } + +--- + +5. Audio Formats, Interleaving & SIMD Memory Alignment +****************************************************** + +Audio samples inside a buffer must adhere to specific bit-depth containerization and channel arrangements to maximize processing efficiency. + +Interleaved vs. Planar (Non-Interleaved) Formats +================================================ + +.. graphviz:: + :caption: Interleaved vs Planar Multi-Channel Audio Packing in Memory + :align: center + + digraph audio_packing { + rankdir=TB; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_interleaved { + label = "Interleaved Stereo Stream (L / R Frame Sequence)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 9; + + i_mem [label="Byte 0: Left[0] | Byte 4: Right[0] | Byte 8: Left[1] | Byte 12: Right[1] | Byte 16: Left[2] | Byte 20: Right[2]", fillcolor="#aed6f1", shape=record]; + i_desc [label="Standard for I2S, SoundWire, HDA DMA, and simple Volume/Mute processing", fillcolor="#d4e6f1"]; + i_mem -> i_desc [style=invis]; + } + + subgraph cluster_planar { + label = "Planar (Non-Interleaved) Multi-Channel Stream"; + style = "filled,rounded"; + color = "#8e44ad"; + fillcolor = "#f4ecf7"; + fontname = "Verdana-Bold"; + fontsize = 9; + + p_left [label="Plane 0 (Left): | Left[0] | Left[1] | Left[2] | Left[3] | Left[4] |", fillcolor="#d7bde2", shape=record]; + p_right [label="Plane 1 (Right): | Right[0] | Right[1] | Right[2] | Right[3] | Right[4] |", fillcolor="#d7bde2", shape=record]; + p_desc [label="Ideal for Frequency-Domain FFTs, Multi-Mic Beamforming, and SIMD Vector Math", fillcolor="#e8daef"]; + p_left -> p_right -> p_desc [style=invis]; + } + } + +Sample Container Formats +======================== + +Audio samples are packaged into standardized container sizes: + +* **16-bit in 16-bit Container (``S16_LE``)**: Compact storage (2 bytes per sample); ideal for low-power voice capture and standard Bluetooth links. +* **24-bit in 32-bit Container (``S24_4LE``)**: High-resolution audio where 24 active bits are placed in the most significant bits (MSB) of a 32-bit word, with the lowest 8 bits zero-padded. This enables direct 32-bit math without pre-shifting. +* **32-bit Fixed-Point (``S32_LE``)**: Full 32-bit dynamic range audio used for professional studio pipelines and high-dynamic-range mixers. +* **32-bit IEEE Floating-Point (``FLOAT``)**: Single-precision floating point used in complex acoustic algorithms (e.g. Valve Steam Audio 3D spatializer, AEC, and neural networks). + +SIMD & DMA Alignment Rules +========================== + +To achieve maximum performance on DSP SIMD engines (Tensilica HiFi 3/4/5, ARM Helium, RISC-V Vector): + +1. **Cacheline Boundary Alignment**: Buffer base addresses and period chunk sizes are aligned to the DSP architecture's cacheline boundary (typically 64 or 128 bytes). This prevents partial cacheline invalidation penalties. +2. **SIMD Vector Alignment**: Digital Signal Processors fetch multiple samples simultaneously using SIMD load instructions (such as 128-bit or 256-bit wide registers). Misaligned buffer offsets force the processor to issue multiple unaligned memory accesses, degrading processing throughput. + +--- + +6. Dynamic Lifecycle, Zero-Copy & Inter-Pipeline Routing +******************************************************** + +The Buffer Lifecycle +==================== + +Buffers progress through an operational lifecycle synchronized with the parent pipeline state machine: + +1. **Instantiation & Allocation**: The buffer structure is created from the topology configuration and assigned an initial capacity in the target memory pool (L2 HP-SRAM or LP-SRAM). +2. **Binding & Connection**: The buffer connects upstream components via their Sink APIs and downstream components via their Source APIs. +3. **Parameter Preparation (``prepare``)**: During the stream prepare phase, the pipeline engine negotiates channel counts, sample rates, and sample containers, configuring the buffer's effective frame size and byte alignment. +4. **Streaming (``ACTIVE``)**: During active playback or capture, the buffer transfers samples, advancing its internal read and write offsets continuously. +5. **Reset & Teardown**: When the stream stops, the buffer resets its offsets to zero and reclaims or re-initializes memory. + +Zero-Copy Optimization +====================== + +In simple pipelines where consecutive components share identical audio formats, SOF employs **In-Place (Zero-Copy) Processing**: + +.. graphviz:: + :caption: In-Place Processing vs Intermediate Double Buffering + :align: center + + digraph zero_copy { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_inplace { + label = "In-Place Zero-Copy Optimization"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 9; + + zp_buf [label="Single Shared Buffer\n(Allocated once in SRAM)", fillcolor="#a9dfbf", shape=cylinder]; + zp_vol [label="Volume Module\n(Modifies samples in-place)", fillcolor="#a9dfbf"]; + zp_mute [label="Mute Module\n(Inspects/modifies same buffer)", fillcolor="#a9dfbf"]; + + zp_buf -> zp_vol [label="Direct Ptr"]; + zp_vol -> zp_mute [label="Passes Same Ptr"]; + } + + subgraph cluster_standard { + label = "Standard Intermediate Buffering"; + style = "filled,rounded"; + color = "#7f8c8d"; + fillcolor = "#f2f4f4"; + fontname = "Verdana-Bold"; + fontsize = 9; + + sb_buf1 [label="Buffer 1", fillcolor="#d5dbdb", shape=cylinder]; + sb_src [label="Sample Rate Converter\n(Produces new rate/size)", fillcolor="#d5dbdb"]; + sb_buf2 [label="Buffer 2", fillcolor="#d5dbdb", shape=cylinder]; + + sb_buf1 -> sb_src [label="Reads"]; + sb_src -> sb_buf2 [label="Writes"]; + } + } + +When components do not alter the sample rate or channel count (e.g. Volume followed by Mute), the downstream module modifies samples directly inside the upstream buffer's memory without allocating an intermediate buffer. Intermediate buffers are only introduced when format transformations occur (such as sample rate conversion, channel mixing, or cross-core routing). + +--- + +7. Buffer Overruns, Underruns (XRUNs) & Self-Healing +***************************************************** + +An **XRUN** is an abnormal streaming state where real-time synchronization breaks down. In audio processing, an XRUN immediately results in audible pops, clicks, or silence. + +The Anatomy of an XRUN +====================== + +* **Buffer Underrun (Starvation)**: + + * Occurs when the consumer (such as the speaker output DMA) arrives to read audio frames, but the producer has not yet delivered them (``Available Data == 0``). + * The hardware DMA engine is forced to replay old samples or emit zeroes, causing an audible drop or glitch. + +* **Buffer Overrun (Overflow)**: + + * Occurs when the producer (such as the microphone input DMA) produces new audio frames, but the consumer has not emptied the buffer (``Free Space < Chunk Size``). + * The new audio frames overwrite unread samples, causing corrupted waveforms or packet loss. + +Automated Self-Healing Recovery +=============================== + +Rather than letting an XRUN destabilize the DSP firmware or hang audio streams, Sound Open Firmware implements an automated **Self-Healing Recovery** mechanism: + +.. graphviz:: + :caption: Automated Buffer XRUN Detection and Self-Healing Recovery Sequence + :align: center + + digraph xrun_recovery { + rankdir=TB; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + s1 [label="1. Normal Streaming State\n(Periodic read and write operations maintain safe latency margin)", fillcolor="#abebc6"]; + s2 [label="2. XRUN Event Triggered\n(Hardware DMA starvation or queue space exhaustion detected)", fillcolor="#fadbd8", style="filled,bold"]; + s3 [label="3. Pipeline Enters XRUN_PAUSED\n(Processing temporarily halted to prevent reading corrupted memory)", fillcolor="#f5b7b1"]; + s4 [label="4. Buffer Flush & Pointer Resynchronization\n(Stale samples cleared; _read_offset and _write_offset reset to initial offset)", fillcolor="#f5b7b1", style="filled,bold"]; + s5 [label="5. Component Re-Preparation\n(Filter delay lines and stream parameters refreshed)", fillcolor="#d4e6f1"]; + s6 [label="6. Automatic Stream Resumption\n(Pipeline triggers START; streaming seamlessly recovers)", fillcolor="#aed6f1"]; + + s1 -> s2 [label="Latency Spike", color="#c0392b", penwidth=1.5]; + s2 -> s3; + s3 -> s4; + s4 -> s5; + s5 -> s6; + s6 -> s1 [label="Stable Audio", color="#27ae60", penwidth=1.5]; + } + +1. **Immediate Detection**: The buffer monitoring logic flags the condition and notifies the parent pipeline engine. +2. **State Freeze (``XRUN_PAUSED``)**: The pipeline transitions into an isolated pause state to protect downstream audio filters from feeding on junk memory. +3. **Pointer Resynchronization**: Read and write offsets are reinitialized to establish a safe initial phase margin (typically one full period offset). +4. **Stale Sample Cleansing**: Corrupted or incomplete frame fragments are zeroed out to eliminate residual pops or speaker thumps. +5. **Seamless Resumption**: The pipeline issues an internal start event, restoring clean audio streaming without requiring application or driver restarts. + +--- + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level C implementation details, data structures, and function prototypes: + +* **Upstream Buffer Specification**: Consult the core buffer architecture documentation in the SOF repository at `thesofproject/sof: src/audio/buffers/README.md `_. +* **Core Source Files**: + + * ``src/audio/buffers/ring_buffer.c``: Implementation of the lockless asynchronous circular ring buffer and double-size modulo offset math. + * ``src/audio/buffers/audio_buffer.c``: Base audio buffer class initialization and format configuration. + * ``src/audio/buffers/comp_buffer.c``: Legacy component buffer connectors and list operations. + * ``src/audio/pipeline/pipeline-xrun.c``: XRUN detection and self-healing recovery handlers. + +* **Core Header Files**: + + * ``src/include/sof/audio/ring_buffer.h``: Ring buffer data structures, SPSC offsets, and modulo wrap-around constants. + * ``src/include/sof/audio/audio_buffer.h``: Base buffer structure and format callback declarations. + * ``src/include/sof/audio/buffer.h``: Comprehensive buffer macros, trace handlers, and legacy ``comp_buffer`` declarations. + * ``src/include/sof/audio/audio_stream.h``: Audio stream configuration descriptors and channel parameters. + +Related Guides +============== + +* :ref:`ipc_infrastructure`: Host-to-DSP messaging, hardware mailbox windows, and dynamic IPC4 compound commands. +* :ref:`pipeline_architecture`: How audio buffers interconnect components into directed acyclic graphs (DAGs). +* :ref:`module_framework`: The standardized module interface that consumes and produces audio samples through Source and Sink APIs. +* :ref:`scheduler_architecture`: Real-time scheduling domains (LL, DP, TWB) that drive buffer read and write intervals. +* :ref:`fw_init_boot`: Boot flow, hardware mailbox FW Ready handshake, and Zephyr initialization. +* :ref:`topology2`: Declaring buffer sizes, capabilities, and period counts in ALSA Topology 2.0 configuration files. diff --git a/developer_guides/firmware/copier_mux_selector.rst b/developer_guides/firmware/copier_mux_selector.rst new file mode 100644 index 00000000..97440ded --- /dev/null +++ b/developer_guides/firmware/copier_mux_selector.rst @@ -0,0 +1,832 @@ +.. _copier_mux_selector: + +Data Routing, Multiplexing & Selection Architecture: Copier, Multiplexer & Selector +#################################################################################### + +In Sound Open Firmware (SOF), audio processing pipelines are decoupled from raw hardware transport and stream topology management. The subsystem responsible for moving audio data across execution boundaries, translating stream formats, routing multiple audio channels, and synchronizing hardware streams consists of three foundational components: + +* **Copier**: The universal boundary data mover and hardware endpoint abstraction module. In IPC4 architectures, the Copier interfaces directly with DMA engines (Host DMA, Digital Audio Interfaces, and Inter-Core IPC Gateways), provides 1-to-N multi-pin stream fan-out, executes dynamic per-sink PCM format conversions, tracks Linear Link Position (LLP) telemetry, and latches DSP wall-clock hardware timestamps. +* **Multiplexer & Demultiplexer (Mux/Demux)**: The multi-stream channel routing crossbar. In IPC3, the Mux/Demux dynamically cross-connects audio channels between :math:`N` inputs and :math:`M` outputs via bitmask routing matrices. In IPC4, the Multiplexer serves as the standardized multi-pin stream aggregator for Echo Cancellation (AEC), fusing primary microphone capture audio with reference playback streams into a synchronized multi-channel stream. +* **Selector**: The intra-stream channel extraction, permutation, and linear downmixing engine. Operating within a single stream, the Selector extracts designated channel subsets (e.g. isolating active microphones from a high-density microphone array), swaps channel assignments, and executes arbitrary :math:`8 \times 8` matrix mixing in :math:`Q10` fixed-point arithmetic. + +Together, these three components establish the complete routing, fan-out, aggregation, and isolation infrastructure required by modern multi-stream audio architectures. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +Executive Architecture Overview: The SOF Data Routing & Endpoint Ecosystem +========================================================================== + +Audio data routing within a modern digital signal processor must reconcile two divergent architectural requirements: + +1. **Hardware Transport Decoupling**: Hardware peripherals (PCIe Host DMA, High Definition Audio links, Serial Synchronous Ports, SoundWire Audio Link Hubs, and PDM digital microphones) operate with rigid FIFO layouts, burst alignments, and hardware frame rates. Internal DSP algorithms, conversely, require uniform circular buffers, predictable frame block sizes, and arbitrary bit depths. +2. **Dynamic Stream Topologies**: Operating systems and audio middleware demand complex routing topologies—including simultaneous media playback, voice assistant capture, acoustic echo cancellation loopback taps, multi-mic spatial beamforming, and offload processing—all sharing concurrent access to shared audio streams without mutual interference. + +SOF resolves these demands through a strict separation of concerns among the Copier, Multiplexer, and Selector components: + +.. list-table:: SOF Routing Component Capability Matrix + :widths: 22 26 26 26 + :header-rows: 1 + + * - Capability / Feature + - Copier Subsystem + - Multiplexer / Demux + - Selector Component + * - **Primary Purpose** + - Hardware endpoint bridging, stream splitting & format conversion. + - Multi-stream channel crossbar & IPC4 AEC stream aggregation. + - Intra-stream channel selection, permutation, and matrix downmixing. + * - **Pin Topology** + - 1 Input Pin, up to 4 Output Pins (Fan-out). + - IPC3: N-in / 1-out (Mux) or 1-in / N-out (Demux). IPC4: 2-in (Mic/Ref), 1-out. + - 1 Input Pin, 1 Output Pin. + * - **Hardware Gateways** + - Direct interface to Host, DAI, & IPC gateways on Pin 0. + - None (Internal DSP stream routing only). + - None (Internal DSP stream processing only). + * - **Format Adaptation** + - Dynamic per-sink format conversion on all sinks. + - Matches input/output stream channel counts & formats. + - Operates on native audio formats with matrix math. + * - **Mathematical Model** + - Bit-depth conversion, frame shift attenuation (:math:`x \gg k`). + - Bitmask matrix cross-wiring (:math:`\text{mask}[\text{ch}]`). + - :math:`8 \times 8` :math:`Q10` fixed-point coefficient matrix. + * - **Timing & Telemetry** + - Linear Link Position (LLP) & DSP Wall-Clock Timestamps. + - Zero-latency sample pass-through with reference sync. + - Frame-synchronized sample selection & mixing. + +--- + +Copier Subsystem Deep Dive: Hardware Endpoint Abstraction +========================================================= + +The **Copier** (UUID ``9ba00c83-ca12-4a83-943c-1fa2e82f9dda``) is the mandatory endpoint and pipeline boundary module in SOF IPC4 architectures. Every pipeline that exchanges audio with the host operating system or external audio codecs begins or terminates with a Copier instance. + +Binding Configurations +---------------------- + +A Copier instance can be instantiated and bound within a pipeline in four distinct topological configurations: + +1. **Input Gateway Ingestion (Case 1)**: + Connects an input hardware gateway to downstream DSP processing modules: + + .. math:: + + \text{InputGateway} \longrightarrow \text{Copier} \longrightarrow \text{DestinationModule} + + Used for host playback pipelines (where the gateway is a Host DMA stream) and audio capture pipelines (where the gateway is a DAI interface receiving from microphones or line-in). + +2. **Output Gateway Transmission (Case 2)**: + Connects upstream DSP processing modules to an output hardware gateway: + + .. math:: + + \text{SourceModule} \longrightarrow \text{Copier} \longrightarrow \text{OutputGateway} + + Used for speaker playback pipelines (delivering processed audio to DAI hardware) and host recording pipelines (delivering captured audio to Host DMA ring buffers). + +3. **Inter-Module Format Bridging (Case 3)**: + Connects two internal DSP modules without a hardware gateway: + + .. math:: + + \text{SourceModule} \longrightarrow \text{Copier} \longrightarrow \text{DestinationModule} + + Used when splitting pipelines across distinct scheduling domains, core boundaries, or when executing complex format adaptations between incompatible processing modules. + +4. **Gateway Transmission with Local Tap (Case 4)**: + Connects upstream DSP modules simultaneously to an output gateway and one or more internal destination modules: + + .. math:: + + \text{SourceModule} \longrightarrow \text{Copier} \begin{cases} \longrightarrow \text{OutputGateway} \\ \longrightarrow \text{DestinationModule} \end{cases} + + Used for hardware loopback taps, where speaker playback audio is transmitted to the physical amplifier while simultaneously being tapped and fed into an Echo Cancellation reference pipeline. + +.. important:: + **The Gateway Pin 0 Invariant**: + In all Copier configurations interfacing with hardware, the gateway is strictly connected to **Pin 0** (Input Pin 0 for input gateways, Output Pin 0 for output gateways). Auxiliary destination modules and loopback taps are bound exclusively to Output Pins 1, 2, or 3. + +.. graphviz:: + :caption: Figure 174: SOF Data Movement and Gateway Interconnect Topology (Host Copier, DAI Copier, and Gateway Copier) + :alt: Architecture of SOF data movement showing Host Copier, DAI Copier, and Gateway Copier binding cases with circular buffers and DMA engines. + + digraph sof_copier_gateway_interconnect { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_host_domain { + label = "Host Operating System & Shared Memory Space"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + host_playback_ring [label="Host Playback Ring Buffer\n(ALSA / AudioFlinger PCM DMA Space)\nCircular Ring Pointer Tracking", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + host_capture_ring [label="Host Capture Ring Buffer\n(ALSA Capture PCM DMA Space)\nUser-Space Ingestion Ring", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + fpi_sync [label="FPI Stream Synchronization Group\n(Period Elapsed & Position Synchronizer)\nSynchronous Multi-Stream Latency Alignment", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.4]; + } + + subgraph cluster_dsp_pipeline { + label = "DSP Firmware Pipeline Architecture"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + host_copier_rx [label="Host Copier (Input Gateway)\nUUID: 9BA00C83-CA12-4A83-943C...\nInput Pin 0: Host DMA FIFO\nManages Host Ring Pointers & Wrap", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + dsp_processing [label="DSP Audio Processing Pipeline\nVolume / Equalizer / DRC / Beamforming\nUniform Periodic Block Processing", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + dai_copier_tx [label="DAI Copier (Output Gateway)\nOutput Pin 0: Hardware DAI Link\nOutput Pin 1: Loopback Reference Tap\nMultichannel Hardware Dispatcher", fillcolor="#FED7D7", color="#C53030", penwidth=1.8]; + gateway_copier [label="IPC Gateway Copier\nInter-Core / Inter-Pipeline DMA\nZero Host Overhead Gateway", fillcolor="#E9D8FD", color="#6B46C1", penwidth=1.6]; + } + + subgraph cluster_hardware_domain { + label = "Hardware Audio Interfaces (DAI & Interconnects)"; + style = "filled,rounded"; + color = "#E2E8F0"; + fillcolor = "#F7FAFC"; + + hw_ssp [label="Intel SSP / I2S Engine\nStereo / TDM Serial Framing", fillcolor="#EDF2F7", color="#4A5568"]; + hw_sndw [label="SoundWire Audio Link Hub (ALH)\nMulti-PDI Aggregation Gateway", fillcolor="#EDF2F7", color="#4A5568"]; + hw_dmic [label="Digital Microphone (DMIC)\nPdm Decimation & Multichannel DMA", fillcolor="#EDF2F7", color="#4A5568"]; + hw_hda [label="High Definition Audio (HDA) Bus\nHD-A Link DMA Tag Controller", fillcolor="#EDF2F7", color="#4A5568"]; + } + + host_playback_ring -> host_copier_rx [label="Host DMA Read", color="#3182CE", penwidth=1.6]; + fpi_sync -> host_copier_rx [label="FPI Sync Signal", style="dashed", color="#4A5568"]; + host_copier_rx -> dsp_processing [label="Pin 0 Audio Stream", color="#276749", penwidth=1.6]; + dsp_processing -> dai_copier_tx [label="Processed Frames", color="#B7791F", penwidth=1.6]; + dai_copier_tx -> hw_ssp [label="Pin 0 (SSP Link)", color="#C53030", penwidth=1.6]; + dai_copier_tx -> hw_sndw [label="Pin 0 (SoundWire ALH)", color="#C53030", penwidth=1.6]; + dai_copier_tx -> hw_hda [label="Pin 0 (HDA Bus)", color="#C53030", penwidth=1.6]; + hw_dmic -> gateway_copier [label="PDM Capture DMA", color="#6B46C1", penwidth=1.6]; + gateway_copier -> host_capture_ring [label="Host DMA Write", color="#3182CE", penwidth=1.6]; + } + +Host Copier Engine +------------------ + +The **Host Copier** connects the DSP memory space to the host operating system's cyclic DMA buffers. In playback mode, it pulls audio data from host memory into DSP local memory; in capture mode, it pushes processed DSP frames to host memory. + +* **Circular Buffer Pointer Tracking**: The Host Copier continuously tracks host read/write pointers. It calculates available space and data counts, handles circular buffer wrap-around, and notifies the host driver when period elapsed events occur. +* **Frame Position Index (FPI) Synchronization Groups**: To prevent phase drift across multi-stream presentations (such as multichannel audio where front, rear, and center/subwoofer channels are split across multiple ALSA substreams), SOF provides FPI update groups (``CONFIG_HOST_DMA_STREAM_SYNCHRONIZATION``). Multiple Host Copiers can be assigned to a common ``fpi_sync_group`` with a shared update period in microseconds. All copiers within the group latch and update their host FIFO position indices synchronously, ensuring perfect phase alignment. + +DAI Copier Engine +----------------- + +The **DAI Copier** bridges DSP audio buffers to external digital audio serial buses: + +* **High Definition Audio (HDA)**: Direct connection to Intel HDA link DMA streams. +* **Serial Synchronous Port (SSP / I2S)**: Interfaces with standard I2S, left-justified, right-justified, or multichannel TDM serial codecs. +* **Digital Microphone (DMIC)**: Interfaces with hardware PDM decimation filters, capturing up to 8 digital microphone channels. +* **SoundWire / Audio Link Hub (ALH)**: Implements multi-gateway aggregation (``is_multi_gateway(node_id)``). When high-channel-count audio (e.g. 4-channel surround or multi-speaker smart amps) is distributed across multiple SoundWire Data Port Interfaces (PDIs), the DAI Copier inspects the ``sof_alh_configuration_blob``, instantiates multiple DAI sub-indices, and automatically multiplexes or demultiplexes the multichannel stream across physical SoundWire data lines using nibble-encoded channel bitmasks. + +IPC Gateway Copier +------------------ + +When audio must traverse pipeline boundaries across heterogeneous DSP cores (such as passing decoded media frames from Primary Core 0 to Secondary Core 1 for post-processing), the **IPC Gateway Copier** uses hardware Inter-Processor Communication (IPC) gateways or shared SRAM FIFO windows. It decouples the scheduling loops of the two pipelines without engaging host DMA channels or triggering host interrupts. + +Copier Fast Mode +---------------- + +Under normal scheduling, a Copier transfers exactly its configured Input Block Size (IBS) or Output Block Size (OBS) per scheduling period. When ``IPC4_COPIER_FAST_MODE`` is enabled in the copier feature mask, the Copier is permitted to burst-transfer multiples of the block size in a single execution tick. Fast Mode is activated during pipeline pre-filling and deep-sleep playback buffer draining, provided all downstream sinks are bound to data-processing queues rather than fixed real-time DAIs. + +--- + +Multi-Pin Fan-Out & Dynamic Per-Sink Format Conversion +====================================================== + +In modern audio architectures, a single audio source must frequently be distributed to multiple consumers operating with distinct sample rates, bit depths, or channel layouts. The Copier natively provides a 1-to-N stream splitter with independent format conversion per output pin. + +Stream Fan-Out Topologies +------------------------- + +The Copier supports up to 4 simultaneous output pins (:math:`\text{Pin}_0, \text{Pin}_1, \text{Pin}_2, \text{Pin}_3`). Each output pin operates with its own circular buffer sink and independently configured audio format: + +.. math:: + + x_{\text{in}}[n] \in \mathcal{F}_{\text{in}} \xrightarrow{\text{Copier}} \begin{cases} + y_0[n] \in \mathcal{F}_{\text{out}, 0} & (\text{Pin 0: Hardware Gateway or Primary Pipeline}) \\ + y_1[n] \in \mathcal{F}_{\text{out}, 1} & (\text{Pin 1: Acoustic Echo Cancellation Reference Tap}) \\ + y_2[n] \in \mathcal{F}_{\text{out}, 2} & (\text{Pin 2: Speech Recognition / Hotword Detector}) \\ + y_3[n] \in \mathcal{F}_{\text{out}, 3} & (\text{Pin 3: Telemetry / Loopback Monitor}) + \end{cases} + +Runtime Per-Sink Format Setup +----------------------------- + +While Pin 0's format is established during initial module instantiation, auxiliary output pins (Pins 1 through 3) can be dynamically configured at runtime via the IPC4 command ``IPC4_COPIER_MODULE_CFG_PARAM_SET_SINK_FORMAT``. The host driver supplies a configuration structure specifying: + +* Target Sink Identifier (Pin Index). +* Upstream Source Audio Format (validating that the input stream format matches expected characteristics). +* Downstream Sink Audio Format (specifying target container bit depth, valid bit resolution, channel count, sample rate, and interleaving scheme). + +Dedicated PCM Converter Execution +--------------------------------- + +When an output pin's target format differs from the input stream, the Copier dynamically binds a specialized PCM converter routine (``pcm_converter_func``) for that specific pin. During every processing period, the Copier reads input audio frames once, pushes un-converted samples directly to sinks with matching formats, and passes the input frames through the dedicated converter routines for sinks requiring transformation: + +* **Container Width Conversion**: 16-bit packed (:math:`S16\_LE`), 24-bit in 32-bit container (:math:`S24\_4LE`), and 32-bit full scale (:math:`S32\_LE`). +* **Bit Depth Formatting**: Arithmetic sign extension, arithmetic left/right shifting, and bit truncation. +* **Channel Layout Adaptation**: Selective channel stripping, channel duplication, or channel remapping according to the runtime channel mask. + +.. graphviz:: + :caption: Figure 175: Copier 4-Way Stream Splitting & Dynamic Per-Sink Format Conversion Pipeline + :alt: Diagram of Copier 4-way stream splitting showing input pin and 4 output pins with independent PCM format conversion engines. + + digraph copier_fanout_format_pipeline { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + input_stream [label="Primary Input Stream\nPin 0 Input\nFormat: 48 kHz / 2-Ch / 32-bit (S32_LE)\nBase Format Reference", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + + subgraph cluster_copier_core { + label = "Copier Multi-Pin Fan-Out Engine (UUID: 9BA00C83...)"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#FFFFFF"; + + copier_rx [label="Stream Ingestion &\nCircular Buffer Dispatcher", fillcolor="#EDF2F7", color="#4A5568"]; + sink0_conv [label="Sink 0 Converter:\nPass-Through Engine\nNo Conversion Required", fillcolor="#E2E8F0", color="#4A5568"]; + sink1_conv [label="Sink 1 Converter:\n32-bit -> 16-bit S16_LE\nDownscale with Rounding", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + sink2_conv [label="Sink 2 Converter:\nChannel Remap & Mask\nIsolate Channel 0 (Mono)", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + sink3_conv [label="Sink 3 Converter:\n32-bit -> 24-bit S24_4LE\nBit Mask & Sign Extend", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + } + + subgraph cluster_sinks { + label = "Output Sink Endpoints"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + sink0_out [label="Output Pin 0 (Hardware Gateway)\nFormat: 48 kHz / 2-Ch / S32_LE\nDestination: Physical Speaker DAI", fillcolor="#FED7D7", color="#C53030", penwidth=1.8]; + sink1_out [label="Output Pin 1 (AEC Reference Tap)\nFormat: 48 kHz / 2-Ch / S16_LE\nDestination: AEC Mux Input Pin 1", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + sink2_out [label="Output Pin 2 (Voice Trigger Tap)\nFormat: 48 kHz / 1-Ch / S16_LE\nDestination: Hotword / Wake Engine", fillcolor="#E9D8FD", color="#6B46C1", penwidth=1.6]; + sink3_out [label="Output Pin 3 (Diagnostic Loopback)\nFormat: 48 kHz / 2-Ch / S24_4LE\nDestination: Host Logging Stream", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.4]; + } + + input_stream -> copier_rx [label="Input Frames", color="#276749", penwidth=1.8]; + copier_rx -> sink0_conv [label="Pin 0 Dispatch", color="#4A5568"]; + copier_rx -> sink1_conv [label="Pin 1 Dispatch", color="#B7791F"]; + copier_rx -> sink2_conv [label="Pin 2 Dispatch", color="#B7791F"]; + copier_rx -> sink3_conv [label="Pin 3 Dispatch", color="#B7791F"]; + + sink0_conv -> sink0_out [label="Unchanged 32-bit", color="#C53030", penwidth=1.8]; + sink1_conv -> sink1_out [label="Converted 16-bit", color="#3182CE", penwidth=1.6]; + sink2_conv -> sink2_out [label="Extracted Mono", color="#6B46C1", penwidth=1.6]; + sink3_conv -> sink3_out [label="Packed 24-in-32", color="#4A5568", penwidth=1.4]; + } + +--- + +Linear Link Position (LLP) Telemetry & DSP Hardware Timestamping Synchronizer +============================================================================= + +In multimedia playback and interactive communications, audio-video synchronization (lip-sync) and low-latency device pairing require precise knowledge of the exact hardware time an audio sample crosses the digital-to-analog boundary. + +Linear Link Position (LLP) Reporting +------------------------------------ + +For High Definition Audio (HDA) links, hardware DMA controllers maintain continuous link position counters accessible to the host controller via standard PCI registers. For non-HDA digital interfaces (such as Serial Synchronous Ports, SoundWire links, and PDM digital microphones), standard hardware counters are unavailable to host software. + +The Copier bridges this architectural gap through the **Linear Link Position (LLP)** telemetry interface: + +* **Telemetry Query Commands**: The host driver sends ``IPC4_COPIER_MODULE_CFG_PARAM_LLP_READING`` or ``IPC4_COPIER_MODULE_CFG_PARAM_LLP_READING_EXTENDED`` via a Large Config Get operation. +* **Cumulative Frame Accumulation**: The Copier maintains 64-bit continuous frame counters tracking the exact number of samples pushed to or pulled from the hardware FIFO: + + .. math:: + + \text{LLP}_{\text{extended}} = \left\{ \text{LLP}_{\text{bytes}}, \text{TotalDataProcessed}_{\text{bytes}}, \text{WallClockTimestamp}_{\mu\text{s}} \right\} + +* **Drift & Jitter Elimination**: By correlating total processed bytes against the hardware interface's sample clock, host drivers calculate link FIFO depth and compensate for clock drift between host system time and the audio crystal oscillator without physical hardware probes. + +DSP Hardware Timestamping Synchronizer +-------------------------------------- + +To eliminate software latency and interrupt jitter during timestamp acquisition, the Copier interfaces directly with dedicated DSP timestamping hardware registers: + +* **Hardware Initialization**: The host initializes timestamping using the parameter ``IPC4_COPIER_MODULE_CFG_PARAM_TIMESTAMP_INIT``, passing the low-level configuration register value ``tsctrl_reg``. +* **Hardware Register Pass-Through**: The Copier programs ``tsctrl_reg`` directly into the local timestamp control register of the physical interface (e.g. SSP local timestamp register). +* **Clock Latching**: Upon the arrival of a hardware frame sync pulse (e.g. I2S word select transition or SoundWire synchronization frame), the hardware automatically latches the current 64-bit DSP wall-clock counter into a shadow register. Software queries read this latched value directly, yielding sub-microsecond timestamp precision completely free of RTOS task scheduling jitter. + +.. graphviz:: + :caption: Figure 176: Linear Link Position (LLP) Telemetry & DSP Wall-Clock Hardware Timestamping Synchronizer + :alt: Architectural diagram of Linear Link Position reporting and hardware wall-clock timestamp latching in the Copier. + + digraph copier_llp_timestamp_telemetry { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_host_query { + label = "Host Operating System Audio Subsystem"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + host_alsa [label="ALSA / PulseAudio / PipeWire Engine\nLip-Sync & Clock Drift Estimator\nIssues Large Config Get (Param ID 4 / 5)", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + } + + subgraph cluster_copier_runtime { + label = "DSP Copier Subsystem Runtime"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + copier_telemetry [label="Copier Telemetry Handler\nEvaluates LLP & Extracted Processed Bytes\nReturns struct ipc4_llp_reading_extended", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + accumulator_64bit [label="64-Bit Continuous Frame Accumulators\nInput Processed: input_total_data_processed\nOutput Processed: output_total_data_processed", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + tsctrl_driver [label="Hardware Timestamp Controller\nProgrammed via tsctrl_reg\nArms Hardware Latching Shadow Registers", fillcolor="#E9D8FD", color="#6B46C1", penwidth=1.6]; + } + + subgraph cluster_hw_registers { + label = "Hardware Interface & Wall-Clock Peripheral Registers"; + style = "filled,rounded"; + color = "#E2E8F0"; + fillcolor = "#F7FAFC"; + + dsp_wall_clock [label="DSP Free-Running Wall Clock\nHigh-Resolution 64-Bit Cycle Counter", fillcolor="#EDF2F7", color="#4A5568"]; + hw_latch_reg [label="Hardware Local Timestamp Register\nAtomic Hardware Latch Register\nLatched on Physical Frame Sync Edge", fillcolor="#FED7D7", color="#C53030", penwidth=1.8]; + dai_fifo [label="Hardware DAI FIFO / Link Serializer\nPhysical Audio Bit Stream Interface", fillcolor="#EDF2F7", color="#4A5568"]; + } + + host_alsa -> copier_telemetry [label="Large Config Get (LLP)", color="#3182CE", penwidth=1.6]; + copier_telemetry -> host_alsa [label="64-Bit LLP Payload", color="#3182CE", penwidth=1.6, constraint=false]; + accumulator_64bit -> copier_telemetry [label="Accumulated Bytes", color="#B7791F"]; + tsctrl_driver -> hw_latch_reg [label="tsctrl_reg Config", color="#6B46C1", penwidth=1.6]; + dsp_wall_clock -> hw_latch_reg [label="Continuous Clock Feed", style="dotted", color="#4A5568"]; + dai_fifo -> hw_latch_reg [label="Frame Sync Pulse Latch", color="#C53030", penwidth=1.8]; + hw_latch_reg -> copier_telemetry [label="Latched Hardware Timestamp", color="#C53030", penwidth=1.6]; + } + +--- + +Integrated Copier Gain & Attenuation Architecture +================================================= + +In addition to routing and format adaptation, the Copier provides integrated sample attenuation and gain management. This capability allows topologies to control audio levels and prevent clipping at boundary interfaces without the memory and scheduling overhead of dedicating an independent Volume processing widget. + +Static Bit-Shift Attenuation +---------------------------- + +For high-bit-depth audio streams, the Copier supports direct hardware-style attenuation via arithmetic bit shifting: + +* **Configuration**: Commanded via ``IPC4_COPIER_MODULE_CFG_ATTENUATION``. +* **Application Scope**: Permitted when the output pin is configured for 32-bit sample containers and the source is bound to a hardware gateway. +* **Mathematical Operation**: For an attenuation parameter :math:`A \in [1..31]`, every output sample is arithmetically right-shifted: + + .. math:: + + y[n] = x[n] \gg A + + This provides rapid, zero-multiplication step attenuation in :math:`6 \text{ dB}` increments (:math:`-6 \text{ dB}, -12 \text{ dB}, -18 \text{ dB}, \dots`), ideal for safeguarding high-power digital amplifier stages during link bring-up. + +Copier Gain Engine +------------------ + +When configured with ``CONFIG_COPIER_GAIN``, the Copier incorporates a dedicated gain sub-engine: + +* **Static Volume Gain**: Applies linear channel-specific scaling factors. +* **Mute Control**: Instantly forces sample values to digital zero without disrupting stream framing or tearing down DMA descriptors. +* **Smooth Transition Ramping**: When changing volume levels or toggling mute, the Copier Gain engine applies smooth linear or exponential sample ramps across configurable millisecond durations. This completely suppresses audible pops, clicks, or zipper noise during stream transitions. + +--- + +Multiplexer & Demultiplexer Architecture: Matrix Bitmask Crossbar +================================================================= + +The **Multiplexer / Demultiplexer** component (UUID ``68:68:b2:c4:30:14:0e:47:a0:89:15:d1:c7:7f:85:1a``) is the channel crossbar router of Sound Open Firmware. Unlike audio mixers (such as Mixin/Mixout), the Multiplexer performs pure channel routing and stream aggregation: it copies, redistributes, or splits individual audio channels without summing or arithmetic scaling. + +Matrix Bitmask Routing Model +---------------------------- + +In IPC3 topologies, routing between input and output streams is defined by an :math:`8 \times 8` binary routing matrix encoded into an array of 8-bit masks: + +.. math:: + + \mathbf{M} \in \{0, 1\}^{8 \times 8} + +* **Multiplexer Mode** (:math:`N` Inputs :math:`\to` 1 Output): + Each stream maintains an array ``mask[PLATFORM_MAX_CHANNELS]``, where each element corresponds to an **input channel**. The bit positions set within ``mask[ch]`` indicate the designated **output channels** to which that input channel must be copied: + + .. math:: + + y[\text{out\_ch}] = x[\text{in\_ch}] \quad \Longleftrightarrow \quad \left( \mathbf{M}_{\text{in\_ch}} \;\&\; (1 \ll \text{out\_ch}) \right) \neq 0 + +* **Demultiplexer Mode** (1 Input :math:`\to` :math:`N` Outputs): + In demultiplexer mode, the mapping is inverted: each element of ``mask[ch]`` corresponds to an **output channel**, and the bit positions indicate which **input channel** provides the source sample. + +.. note:: + **Zero Mixing Invariant**: + The Multiplexer/Demultiplexer component strictly forbids audio mixing. If a configuration specifies multiple input channels mapped to the same output channel bit, the component rejects the configuration during initialization with an error. + +Pre-Computed Lookup Tables +-------------------------- + +To achieve zero-overhead execution during real-time processing, the component compiles the binary bitmask matrix into a pre-computed lookup table (``mux_look_up``) during the pipeline ``prepare`` phase. The lookup table resolves source and destination memory pointers, buffer offsets, channel stride increments (``src_inc``, ``dest_inc``), and element counts. During inner processing loops, the DSP executes direct assembly copy operations without evaluating conditional branches or computing bit shifts. + +.. graphviz:: + :caption: Figure 177: Multiplexer (Mux) & Demultiplexer (Demux) Channel Routing Matrix & Bitmask Architecture + :alt: Diagram of Mux and Demux channel routing showing 8x8 binary bitmask matrices and zero-overhead lookup table dispatch. + + digraph mux_demux_routing_matrix { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_inputs { + label = "Input Audio Channels"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + in_s0_c0 [label="Stream 0: Channel 0\n(Left Channel)", fillcolor="#EBF8FF", color="#3182CE"]; + in_s0_c1 [label="Stream 0: Channel 1\n(Right Channel)", fillcolor="#EBF8FF", color="#3182CE"]; + in_s1_c0 [label="Stream 1: Channel 0\n(Auxiliary Mic / Ref 0)", fillcolor="#FEFCBF", color="#B7791F"]; + in_s1_c1 [label="Stream 1: Channel 1\n(Auxiliary Mic / Ref 1)", fillcolor="#FEFCBF", color="#B7791F"]; + } + + subgraph cluster_matrix_core { + label = "8x8 Channel Routing Matrix & Compiled Lookup Table"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + matrix_eval [label="Matrix Bitmask Mapping\nStream 0: mask[0]=0x01, mask[1]=0x02\nStream 1: mask[0]=0x04, mask[1]=0x08\nStrict No-Summing Invariant", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + lookup_tbl [label="Compiled Lookup Table (mux_look_up)\nDirect Stride & Pointer Offsets\nZero Conditional Branching Inner Loop", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.6]; + } + + subgraph cluster_outputs { + label = "Aggregated Output Stream"; + style = "filled,rounded"; + color = "#FED7D7"; + fillcolor = "#F7FAFC"; + + out_c0 [label="Output Slot 0 (Left)", fillcolor="#EBF8FF", color="#3182CE"]; + out_c1 [label="Output Slot 1 (Right)", fillcolor="#EBF8FF", color="#3182CE"]; + out_c2 [label="Output Slot 2 (Ref 0)", fillcolor="#FEFCBF", color="#B7791F"]; + out_c3 [label="Output Slot 3 (Ref 1)", fillcolor="#FEFCBF", color="#B7791F"]; + } + + in_s0_c0 -> matrix_eval [label="Map to Bit 0 (0x01)", color="#3182CE"]; + in_s0_c1 -> matrix_eval [label="Map to Bit 1 (0x02)", color="#3182CE"]; + in_s1_c0 -> matrix_eval [label="Map to Bit 2 (0x04)", color="#B7791F"]; + in_s1_c1 -> matrix_eval [label="Map to Bit 3 (0x08)", color="#B7791F"]; + + matrix_eval -> lookup_tbl [label="Compile Table", color="#276749", penwidth=1.6]; + + lookup_tbl -> out_c0 [label="Copy Slot 0", color="#3182CE", penwidth=1.6]; + lookup_tbl -> out_c1 [label="Copy Slot 1", color="#3182CE", penwidth=1.6]; + lookup_tbl -> out_c2 [label="Copy Slot 2", color="#B7791F", penwidth=1.6]; + lookup_tbl -> out_c3 [label="Copy Slot 3", color="#B7791F", penwidth=1.6]; + } + +--- + +IPC4 Echo Cancellation (AEC) Reference Stream Aggregator +======================================================== + +In SOF IPC4 topologies, the Multiplexer component assumes a critical, standardized role: the **Acoustic Echo Cancellation (AEC) Reference Stream Aggregator**. + +Speech processing algorithms, beamformers, and voice recognition engines require two synchronized audio inputs: + +1. The acoustic capture stream picked up by physical microphones (containing the user's speech plus echo from the device's loudspeakers). +2. The reference playback stream sent to the loudspeakers (the pure echo source). + +To pass both streams into a single processing algorithm via standard single-input module adapters, the Multiplexer aggregates them into a composite multi-channel stream. + +Deterministic Channel Allocation +-------------------------------- + +In IPC4, the Multiplexer defines a deterministic pin mapping: + +* **Input Pin 0 (Primary Capture Stream)**: + Contains :math:`M` channels (:math:`\text{Ch}_0 \dots \text{Ch}_{M-1}`, where :math:`M \le 4`) representing the physical microphone signals. These channels are mapped directly to output channels :math:`0 \dots M-1`: + + .. math:: + + y[\text{ch}] = x_0[\text{ch}], \quad \forall \; \text{ch} \in [0, M-1] + +* **Input Pin 1 (Reference Stream)**: + Contains :math:`N` channels (:math:`\text{Ch}_0 \dots \text{Ch}_{N-1}`, where :math:`N \le 2`) representing the loudspeaker playback signals tapped from the output Copier. These channels are appended immediately following the capture channels: + + .. math:: + + y[M + \text{ch}] = x_1[\text{ch}], \quad \forall \; \text{ch} \in [0, N-1] + +Total output channel count is therefore exactly :math:`M + N`. For example, a 2-channel microphone array combined with a 2-channel speaker reference yields a 4-channel output stream where channels 0 and 1 represent microphones and channels 2 and 3 represent reference audio. + +Fault-Tolerant Zero-Padding Mechanics +------------------------------------- + +In real-time operating systems, playback streams can start, stop, or pause independently of microphone capture. If the loudspeaker playback pipeline stops, Input Pin 1 ceases delivering data. + +To prevent pipeline stalling or algorithmic crashes in downstream AEC algorithms, the IPC4 Multiplexer implements autonomous fault tolerance: + +* **Primary Stream Invariant**: If Input Pin 0 (microphone capture) is disconnected or starving, the Multiplexer produces no output. Capture pipelines only execute when microphone data is actively present. +* **Reference Stream Zero-Padding**: If Input Pin 1 (echo reference) is disconnected, paused, or starving, the Multiplexer does **not** stall. Instead, it processes microphone frames normally and automatically pads the reference output slots (:math:`M \dots M+N-1`) with digital zeros: + + .. math:: + + y[M + \text{ch}] = 0, \quad \forall \; \text{ch} \in [0, N-1] + +This zero-padding ensures that downstream AEC algorithms maintain continuous frame synchronization without experiencing pipeline underflow, allowing transparent adaptation when media playback starts and stops. + +.. graphviz:: + :caption: Figure 178: IPC4 Echo Cancellation (AEC) Reference Stream Aggregation via Multiplexer + :alt: Architecture of IPC4 Echo Cancellation stream aggregation showing Pin 0 mic capture, Pin 1 speaker reference tap, and zero-padding fallback. + + digraph ipc4_aec_mux_aggregation { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_sources { + label = "Input Stream Sources"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + mic_stream [label="Microphone Capture Stream\nInput Pin 0 (M Channels)\nM = 2 Channels (Mic Left, Mic Right)\nContinuous Capture Source", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + ref_stream [label="Speaker Playback Reference Tap\nInput Pin 1 (N Channels)\nN = 2 Channels (Spk Left, Spk Right)\nDynamic / Intermittent Stream", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.8]; + } + + subgraph cluster_mux_core { + label = "IPC4 Multiplexer Core (UUID: MUX4_UUID)"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + pin_eval [label="Input Pin Monitor\nCheck Pin 0 & Pin 1 Status", fillcolor="#EDF2F7", color="#4A5568"]; + channel_align [label="Channel Aggregator\nSlot 0..1: Mic Channels\nSlot 2..3: Reference Channels", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + zero_pad [label="Autonomous Zero-Padding Engine\nFills Slots 2..3 with 0x00000000\nif Reference Pin is Disconnected", fillcolor="#FED7D7", color="#C53030", penwidth=1.6]; + } + + subgraph cluster_downstream { + label = "Composite Multi-Channel Destination"; + style = "filled,rounded"; + color = "#E2E8F0"; + fillcolor = "#F7FAFC"; + + aec_input [label="Acoustic Echo Cancellation / TDFB Module\n4-Channel Composite Stream Input\n[Mic L, Mic R, Ref L, Ref R]\nContinuous Real-Time Processing", fillcolor="#E9D8FD", color="#6B46C1", penwidth=1.8]; + } + + mic_stream -> pin_eval [label="Pin 0 Frames", color="#276749", penwidth=1.8]; + ref_stream -> pin_eval [label="Pin 1 Frames", color="#3182CE", penwidth=1.8]; + + pin_eval -> channel_align [label="Reference Active", color="#276749", penwidth=1.6]; + pin_eval -> zero_pad [label="Reference Inactive / Stalled", color="#C53030", style="dashed", penwidth=1.6]; + + channel_align -> aec_input [label="Composite 4-Ch Output", color="#6B46C1", penwidth=1.8]; + zero_pad -> aec_input [label="Zero-Padded 4-Ch Output", color="#C53030", style="dashed", penwidth=1.6]; + } + +--- + +Selector Component: Dynamic Channel Extraction, Permutation & Matrix Swapping +============================================================================= + +While the Multiplexer routes audio channels across multiple streams, the **Selector** component (UUID ``c1:92:fe:32:17:1e:c2:4f:97:58:c7:f3:54:2e:98:0a``) operates inside a single stream to isolate, rearrange, or downmix channels. + +Channel Extraction and Dropping +------------------------------- + +High-density audio interfaces frequently deliver more channels than required by downstream processing. For example, a digital microphone controller may provide an 8-channel TDM capture stream, whereas a voice assistant module requires only 2 primary microphone signals. + +The Selector extracts the designated channels and drops the remainder: + +.. math:: + + \mathbf{y}[n] = \begin{bmatrix} x_{\text{sel}[0]}[n] \\ x_{\text{sel}[1]}[n] \end{bmatrix}, \quad \text{where } \mathbf{x}[n] \in \mathbb{R}^8, \; \mathbf{y}[n] \in \mathbb{R}^2 + +In IPC3 mode, this is controlled by the configuration parameters ``in_channels_count``, ``out_channels_count``, and ``sel_channel``. + +IPC4 Fixed-Point Matrix Mixing Model +------------------------------------ + +In IPC4 architectures, the Selector evolves into a general-purpose linear matrix mixer. Channel routing, permutation, and downmixing are defined by an :math:`8 \times 8` matrix of 16-bit signed coefficients in :math:`Q10` fixed-point format (``struct ipc4_selector_coeffs_config``): + +.. math:: + + y_i[n] = \sum_{j=0}^{M-1} c_{i,j} \cdot x_j[n], \quad i \in [0, N-1] + +where :math:`M` is the input channel count, :math:`N` is the output channel count, and :math:`c_{i,j}` are the :math:`Q10` mixing coefficients. In :math:`Q10` arithmetic: + +* Unity gain (:math:`1.0`) is represented by :math:`1024` (``SEL_COEF_ONE_Q10``). +* Complete attenuation (:math:`0.0`) is represented by :math:`0`. +* Half gain (:math:`-6.02 \text{ dB}`) is represented by :math:`512`. + +This matrix formulation enables diverse audio transformations: + +* **Channel Permutation & Swapping**: Setting off-diagonal coefficients to 1024 swaps channels (e.g. reversing Left and Right channels): + + .. math:: + + \mathbf{C}_{\text{swap}} = \begin{bmatrix} 0 & 1024 \\ 1024 & 0 \end{bmatrix} + +* **Stereo-to-Mono Downmixing**: Summing Left and Right channels with equal weighting (:math:`-6 \text{ dB}` per channel) prevents arithmetic overflow: + + .. math:: + + \mathbf{C}_{\text{downmix}} = \begin{bmatrix} 512 & 512 \end{bmatrix} + +* **5.1 Surround Downmixing**: Converting 6-channel surround sound to 2-channel stereo with standard psychoacoustic ITU coefficients: + + .. math:: + + \begin{aligned} + L_{\text{out}} &= L + 0.707 C + 0.707 L_s \\ + R_{\text{out}} &= R + 0.707 C + 0.707 R_s + \end{aligned} + +Multi-Profile Configuration Caching +----------------------------------- + +A single Selector widget can store up to 8 distinct configuration profiles in memory (``SEL_MAX_NUM_CONFIGS = 8``). When stream parameters change dynamically (such as switching from stereo to quad-channel microphone capture), the Selector matches the active stream's channel count and channel configuration against its cached profiles, applying the corresponding mixing coefficients instantly without issuing new IPC round-trips to the host driver. + +.. graphviz:: + :caption: Figure 179: Selector Component: Dynamic Channel Extraction, Permutation & Matrix Swapping + :alt: Diagram of Selector component demonstrating 8x8 Q10 matrix mixing, channel extraction, channel swapping, and downmixing. + + digraph selector_matrix_permutation { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_source_channels { + label = "Multi-Channel Input Stream (e.g. 8-Ch DMIC)"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + ch0 [label="Ch 0: Mic 1 (Front Left)", fillcolor="#EBF8FF", color="#3182CE"]; + ch1 [label="Ch 1: Mic 2 (Front Right)", fillcolor="#EBF8FF", color="#3182CE"]; + ch2 [label="Ch 2: Mic 3 (Rear Left)", fillcolor="#EDF2F7", color="#4A5568"]; + ch3 [label="Ch 3: Mic 4 (Rear Right)", fillcolor="#EDF2F7", color="#4A5568"]; + ch_unused [label="Ch 4..7: Unused Sensors\n(To Be Dropped)", fillcolor="#FED7D7", color="#C53030"]; + } + + subgraph cluster_selector_core { + label = "Selector Core (UUID: MICSEL_UUID)"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + matrix_q10 [label="8x8 Q10 Coefficient Matrix\nc[0][0] = 1024 (1.0x)\nc[1][1] = 1024 (1.0x)\nc[i][j] = 0 (Unused/Dropped)", fillcolor="#C6F6D5", color="#276749", penwidth=1.8]; + profile_cache [label="Configuration Cache\nStores up to 8 Profiles\nDynamic Topology Matching", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + } + + subgraph cluster_sink_channels { + label = "Selected Output Stream (Stereo Clean)"; + style = "filled,rounded"; + color = "#E2E8F0"; + fillcolor = "#F7FAFC"; + + out_left [label="Out Ch 0: Primary Mic L", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + out_right [label="Out Ch 1: Primary Mic R", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.6]; + } + + ch0 -> matrix_q10 [label="Gain 1024 (Unity)", color="#3182CE", penwidth=1.6]; + ch1 -> matrix_q10 [label="Gain 1024 (Unity)", color="#3182CE", penwidth=1.6]; + ch2 -> matrix_q10 [label="Gain 0 (Drop)", color="#4A5568", style="dotted"]; + ch3 -> matrix_q10 [label="Gain 0 (Drop)", color="#4A5568", style="dotted"]; + ch_unused -> matrix_q10 [label="Gain 0 (Drop)", color="#C53030", style="dotted"]; + + matrix_q10 -> out_left [label="Channel 0 Stream", color="#3182CE", penwidth=1.6]; + matrix_q10 -> out_right [label="Channel 1 Stream", color="#3182CE", penwidth=1.6]; + profile_cache -> matrix_q10 [label="Active Profile", style="dashed", color="#B7791F"]; + } + +--- + +ALSA Topology 2 Integration & Widget Declarations +================================================= + +In ALSA Topology 2 (``topology2``), the Copier, Multiplexer, and Selector are instantiated as declarative widget objects. + +Copier Widget Declarations +-------------------------- + +Copiers are defined using dedicated configuration templates in ``tools/topology/topology2/include/components/``: + +* ``dai-copier.conf``: Declares hardware interface copiers (HDA, SSP, DMIC, ALH) bound to physical DAIs. Attributes include ``copier_type``, ``direction``, ``node_type``, and ``cpc`` (cycles per chunk). +* ``host-copier.conf``: Declares host PCM endpoint copiers interfacing with host DMA streams. +* ``module-copier.conf``: Declares inter-pipeline or inter-core boundary copiers. + +All Copier widgets share the standardized UUID: + +.. code-block:: text + + UUID: 83:0c:a0:9b:12:ca:83:4a:94:3c:1f:a2:e8:2f:9d:da + +Multiplexer / Demultiplexer Widget Declarations +----------------------------------------------- + +Multiplexers and Demultiplexers are declared using ``muxdemux.conf`` with widget type ``effect``: + +.. code-block:: text + + Class.Widget."muxdemux" { + UUID: "68:68:b2:c4:30:14:0e:47:a0:89:15:d1:c7:7f:85:1a" + type: "effect" + num_input_pins: 2 + num_output_pins: 1 + } + +The widget includes an ALSA byte control used to upload runtime routing matrices or AEC reference mappings. + +Selector Widget Declarations +---------------------------- + +The Selector is declared using ``micsel.conf`` with widget type ``effect``: + +.. code-block:: text + + Class.Widget."micsel" { + UUID: "c1:92:fe:32:17:1e:c2:4f:97:58:c7:f3:54:2e:98:0a" + type: "effect" + num_input_pins: 1 + num_output_pins: 1 + } + +Its configuration blob carries the :math:`8 \times 8` :math:`Q10` coefficient tables and channel selection masks. + +--- + +End-to-End System Audio Graph: Component Synergy +================================================ + +In production systems, Copier, Multiplexer, and Selector do not operate in isolation; they interact seamlessly across concurrent playback, capture, and voice assistant pipelines. + +The following architectural graph illustrates how these components interlock in a complete PC audio topology featuring simultaneous media playback, acoustic echo cancellation, and beamformed voice capture: + +.. graphviz:: + :caption: Figure 180: End-to-End System Audio Graph: Media Playback, Voice Capture, AEC Muxing, and Loopback Monitoring + :alt: Complete end-to-end audio graph showing Host Copier, Volume, DRC, DAI Copier, Selector, AEC Multiplexer, and Voice Pipeline. + + digraph end_to_end_system_audio_graph { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_playback_pipeline { + label = "Media Playback Pipeline (Core 0)"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + host_play_copier [label="Host Copier (Playback)\nIngests Stereo Media from OS\nTracks Host Ring Pointers", fillcolor="#C6F6D5", color="#276749", penwidth=1.6]; + pb_vol [label="Volume / EQ / DRC\nDynamic Processing & Protection", fillcolor="#FEFCBF", color="#B7791F"]; + dai_play_copier [label="DAI Copier (Speaker Output)\nPin 0: Hardware Speaker Link\nPin 1: AEC Loopback Tap (48 kHz)", fillcolor="#FED7D7", color="#C53030", penwidth=1.8]; + hw_speakers [label="Physical Speakers / Codec\nStereo Acoustic Output", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.4]; + } + + subgraph cluster_capture_pipeline { + label = "Microphone Capture & Voice Pre-Processing Pipeline (Core 0)"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#FFFFFF"; + + hw_dmic_in [label="Hardware DMIC Array\n4-Channel Raw PDM Capture", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.4]; + dai_cap_copier [label="DAI Copier (DMIC Capture)\nIngests 4-Channel PDM Stream\nProduces 48 kHz / 32-bit Audio", fillcolor="#C6F6D5", color="#276749", penwidth=1.6]; + mic_selector [label="Selector Widget (Channel Isolation)\nExtracts Primary 2 Voice Mics\nDrops 2 Auxiliary Channels", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + aec_mux [label="IPC4 Multiplexer Widget (AEC Aggregator)\nPin 0: 2-Ch Clean Voice Mics\nPin 1: 2-Ch Speaker Reference Tap\nOutputs 4-Ch Synchronized Stream", fillcolor="#E9D8FD", color="#6B46C1", penwidth=2.0]; + aec_tdfb [label="Acoustic Echo Cancellation &\nTDFB Beamforming Engine\nCancels Echo & Enhances Target Voice", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.6]; + host_cap_copier [label="Host Copier (Voice Capture)\nPushes Clean Enhanced Voice to OS\n(PipeWire / Google Meet / Teams)", fillcolor="#C6F6D5", color="#276749", penwidth=1.6]; + } + + host_play_copier -> pb_vol [label="Stereo Audio", color="#276749", penwidth=1.6]; + pb_vol -> dai_play_copier [label="Processed Frames", color="#B7791F", penwidth=1.6]; + dai_play_copier -> hw_speakers [label="Pin 0 (DAI Link)", color="#C53030", penwidth=1.8]; + + hw_dmic_in -> dai_cap_copier [label="4-Ch PDM DMA", color="#4A5568", penwidth=1.6]; + dai_cap_copier -> mic_selector [label="4-Ch Raw Audio", color="#276749", penwidth=1.6]; + mic_selector -> aec_mux [label="Pin 0: 2-Ch Selected Mics", color="#B7791F", penwidth=1.8]; + + dai_play_copier -> aec_mux [label="Pin 1: 2-Ch Speaker Echo Reference", color="#3182CE", penwidth=1.8, style="dashed"]; + + aec_mux -> aec_tdfb [label="4-Ch Composite Stream\n[Mics + Ref]", color="#6B46C1", penwidth=2.0]; + aec_tdfb -> host_cap_copier [label="Clean Enhanced Voice", color="#276749", penwidth=1.8]; + } + +Workflow Walkthrough +-------------------- + +1. **Host Ingestion**: The Host Copier pulls stereo audio from user space and feeds the volume, equalizer, and DRC modules. +2. **Playback Delivery & Loopback Tapping**: The DAI Copier transmits audio to physical speakers via Pin 0 while simultaneously tapping the identical signal onto Output Pin 1. +3. **Microphone Capture & Selection**: The DAI Capture Copier ingests 4 channels from the digital microphone array. The Selector isolates the two primary front-facing microphones and drops the auxiliary background channels. +4. **Echo Reference Aggregation**: The Multiplexer fuses the 2-channel microphone audio on Pin 0 with the 2-channel speaker loopback reference on Pin 1 into a synchronized 4-channel composite stream. +5. **Speech Enhancement & Delivery**: Downstream Acoustic Echo Cancellation (AEC) and Time-Domain Fixed Beamforming (TDFB) cancel the speaker echo and beamform the user's speech. The final clean audio stream is written into host memory by the Host Capture Copier. + +Through this coordinated division of labor, Sound Open Firmware delivers modular, high-performance, and mathematically robust audio graphs across desktop, mobile, and embedded platforms. diff --git a/developer_guides/firmware/crossover.rst b/developer_guides/firmware/crossover.rst new file mode 100644 index 00000000..9de4dd37 --- /dev/null +++ b/developer_guides/firmware/crossover.rst @@ -0,0 +1,588 @@ +.. _crossover: + +Crossover Filter Architecture +############################# + +The **Crossover Filter** subsystem in Sound Open Firmware provides spectral band splitting, multi-driver transducer routing, and frequency-domain decomposition across active loudspeaker systems and multi-band audio processing pipelines. + +In acoustic engineering, physical speaker transducers are bounded by rigid physical and mechanical constraints: large-diameter woofers excel at moving large volumes of air to reproduce low-frequency bass but possess too much cone inertia to oscillate rapidly at high frequencies without severe breakup distortion. Conversely, miniature, lightweight tweeters reproduce delicate high-frequency transients effortlessly, but undergo destructive excursion and voice coil burnout if subjected to high-energy bass frequencies. + +To overcome these constraints, high-fidelity audio systems employ **Multi-Way Loudspeakers** (such as 2-way woofer/tweeter systems, 3-way sub/mid/tweeter setups, or 4-way full-range towers). The SOF Crossover component acts as the digital frequency division engine, cleanly partitioning a wideband input audio stream into multiple dedicated frequency bands tailored to individual acoustic drivers or downstream processing components. + +SOF implements active digital crossovers using **Linkwitz-Riley 4th-Order (LR4)** filter networks configured in 2-way, 3-way, and 4-way topologies, providing steep 24 dB/octave attenuation slopes, flat magnitude summation, and in-phase acoustic alignment without comb filtering. + +This guide provides a comprehensive, high-level architectural walkthrough of the Crossover filter subsystem, Linkwitz-Riley filter theory, multi-way splitting topologies with all-pass phase alignment, 1-to-N multi-sink buffer distribution, and SIMD hardware acceleration without delving into low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +.. _crossover_principles: + +1. Electro-Acoustic Motivations & Crossover Principles +****************************************************** + +An audio crossover is an electrical or digital filter network that splits an incoming wideband audio signal into multiple frequency bands tailored to specific transducers or processors: + +* **Subwoofer Band (< 80 Hz)**: Extremely high excursion, omnidirectional deep bass reproduction. +* **Woofer / Bass Band (80 Hz – 1 kHz)**: Low-to-midrange bass punch, drum transients, and lower vocal registers. +* **Midrange Band (1 kHz – 4 kHz)**: Critical human vocal fundamentals, speech clarity, and instrumental harmonics. +* **Tweeter / High Band (> 4 kHz)**: High-frequency sibilance, cymbal brilliance, and spatial airiness. + +Passive Analog Crossovers vs Active Digital Crossovers +====================================================== + +Traditionally, multi-driver speaker cabinets rely on **passive analog crossovers** placed inside the loudspeaker cabinet between a single power amplifier and the physical drivers: + +* **Limitations of Passive Analog Crossovers**: + + - **Power Dissipation & Thermal Drift**: Passive crossovers utilize large inductors with high DC resistance and electrolytic capacitors that dissipate amplifier power as heat. Component heating causes filter values to drift significantly during loud listening sessions. + - **Damping Factor Loss**: Inductors placed in series with woofers degrade the amplifier's electrical damping factor, resulting in loose, uncontrolled bass ringing. + - **Component Tolerances & Phase Smearing**: Real-world passive component tolerances (often 5% to 10%) cause unpredictable phase shifts, irregular impedance curves, and destructive acoustic notches at the crossover frequency. + +* **Advantages of Active DSP Crossovers in SOF**: + + - **Pristine Digital Domain Splitting**: Frequency division occurs inside the DSP firmware before digital-to-analog conversion and power amplification (bi-amping, tri-amping, or quad-amping). + - **Zero Power Loss & Perfect Damping**: Power amplifiers connect directly to driver voice coils with zero intervening passive circuitry, maximizing electrical damping and acoustic efficiency. + - **Mathematical Precision & Stability**: Digital filter coefficients operate with mathematical exactness, unaffected by temperature, component aging, or electrical tolerances. + - **Steep 24 dB/Octave Roll-Offs**: Active DSP filters easily achieve steep 4th-order Linkwitz-Riley slopes that would require prohibitively bulky, expensive, and lossy passive components. + +.. graphviz:: + :caption: Active DSP Crossover vs Passive Analog Crossover Architectures in Multi-Driver Loudspeakers + + digraph crossover_taxonomy { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_passive { + label="Legacy Passive Crossover (Post-Amplifier Analog Domain)"; + style="filled,rounded"; + fillcolor="#FED7D7"; + color="#E53E3E"; + + p_src [label="Host Audio Stream", fillcolor="#FFFFFF", color="#CBD5E0"]; + p_dac [label="Single DAC & Pre-Amp", fillcolor="#FFFFFF", color="#CBD5E0"]; + p_amp [label="Single Power Amplifier\n(Must amplify entire wideband spectrum)", fillcolor="#FED7D7", color="#E53E3E"]; + p_xov [label="Passive LC Filter Network\nBulky inductors & capacitors\nPower loss & thermal drift", fillcolor="#FED7D7", color="#E53E3E", fontcolor="#742A2A"]; + p_spk1 [label="Woofer Driver", fillcolor="#FFFFFF", color="#CBD5E0"]; + p_spk2 [label="Tweeter Driver", fillcolor="#FFFFFF", color="#CBD5E0"]; + + p_src -> p_dac -> p_amp -> p_xov; + p_xov -> p_spk1 [label="Lows (Damping Lost)"]; + p_xov -> p_spk2 [label="Highs"]; + } + + subgraph cluster_active { + label="SOF Active DSP Crossover (Pre-Amplifier Digital Domain)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + a_src [label="Host Audio Stream", fillcolor="#FFFFFF", color="#CBD5E0"]; + a_xov [label="SOF Crossover Component (crossover.c)\nLinkwitz-Riley 4th-Order (LR4) Digital Engine\nSteep 24 dB/oct slope, 0 dB flat sum, in-phase", fillcolor="#BEE3F8", color="#3182CE"]; + a_amp1 [label="Dedicated Woofer DAC & Amp\nDirect voice coil connection\nMaximum electrical damping", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + a_amp2 [label="Dedicated Tweeter DAC & Amp\nLow-noise linear amplification\nZero bass excursion hazard", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + a_spk1 [label="Woofer Driver\n(Tight, punchy bass)", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + a_spk2 [label="Tweeter Driver\n(Crisp, distortion-free highs)", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + a_src -> a_xov; + a_xov -> a_amp1 [label="Low Band"]; + a_xov -> a_amp2 [label="High Band"]; + a_amp1 -> a_spk1; + a_amp2 -> a_spk2; + } + } + +--- + +.. _lr4_filter_theory: + +2. Linkwitz-Riley 4th-Order (LR4) Filter Theory & Phase Alignment +***************************************************************** + +Selecting the mathematical filter topology for an active acoustic crossover is critical. In audio textbooks, Butterworth filters are renowned for their maximally flat passband response. However, when applied to multi-driver acoustic crossovers, traditional Butterworth filters exhibit severe acoustic flaws. + +The Flaws of Conventional Butterworth Crossovers +================================================ + +* **The +3 dB Acoustic Bump**: Standard Butterworth low-pass and high-pass filters intersect at their :math:`-3\text{ dB}` half-power points. While uncorrelated signals (such as independent white noise sources) sum flat, coherent audio signals (such as musical notes spanning the crossover frequency) sum in voltage: :math:`1/\sqrt{2} + 1/\sqrt{2} = \sqrt{2} \approx +3\text{ dB}`. This produces an unnatural, audible acoustic peak at the crossover frequency :math:`f_c`. +* **Phase Quadrature & Acoustic Lobing**: Butterworth filters produce a :math:`90^\circ` phase difference between their low-pass and high-pass outputs at :math:`f_c`. When sound radiates into a room from physically separated speaker drivers, this :math:`90^\circ` phase disparity causes the primary acoustic radiation lobe to tilt off-axis, creating destructive comb filtering and acoustic notches whenever the listener moves vertically. + +The Linkwitz-Riley (LR4) Innovation +=================================== + +To solve these acoustic dilemmas, acoustic pioneers Siegfried Linkwitz and Russ Riley designed the **Linkwitz-Riley** filter topology. In Sound Open Firmware, all active crossovers are implemented as **4th-Order Linkwitz-Riley (LR4)** networks: + +1. **Cascaded Butterworth Pairs**: An LR4 filter is constructed by cascading two identical 2nd-order Butterworth filters in series: + + .. math:: + + H_{\text{LR4}}(z) = \Big( H_{\text{Butterworth 2nd}}(z) \Big)^2 + +2. **Flat 0 dB Magnitude Summation**: Because each 2nd-order stage contributes :math:`-3\text{ dB}` of attenuation at :math:`f_c`, the cascaded LR4 low-pass and high-pass filters are both down by exactly :math:`-6\text{ dB}` at the crossover frequency: + + .. math:: + + |H_{\text{LP}}(j\omega_c)| = 0.5 \quad (-6\text{ dB}), \qquad |H_{\text{HP}}(j\omega_c)| = 0.5 \quad (-6\text{ dB}) + + When the low-pass and high-pass acoustic outputs sum in the air, their coherent combination is mathematically flat: + + .. math:: + + |H_{\text{LP}}(j\omega) + H_{\text{HP}}(j\omega)| = 1.0 \quad (0\text{ dB}) \quad \forall \omega + +3. **Strict In-Phase Acoustic Alignment**: The phase difference between the low-pass and high-pass outputs of an LR4 filter is exactly :math:`360^\circ` (or :math:`0^\circ` modulo :math:`360^\circ`) across all frequencies. Because the drivers operate perfectly in phase across the transition band, the acoustic radiation pattern remains centered along the horizontal listening axis with zero vertical lobing tilt. +4. **Steep 24 dB/Octave Roll-Off**: The 4th-order slope provides rapid attenuation outside the passband, shielding fragile tweeters from low-frequency excursion damage and eliminating high-frequency woofer cone breakup resonances. + +.. graphviz:: + :caption: Linkwitz-Riley 4th-Order (LR4) Magnitude Summation (-6 dB at fc) and In-Phase Acoustic Alignment + + digraph lr4_theory { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_mag { + label="LR4 Magnitude & Phase Alignment Characteristics"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + p_lp [label="Low-Pass LR4 Branch (Woofer)\n-6 dB Cutoff at fc\n24 dB / Octave Attenuation Slope", fillcolor="#BEE3F8", color="#3182CE"]; + p_hp [label="High-Pass LR4 Branch (Tweeter)\n-6 dB Cutoff at fc\n24 dB / Octave Attenuation Slope", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + p_sum [label="Acoustic Magnitude Summation\n0.5 + 0.5 = 1.0 -> Perfectly Flat 0 dB Response\nZero passband ripple, zero crossover bump", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + p_pha [label="Phase Alignment & Spatial Polar Symmetry\nPhase Difference = 360° (Strictly In-Phase)\nZero off-axis lobing tilt, zero comb filtering notches", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + p_lp -> p_sum [label="-6 dB at fc"]; + p_hp -> p_sum [label="-6 dB at fc"]; + p_sum -> p_pha [label="Coherent Radiation", style="bold", color="#276749"]; + } + } + +--- + +.. _crossover_topologies: + +3. Crossover Topologies: 2-Way, 3-Way, and 4-Way Splitting +********************************************************** + +Sound Open Firmware supports three fundamental crossover topologies configured via parameter blobs and topology tokens: + +2-Way Crossover Topology (Woofer + Tweeter) +=========================================== + +* **Structure**: Splits wideband audio at a single cutoff frequency :math:`f_c` using one low-pass LR4 filter and one high-pass LR4 filter. +* **Filter Count**: 2 LR4 filters (each composed of 2 biquads in series, totaling 4 biquads per channel). +* **Outputs**: Output 0 (Low / Woofer) and Output 1 (High / Tweeter). +* **Use Cases**: Standard stereo bookshelf speakers, two-way studio monitors, and dual-driver laptop audio systems. + +3-Way Crossover Topology & The All-Pass Phase Equalization Trick +================================================================ + +In a 3-way crossover, audio is partitioned into three bands: Low (Sub/Woofer), Mid (Midrange driver), and High (Tweeter) across two cutoff frequencies (:math:`f_{c1}, f_{c2}`): + +* **The Asymmetric Phase Dilemma**: + + - The incoming signal is first split into a Low branch and a High branch at :math:`f_{c1}` using LR4 pair 0 (``LP0`` and ``HP0``). + - The high branch is subsequently split at :math:`f_{c2}` using LR4 pair 2 (``LP2`` and ``HP2``), yielding the Midrange and Tweeter outputs. + - Notice that the Midrange and Tweeter signals pass through **two sequential LR4 filters**, while the Low signal only passes through **one LR4 filter** (``LP0``). + - Because each LR4 filter introduces a phase shift, passing through two filters rotates the phase of Mid and High by :math:`360^\circ` relative to Low, causing a catastrophic :math:`180^\circ` phase inversion between Low and Mid! + +* **The All-Pass Merger Solution**: + + - To restore phase coherence, SOF routes the Low branch through an auxiliary LR4 filter pair (``LP1`` and ``HP1``) and immediately sums their outputs back together (``crossover_generic_lr4_merge()``). + - Because an LR4 low-pass and high-pass sum to a flat magnitude of 1.0, this operation acts as a pure **all-pass filter**: it leaves the magnitude of the Low band completely unaltered while introducing the exact phase shift and group delay of an additional LR4 stage! + - Consequently, all three output bands pass through exactly two LR4 stages, guaranteeing strict phase alignment across all crossover regions. + +4-Way Crossover Topology (Sub + Woofer + Mid + Tweeter) +======================================================= + +* **Structure**: A fully symmetrical 2-stage tree decomposition across three cutoff frequencies (:math:`f_{c1}, f_{c2}, f_{c3}`): + - Stage 1: Splits the wideband signal into Low-Mid and Mid-High branches using LR4 pair 1 (``LP1``, ``HP1``). + - Stage 2: Low-Mid is split into Sub and Woofer using LR4 pair 0 (``LP0``, ``HP0``); Mid-High is split into Midrange and Tweeter using LR4 pair 2 (``LP2``, ``HP2``). +* **Filter Count**: 6 LR4 filters (12 biquads per channel). +* **Inherent Phase Alignment**: Because every signal path traverses exactly two sequential LR4 stages, phase delays are inherently identical across all four bands without requiring auxiliary phase-correction networks. + +.. graphviz:: + :caption: Crossover Split Topologies: 2-Way, 3-Way (with All-Pass Phase Merger), and 4-Way Tree Decomposition + + digraph topologies { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.45]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_2way { + label="2-Way Crossover (1 Cutoff fc)"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + x2_in [label="Input x[n]", fillcolor="#FFFFFF", color="#CBD5E0"]; + x2_lp0 [label="LR4 LP0 (fc)", fillcolor="#BEE3F8", color="#3182CE"]; + x2_hp0 [label="LR4 HP0 (fc)", fillcolor="#BEE3F8", color="#3182CE"]; + x2_out0 [label="LOW (Woofer)", fillcolor="#C6F6D5", color="#38A169"]; + x2_out1 [label="HIGH (Tweeter)", fillcolor="#C6F6D5", color="#38A169"]; + + x2_in -> x2_lp0 -> x2_out0; + x2_in -> x2_hp0 -> x2_out1; + } + + subgraph cluster_3way { + label="3-Way Crossover (2 Cutoffs: fc1, fc2) with All-Pass Phase Merger"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + x3_in [label="Input x[n]", fillcolor="#FFFFFF", color="#CBD5E0"]; + x3_lp0 [label="LR4 LP0 (fc1)", fillcolor="#FAF089", color="#B7791F"]; + x3_hp0 [label="LR4 HP0 (fc1)", fillcolor="#FAF089", color="#B7791F"]; + + x3_mrg [label="All-Pass Phase Merger\n(LP1 + HP1 Summation)\nEqualizes group delay", fillcolor="#FED7D7", color="#E53E3E", fontcolor="#742A2A"]; + x3_lp2 [label="LR4 LP2 (fc2)", fillcolor="#FAF089", color="#B7791F"]; + x3_hp2 [label="LR4 HP2 (fc2)", fillcolor="#FAF089", color="#B7791F"]; + + x3_out0 [label="LOW (Sub/Woofer)", fillcolor="#C6F6D5", color="#38A169"]; + x3_out1 [label="MID (Midrange)", fillcolor="#C6F6D5", color="#38A169"]; + x3_out2 [label="HIGH (Tweeter)", fillcolor="#C6F6D5", color="#38A169"]; + + x3_in -> x3_lp0 -> x3_mrg -> x3_out0; + x3_in -> x3_hp0; + x3_hp0 -> x3_lp2 -> x3_out1; + x3_hp0 -> x3_hp2 -> x3_out2; + } + + subgraph cluster_4way { + label="4-Way Crossover (3 Cutoffs: fc1, fc2, fc3) Symmetrical Tree"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + x4_in [label="Input x[n]", fillcolor="#FFFFFF", color="#CBD5E0"]; + x4_lp1 [label="LR4 LP1 (fc2)", fillcolor="#C6F6D5", color="#38A169"]; + x4_hp1 [label="LR4 HP1 (fc2)", fillcolor="#C6F6D5", color="#38A169"]; + + x4_lp0 [label="LR4 LP0 (fc1)", fillcolor="#C6F6D5", color="#38A169"]; + x4_hp0 [label="LR4 HP0 (fc1)", fillcolor="#C6F6D5", color="#38A169"]; + x4_lp2 [label="LR4 LP2 (fc3)", fillcolor="#C6F6D5", color="#38A169"]; + x4_hp2 [label="LR4 HP2 (fc3)", fillcolor="#C6F6D5", color="#38A169"]; + + x4_out0 [label="SUB", fillcolor="#68D391", color="#276749"]; + x4_out1 [label="WOOFER", fillcolor="#68D391", color="#276749"]; + x4_out2 [label="MID", fillcolor="#68D391", color="#276749"]; + x4_out3 [label="TWEETER", fillcolor="#68D391", color="#276749"]; + + x4_in -> x4_lp1; + x4_in -> x4_hp1; + x4_lp1 -> x4_lp0 -> x4_out0; + x4_lp1 -> x4_hp0 -> x4_out1; + x4_hp1 -> x4_lp2 -> x4_out2; + x4_hp1 -> x4_hp2 -> x4_out3; + } + } + +--- + +.. _df1_mechanics: + +4. Direct Form I Biquad Cascade Implementation Mechanics +******************************************************** + +Each 4th-order Linkwitz-Riley filter is implemented in DSP firmware by cascading two identical 2nd-order Direct Form I (DF1) biquad stages in series. + +Direct Form I Difference Equations +================================== + +For each biquad section, the output is computed via the standard difference equation: + +.. math:: + + y[n] = b_0 x[n] + b_1 x[n-1] + b_2 x[n-2] - a_1 y[n-1] - a_2 y[n-2] + +State Variables & Accumulator Precision +======================================= + +* **Independent Delay States**: Direct Form I stores two input state variables (:math:`x[n-1], x[n-2]`) and two output state variables (:math:`y[n-1], y[n-2]`). For an LR4 filter (two biquads), exactly 4 delay slots are allocated per filter (``CROSSOVER_NUM_DELAYS_LR4 = 4``). +* **64-Bit Internal Accumulation**: All product terms accumulate into a 64-bit register with guard bits before rounding and shifting. This prevents internal overflow and avoids limit cycle oscillations near low-frequency cutoff points. +* **Fixed-Point Formatting**: + - Filter coefficients (:math:`a_1, a_2, b_0, b_1, b_2`) are represented in high-precision :math:`Q2.30` fixed-point format. + - Headroom and gain normalization are controlled via per-section ``output_shift`` and ``output_gain`` (:math:`Q2.14`). + +.. graphviz:: + :caption: Cascaded Biquad Implementation of an LR4 Filter with 64-Bit Accumulation + + digraph biquad_cascade { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_x [label="Audio Input x[n]", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_bq1 { + label="Biquad Stage 1 (2nd-Order Butterworth)"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + bq1_core [label="Direct Form I Engine\nFeedforward (b0, b1, b2)\nFeedback (-a1, -a2)\n64-Bit Accumulator", fillcolor="#BEE3F8", color="#3182CE"]; + } + + subgraph cluster_bq2 { + label="Biquad Stage 2 (2nd-Order Butterworth)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + bq2_core [label="Direct Form I Engine\nIdentical Coefficients\nHeadroom Scaler (out_shift)\n64-Bit Accumulator", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + out_y [label="LR4 Output y[n]\n(24 dB / Octave Slope)", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + in_x -> bq1_core; + bq1_core -> bq2_core [label="Intermediate z[n]"]; + bq2_core -> out_y; + } + +--- + +.. _multisink_topology_ipc4: + +5. Multi-Sink Routing, ALSA Topology 2 & IPC4 Pin Indexing +********************************************************** + +Unlike standard 1-in-1-out audio effect widgets (such as Volume or Equalizer), the Crossover module is an inherently **1-to-N multi-sink stream splitter**: it consumes a single wideband input stream and simultaneously drives multiple independent sink buffers. + +Multi-Sink Buffer Management +============================ + +* **Sink Array (``bsinks[]``)**: The crossover processing function receives an array of output stream buffers corresponding to the number of configured bands (2, 3, or 4). +* **Sink Assignment Vector (``assign_sinks[]``)**: A parameter array maps logical crossover frequency outputs to destination sink pipeline IDs: + + .. code-block:: text + + assign_sinks[0] = 0 # Logical Low band -> Sink Buffer 0 (Woofer Pipeline) + assign_sinks[1] = 1 # Logical High band -> Sink Buffer 1 (Tweeter Pipeline) + +* **Passthrough Fallback Mode**: When ``num_sinks == 1`` or when the component is disabled via ALSA mixer controls, the module operates in passthrough mode (``crossover_default_pass()``), replicating input frames across output buffers with zero filtering overhead. + +IPC4 Dynamic Pin Indexing +========================= + +In SOF IPC4, modules are dynamically bound by connecting source pins to sink pins across independent processing modules. Because the Crossover component produces multiple output pins dynamically, the IPC4 firmware requires upfront knowledge of output pin indices before pipeline instantiation: + +* **Early Initialization Config (``init_config = 1``)**: In ``crossover.toml``, the Crossover module sets ``init_config = 1``, instructing the build system to append the extended base configuration (``base_cfg_ext``) to the module initialization IPC payload. +* **Pin Binding**: This upfront payload informs the IPC4 runtime how many output pins are active, enabling the host driver to bind downstream pipeline widgets directly to individual crossover frequency bands. + +ALSA Topology 2 Integration +=========================== + +The Crossover widget is declared in ALSA Topology 2 configuration files using ``tools/topology/topology2/include/components/crossover.conf``: + +* **Widget Type**: ``effect`` +* **Component UUID**: ``d1:9a:8c:94:6a:80:31:41:ad:6c:b2:bd:a9:e3:5a:9f`` +* **Static ROM Initialization**: Default crossover cutoff frequencies, biquad coefficients, and sink routing maps can be compiled directly into the topology binary (``.bin``), establishing active speaker frequency division immediately upon hardware boot. + +.. graphviz:: + :caption: 1-to-N Multi-Sink Buffer Distribution and ALSA Topology 2 / IPC4 Output Pin Binding + + digraph multisink_binding { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_buf [label="Single Wideband Input Stream\n(Pipeline Buffer)", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_comp { + label="Crossover Splitter Widget (crossover.conf)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + x_eng [label="Crossover Engine (crossover.c)\nLR4 Filter Bank Splitter\nassign_sinks[] Routing Table", fillcolor="#FAF089", color="#B7791F"]; + p_out0 [label="Output Pin 0 (Low Band)", fillcolor="#FFFFFF", color="#B7791F"]; + p_out1 [label="Output Pin 1 (High Band)", fillcolor="#FFFFFF", color="#B7791F"]; + + x_eng -> p_out0; + x_eng -> p_out1; + } + + subgraph cluster_sinks { + label="Downstream Sink Pipelines / DAI Endpoints"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + pipe_w [label="Woofer Pipeline / DAI\n(Smart Amp I2S Channel 0)", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + pipe_t [label="Tweeter Pipeline / DAI\n(Smart Amp I2S Channel 1)", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + in_buf -> x_eng; + p_out0 -> pipe_w [label="IPC4 Pin Binding 0", color="#38A169", style="bold"]; + p_out1 -> pipe_t [label="IPC4 Pin Binding 1", color="#38A169", style="bold"]; + } + +--- + +.. _system_integration_multiband: + +6. System-Level Deployment: Multi-Amp Systems vs Multi-Band DRC +*************************************************************** + +The SOF Crossover engine serves two primary architectural deployment models across audio products: + +Model A: Standalone Multi-Amplifier Loudspeaker Systems +======================================================== + +In high-end laptops, automotive audio, and smart speakers, the Crossover operates as an autonomous 1-to-N stream splitter: + +* The input audio stream is split into discrete bands that exit the DSP through separate digital audio interfaces (e.g. multi-channel SoundWire or TDM I2S). +* Each band is routed to a dedicated physical amplifier chip optimized for that specific driver (e.g. high-current Class-D amplifier for woofers, ultra-low-noise amplifier for tweeters). +* Features per-channel independent volume ramping, limiter protection, and speaker EQ. + +Model B: Embedded Spectral Splitting within Multi-Band DRC +========================================================== + +In compact single-amplifier systems, the Crossover operates as an internal component embedded inside the **Multi-Band Dynamic Range Compressor** (``src/audio/multiband_drc/``): + +* The LR4 crossover filter bank partitions the signal into sub-bands internally without exposing multiple external sink pins. +* Each band is compressed independently by parallel DRC instances to eliminate spectral pumping. +* The bands are recombined into a single wideband output stream delivered to a single shared speaker amplifier. + +.. graphviz:: + :caption: System-Level Acoustic Deployment: Standalone Multi-Amping vs Multi-Band DRC Subsystem + + digraph system_deployment { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_dep_a { + label="Deployment Model A: Standalone Multi-Amping (Multi-Sink Architecture)"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + a_in [label="Media Playback Stream", fillcolor="#FFFFFF", color="#CBD5E0"]; + a_xov [label="Crossover Widget (1-to-N Splitter)\nMultiple Output Pins", fillcolor="#BEE3F8", color="#3182CE"]; + a_amp0 [label="Hardware Amp 0: Woofer", fillcolor="#C6F6D5", color="#38A169"]; + a_amp1 [label="Hardware Amp 1: Tweeter", fillcolor="#C6F6D5", color="#38A169"]; + + a_in -> a_xov; + a_xov -> a_amp0 [label="Low Pin"]; + a_xov -> a_amp1 [label="High Pin"]; + } + + subgraph cluster_dep_b { + label="Deployment Model B: Embedded Crossover in Multi-Band DRC (Single-Sink)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + b_in [label="Media Playback Stream", fillcolor="#FFFFFF", color="#CBD5E0"]; + b_mdrc [label="Multi-Band DRC Widget (multiband_drc.c)\nInternal LR4 Crossover -> Parallel DRCs -> Summation", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + b_amp [label="Single Shared Hardware Amplifier & Speaker", fillcolor="#C6F6D5", color="#38A169"]; + + b_in -> b_mdrc -> b_amp [label="Single Wideband Output"]; + } + } + +--- + +.. _simd_crossover_acceleration: + +7. SIMD Vector Acceleration Across DSP Architectures +**************************************************** + +Processing multi-channel audio through up to 6 LR4 filters (12 cascaded biquads per channel) imposes significant computational demands on embedded DSP cores. + +SOF optimizes the crossover filtering pipeline through dedicated vector implementations: + +* **Cadence Tensilica Xtensa HiFi 3 & HiFi 4**: + + - Vectorized biquad filtering utilizing 64-bit dual multiply-accumulate instructions (``AE_MULAA32RA``). + - Processes multiple audio channels or biquad sections in parallel with hardware saturation. + - Automatic circular delay indexing without scalar pointer branching. + +* **Cadence Tensilica Xtensa HiFi 5**: + + - 8-way 32-bit vector processing engine (256-bit bus) accelerating parallel multi-channel crossover splits. + - Dual 128-bit memory load buses allow simultaneously fetching filter coefficients and audio delay buffers in a single clock cycle. + +* **Generic Portable Scalar C (``crossover_generic.c``)**: + + - Clean, portable scalar C implementations designed for non-Xtensa platforms (e.g. ARM Cortex-M7 on Teensy 4.1, RISC-V on ESP32-P4). + +.. graphviz:: + :caption: SIMD Vector Processing across Hardware Architectures + + digraph simd_crossover { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_gen { + label="Generic Scalar C (crossover_generic.c)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + g_core [label="Portable Scalar C Loop\n1 sample per iteration\nTarget: ARM Cortex-M, RISC-V, Simulator", fillcolor="#FFFFFF", color="#CBD5E0"]; + } + + subgraph cluster_hf3 { + label="Xtensa HiFi 3 / HiFi 4"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + h3_core [label="Dual / Quad 32-bit Vector Engine\nParallel Direct Form I biquads\n64-bit dual MAC instructions", fillcolor="#BEE3F8", color="#3182CE"]; + } + + subgraph cluster_hf5 { + label="Xtensa HiFi 5 (Octa Vector Engine)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + h5_core [label="Octa 32-bit Vector Engine (256-bit bus)\n8 samples processed per cycle\nDual 128-bit memory buses for coefficients & delays", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + g_core -> h3_core [label="2x - 4x Speedup", color="#3182CE"]; + h3_core -> h5_core [label="2x Speedup (8x Total)", color="#38A169", style="bold"]; + } + +--- + +.. _upstream_crossover_references: + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level implementation details, mathematical structures, and tuning scripts: + +* **Upstream Component Specifications**: + - `thesofproject/sof: src/audio/crossover/README.md `_ +* **Crossover Firmware Source Files**: + - ``src/audio/crossover/crossover.c``: Component initialization, multi-sink dispatch, and lifecycle. + - ``src/audio/crossover/crossover.h``: Crossover state definitions (``struct comp_data``) and function map prototypes. + - ``src/audio/crossover/crossover_user.h``: User parameter definitions (``struct sof_crossover_config``). + - ``src/include/module/crossover/crossover_common.h``: Common crossover state definitions (``struct crossover_state``) shared with Multi-Band DRC. + - ``src/audio/crossover/crossover_generic.c``: Portable scalar C splitting implementations (``split_2way``, ``split_3way``, ``split_4way``, and ``lr4_merge``). +* **Topology Definitions**: + - ``tools/topology/topology2/include/components/crossover.conf``: ALSA Topology 2 configuration class for Crossover widgets. +* **MATLAB / Octave Tuning Scripts**: + - ``src/audio/crossover/tune/sof_example_crossover.m``: Interactive script for generating Linkwitz-Riley crossover biquad coefficients across 2-way, 3-way, and 4-way configurations. + - ``src/audio/crossover/tune/sof_crossover_gen_coefs.m``: Low-level coefficient calculation and quantization functions. + +Related Subsystem Architecture Guides +===================================== + +* :ref:`drc_multiband_drc`: Single-band and multi-band dynamic range compression utilizing Linkwitz-Riley crossovers for spectral isolation. +* :ref:`eq_fir_iir`: Finite and Infinite Impulse Response equalizers, linear-phase filtering, and biquad cascades. +* :ref:`volume_module`: Per-channel gain scaling, smooth volume ramping, and zero-crossing muting. +* :ref:`src_asrc`: Sample rate conversion architecture handling fixed and drifting clocks across heterogeneous audio interfaces. +* :ref:`mixin_mixout`: Multi-stream audio mixing and distribution across post-crossover loudspeaker and headphone buses. +* :ref:`module_framework`: The standardized module interface, Source/Sink APIs, and memory sandboxing wrapping Crossover components. +* :ref:`pipeline_architecture`: How Crossover widgets are integrated into directed acyclic audio graphs (DAGs). diff --git a/developer_guides/firmware/dcblock.rst b/developer_guides/firmware/dcblock.rst new file mode 100644 index 00000000..1a9c1ab9 --- /dev/null +++ b/developer_guides/firmware/dcblock.rst @@ -0,0 +1,822 @@ +.. _dcblock: + +DC Blocker Architecture +####################### + +The **DC Blocker** subsystem in Sound Open Firmware removes direct current (0 Hz DC) bias and infrasonic baseline drift from digital audio streams across microphone capture pipelines, loudspeaker playback paths, and non-linear audio processing blocks. + +In mixed-signal hardware and digital signal processing, DC offset is an insidious artifact: analog-to-digital converter (ADC) operational amplifier offsets, PDM microphone decimation leakage, grounding thermal drift, and synthetic non-linear processing algorithms all introduce static DC biases into audio signals. In digital audio pipelines, a DC bias robs signals of fixed-point dynamic range headroom, causes asymmetric waveform clipping, generates audible clicks and pops during stream transitions, threatens moving-coil loudspeaker voice coils with destructive resistive heating, and impairs downstream adaptive algorithms such as acoustic echo cancellers, beamformers, dynamic range compressors, and keyword spotters. + +Sound Open Firmware integrates a dedicated, highly optimized **first-order recursive high-pass DC blocking filter** (:math:`H(z) = \frac{1 - z^{-1}}{1 - R z^{-1}}`) providing complete 0 Hz transmission nulling, mathematically flat passband response across the human audible spectrum, configurable cutoff frequencies, 64-bit fixed-point accumulation, and architecture-specific SIMD vector acceleration across Cadence Tensilica Xtensa HiFi 3, HiFi 4, and HiFi 5 DSPs, alongside portable scalar implementations for ARM Cortex-M and RISC-V cores. + +This guide provides a comprehensive, high-level architectural walkthrough of the DC Blocker subsystem, analyzing the physical origins of DC bias, pole-zero digital filter mechanics, transient step responses, fixed-point precision and limit cycle avoidance, multi-channel stream processing, ALSA Topology 2 / IPC dynamic configuration, and SIMD hardware acceleration without delving into low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +.. _dcblock_origins_hazards: + +1. Physical Origins & Hazards of DC Offset in Audio Systems +*********************************************************** + +Direct current (DC) in audio refers to a constant, non-zero static voltage or digital baseline offset (:math:`0\text{ Hz}`) added to an alternating audio waveform. While humans cannot hear a static 0 Hz offset directly, its presence within digital audio pipelines creates severe acoustic, electrical, and algorithmic degradations. + +Physical and Algorithmic Sources of DC Bias +=========================================== + +DC offset enters digital audio pipelines through both hardware imperfections and non-linear digital algorithms: + +* **ADC Front-End Operational Amplifier Offset**: Real-world analog preamplifiers and delta-sigma ADCs exhibit slight differential transistor mismatches and input bias currents, producing a persistent analog DC voltage that digitizes into a non-zero digital mean value. +* **PDM Digital Microphone Decimation Leakage**: Digital MEMS microphones outputting Pulse Density Modulation (PDM) streams rely on internal sigma-delta modulators. Imperfections in internal integrator feedback loops and decimation sinc filters can pass residual DC offsets into the decimated PCM output. +* **Ground Drift & Thermal Asymmetry**: Single-ended analog inputs, long microphone cables, and uneven chassis heating introduce ground potential shifts and thermal gradients that appear as slow-moving DC wander. +* **Synthetic Non-Linear Audio Algorithms**: Non-linear signal processing operations—such as half-wave rectification in envelope detectors, asymmetric waveshapers, harmonic exciters, and non-linear dynamic bass synthesis—produce non-zero average DC components as an unavoidable mathematical byproduct of harmonic generation. + +Acoustic and Algorithmic Hazards +================================ + +Uncorrected DC offsets inflict severe degradation across both playback and capture pipelines: + +* **Dynamic Range Loss & Asymmetric Clipping**: In fixed-point PCM representation (:math:`Q1.15` or :math:`Q1.31`), signal amplitude is bounded within :math:`[-1.0, +1.0)`. A DC bias shifts the resting baseline away from zero, disproportionately reducing available headroom in one direction. For example, a :math:`+0.1` DC offset reduces positive headroom to :math:`+0.9` (a loss of nearly :math:`1\text{ dB}` of dynamic range). When loud peaks occur, the signal clips asymmetrically, introducing harsh even-order harmonic distortion. +* **Loudspeaker Voice Coil Thermal Destruction**: In playback pipelines, passing DC through a power amplifier into a moving-coil loudspeaker causes a continuous, unvarying electrical current (:math:`I_{dc} = V_{dc} / R_e`) to flow through the voice coil. Because the voice coil cannot radiate 0 Hz acoustic energy into the air, 100% of this electrical power dissipates as resistive heat (:math:`P = I^2 R`). In compact mobile speakers and headphones, continuous DC dissipation rapidly overheats voice coil adhesives, causing voice coil warping, bobbin rubbing, and permanent open-circuit burnout. +* **Permanent Speaker Cone Displacement & Intermodulation Distortion**: DC current generates a static Lorentz force (:math:`F = B \cdot l \cdot I_{dc}`), holding the speaker cone permanently displaced away from its neutral mechanical resting position (:math:`x_{dc} = F / k_s`). In this displaced state, the spider and surround suspensions operate in their non-linear mechanical compliance region. This restricts allowable linear excursion, produces premature bottoming-out, and generates severe intermodulation distortion (IMD) between low-frequency and high-frequency content. +* **Audible Clicks, Pops, and Thumps**: When starting, stopping, pausing, or gating an audio stream with a DC offset, the signal value abruptly steps between zero and the DC level. In the frequency domain, an instantaneous step function generates a wideband acoustic burst, perceived by the user as an annoying and unprofessional click, pop, or low-frequency thump. +* **Downstream DSP Algorithm Corruption**: Modern audio algorithms assume that input signals have zero mean (:math:`E[x] = 0`): + + - **Acoustic Echo Cancellation (AEC) & Beamforming (TDFB)**: Adaptive FIR filters adjust their weights via gradient descent (LMS/NLMS). A static DC offset skews gradient estimates, slows filter convergence, and causes adaptive cancellation filters to diverge. + - **Dynamic Range Compression (DRC)**: Envelope detectors compute signal energy via rectification or squaring. DC bias artificially elevates the measured signal energy, causing the compressor to continuously duck gain even during complete acoustic silence. + - **Voice Activity Detection (VAD) & Keyword Spotters (TFLM)**: Neural networks and energy-based detectors misinterpret DC energy as acoustic voice activity, preventing DSP power islands from entering low-power sleep states. + +.. graphviz:: + :caption: DC Offset Origins and Acoustic / DSP Hazards in Audio Pipelines + + digraph dc_hazards { + bgcolor="transparent"; + rankdir=LR; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_origins { + label="DC Offset Origins"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + adc_bias [label="ADC Preamplifier Offset\n& Transistor Mismatch", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + pdm_leak [label="PDM Digital MEMS\nDecimation Filter Leakage", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + nonlinear [label="Non-Linear Audio Effects\n(Waveshapers / Exciters)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + } + + sum_node [label="Audio Stream\nwith DC Bias\n(Non-Zero Mean)", fillcolor="#C53030", fontcolor="#FFFFFF", shape=ellipse]; + + subgraph cluster_hazards { + label="System Hazards Without DC Blocker"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + headroom [label="Headroom Loss &\nAsymmetric Clipping", fillcolor="#742A2A", fontcolor="#FFFFFF"]; + thermal [label="Voice Coil Thermal Burnout\n(Resistive Heating: P = I²R)", fillcolor="#742A2A", fontcolor="#FFFFFF"]; + excursion [label="Cone Offset Displacement\n& Intermodulation Distortion", fillcolor="#742A2A", fontcolor="#FFFFFF"]; + clicks [label="Audible Pops & Thumps\non Play/Pause/Mute", fillcolor="#742A2A", fontcolor="#FFFFFF"]; + dsp_error [label="AEC Divergence, DRC Ducking\n& False VAD Triggers", fillcolor="#742A2A", fontcolor="#FFFFFF"]; + } + + adc_bias -> sum_node; + pdm_leak -> sum_node; + nonlinear -> sum_node; + + sum_node -> headroom; + sum_node -> thermal; + sum_node -> excursion; + sum_node -> clicks; + sum_node -> dsp_error; + } + +--- + +.. _dcblock_filter_theory: + +2. Digital DC Blocker Filter Theory & Pole-Zero Mechanics +********************************************************* + +Sound Open Firmware eliminates DC bias using a classic, computationally efficient **first-order recursive digital high-pass filter**. + +Difference Equation & Z-Domain Transfer Function +================================================ + +The time-domain difference equation of the DC Blocker filter is expressed as: + +.. math:: + + y[n] = x[n] - x[n-1] + R \cdot y[n-1] + +where: + +* :math:`x[n]` is the current input audio sample. +* :math:`x[n-1]` is the previous input audio sample (feedforward delay). +* :math:`y[n-1]` is the previous filter output sample (feedback recursive delay). +* :math:`R` is the pole radius parameter (:math:`0 < R < 1`, typically :math:`0.98 \le R < 1.0`). +* :math:`y[n]` is the DC-free output audio sample. + +Taking the Z-transform of both sides: + +.. math:: + + Y(z) = X(z) - z^{-1} X(z) + R \cdot z^{-1} Y(z) + +.. math:: + + Y(z)(1 - R z^{-1}) = X(z)(1 - z^{-1}) + +yielding the discrete-time transfer function: + +.. math:: + + H(z) = \frac{Y(z)}{X(z)} = \frac{1 - z^{-1}}{1 - R z^{-1}} = \frac{z - 1}{z - R} + +Pole-Zero Geometry on the Complex Z-Plane +========================================= + +The transfer function reveals an exceptionally elegant geometric placement of poles and zeros: + +* **Transmission Zero at :math:`z = 1`**: The numerator :math:`(z - 1)` places an exact transmission zero on the unit circle at angle :math:`\omega = 0` (:math:`0\text{ Hz}` / DC). Evaluating the frequency response at DC (:math:`z = e^{j 0} = 1`): + + .. math:: + + H(1) = \frac{1 - 1}{1 - R} = 0 \quad (-\infty\text{ dB}) + + This mathematical null guarantees **100% complete rejection of any constant DC bias**. + +* **Stabilizing Pole at :math:`z = R`**: The denominator :math:`(z - R)` places a single pole on the positive real axis at radius :math:`R`. Because :math:`0 < R < 1`, the pole lies strictly inside the unit circle, guaranteeing **Bounded-Input Bounded-Output (BIBO) stability**. + + As frequency :math:`\omega` increases away from DC, the distance from the evaluation point :math:`e^{j \omega}` on the unit circle to the pole at :math:`z=R` rapidly approaches the distance to the zero at :math:`z=1`. The pole effectively cancels out the attenuation of the zero across higher frequencies, restoring the magnitude response back to unity (:math:`0\text{ dB}`). + +Frequency Response & Cutoff Frequency Formulation +================================================= + +At the Nyquist frequency (:math:`z = e^{j \pi} = -1`, corresponding to :math:`f_s / 2`): + +.. math:: + + H(-1) = \frac{1 - (-1)}{1 - R(-1)} = \frac{2}{1 + R} + +Since :math:`R` is very close to :math:`1.0` (for example, :math:`R = 0.995`), :math:`\frac{2}{1 + R} \approx \frac{2}{1.995} \approx 1.0025` (:math:`+0.02\text{ dB}`). Across the vast majority of the audible band (from :math:`\approx 100\text{ Hz}` to :math:`20\text{ kHz}`), the filter behaves as a virtually perfect flat wire with :math:`0\text{ dB}` gain and negligible phase distortion. + +The -3 dB cutoff frequency :math:`f_c` (the frequency at which :math:`|H(e^{j \omega_c})|^2 = \frac{1}{2}`) is derived analytically: + +.. math:: + + \cos\left(\frac{2\pi f_c}{f_s}\right) = \frac{2R}{1 + R^2} + +For values of :math:`R` close to :math:`1.0` and cutoff frequencies much lower than the sampling rate (:math:`f_c \ll f_s`), this relationship simplifies with high accuracy to the first-order approximation: + +.. math:: + + f_c \approx \frac{(1 - R) \cdot f_s}{2\pi} \quad \iff \quad R \approx 1 - \frac{2\pi f_c}{f_s} + +.. graphviz:: + :caption: Z-Domain Pole-Zero Constellation and Normalized Magnitude Frequency Response + + digraph dc_theory { + bgcolor="transparent"; + rankdir=LR; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_zplane { + label="Z-Domain Pole-Zero Constellation"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + unit_circle [label="Unit Circle (|z| = 1)\nStability Boundary", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + zero_dc [label="Transmission Zero at z = 1.0\n(Exact Rejection at 0 Hz / DC)", fillcolor="#C53030", fontcolor="#FFFFFF", shape=ellipse]; + pole_r [label="Stabilizing Pole at z = R\n(0 < R < 1, Real Axis)", fillcolor="#2B6CB0", fontcolor="#FFFFFF", shape=diamond]; + + unit_circle -> zero_dc [label="Placed on boundary"]; + unit_circle -> pole_r [label="Placed inside boundary"]; + } + + subgraph cluster_response { + label="Frequency Magnitude Response |H(f)|"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + dc_notch [label="0 Hz (DC):\n-∞ dB (Infinite Null)", fillcolor="#742A2A", fontcolor="#FFFFFF"]; + fc_point [label="Cutoff fc (-3.01 dB):\nfc ≈ (1 - R)·fs / (2π)", fillcolor="#D69E2E", fontcolor="#FFFFFF"]; + passband [label="Audible Passband (> fc):\nFlat 0.0 dB Unity Gain", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + dc_notch -> fc_point [label="Steep +6 dB/oct roll-off"]; + fc_point -> passband [label="Flattens to unity"]; + } + + zero_dc -> dc_notch [label="Enforces null", style="bold", color="#E53E3E"]; + pole_r -> passband [label="Restores passband gain", style="bold", color="#3182CE"]; + } + +--- + +.. _dcblock_step_response: + +3. Transient Step Response & The Cutoff Frequency Trade-Off +*********************************************************** + +While frequency-domain analysis shows how effectively the DC Blocker suppresses 0 Hz steady-state signals, time-domain transient analysis determines how fast the filter recovers from abrupt DC shifts. + +Time-Domain Step Response +========================= + +When an instantaneous DC offset step of magnitude :math:`\Delta_{dc}` enters the filter at sample :math:`n = 0` (such as during microphone power-on, stream unmuting, or an abrupt analog bias jump), the recursive difference equation produces: + +* At :math:`n = 0`: :math:`y[0] = \Delta_{dc} - 0 + 0 = \Delta_{dc}`. +* At :math:`n = 1`: :math:`y[1] = \Delta_{dc} - \Delta_{dc} + R \cdot y[0] = R \cdot \Delta_{dc}`. +* At :math:`n = 2`: :math:`y[2] = \Delta_{dc} - \Delta_{dc} + R \cdot y[1] = R^2 \cdot \Delta_{dc}`. +* At arbitrary sample :math:`n \ge 1`: + +.. math:: + + y[n] = \Delta_{dc} \cdot R^n + +The filter output decays toward zero along an exponential decay curve governed by the pole radius :math:`R`. + +Exponential Decay Envelope and Time Constant +============================================ + +Expressing the discrete decay in continuous time (:math:`t = n / f_s`): + +.. math:: + + R^n = e^{n \ln R} = e^{-t / \tau} + +The **decay time constant** :math:`\tau` (the duration required for the DC offset to decay to :math:`1/e \approx 36.8\%` of its initial amplitude) is: + +.. math:: + + \tau = -\frac{1}{f_s \ln R} \approx \frac{1}{f_s (1 - R)} \approx \frac{1}{2\pi f_c} + +The **settling time** :math:`t_s` required for the DC offset to decay to less than 1% (-40 dB) of its initial magnitude is approximately :math:`4.6 \cdot \tau`: + +.. math:: + + t_s \approx 4.6 \cdot \tau \approx \frac{4.6}{2\pi f_c} \approx \frac{0.73}{f_c} + +The Fundamental Engineering Dilemma +=================================== + +The relationship :math:`t_s \approx 0.73 / f_c` exposes a fundamental, inescapable engineering compromise in DC blocker design: + +1. **Ultralow Cutoff (:math:`f_c \le 20\text{ Hz}`, :math:`R \ge 0.997` at 48 kHz)**: + + - *Acoustic Advantage*: Preserves deep sub-bass musical reproduction (e.g. pipe organs, kick drums, 5-string bass guitars) with negligible amplitude attenuation and minimal low-frequency phase rotation. + - *Transient Penalty*: Settling time is long (:math:`t_s \approx 37\text{ ms}` at 20 Hz; :math:`t_s \approx 150\text{ ms}` at 5 Hz). When an abrupt DC transient or microphone handling thump occurs, a low-frequency damped transient tail lingers in the audio stream for hundreds of milliseconds. Furthermore, when :math:`R` is exceptionally close to :math:`1.0`, arithmetic truncation errors require 64-bit precision to prevent quantization hum. + +2. **Elevated Cutoff (:math:`f_c \ge 100\text{ Hz}`, :math:`R \le 0.987` at 48 kHz)**: + + - *Acoustic Advantage*: Blisteringly fast transient recovery (:math:`t_s < 7\text{ ms}`). DC offsets, microphone handling clicks, and ADC startup thumps are extinguished almost instantaneously. It also provides beneficial attenuation of infrasonic air conditioning rumble, wind noise, and physical mechanical vibrations. + - *Acoustic Penalty*: Audible roll-off of low-frequency musical bass. While unacceptable for full-range high-fidelity music playback, this response is **ideal for speech capture pipelines, teleconferencing, and voice trigger detection** where human vocal fundamentals lie above 80–100 Hz. + +Standard Configuration Presets +============================== + +Sound Open Firmware provides standard tuning presets configured for common sampling rates (16 kHz and 48 kHz): + +.. list-table:: + :widths: 15 15 15 20 35 + :header-rows: 1 + + * - Cutoff (:math:`f_c`) + - :math:`R` (@ 16 kHz) + - :math:`R` (@ 48 kHz) + - Settling Time (:math:`t_s`) + - Target Deployment Application + * - **20 Hz** + - 0.9922 + - 0.9974 + - ~37 ms + - High-fidelity music playback, studio monitors, audio post-production pipelines. + * - **40 Hz** + - 0.9844 + - 0.9948 + - ~18 ms + - Consumer multimedia playback, laptop speakers with limited bass extension. + * - **80 Hz** + - 0.9691 + - 0.9896 + - ~9 ms + - General communications capture, teleconferencing headsets. + * - **100 Hz** + - 0.9615 + - 0.9870 + - ~7 ms + - Voice assistant capture, keyword spotters, noisy mobile microphones. + * - **150 Hz** + - 0.9431 + - 0.9804 + - ~5 ms + - **SOF Default Preset**: Aggressive rumble suppression and ultra-fast DC settling. + +.. graphviz:: + :caption: Transient Step Response and DC Settling Times Across Filter Radius R + + digraph dc_step { + bgcolor="transparent"; + rankdir=TB; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + step_in [label="Input DC Transient Step (Δdc = +1.0 at n = 0)", fillcolor="#C53030", fontcolor="#FFFFFF"]; + + subgraph cluster_decay { + label="Decay Envelopes: y[n] = Δdc · Rⁿ"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + fast_decay [label="High Cutoff (R = 0.980, fc ≈ 150 Hz)\n• τ ≈ 1.0 ms\n• Settling time ts ≈ 5 ms\n• Rapid recovery; attenuates sub-bass\n• Ideal for Speech & Mic Capture", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + med_decay [label="Medium Cutoff (R = 0.990, fc ≈ 80 Hz)\n• τ ≈ 2.0 ms\n• Settling time ts ≈ 9 ms\n• Balanced voice & communications profile", fillcolor="#3182CE", fontcolor="#FFFFFF"]; + slow_decay [label="Low Cutoff (R = 0.997, fc ≈ 20 Hz)\n• τ ≈ 8.0 ms\n• Settling time ts ≈ 37 ms\n• Preserves full sub-bass musical fidelity\n• Ideal for Hi-Fi Playback", fillcolor="#805AD5", fontcolor="#FFFFFF"]; + } + + step_in -> fast_decay [label="R = 0.980"]; + step_in -> med_decay [label="R = 0.990"]; + step_in -> slow_decay [label="R = 0.997"]; + } + +--- + +.. _dcblock_fixed_point: + +4. Fixed-Point Arithmetic, Precision & Limit Cycle Elimination +************************************************************** + +In textbook floating-point arithmetic, evaluating :math:`y[n] = x[n] - x[n-1] + R \cdot y[n-1]` is straightforward. However, Sound Open Firmware operates predominantly on energy-efficient embedded digital signal processors utilizing fixed-point integer mathematics. Implementing recursive filters with poles close to the unit circle under fixed-point arithmetic introduces severe hazards that require rigorous numerical engineering. + +The Hazards of Fixed-Point Recursion +==================================== + +When the pole radius :math:`R` approaches :math:`1.0` (e.g. :math:`R = 0.997`): + +* **Limit Cycle Oscillations**: In a recursive filter, the product :math:`R \cdot y[n-1]` must be rounded to fit back into the state variable format. If naive truncation (floor) or rounding is applied, the state variable can become trapped in a non-zero repeating state even when the input signal has dropped to absolute zero (:math:`x[n] = 0`). These self-sustaining limit cycles manifest as an audible low-level whine, quantization hum, or persistent phantom DC drift. +* **Coefficient Quantization Drift**: If the coefficient :math:`R` lacks sufficient fractional bit depth, rounding :math:`R` can shift the pole position. Under coarse quantization, an intended :math:`R = 0.999` might round up to :math:`1.0` (turning the filter into a pure integrator that accumulates numerical overflow until saturation) or round down significantly (shifting :math:`f_c` from 20 Hz up to 150 Hz). + +High-Precision Data Path in SOF +=============================== + +To eliminate limit cycles and preserve mathematical precision, SOF implements the DC Blocker using a high-precision fixed-point architecture: + +* **Coefficient Representation (:math:`Q2.30`)**: The coefficient :math:`R` is stored as a 32-bit signed integer in :math:`Q2.30` format (2 integer bits including sign, 30 fractional bits). This yields a fractional quantization resolution of: + + .. math:: + + \Delta Q = 2^{-30} \approx 9.31 \times 10^{-10} + + Unity gain ($1.0$) is defined as `ONE_Q2_30` ($0x40000000 = 1073741824$). This immense fractional depth allows exact placement of poles arbitrarily close to the unit circle without quantization rounding error. + +* **State Variables (:math:`Q1.31`)**: The delay line states `x_prev` (:math:`x[n-1]`) and `y_prev` (:math:`y[n-1]`) are maintained as 32-bit signed integers in :math:`Q1.31` format, matching the DSP native audio sample depth. + +* **64-Bit Multiplication & Accumulation (:math:`Q3.61`)**: + + Multiplying the coefficient :math:`R` (:math:`Q2.30`) by the recursive state :math:`y[n-1]` (:math:`Q1.31`) yields a 64-bit product in :math:`Q3.61` format: + + .. math:: + + \text{Format}(R \cdot y[n-1]) = Q(2 + 1) . (30 + 31) = Q3.61 + +* **Symmetric Rounding and Shifting**: + + To recombine the recursive product with the feedforward difference :math:`(x[n] - x[n-1])`, the 64-bit product is scaled and rounded back to 32-bit resolution. SOF utilizes symmetric rounding (`Q_SHIFT_RND` or `AE_ROUND32F64SSYM`), adding a half-LSB rounding bias (:math:`2^{29}`) before arithmetic right-shifting. This completely eliminates DC bias accumulation and suppresses limit cycle oscillations into inaudibility below :math:`-140\text{ dB}`. + +* **Saturated Clamping**: + + The final output is passed through 32-bit saturation (`sat_int32()`). If transient numerical overshoot occurs, the output smoothly clamps to :math:`[-2^{31}, 2^{31}-1]` rather than wrapping around to the opposite polarity, preventing catastrophic full-scale crackles. + +Passthrough Bypass Mode +======================= + +When the DC Blocker is unconfigured or disabled via ALSA mixer controls, setting :math:`R = \text{ONE\_Q2\_30} = 1.0` transforms the transfer function into: + +.. math:: + + H(z) = \frac{1 - z^{-1}}{1 - 1 \cdot z^{-1}} = 1.0 + +In this state, the recursive pole perfectly cancels the feedforward zero, transforming the filter into a mathematically bit-exact, zero-attenuation passthrough. + +.. graphviz:: + :caption: Fixed-Point Arithmetic Data Path and 64-Bit Intermediate Accumulation + + digraph dc_arithmetic { + bgcolor="transparent"; + rankdir=LR; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + x_in [label="Input Sample x[n]\n(32-bit Q1.31)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + sub_diff [label="Feedforward Difference\nx[n] - x[n-1]\n(64-bit)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + z_x [label="Unit Delay\nx[n-1]\n(struct dcblock_state)", fillcolor="#718096", fontcolor="#FFFFFF"]; + + mul_r [label="64-Bit Multiplier\nR (Q2.30) × y[n-1] (Q1.31)\nProduct: Q3.61", fillcolor="#D69E2E", fontcolor="#FFFFFF"]; + r_coef [label="Pole Radius R\n(32-bit Q2.30)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + z_y [label="Recursive State\ny[n-1]\n(struct dcblock_state)", fillcolor="#718096", fontcolor="#FFFFFF"]; + + acc_sum [label="64-Bit Accumulator\n(Diff + R·y[n-1])", fillcolor="#D69E2E", fontcolor="#FFFFFF"]; + shift_rnd [label="Symmetric Rounding Shift\nQ_SHIFT_RND(61, 31)\n(Eliminates Limit Cycles)", fillcolor="#805AD5", fontcolor="#FFFFFF"]; + sat_out [label="32-Bit Saturation Clamp\nsat_int32()\nOutput y[n]", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + x_in -> sub_diff [label="Positive (+)", color="#3182CE"]; + x_in -> z_x [label="Store state"]; + z_x -> sub_diff [label="Negative (-)", color="#E53E3E"]; + + r_coef -> mul_r; + z_y -> mul_r; + + sub_diff -> acc_sum [label="64-bit diff"]; + mul_r -> acc_sum [label="64-bit prod"]; + + acc_sum -> shift_rnd; + shift_rnd -> sat_out; + sat_out -> z_y [label="Update y[n-1] feedback", style="dashed", color="#38A169"]; + } + +--- + +.. _dcblock_multichannel: + +5. Multi-Channel Processing & Buffer Stream Traversal +***************************************************** + +Audio streams in SOF frequently carry multi-channel audio—ranging from stereo playback (2 channels) up to dense microphone arrays (4, 6, or 8 channels for beamforming and speech recognition). The DC Blocker provides multi-channel stream processing with state isolation. + +Per-Channel Independent State Tracking +====================================== + +Because each physical microphone and audio channel possesses unique analog DC offsets and distinct signal histories, filter state variables must be strictly isolated. Cross-channel state contamination would destroy stereo imaging and introduce cross-channel phase distortion. + +SOF defines dedicated state tracking in private component data: + +.. code-block:: text + + struct comp_data { + struct dcblock_state state[PLATFORM_MAX_CHANNELS]; + int32_t R_coeffs[PLATFORM_MAX_CHANNELS]; + ... + }; + +* `state[ch].x_prev`: Tracks the prior input sample :math:`x[n-1]` independently for channel `ch`. +* `state[ch].y_prev`: Tracks the prior recursive output sample :math:`y[n-1]` independently for channel `ch`. +* `R_coeffs[ch]`: Stores the independent pole coefficient for channel `ch`. This enables **heterogeneous channel configurations**—for example, applying an aggressive :math:`150\text{ Hz}` cutoff on primary voice capture microphones while maintaining a gentle :math:`20\text{ Hz}` cutoff on an acoustic echo cancellation reference loopback channel. + +Interleaved Stream Traversal Mechanics +====================================== + +Audio buffers in SOF are formatted as interleaved PCM frames (:math:`L, R, L, R...` or :math:`C_0, C_1, C_2...`). Processing interleaved multi-channel buffers requires stepping through memory with a channel stride: + +1. **Outer Channel / Inner Frame Loop**: The processing routine iterates across channels :math:`ch \in [0, nch-1]`. For each channel, the filter loads `state[ch].x_prev`, `state[ch].y_prev`, and `R_coeffs[ch]`. +2. **Channel-Strided Pointer Stepping**: Pointers advance across interleaved frames using a stride increment: + + .. math:: + + \text{stride} = nch \times \text{sizeof}(\text{sample}) + +3. **Buffer Wrap Boundary Handling**: To prevent pointer corruption across circular ring buffers, the processing loop checks available non-wrapping frames using `audio_stream_samples_without_wrap()`, process chunks up to the buffer boundary, and then invokes `audio_stream_wrap()` to seamlessly loop pointers back to the buffer base. + +Format Adaptability Across Audio Depths +======================================= + +The DC Blocker supports all standard SOF PCM frame formats via dedicated inner processing routines: + +* **S16_LE (16-bit)**: Samples are loaded and sign-extended by 16 bits to :math:`Q1.31` for filtering, then scaled and saturated back to 16 bits via `sat_int16(Q_SHIFT_RND(y, 31, 15))`. +* **S24_4LE (24-bit in 32-bit container)**: Samples are shifted by 8 bits to :math:`Q1.31`, processed through the 64-bit accumulator, and rounded back to 24 bits with `sat_int24(Q_SHIFT_RND(y, 31, 23))`. +* **S32_LE (32-bit native)**: Samples undergo full 32-bit direct processing with zero bit-depth truncation. + +.. graphviz:: + :caption: Multi-Channel Interleaved Buffer Traversal and Independent State Isolation + + digraph dc_multichannel { + bgcolor="transparent"; + rankdir=TB; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_interleaved_in { + label="Source Stream Buffer (Interleaved Frames)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + in_c0 [label="Frame 0: Ch 0 (Left)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + in_c1 [label="Frame 0: Ch 1 (Right)", fillcolor="#805AD5", fontcolor="#FFFFFF"]; + in_c2 [label="Frame 1: Ch 0 (Left)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + in_c3 [label="Frame 1: Ch 1 (Right)", fillcolor="#805AD5", fontcolor="#FFFFFF"]; + + in_c0 -> in_c1 -> in_c2 -> in_c3 [style="invis"]; + } + + subgraph cluster_states { + label="Component Private Data: Isolated Channel States"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + state_c0 [label="Channel 0 State Structure:\n• x_prev[0], y_prev[0]\n• R_coeffs[0] (fc = 100 Hz)", fillcolor="#2C5282", fontcolor="#FFFFFF"]; + state_c1 [label="Channel 1 State Structure:\n• x_prev[1], y_prev[1]\n• R_coeffs[1] (fc = 100 Hz)", fillcolor="#553C9A", fontcolor="#FFFFFF"]; + } + + subgraph cluster_interleaved_out { + label="Sink Stream Buffer (DC-Free Audio)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + out_c0 [label="Frame 0: Ch 0 Clean", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + out_c1 [label="Frame 0: Ch 1 Clean", fillcolor="#38A169", fontcolor="#FFFFFF"]; + out_c2 [label="Frame 1: Ch 0 Clean", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + out_c3 [label="Frame 1: Ch 1 Clean", fillcolor="#38A169", fontcolor="#FFFFFF"]; + + out_c0 -> out_c1 -> out_c2 -> out_c3 [style="invis"]; + } + + in_c0 -> state_c0 [label="Stride load Ch 0", color="#3182CE"]; + in_c2 -> state_c0 [label="Stride load Ch 0", color="#3182CE"]; + + in_c1 -> state_c1 [label="Stride load Ch 1", color="#805AD5"]; + in_c3 -> state_c1 [label="Stride load Ch 1", color="#805AD5"]; + + state_c0 -> out_c0 [label="Write Ch 0", color="#38A169"]; + state_c0 -> out_c2 [label="Write Ch 0", color="#38A169"]; + + state_c1 -> out_c1 [label="Write Ch 1", color="#38A169"]; + state_c1 -> out_c3 [label="Write Ch 1", color="#38A169"]; + } + +--- + +.. _dcblock_simd: + +6. SIMD Vector Acceleration Across DSP Architectures +**************************************************** + +To achieve ultra-low power consumption and minimize DSP clock cycle consumption (MIPS), Sound Open Firmware implements specialized hardware vector optimizations across multiple DSP architectures. + +Cadence Tensilica Xtensa HiFi 3 Optimization +============================================ + +On Cadence Tensilica Xtensa HiFi 3 DSP cores: + +* **64-Bit Vector Accumulation (`AE_MULF32S_LL`)**: Multiplies the 32-bit :math:`Q2.30` coefficient :math:`R` by the 32-bit :math:`Q1.31` recursive state :math:`y[n-1]` using the lower 32 bits of 64-bit vector registers, generating a 64-bit product in :math:`Q2.62` representation. +* **Vector Subtraction & Addition (`AE_SUB64`, `AE_ADD64S`)**: Performs 64-bit subtraction :math:`(x[n] - x[n-1])` and 64-bit addition in single-cycle operations. +* **Symmetric Rounding (`AE_ROUND32F64SSYM`)**: Symmetrically rounds the 64-bit accumulated result back to 32 bits in a single hardware cycle. +* **Hardware Circular Buffer Addressing (`AE_SETCBEGIN0`, `AE_SETCEND0`)**: Programs the hardware circular address register `CBEGIN0` and `CEND0` with the source buffer boundary. The DSP automatically wraps input read pointers (`AE_L16_XC`, `AE_L32_XC`) in hardware with zero branching overhead. + +Cadence Tensilica Xtensa HiFi 4 Optimization: Dual Circular Registers +===================================================================== + +Cadence Tensilica Xtensa HiFi 4 cores introduce dual independent circular address registers, enabling a higher tier of throughput optimization: + +* **Simultaneous Source and Sink Circular Auto-Wrapping**: + + - Source buffer boundaries are bound to circular register 0 (`AE_SETCBEGIN0`, `AE_SETCEND0`). + - Sink buffer boundaries are bound to circular register 1 (`AE_SETCBEGIN1`, `AE_SETCEND1`). + +* **Branchless Inner Loop Execution**: + + In HiFi 3 or scalar C, the firmware must subdivide execution into chunks bounded by the closest wrap boundary between source and sink buffers. On HiFi 4, hardware automatically wraps both read pointers (`AE_L16_XC`, `AE_L32_XC`) and write pointers (`AE_S16_0_XC1`, `AE_S32_L_XC1`) simultaneously. As a result, the entire buffer of `frames` executes in a **single, unfragmented, branchless loop**, maximizing instruction cache efficiency and minimizing pipeline stalls. + +Xtensa HiFi 5 & Vector SIMD +=========================== + +On Cadence Tensilica Xtensa HiFi 5 cores, 256-bit SIMD registers execute 8 parallel 32-bit fixed-point operations concurrently. In multi-microphone array pipelines (such as 8-channel microphone arrays on smart speakers and conference room bars), HiFi 5 processes all 8 channels simultaneously across vector lanes. + +Portable Generic Scalar C +========================= + +For embedded microcontrollers lacking proprietary DSP extensions—such as the PJRC Teensy 4.1 (ARM Cortex-M7) and Espressif ESP32-P4 (RISC-V)—SOF provides a clean, portable scalar C implementation (`dcblock_generic.c`). The compiler maps the 64-bit accumulation and `Q_SHIFT_RND` macros to native hardware 32-bit multiplier pairs with zero precision loss. + +.. graphviz:: + :caption: SIMD Execution Pipelines on Xtensa HiFi 3, HiFi 4 (Dual Circular Buffers), and Scalar Architectures + + digraph dc_simd { + bgcolor="transparent"; + rankdir=LR; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_hifi3 { + label="Tensilica Xtensa HiFi 3"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + hifi3_circ [label="Circular Source Reg 0\nAE_SETCBEGIN0 / CEND0", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + hifi3_mac [label="Vector MAC Pipeline:\n• AE_MULF32S_LL (Q2.62)\n• AE_ADD64S / AE_SUB64\n• AE_ROUND32F64SSYM", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + hifi3_loop [label="Software Chunk Loop\n(Bounded by sink wrap)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + + hifi3_circ -> hifi3_mac -> hifi3_loop; + } + + subgraph cluster_hifi4 { + label="Tensilica Xtensa HiFi 4 (Dual Circular)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + hifi4_circ [label="Dual Hardware Circular Regs:\n• CBEGIN0: Source Read (AE_L32_XC)\n• CBEGIN1: Sink Write (AE_S32_L_XC1)", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + hifi4_loop [label="Flat Branchless Loop\n(Processes all frames in 1 pass\nwith zero wrap checks)", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + hifi4_circ -> hifi4_loop [label="Hardware auto-wrap"]; + } + + subgraph cluster_generic { + label="Generic Scalar C (ARM / RISC-V)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + scalar_code [label="Standard C Implementation:\n• int64_t 64-bit math\n• Q_SHIFT_RND rounding\n• audio_stream_wrap()", fillcolor="#718096", fontcolor="#FFFFFF"]; + } + } + +--- + +.. _dcblock_pipeline_ipc: + +7. Pipeline Integration, ALSA Topology 2 & IPC Interfaces +********************************************************* + +The DC Blocker component conforms to the standardized Sound Open Firmware **Module Adapter** interface and integrates into audio pipelines defined via ALSA Topology 2. + +ALSA Topology 2 Component Widget +================================ + +In ALSA Topology 2 (`tools/topology/topology2/include/components/dcblock.conf`), the DC Blocker is defined as a specialized processing effect widget: + +* **Widget Class**: `Class.Widget."dcblock"` +* **Widget Type**: `effect` +* **UUID**: `af:ef:09:b8:81:56:b1:42:9e:d6:04:bb:01:2d:d3:84` +* **Pin Configuration**: Exactly 1 input pin (`num_input_pins 1`) and 1 output pin (`num_output_pins 1`). +* **Power Management**: `no_pm "true"` (synchronous in-place audio stream processing without autonomous power gating). + +Topology instantiation is simple and declarative: + +.. code-block:: text + + Object.Widget.dcblock."1" { + index 1 + instance 0 + } + +Module Adapter & LLEXT Runtime Dynamic Linking +============================================== + +The DC Blocker implements the standard `struct module_interface` API: + +* `init`: Allocates private component data (`struct comp_data`), zeroes state delay lines, and creates a `comp_data_blob_handler` for dynamic control configuration. +* `prepare`: Validates that exactly one source buffer and one sink buffer are connected, negotiates frame formats (:math:`S16\_LE`, :math:`S24\_4LE`, or :math:`S32\_LE`), resolves the matching SIMD processing function from `dcblock_fnmap[]`, and extracts initial coefficients from the topology configuration blob. +* `process_audio_stream`: Calls the selected architecture-optimized processing function to transform input frames into DC-free sink audio. +* `reset`: Flushes internal delay line states (`x_prev = 0, y_prev = 0`) to prevent state discontinuities across stream restarts. +* `free`: Releases private memory and frees the blob handler. + +For platforms leveraging modular firmware packaging, the DC Blocker exports a standard Loadable Extension manifest (`SOF_LLEXT_MODULE_MANIFEST("DCBLOCK", ...)`), enabling dynamic loading into DSP SRAM on demand. + +Dynamic IPC Configuration Blobs (IPC3 & IPC4) +============================================= + +Cutoff frequencies can be updated dynamically at runtime without interrupting active audio playback or capture: + +* **IPC3**: Delivered via `SOF_IPC_COMP_SET_DATA` carrying a serialized binary configuration payload. +* **IPC4**: Delivered via `SET_LARGE_CONFIG` messages using the standard multi-fragment data blob protocol. The `comp_data_blob_handler` handles fragment reassembly, bounds validation, and atomic pointer assignment to `cd->config`. + +End-to-End Pipeline Deployments +=============================== + +The DC Blocker occupies critical strategic positions across SOF audio processing graphs: + +1. **Capture Pipeline (Microphone Ingestion)**: Positioned immediately after the hardware DAI Copier or PDM Receiver. Removing ADC DC offset before the signal reaches downstream processing prevents divergence in Acoustic Echo Cancellation (AEC), eliminates false energy triggers in Voice Activity Detectors (VAD), and stabilizes beamforming weights in the Time-Domain Fixed Beamformer (TDFB). +2. **Playback Pipeline (Amplifier & Driver Protection)**: Positioned before Volume Control, Dynamic Range Compression (DRC), and Smart Amp. Suppressing DC offsets protects speaker voice coils against thermal burning, prevents cone resting displacement, maximizes positive/negative dynamic headroom, and eliminates pops during play/pause transitions. +3. **Inter-Stage DC Decoupling**: Placed downstream of non-linear DSP algorithms (such as harmonic exciters, waveshapers, or soft clippers) to strip away artificial DC biases generated by non-linear distortion. + +.. graphviz:: + :caption: System Pipeline Topology: Capture Path Pre-Processing and Playback Protection Deployments + + digraph dc_pipeline { + bgcolor="transparent"; + rankdir=TB; + node [fontname="Helvetica", fontsize=10, shape=box, style="filled,rounded", color="#4A5568", penwidth=1.5]; + edge [fontname="Helvetica", fontsize=9, color="#A0AEC0", penwidth=1.2]; + + subgraph cluster_capture { + label="Capture Pipeline (Microphone Ingestion & Pre-Processing)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + pdm_mic [label="PDM Digital Mics /\nAnalog ADC Front-End", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + copier_rx [label="DAI Copier (RX)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + dcb_cap [label="DC Blocker\n(fc = 100 Hz / 150 Hz)\n• Strips ADC offset\n• Rejects wind/handling", fillcolor="#C53030", fontcolor="#FFFFFF"]; + aec [label="Acoustic Echo Canceller\n(AEC)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + tdfb [label="Beamformer\n(TDFB)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + vad [label="Voice Activity Detector\n& Keyword Spotter (TFLM)", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + pdm_mic -> copier_rx -> dcb_cap; + dcb_cap -> aec [label="Zero-mean audio"]; + aec -> tdfb -> vad; + } + + subgraph cluster_playback { + label="Playback Pipeline (Amplifier & Transducer Protection)"; + color="#E2E8F0"; + style="dashed,rounded"; + fillcolor="#2D3748"; + fontname="Helvetica"; + fontsize=11; + fontcolor="#CBD5E0"; + + host_tx [label="Host Audio Stream\n(Decoder / Media Stream)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + dcb_play [label="DC Blocker\n(fc = 20 Hz / 40 Hz)\n• Preserves sub-bass\n• Prevents voice coil heat", fillcolor="#C53030", fontcolor="#FFFFFF"]; + eq [label="Equalizer\n(EQ FIR / IIR)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + drc [label="Dynamic Range\nCompressor (DRC)", fillcolor="#2B6CB0", fontcolor="#FFFFFF"]; + smart_amp [label="Smart Amp /\nDAI Copier (TX)", fillcolor="#4A5568", fontcolor="#FFFFFF"]; + speaker [label="Loudspeaker Driver\n(Zero DC Current / P=0W)", fillcolor="#2F855A", fontcolor="#FFFFFF"]; + + host_tx -> dcb_play; + dcb_play -> eq -> drc -> smart_amp -> speaker; + } + } + +--- + +.. _dcblock_tuning_references: + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level implementation details, mathematical tuning scripts, and topology configurations: + +* **Upstream Component Source Files**: + + - `thesofproject/sof: src/audio/dcblock/README.md `_: Component overview, directory layout, and architecture summary. + - `src/audio/dcblock/dcblock.c `_: Module lifecycle management (`init`, `prepare`, `process`, `reset`, `free`). + - `src/audio/dcblock/dcblock.h `_: Component private data structures (`struct dcblock_state`, `struct comp_data`), format map dispatch, and function declarations. + - `src/audio/dcblock/dcblock_generic.c `_: Portable scalar C fixed-point implementation with 64-bit accumulation and symmetric rounding. + - `src/audio/dcblock/dcblock_hifi3.c `_: Xtensa HiFi 3 SIMD vector optimizations and single circular source addressing. + - `src/audio/dcblock/dcblock_hifi4.c `_: Xtensa HiFi 4 optimizations featuring simultaneous dual circular buffer registers for branchless streaming. + - `src/audio/dcblock/dcblock_ipc3.c `_ & `dcblock_ipc4.c `_: Protocol-specific IPC handlers and stream parameter negotiation. + +* **Topology Definitions**: + + - `tools/topology/topology2/include/components/dcblock.conf `_: ALSA Topology 2 class definition for the DC Blocker widget. + +* **GNU Octave / MATLAB Tuning Scripts**: + + - `src/audio/dcblock/tune/sof_example_dcblock.m `_: Interactive script calculating optimal :math:`R` coefficients for target cutoff frequencies (20–200 Hz across 16 kHz and 48 kHz rates), exporting topology `.conf`, `.m4`, and binary `.bin` configuration blobs. + - `src/audio/dcblock/tune/sof_dcblock_plot_transferfn.m `_: Evaluates and plots the filter frequency magnitude transfer function :math:`H(z)`. + - `src/audio/dcblock/tune/sof_dcblock_plot_stepfn.m `_: Simulates and plots the transient time-domain step response to verify settling time and decay envelopes. + +Related Subsystem Architecture Guides +===================================== + +* :ref:`crossover`: Linkwitz-Riley 4th-order multi-way frequency division across active loudspeaker drivers. +* :ref:`drc_multiband_drc`: Dynamic range compression, soft knee limiting, lookahead delays, and multi-band dynamics control. +* :ref:`eq_fir_iir`: Finite and Infinite Impulse Response equalizers, cascaded biquads, and parametric speaker compensation. +* :ref:`volume_module`: Per-channel gain scaling, smooth volume ramping, and zero-crossing detection. +* :ref:`module_framework`: Standardized module lifecycle, Source/Sink APIs, and memory management. +* :ref:`pipeline_architecture`: How processing modules interconnect into directed acyclic audio graphs. diff --git a/developer_guides/firmware/drc_multiband_drc.rst b/developer_guides/firmware/drc_multiband_drc.rst new file mode 100644 index 00000000..96df83ea --- /dev/null +++ b/developer_guides/firmware/drc_multiband_drc.rst @@ -0,0 +1,577 @@ +.. _drc_multiband_drc: + +Dynamic Range Compression Architecture (DRC & Multi-Band DRC) +############################################################# + +The **Dynamic Range Compression** subsystem in Sound Open Firmware provides real-time acoustic loudness management, speaker excursion protection, dialogue intelligibility enhancement, and audio leveling across heterogeneous playback and capture streams. + +Audio signals in real-world environments present extreme dynamic variations: whisper-quiet dialogue alternating with deafening explosions in movie soundtracks, wide acoustic swings in digital microphone voice capture, and high-energy bass peaks that overdrive compact micro-speaker diaphragms. Without dynamic management, high-amplitude transients cause severe acoustic distortion, amplifier clipping, and voice coil thermal damage, while low-amplitude nuances remain inaudible. + +SOF addresses these dynamics through two specialized, complementary components: + +1. **Dynamic Range Compressor (DRC)** (``src/audio/drc/``): A full-featured single-band compressor featuring lookahead pre-delay buffering, quadratic soft-knee smoothing, adaptive multi-segment release ballistics, and division-based sub-block envelope processing. +2. **Multi-Band Dynamic Range Compressor (Multi-Band DRC)** (``src/audio/multiband_drc/``): A compound multi-stage processing component that splits the audio spectrum into 2, 3, or 4 discrete frequency bands using Linkwitz-Riley 4th-order (LR4) crossover filters, compresses each band independently to eliminate spectral pumping, and recombines the bands through emphasis and de-emphasis equalization. + +This guide provides a comprehensive, high-level architectural walkthrough of single-band DRC, lookahead mechanics, envelope ballistics, multi-band Linkwitz-Riley splitting, dynamic IPC configuration, and SIMD hardware acceleration without delving into low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +.. _drc_principles: + +1. Dynamic Range Compression in Audio Systems +********************************************* + +Dynamic range compression narrows the span between the quietest and loudest portions of an audio signal. Unlike static gain or volume scaling, compression is an active, level-dependent non-linear operation: low-level signals pass through unmodified (or amplified), while signals exceeding a predetermined threshold are attenuated according to a mathematical transfer function. + +Core Audio Use Cases in SOF +=========================== + +* **Micro-Speaker Protection & Excursion Limiting**: Compact transducers in laptops, smartphones, and monitors have strict physical excursion limits. High-energy low-frequency bursts can force the voice coil beyond its linear magnetic gap, causing harsh bottoming-out distortion or permanent mechanical failure. DRC applies peak limiting and compression to tame dangerous transients. +* **Speech Intelligibility & Dialogue Leveling**: In movies, podcasts, and teleconferencing, listeners frequently struggle to hear soft voices without cranking the volume—only to be overwhelmed when sound effects or loud participants speak. DRC compresses peak levels and applies makeup gain to lift quiet speech into an audible, comfortable zone. +* **Microphone Voice Capture Dynamics**: Digital and analog microphones capture signals ranging from soft ambient whispers to loud vocal shouts. DRC prevents analog-to-digital converter (ADC) saturation and clipping while maintaining consistent speech levels for automatic speech recognition (ASR) engines. + +Static Transfer Characteristic & Parameters +=========================================== + +The static compression curve defines the relationship between input level (:math:`X_{\text{dB}}`) and output level (:math:`Y_{\text{dB}}`): + +1. **Threshold (:math:`T_{\text{dB}}`)**: The input level above which compression begins. Below the threshold, the transfer function has a 1:1 slope (linear unity gain). +2. **Soft Knee (:math:`W_{\text{dB}}`)**: A smooth transition region surrounding the threshold. Rather than transitioning abruptly from unity gain to compression (a "hard knee"), SOF employs a quadratic polynomial curve over a knee width of :math:`W_{\text{dB}}`. This eliminates sharp slope discontinuities that produce audible harmonic distortion. +3. **Compression Ratio (:math:`R:1`)**: The degree of attenuation applied to signals above the knee. A ratio of :math:`4:1` means that for every 4 dB increase in input level above the threshold, the output level only increases by 1 dB (slope :math:`1/R = 0.25`). Very high ratios (e.g. :math:`20:1` to :math:`\infty:1`) configure the compressor as a brickwall limiter. +4. **Main Makeup Gain**: Post-compression linear amplification applied to restore overall average loudness lost during peak reduction. + +.. graphviz:: + :caption: Static Dynamic Range Compression Transfer Function: Threshold, Soft Knee, Ratio, and Makeup Gain + + digraph drc_curve { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_regions { + label="Compression Characteristic Curve (Input dB vs Output dB)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + reg_lin [label="Linear Region (Below Threshold)\nInput < Threshold\nSlope = 1:1 (Unity Gain, No Compression)", fillcolor="#FFFFFF", color="#CBD5E0"]; + reg_knee [label="Soft Knee Region (Threshold ± Knee/2)\nQuadratic Spline Interpolation\nSmooth parabolic transition, zero slope discontinuity", fillcolor="#FEFCBF", color="#D69E2E", fontcolor="#744210"]; + reg_comp [label="Compressed Region (Above Knee)\nInput > Threshold + Knee/2\nSlope = 1 / Ratio (e.g. 4:1 or 20:1 Limiting)", fillcolor="#BEE3F8", color="#3182CE"]; + reg_gain [label="Main Makeup Gain\nPost-compression linear amplification\nRestores perceived audio loudness", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + reg_lin -> reg_knee -> reg_comp -> reg_gain; + } + } + +--- + +.. _single_band_drc_architecture: + +2. Single-Band DRC Processing Architecture +****************************************** + +The single-band DRC component (``src/audio/drc/drc.c``) decouples audio streaming from level detection by utilizing a dedicated sidechain detector path and a lookahead pre-delay buffer. + +Signal Path vs Sidechain Detector Path +====================================== + +The compressor splits incoming audio into two parallel branches: + +1. **The Signal Path**: Carries the audio samples that will eventually be delivered to the output. These samples pass through a circular lookahead pre-delay buffer before being scaled by the calculated compressor gain. +2. **The Sidechain Detector Path**: Analyzes the instantaneous amplitude of the audio signal, evaluates peak and RMS signal envelopes, maps levels through the static compression curve, and calculates the target attenuation. + +Lookahead Pre-Delay Buffering +============================= + +A fundamental challenge in dynamic range compression is that loud acoustic transients (such as the initial crack of a snare drum or gun shot) rise in a fraction of a millisecond. If the compressor only reacts after detecting the transient, the leading edge of the burst leaks through unattenuated, causing amplifier clipping: + +* **Pre-Delay Circular Buffer (``pre_delay_buffers``)**: SOF introduces a small, configurable delay into the signal path (up to 512 frames, typically 5 to 10 ms at 48 kHz). +* **Transient Anticipation**: Because the sidechain detector inspects incoming samples before they exit the pre-delay buffer, the envelope generator begins ramping down compressor gain *before* the transient peak reaches the output gain multiplier. +* **Overshoot Prevention**: Transient peaks are smoothly captured and compressed without requiring harsh, zero-attack brickwall clipping. + +Division-Based Sub-Block Processing +=================================== + +Calculating logarithmic decibel conversions, exponential envelope decay curves, and quadratic knee formulas for every single audio sample would impose prohibitive MIPS overhead on embedded DSP cores. + +SOF optimizes this via **Division-Based Processing**: + +* **Division Frames (``DRC_DIVISION_FRAMES = 32``)**: Heavy envelope calculations (such as target gain and exponential attack/release rates) execute once every 32 audio frames (~0.67 ms at 48 kHz). +* **Sample-by-Sample Linear Interpolation**: Across the 32 frames of each division, the compressor applies smooth linear interpolation between the current gain and the target gain, delivering artifact-free volume modulation with minimal processing overhead. + +.. graphviz:: + :caption: Single-Band DRC Processing Architecture: Lookahead Pre-Delay, Sidechain Detector, and Envelope Gain Application + + digraph drc_architecture { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_pcm [label="Input Audio x[n]\n(From Pipeline Buffer)", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_sidechain { + label="Sidechain Detector & Gain Computer Path"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + det_peak [label="Peak / Envelope Detector\ndrc_update_detector_average()\nEvaluates signal energy in dB", fillcolor="#FAF089", color="#B7791F"]; + det_calc [label="Compression Curve & Knee\nEvaluates Threshold, Knee, Ratio\nDetermines Target Gain (Q2.30)", fillcolor="#FAF089", color="#B7791F"]; + det_ball [label="Ballistics Generator (Division)\ndrc_update_envelope() (Every 32 Frames)\nComputes Attack / Adaptive Release Rate", fillcolor="#FAF089", color="#B7791F"]; + + det_peak -> det_calc -> det_ball; + } + + subgraph cluster_signal { + label="Delayed Audio Signal Path"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + p_delay [label="Lookahead Pre-Delay Buffer\npre_delay_buffers[ch] (up to 512 frames)\nAnticipates incoming transients", fillcolor="#BEE3F8", color="#3182CE"]; + } + + vca_gain [label="Gain Multiplier (VCA)\ndrc_compress_output()\nSmooth interpolated sample scaling", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + out_pcm [label="Compressed Audio y[n]\n(Zero Transient Overshoot)", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + in_pcm -> p_delay; + in_pcm -> det_peak; + det_ball -> vca_gain [label="Interpolated Gain", color="#D69E2E", style="bold"]; + p_delay -> vca_gain [label="Delayed Audio"]; + vca_gain -> out_pcm; + } + +--- + +.. _envelope_ballistics: + +3. Envelope Ballistics & Adaptive Release Mechanics +*************************************************** + +The dynamic response of a compressor over time is governed by its **ballistics**: how quickly it attenuates the signal when a loud sound occurs (**Attack**), and how smoothly it restores gain once the loud sound ceases (**Release**). + +Attack Ballistics (Transient Capture) +===================================== + +* **Attack Time**: The duration required for the compressor to apply gain reduction after the input crosses above the threshold. +* **Fast Response**: Attack times are typically fast (1 ms to 10 ms) to prevent high-amplitude peaks from damaging speaker hardware or clipping downstream DACs. +* **Logarithmic Envelope Tracking**: Gain reduction follows an exponential decay towards the target attenuation, ensuring rapid initial clamping. + +The Pitfalls of Conventional Static Release +=========================================== + +Selecting a static release time constant involves a severe compromise: + +* **If Release is Too Fast**: Following a bass note or vocal peak, the gain recovers so rapidly that it amplifies the low-frequency waveform cycles themselves, introducing severe harmonic distortion and audible "breathing" or noise-pumping artifacts. +* **If Release is Too Slow**: A single brief snare drum crack causes the entire audio track to drop in volume and remain suppressed for hundreds of milliseconds, creating a sluggish, muffled presentation. + +SOF Adaptive Multi-Segment Release Curve +======================================== + +SOF addresses this challenge by implementing an **adaptive non-linear release curve** governed by parameterized polynomial coefficients (:math:`kA, kB, kC, kD, kE`): + +1. **Short-Duration Transients**: If a loud peak lasts only a few milliseconds, the release curve executes a rapid recovery, instantly restoring natural volume without sluggishness. +2. **Sustained Loud Passages**: If the audio signal remains consistently loud over an extended period, the compressor smoothly transitions into a slower, gentler release mode. This prevents rapid gain fluctuations across low-frequency cycles, eliminating distortion while maintaining transparent acoustic leveling. + +.. graphviz:: + :caption: Envelope Ballistics: Fast Attack Transient Protection vs Adaptive Non-Linear Release Recovery + + digraph ballistics { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_attack { + label="Attack Phase (Transient Onset)"; + style="filled,rounded"; + fillcolor="#FED7D7"; + color="#E53E3E"; + + atk_det [label="Signal Crosses Above Threshold\nImmediate transient detection via lookahead", fillcolor="#FFFFFF", color="#CBD5E0"]; + atk_drp [label="Rapid Gain Attenuation (1 - 10 ms)\nSuppresses peak energy before speaker overload", fillcolor="#FED7D7", color="#E53E3E", fontcolor="#742A2A"]; + + atk_det -> atk_drp; + } + + subgraph cluster_release { + label="Adaptive Release Phase (Post-Transient Recovery)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + rel_eval [label="Adaptive Release Evaluator (kA, kB, kC, kD, kE)\nMeasures duration and depth of gain compression", fillcolor="#FFFFFF", color="#CBD5E0"]; + rel_fast [label="Fast Release Branch\nShort transient burst -> Rapid gain recovery\nPrevents muffled audio and restores clarity", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + rel_slow [label="Slow Release Branch\nSustained loud passage -> Gentle smooth recovery\nEliminates harmonic distortion & breathing artifacts", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + + rel_eval -> rel_fast [label="Short Peak"]; + rel_eval -> rel_slow [label="Sustained Passage"]; + } + + atk_drp -> rel_eval [label="Signal Drops Below Threshold", color="#4A5568", style="dashed"]; + } + +--- + +.. _multiband_drc_paradigm: + +4. The Multi-Band DRC Paradigm & Spectral Pumping Elimination +************************************************************* + +While single-band DRC provides effective dynamics control for speech and narrow-band sources, wideband complex audio (such as contemporary music, gaming, and cinematic soundtracks) reveals its inherent limitation: **Spectral Pumping**. + +The Spectral Pumping Hazard +=========================== + +In a single-band compressor, gain reduction is governed by the total wideband signal energy: + +* In almost all acoustic genres, low-frequency sounds (bass guitars, kick drums, synthetic sub-bass) carry vastly more physical energy than mid-frequency vocals or high-frequency cymbals. +* When a heavy kick drum hits, the single-band detector detects a massive energy surge and aggressively attenuates the compressor gain across the entire audio spectrum. +* Consequently, the mid-range vocals and high-frequency hi-hats are audibly "ducked" and dragged down in volume with every bass drum hit. This unmusical breathing effect is known as **spectral pumping**. + +The Multi-Band Solution +======================= + +Multi-Band Dynamic Range Compression (``src/audio/multiband_drc/``) eliminates spectral pumping by partitioning the continuous audio spectrum into distinct, isolated frequency bands: + +1. **Acoustic Isolation**: The low-frequency bass energy is separated from mid-frequency vocals and high-frequency cymbals. +2. **Independent Compressor Engines**: Each band processes audio through its own dedicated DRC instance with specialized parameter tuning: + - **Low Band (Bass)**: Configured with a low threshold, high ratio, and fast attack to clamp speaker-damaging diaphragm excursions. + - **Mid Band (Vocals & Instruments)**: Configured with a gentle ratio and transparent release to lift dialogue without altering musical warmth. + - **High Band (Treble & Cymbals)**: Tuned as a fast limiter/de-esser to eliminate harsh sibilance without affecting midrange presence. +3. **Transparent Recombination**: The independently compressed bands are mixed together, preserving full dynamic punch and vocal clarity simultaneously. + +.. graphviz:: + :caption: The Spectral Pumping Hazard: Wideband Compression Ducking vs Multi-Band Frequency Isolation + + digraph spectral_pumping { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_single { + label="Single-Band Compressor (Spectral Pumping Hazard)"; + style="filled,rounded"; + fillcolor="#FED7D7"; + color="#E53E3E"; + + s_in [label="Input: Heavy Bass Drum + Quiet Vocal + High Cymbals", fillcolor="#FFFFFF", color="#CBD5E0"]; + s_det [label="Wideband Energy Detector\nDominated by massive low-frequency bass energy", fillcolor="#FAF089", color="#B7791F"]; + s_gain [label="Single Wideband Gain Attenuation\nPulls down ENTIRE audio spectrum", fillcolor="#FED7D7", color="#E53E3E", fontcolor="#742A2A"]; + s_out [label="Output: Vocal and cymbals audibly duck and pump with each bass kick", fillcolor="#FFFFFF", color="#E53E3E"]; + + s_in -> s_det -> s_gain -> s_out; + } + + subgraph cluster_multi { + label="Multi-Band Compressor (Isolated Dynamic Control)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + m_in [label="Input: Heavy Bass Drum + Quiet Vocal + High Cymbals", fillcolor="#FFFFFF", color="#CBD5E0"]; + m_split [label="Linkwitz-Riley (LR4) Crossover Splitter\nSeparates Bass, Mids, and Highs into isolated paths", fillcolor="#BEE3F8", color="#3182CE"]; + m_b0 [label="Low Band DRC\nTames heavy bass excursion", fillcolor="#C6F6D5", color="#38A169"]; + m_b1 [label="Mid Band DRC\nPreserves crystal clear vocals (No Ducking)", fillcolor="#C6F6D5", color="#38A169"]; + m_b2 [label="High Band DRC\nTames harsh cymbal sibilance", fillcolor="#C6F6D5", color="#38A169"]; + m_sum [label="Output Summation\nNatural, punchy, uncompromised audio reproduction", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + m_in -> m_split; + m_split -> m_b0 -> m_sum; + m_split -> m_b1 -> m_sum; + m_split -> m_b2 -> m_sum; + } + } + +--- + +.. _multiband_drc_pipeline: + +5. Multi-Band DRC Compound Pipeline Architecture +************************************************ + +The Multi-Band DRC component (``src/audio/multiband_drc/multiband_drc.c``) is structured as a **compound 4-stage processing pipeline**: + +Stage 1: Emphasis Equalizer (Pre-Filter) +======================================== + +Before splitting the signal into frequency bands, audio passes through an **Emphasis Equalizer** consisting of two cascaded IIR biquad filters: + +* Shapes the spectral distribution to compensate for frequency-dependent acoustic anomalies in the physical enclosure. +* Pre-conditions the signal to optimize crossover splitting efficiency. +* Can be bypassed (set to neutral passthrough) if external upstream equalization is present. + +Stage 2: Linkwitz-Riley 4th-Order (LR4) Crossover Bank +====================================================== + +The audio spectrum is split into 2, 3, or 4 discrete bands using a **Linkwitz-Riley 4th-order (LR4)** crossover filter bank: + +* **Acoustic Summation Perfection**: An LR4 crossover is formed by cascading two 2nd-order Butterworth filters. At the crossover frequency :math:`f_c`, both the low-pass and high-pass branches are attenuated by exactly :math:`-6\text{ dB}`, resulting in a perfectly flat combined magnitude response (:math:`0\text{ dB}`) upon summation. +* **Zero Phase Difference**: The low-pass and high-pass outputs are strictly in phase (:math:`0^\circ` or :math:`360^\circ` phase difference) across the transition band, completely eliminating destructive comb filtering, phase cancellation notches, or acoustic lobing. + +Stage 3: Parallel Independent DRC Engines +========================================= + +Each frequency band feeds an independent instance of the single-band DRC engine: + +* Each band maintains its own lookahead pre-delay buffer, threshold, knee, ratio, attack time, and adaptive release curves. +* Bands operate in parallel, independently modulating their respective frequency slices. + +Stage 4: Summation & De-Emphasis Equalizer +========================================== + +The outputs of the parallel DRC engines are summed sample-by-sample and routed through a **De-Emphasis Equalizer**: + +* A 2-biquad IIR filter network that mirrors the pre-emphasis curve, restoring the overall tonal balance. +* Delivers a single, cohesive, high-dynamic output stream to downstream audio endpoints. + +.. graphviz:: + :caption: Multi-Band DRC Compound Pipeline Architecture: Emphasis, LR4 Crossover, Parallel DRC Engines, Summation, and De-Emphasis + + digraph multiband_pipeline { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_audio [label="Input Stream x[n]\n(Single Source)", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_emp { + label="Stage 1: Emphasis EQ"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + eq_emp [label="Emphasis Equalizer\n2-Biquad IIR Cascade\nSpectral Pre-Conditioning", fillcolor="#BEE3F8", color="#3182CE"]; + } + + subgraph cluster_xover { + label="Stage 2: LR4 Crossover Splitter"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + xo_bank [label="Linkwitz-Riley (LR4) Bank\nCascaded Butterworth pairs\nFlat 0 dB magnitude sum\nZero inter-band phase error", fillcolor="#FAF089", color="#B7791F"]; + } + + subgraph cluster_drcs { + label="Stage 3: Parallel DRC Band Engines"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + drc_b0 [label="Band 0 DRC (Lows / Bass)\nLookahead + High Ratio\nExcursion Protection", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + drc_b1 [label="Band 1 DRC (Midrange)\nGentle Ratio + Soft Knee\nVocal Transparency", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + drc_b2 [label="Band 2 DRC (Highs / Treble)\nFast Limiting & De-Esser\nHigh-Frequency Smoothing", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + drc_b0 -> drc_b1 -> drc_b2 [style="invis"]; + } + + sum_node [label="Sample-by-Sample\nBand Summation (+)", fillcolor="#EDF2F7", color="#A0AEC0"]; + + subgraph cluster_deemp { + label="Stage 4: De-Emphasis EQ"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + eq_deemp [label="De-Emphasis Equalizer\n2-Biquad IIR Cascade\nTonal Balance Restoration", fillcolor="#BEE3F8", color="#3182CE"]; + } + + out_audio [label="Output Stream y[n]\n(Single Sink)", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + in_audio -> eq_emp -> xo_bank; + xo_bank -> drc_b0 [label="Low Band"]; + xo_bank -> drc_b1 [label="Mid Band"]; + xo_bank -> drc_b2 [label="High Band"]; + drc_b0 -> sum_node; + drc_b1 -> sum_node; + drc_b2 -> sum_node; + sum_node -> eq_deemp -> out_audio; + } + +--- + +.. _drc_dynamic_updates_topology: + +6. Dynamic Parameter Updates & ALSA Topology 2 Integration +********************************************************** + +Dynamic range compressors must accommodate runtime adjustments from host applications, such as switching between "Movie", "Night Mode", and "Voice" audio presets in userspace sound managers. + +Component Configuration Blobs +============================= + +Compressor parameters are packaged into serialized binary blobs managed by the ``comp_data_blob_handler`` framework: + +* **Single-Band DRC Config (``struct sof_drc_config``)**: Contains the single-band threshold, knee width, compression ratio, lookahead pre-delay time, division frames, and adaptive release coefficients (:math:`kA` through :math:`kE`). +* **Multi-Band DRC Config (``struct sof_multiband_drc_config``)**: A compound configuration structure encompassing the number of active bands (up to 4), emphasis/de-emphasis biquad coefficients, Linkwitz-Riley crossover biquad coefficients, and an array of independent DRC parameter blocks (one for each active frequency band). +* **Multi-Packet Staging**: Large multi-band configuration blobs exceeding a single IPC mailbox window are transparently reassembled in memory before being validated and applied atomically between audio periods. + +ALSA Topology 2 Integration +=========================== + +Both DRC components are declared as native audio effect widgets in ALSA Topology 2: + +* **Single-Band DRC Widget (``drc.conf``)**: + + - Widget Type: ``effect`` + - UUID: ``da:e4:6e:b3:6f:00:f9:47:a0:6d:fe:cb:e2:d8:b6:ce`` + - Control Binding: Features an ALSA mixer switch control (control index 0) allowing userspace to dynamically enable or bypass the compressor (``drc_default_pass()``). + +* **Multi-Band DRC Widget (``multiband_drc.conf``)**: + + - Widget Type: ``effect`` + - UUID: ``56:22:9f:0d:4f:8e:b3:47:84:48:23:9a:33:4f:11:91`` + - Control Binding: Exposes switch controls for global bypass (``multiband_drc_default_pass()``) and crossover configuration. + +.. graphviz:: + :caption: Dynamic IPC Configuration Blob Handling, Parameter Staging, and ALSA Topology 2 Integration + + digraph drc_topology_flow { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + host_drv [label="Host ALSA Driver / Userspace Audio Server\nSends DRC / Multi-Band Preset via IPC", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_blob { + label="Blob Handler & Parameter Staging"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + b_rx [label="comp_data_blob_handler\nFragment Reassembly & Offset Tracking", fillcolor="#FAF089", color="#B7791F"]; + b_val [label="Parameter Validation Hook\nCheck Band Counts (<= 4), Buffer Limits & Taps\nVerify Stability of Crossover & EQ Biquads", fillcolor="#FAF089", color="#B7791F"]; + b_swap [label="Atomic State Transition\nRe-allocate lookahead pre-delay buffers if needed\nAtomic pointer swap at period boundary", fillcolor="#FAF089", color="#B7791F"]; + + b_rx -> b_val -> b_swap; + } + + subgraph cluster_topo { + label="ALSA Topology 2 Effect Widgets"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + w_drc [label="drc.conf Widget\nUUID: da:e4:6e:b3:...\nBypass Switch Control", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + w_mdrc [label="multiband_drc.conf Widget\nUUID: 56:22:9f:0d:...\nMulti-Band Compound Effect", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + w_drc -> w_mdrc [style="invis"]; + } + + host_drv -> b_rx; + b_swap -> w_drc [color="#38A169", style="bold"]; + b_swap -> w_mdrc [color="#38A169", style="bold"]; + } + +--- + +.. _simd_drc_acceleration: + +7. SIMD Vector Acceleration Across DSP Architectures +**************************************************** + +Processing multi-channel audio through dynamic compressors involves significant mathematical throughput: circular lookahead buffer indexing, logarithmic decibel energy extraction, exponential envelope smoothing, crossover filtering, and multi-band gain scaling. + +SOF provides architecture-specific SIMD vector acceleration: + +* **Tensilica Xtensa HiFi 3 & HiFi 4 (``drc_hifi4.c``, ``drc_math_hifi3.c``)**: + + - Vectorized lookahead buffer read and write operations advancing circular indices without scalar address math. + - SIMD vector gain application multiplying four 32-bit audio samples concurrently with hardware saturation. + - Fast fixed-point mathematical approximations for logarithmic decibel calculation and exponential decay curves using CORDIC and polynomial lookup tables (LUTs). + +* **Tensilica Xtensa HiFi 5**: + + - 8-way 32-bit vector processing engine (256-bit bus) accelerating multi-band crossover splitting and parallel DRC compression stages. + - Dual memory load buses allow simultaneously loading audio delay lines and compressor gain coefficients in a single clock cycle. + +* **Generic Portable Scalar C (``drc_generic.c``, ``multiband_drc_generic.c``)**: + + - Clean, portable scalar C implementations designed for non-Xtensa platforms (e.g. ARM Cortex-M7 on Teensy 4.1, RISC-V on ESP32-P4). + +.. graphviz:: + :caption: SIMD Vector Acceleration and Fixed-Point Math Approximations across DSP Architectures + + digraph simd_drc { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_gen { + label="Generic Scalar C (drc_generic.c / multiband_drc_generic.c)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + g_core [label="Portable Scalar C Loop\n1 sample per iteration\nTarget: ARM Cortex-M, RISC-V, Simulator", fillcolor="#FFFFFF", color="#CBD5E0"]; + } + + subgraph cluster_hf3 { + label="Xtensa HiFi 3 / HiFi 4 (drc_hifi4.c / drc_math_hifi3.c)"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + h3_core [label="Quad 32-bit Vector Engine\nVector circular pre-delay buffering\nCORDIC / LUT log-exp approximations", fillcolor="#BEE3F8", color="#3182CE"]; + } + + subgraph cluster_hf5 { + label="Xtensa HiFi 5 (Octa Vector Acceleration)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + h5_core [label="Octa 32-bit Vector Engine (256-bit bus)\n8 samples processed concurrently\nAccelerates parallel multi-band DRC filtering", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + g_core -> h3_core [label="2x - 4x Speedup", color="#3182CE"]; + h3_core -> h5_core [label="2x Speedup (8x Total)", color="#38A169", style="bold"]; + } + +--- + +.. _upstream_drc_references: + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level implementation details, mathematical structures, and tuning scripts: + +* **Upstream Component Specifications**: + - `thesofproject/sof: src/audio/drc/README.md `_ + - `thesofproject/sof: src/audio/multiband_drc/README.md `_ +* **Single-Band DRC Source Files**: + - ``src/audio/drc/drc.c``: Component initialization, lifecycle, and buffer dispatch. + - ``src/audio/drc/drc.h``: DRC state definitions (``struct drc_state``), pre-delay buffer management, and division masks. + - ``src/audio/drc/drc_algorithm.h``: Core algorithm prototypes (detector averaging, envelope updating, and compression scaling). + - ``src/audio/drc/drc_user.h``: Parameter definitions (``struct sof_drc_params``, ``struct sof_drc_config``). + - ``src/audio/drc/drc_generic.c``: Portable scalar C compression kernel. + - ``src/audio/drc/drc_hifi4.c``: Tensilica Xtensa HiFi 4 SIMD vector implementation. + - ``src/audio/drc/drc_math_hifi3.c``: Xtensa HiFi 3 fixed-point math acceleration. +* **Multi-Band DRC Source Files**: + - ``src/audio/multiband_drc/multiband_drc.c``: Compound component lifecycle, state reset, and memory allocation. + - ``src/audio/multiband_drc/multiband_drc.h``: Multi-band state (``struct multiband_drc_state``) encompassing emphasis, crossover, DRCs, and deemphasis. + - ``src/audio/multiband_drc/user/multiband_drc.h``: Multi-band configuration structures (``struct sof_multiband_drc_config``). + - ``src/audio/multiband_drc/multiband_drc_generic.c``: Compound pipeline execution loop. +* **Topology Definitions**: + - ``tools/topology/topology2/include/components/drc.conf``: ALSA Topology 2 configuration class for single-band DRC widgets. + - ``tools/topology/topology2/include/components/multiband_drc.conf``: ALSA Topology 2 configuration class for multi-band DRC widgets. +* **MATLAB / Octave Tuning Scripts**: + - ``src/audio/drc/tune/sof_example_drc.m``: Interactive script for tuning threshold, knee, ratio, and attack/release curves. + - ``src/audio/multiband_drc/tune/sof_example_multiband_drc.m``: Tuning script for configuring multi-band crossover frequencies and per-band compressor profiles. + +Related Subsystem Architecture Guides +===================================== + +* :ref:`smart_amp`: Adaptive speaker protection, real-time current/voltage (I/V) sense telemetry, and excursion/thermal limiters. +* :ref:`sound_dose`: Auditory safety evaluation, IEC 61672-1 A-weighting integration, and dynamic gain limiting. +* :ref:`volume_module`: Per-channel gain scaling, smooth volume ramping, and zero-crossing muting. +* :ref:`eq_fir_iir`: Finite and Infinite Impulse Response equalizers, linear-phase filtering, and biquad cascades. +* :ref:`src_asrc`: Sample rate conversion architecture handling fixed and drifting clocks across heterogeneous audio interfaces. +* :ref:`mixin_mixout`: Multi-stream mixing and audio distribution preceding or following dynamics processing. +* :ref:`module_framework`: The standardized module interface, Source/Sink APIs, and memory sandboxing wrapping DRC components. +* :ref:`pipeline_architecture`: How DRC widgets are integrated into directed acyclic audio graphs (DAGs). diff --git a/developer_guides/firmware/eq_fir_iir.rst b/developer_guides/firmware/eq_fir_iir.rst new file mode 100644 index 00000000..a2369ca0 --- /dev/null +++ b/developer_guides/firmware/eq_fir_iir.rst @@ -0,0 +1,573 @@ +.. _eq_fir_iir: + +Equalizer Architecture (EQ FIR & EQ IIR) +######################################## + +The **Equalization** subsystem in Sound Open Firmware provides real-time frequency response shaping, acoustic correction, and dynamic tone control across heterogeneous audio pipelines and physical transducers. + +In modern audio systems, physical transducers—such as laptop micro-speakers, smartphone earpieces, and digital microphone arrays—inevitably suffer from non-ideal acoustical characteristics: mechanical cavity resonances, enclosure-induced high-frequency roll-off, and limited low-frequency bass extension. Furthermore, room acoustics, listener preferences, and voice intelligibility algorithms require precise, low-latency spectral filtering. + +SOF addresses these challenges through two specialized, complementary equalizer components: + +1. **Finite Impulse Response (FIR) Equalizer** (``eq_fir``, ``src/audio/eq_fir/``): Feedforward transversal filter engine providing exact linear-phase response, constant group delay, and arbitrary magnitude shaping without phase distortion. +2. **Infinite Impulse Response (IIR) Equalizer** (``eq_iir``, ``src/audio/eq_iir/``): Recursive feedback filter engine implementing cascaded second-order sections (biquads) for ultra-low algorithmic latency, minimal memory footprint, and classic parametric tone shaping (peaking bells, shelves, and passbands). + +This guide provides a comprehensive, high-level architectural walkthrough of the EQ FIR and EQ IIR subsystems, filter topologies, biquad cascades, dynamic parameter updates, and SIMD hardware acceleration without delving into low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +.. _eq_taxonomy: + +1. Equalization in Audio Systems & Filter Taxonomy +************************************************** + +Audio equalization modifies the balance of frequency components within an audio signal. In SOF, equalization is applied across several core audio use cases: + +* **Speaker Frequency Response Correction**: Flattening peaky mechanical resonances and boosting attenuated frequency bands to produce natural, transparent sound reproduction within mass-market industrial designs. +* **Microphone Acoustic Flattening**: Correcting frequency deviations across MEMS digital and analog microphone capsules prior to Acoustic Echo Cancellation (AEC) and directional beamforming. +* **Parametric User Tone Controls**: Implementing interactive user-facing equalizers (e.g. 10-band graphic equalizers, bass boost, speech enhancement, and treble tone controls). +* **Driver Protection & Rumble Filtering**: Rolling off sub-audible frequencies below speaker excursion limits to prevent mechanical damage and voice coil burnout. + +FIR vs IIR Architectural Taxonomy +================================= + +The choice between FIR and IIR equalizers involves architectural trade-offs between phase linearity, algorithmic latency, computational complexity, and memory utilization: + +* **Finite Impulse Response (FIR) Equalizer**: + + - **Structure**: Feedforward transversal delay line with no feedback paths. The impulse response settles to exactly zero after :math:`L` samples. + - **Phase Response**: Exact linear phase with constant group delay :math:`\tau = (L - 1) / 2` samples across all frequencies. Preserves transient waveforms without phase dispersion. + - **Stability**: Unconditionally stable. All transfer function poles reside at the origin (:math:`z = 0`). + - **Resource Cost**: Higher computational load (requires :math:`L` multiply-accumulate operations per sample) and larger delay line memory. Algorithmic latency is proportional to filter length. + +* **Infinite Impulse Response (IIR) Equalizer**: + + - **Structure**: Recursive feedback network where the current output depends on both past inputs and past outputs. The impulse response decays asymptotically over time. + - **Phase Response**: Minimum-phase response with frequency-dependent group delay, mimicking analog RC/RLC active filter circuits. + - **Latency**: Ultra-low algorithmic latency (typically a fraction of a sample), making it ideal for interactive communications and gaming. + - **Resource Cost**: Exceptionally efficient (only 5 coefficients and 4 state variables per second-order biquad). However, poles must be carefully bounded within the unit circle to guarantee stability. + +.. graphviz:: + :caption: Architectural Taxonomy: Feedforward Transversal FIR vs Recursive Cascaded Biquad IIR + + digraph eq_taxonomy { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_fir { + label="FIR Equalizer (Feedforward Transversal Architecture)"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + fir_in [label="Input Sample x[n]", fillcolor="#FFFFFF", color="#CBD5E0"]; + fir_delay [label="Tapped Delay Line\nx[n-1], x[n-2], ..., x[n-L+1]\nUnconditionally Stable (Poles at Origin)", fillcolor="#BEE3F8", color="#3182CE"]; + fir_mac [label="Tap Multipliers (h[0] .. h[L-1])\nSymmetric Tap Pre-Addition\nConstant Group Delay (Linear Phase)", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + fir_out [label="Output Sample y[n]\nPreserved Transient Waveforms", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + fir_in -> fir_delay -> fir_mac -> fir_out [color="#3182CE"]; + } + + subgraph cluster_iir { + label="IIR Equalizer (Recursive Biquad Cascade Architecture)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + iir_in [label="Input Sample x[n]", fillcolor="#FFFFFF", color="#D69E2E"]; + iir_biquad [label="Cascaded Biquad Stages (SOS)\nFeedforward Zeros (b0, b1, b2)\nFeedback Poles (a1, a2)\nPoles bounded inside Unit Circle (|z| < 1)", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + iir_out [label="Output Sample y[n]\nUltra-Low Algorithmic Latency\nAnalog Emulation (Minimum Phase)", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + iir_in -> iir_biquad -> iir_out [color="#B7791F"]; + } + } + +--- + +.. _fir_architecture: + +2. Finite Impulse Response (FIR) Equalizer Architecture +******************************************************* + +The FIR Equalizer component (``src/audio/eq_fir/eq_fir.c``) applies digital filtering by computing discrete-time convolution between the incoming audio stream and a pre-designed impulse response vector :math:`h[k]`: + +.. math:: + + y[n] = \sum_{k=0}^{L-1} h[k] \cdot x[n-k] + +where :math:`L` represents the filter length (number of taps). + +Circular Delay Line Management +============================== + +To compute convolution across successive audio frames without copying memory blocks, SOF maintains a circular delay line for each audio channel: + +1. **Circular Addressing**: Historical input samples are stored in a contiguous RAM buffer. When new samples arrive, they overwrite the oldest samples using circular pointer indexing. +2. **Multi-Channel Separation**: Each audio channel maintains its own independent delay line buffer, sized according to the longest configured filter across the system. +3. **Zero Overhead**: Circular buffer pointer arithmetic avoids memory shift operations (``memmove``), keeping memory bus activity strictly proportional to audio frame sizes. + +Linear Phase Symmetry Optimization +================================== + +Most acoustic equalization curves require linear phase to prevent phase smearing across stereo and surround sound fields. A filter has linear phase if and only if its impulse response exhibits even symmetry (:math:`h[k] = h[L-1-k]`) or odd anti-symmetry (:math:`h[k] = -h[L-1-k]`). + +SOF exploits this mathematical property through **symmetric tap folding**: + +.. math:: + + y[n] = h\left[\frac{L-1}{2}\right] \cdot x\left[n - \frac{L-1}{2}\right] + \sum_{k=0}^{\frac{L-3}{2}} h[k] \cdot \Big( x[n-k] + x[n - L + 1 + k] \Big) + +By pre-adding the symmetric past and current input samples before multiplying by the shared coefficient :math:`h[k]`, the total number of multiplication operations is reduced by **50%** (from :math:`L` down to :math:`L/2` multiplications per sample). + +.. graphviz:: + :caption: FIR Transversal Tap Delay Line with Linear Phase Symmetric Tap Folding Optimization + + digraph fir_structure { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_x [label="Input Audio x[n]\n(Current Sample)", fillcolor="#EDF2F7", color="#CBD5E0"]; + d_line [label="Circular Delay Line Buffer\nx[n], x[n-1], x[n-2], ... x[n-L+1]", fillcolor="#BEE3F8", color="#3182CE"]; + + subgraph cluster_fold { + label="Symmetric Tap Folding Engine (50% Multiplication Reduction)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + pre_add [label="Pairwise Pre-Adders:\n(x[n-k] + x[n-L+1+k])", fillcolor="#FAF089", color="#B7791F"]; + mult [label="Coefficient Multipliers:\nh[k] * (Sum)\nQ1.15 Fixed-Point Coeffs", fillcolor="#FAF089", color="#B7791F"]; + acc [label="64-Bit Accumulator\nSum across all folded taps", fillcolor="#FAF089", color="#B7791F"]; + + pre_add -> mult -> acc; + } + + post_sh [label="Output Scaler & Shift\nApply out_shift & Saturation", fillcolor="#E2E8F0", color="#A0AEC0"]; + out_y [label="Equalized Audio y[n]\nLinear Phase (Zero Distortion)", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + in_x -> d_line; + d_line -> pre_add; + acc -> post_sh -> out_y; + } + +--- + +.. _iir_architecture: + +3. Infinite Impulse Response (IIR) Equalizer Architecture & Biquad Cascades +*************************************************************************** + +The IIR Equalizer component (``src/audio/eq_iir/eq_iir.c``) implements frequency shaping using recursive difference equations. The general transfer function of an :math:`N`-th order IIR filter is a ratio of polynomials: + +.. math:: + + H(z) = \frac{\sum_{k=0}^{N} b_k z^{-k}}{1 + \sum_{k=1}^{N} a_k z^{-k}} + +The Sensitivity Hazard of High-Order Monolithic Filters +======================================================= + +Directly implementing a high-order polynomial filter (e.g. 10th or 20th order) in digital signal processing hardware is notoriously dangerous: + +* The roots of high-order polynomials are hypersensitive to small perturbations in filter coefficients caused by fixed-point quantization. +* Tiny round-off errors can push poles outside the complex unit circle (:math:`|z| \ge 1.0`), causing catastrophic instability, oscillation, and rail-to-rail digital clipping. + +Cascaded Second-Order Sections (SOS / Biquads) +============================================== + +To ensure absolute numerical stability, SOF factors all high-order IIR filters into a cascade of independent **Second-Order Sections (SOS)**, commonly known as **Biquads**: + +.. math:: + + H(z) = \prod_{k=1}^{K} H_k(z) = \prod_{k=1}^{K} \frac{b_{0,k} + b_{1,k} z^{-1} + b_{2,k} z^{-2}}{1 + a_{1,k} z^{-1} + a_{2,k} z^{-2}} + +Each biquad section isolates a single conjugate pair of poles and zeros: + +* **Poles within Unit Circle**: Stability is verified algebraically for each biquad individually by checking that :math:`|a_{2,k}| < 1` and :math:`|a_{1,k}| < 1 + a_{2,k}`. +* **Octave Band Coverage**: SOF supports cascading up to 11 biquads in series (a 22nd-order filter), sufficient to cover all 11 octave bands across the 20 Hz – 20 kHz audio spectrum. + +Direct Form I (DF1) Implementation Mechanics +============================================ + +SOF implements biquads using **Direct Form I (DF1)** with 64-bit accumulators: + +1. **Independent State Variables**: Direct Form I maintains separate delay histories for input samples (:math:`x[n-1], x[n-2]`) and output samples (:math:`y[n-1], y[n-2]`). +2. **64-Bit Internal Accumulation**: All five product terms (:math:`b_0 x[n] + b_1 x[n-1] + b_2 x[n-2] - a_1 y[n-1] - a_2 y[n-2]`) accumulate into a high-precision 64-bit accumulator with guard bits before rounding and shifting. +3. **Limit Cycle Immunity**: In low-frequency narrow-band equalization (such as deep bass boosts at 40 Hz), poles lie extremely close to :math:`z = 1.0`. Direct Form II structures can suffer from internal node overflow and limit cycle oscillations. Direct Form I with 64-bit accumulation completely avoids internal node overflow. + +.. graphviz:: + :caption: Cascaded Direct Form I (DF1) Second-Order Section (Biquad) Processing Chain with 64-bit Accumulator + + digraph iir_biquad_cascade { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_pcm [label="Input Audio x[n]\n(From Pipeline Buffer)", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_bq0 { + label="Biquad Stage 0 (e.g. Bass Shelf / Low Cut)"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + df1_x0 [label="Input State\nx0[n-1], x0[n-2]", fillcolor="#FFFFFF", color="#BEE3F8"]; + df1_c0 [label="Feedforward (b0, b1, b2)\nFeedback (-a1, -a2)\n64-Bit Accumulator", fillcolor="#BEE3F8", color="#3182CE"]; + df1_y0 [label="Output State\ny0[n-1], y0[n-2]", fillcolor="#FFFFFF", color="#BEE3F8"]; + + df1_x0 -> df1_c0 -> df1_y0; + } + + subgraph cluster_bq1 { + label="Biquad Stage 1 (e.g. Parametric Peaking Bell)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + df1_c1 [label="Biquad 1 DF1 Engine\nIndependent Poles/Zeros\n64-Bit Accumulator", fillcolor="#FAF089", color="#B7791F"]; + } + + subgraph cluster_bqk { + label="Biquad Stage K-1 (e.g. Treble Shelf)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + df1_ck [label="Biquad K-1 DF1 Engine\nFinal Shaping Section\nHeadroom Scaler", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + out_pcm [label="Equalized Audio y[n]\n(Ultra-Low Latency)", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + in_pcm -> df1_x0; + df1_y0 -> df1_c1 [label="Intermediate SOS"]; + df1_c1 -> df1_ck [label="Cascaded SOS", style="dashed"]; + df1_ck -> out_pcm; + } + +--- + +.. _parametric_eq_topologies: + +4. Parametric Equalizer Topologies & Biquad Filter Types +******************************************************** + +By configuring the five coefficients (:math:`b_0, b_1, b_2, a_1, a_2`) of each biquad section, the SOF IIR equalizer implements all classic parametric filter types defined in the Audio EQ Cookbook: + +* **Peaking / Bell Filter**: + + - Provides selective boost or attenuation centered around a target frequency :math:`f_0`. + - Configured via Center Frequency (:math:`f_0`), Quality Factor (:math:`Q` or bandwidth in octaves), and Gain (:math:`G` in dB). + - Primary tool for eliminating sharp speaker resonance peaks and acoustic cavity dips. + +* **Low-Shelf & High-Shelf Filters**: + + - Boosts or attenuates all frequencies below (low-shelf) or above (high-shelf) a transition corner frequency with a smooth plateau response. + - Used for classic bass and treble tone controls. + +* **High-Pass Filter (HPF) & Low-Pass Filter (LPF)**: + + - 12 dB/octave attenuation slope per biquad (cascaded to form 24 dB/oct or 48 dB/oct Butterworth, Linkwitz-Riley, or Chebyshev filters). + - HPF blocks sub-audible DC offsets and speaker rumble; LPF blocks ultrasonic noise above the audible band. + +* **Band-Pass (BPF) & Notch (Band-Stop) Filters**: + + - BPF isolates a specific frequency band for feature detection or wake-word preprocessing. + - Notch filters provide deep, surgical attenuation (e.g. -40 dB) at a specific frequency to eliminate electrical mains hum (50 Hz / 60 Hz) or microphone feedback howling. + +* **Flat / Neutral Biquad**: + + - Configured with :math:`b_0 = 1.0, \text{gain} = 1.0` and all other coefficients zero. + - Acts as a zero-overhead passthrough section for unused biquad slots in a generic configuration. + +.. graphviz:: + :caption: Parametric EQ Biquad Filter Types and Characteristic Frequency Response Curves + + digraph parametric_types { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_types { + label="Parametric Biquad Library (Audio EQ Cookbook Topologies)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + t_bell [label="Peaking / Bell Filter\nBoost/Cut around Center Frequency f0\nAdjustable Q (Bandwidth) & Gain (dB)", fillcolor="#BEE3F8", color="#3182CE"]; + t_shelf [label="Low / High Shelving Filters\nSmooth plateau boost/attenuation\nBass & Treble User Tone Controls", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + t_pass [label="High-Pass (HPF) & Low-Pass (LPF)\n12 dB / 24 dB / 48 dB per octave slopes\nRumble filtering & Tweeter protection", fillcolor="#FED7D7", color="#E53E3E", fontcolor="#742A2A"]; + t_notch [label="Band-Stop / Notch Filter\nSurgical high-Q narrow attenuation\n50/60 Hz Mains Hum & Howl Suppression", fillcolor="#E9D8FD", color="#805AD5", fontcolor="#44337A"]; + t_flat [label="Flat / Passthrough Section\nb0 = 1.0, Gain = 1.0 (Neutral)\nUnused cascade slots bypass", fillcolor="#FFFFFF", color="#CBD5E0"]; + + t_pass -> t_shelf -> t_bell -> t_notch -> t_flat [style="invis"]; + } + } + +--- + +.. _dynamic_updates: + +5. Dynamic Parameter Updates & Configuration Blobs +************************************************** + +Equalizers must adapt dynamically to user actions (e.g. moving a graphic equalizer slider in an audio control panel) and environmental context (e.g. switching between built-in laptop speakers and an external dock). + +The Component Blob Handler Framework +==================================== + +SOF delivers equalizer parameters from the Linux host driver using **Component Configuration Blobs** managed by the ``comp_data_blob_handler`` infrastructure: + +1. **IPC Delivery**: The host sends serialized configuration blobs via IPC3 (``SOF_IPC_COMP_SET_DATA``) or IPC4 (``SET_LARGE_CONFIG`` with dedicated component UUID). +2. **Fragmented Assembly**: If a filter configuration exceeds the maximum single IPC mailbox window, the blob handler transparently reassembles incoming multi-part packet fragments. +3. **Pre-Validation Hook**: Before applying any changes to the running audio stream, the blob handler invokes the component's validator callback (``eq_fir_init_coef()`` with ``fir == NULL`` or ``eq_iir_validate_config()``). + +Atomic Swapping & Glitchless Transitions +======================================== + +Applying an unvalidated or corrupt filter configuration can crash the DSP or generate destructive acoustic pops. SOF enforces strict atomic updating: + +* **Validation Bounds**: The validator checks payload byte length, verifies channel count matches active stream configuration, and ensures filter taps or biquad counts do not exceed hardware limits. +* **Delay Line Reallocation**: If the new configuration requires more taps or biquads than currently allocated, new RAM buffers are allocated before releasing the previous ones. +* **Atomic State Pointer Swap**: The running audio thread continues executing using the existing filter configuration until the new configuration is fully prepared in memory. Once ready, active state pointers are swapped atomically between audio periods. +* **Glitchless Crossfading**: Filter states prevent DC discontinuities and audible pops during runtime adjustment. + +.. graphviz:: + :caption: Dynamic IPC Configuration Blob Handling, Safe Validation, and Active Coefficient Swapping + + digraph dynamic_config_flow { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + host_ipc [label="Host ALSA / PipeWire User Interface\nSends EQ Profile via IPC Blob", fillcolor="#EDF2F7", color="#CBD5E0"]; + blob_mgr [label="Component Blob Handler (comp_data_blob_handler)\nFragment Reassembly & Staging", fillcolor="#BEE3F8", color="#3182CE"]; + + subgraph cluster_val { + label="Pre-Validation & Safety Checks"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + v_check [label="Validation Callback (eq_validate)\nCheck Payload Sizing & Header Magic\nVerify Channel Bounds & Stability Limits", fillcolor="#FAF089", color="#B7791F"]; + v_alloc [label="Shadow Allocation\nAllocate new delay lines in DSP RAM\nPre-compute Q2.30 / Q1.15 coefficient tables", fillcolor="#FAF089", color="#B7791F"]; + + v_check -> v_alloc [label="Valid"]; + } + + subgraph cluster_exec { + label="Active Audio Processing Loop"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + swap_ptr [label="Atomic State Swap\nSwap active coefficient & delay pointers\nZero pipeline interruption", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + run_eng [label="Active Filtering Engine\nExecutes with updated EQ curve\nGlitchless acoustic transition", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + swap_ptr -> run_eng; + } + + host_ipc -> blob_mgr -> v_check; + v_alloc -> swap_ptr [label="Atomic Swap Trigger", color="#38A169", style="bold"]; + } + +--- + +.. _multichannel_topology: + +6. Multi-Channel Processing & ALSA Topology Integration +******************************************************* + +Real-world consumer hardware rarely features acoustically identical speaker channels. In thin laptops, the left speaker is often constrained by the internal battery while the right speaker sits next to a thermal exhaust vent, causing significant differences in frequency response. + +Independent Channel Response Assignment +======================================= + +SOF equalizers solve this via **Channel Response Mapping**: + +* **Response Definition Pool**: A single configuration blob can define multiple distinct filter responses (up to 8 independent FIR or IIR responses). +* **Channel Assignment Vector (``assign_response[]``)**: A mapping array assigns which response curve applies to each audio channel: + + .. code-block:: text + + assign_response = [0, 1] # Left channel -> Curve 0, Right channel -> Curve 1 + +* **Selective Passthrough**: Channels assigned an index of ``-1`` bypass the filter engine entirely, passing unmodified audio through high-speed memory copies (``audio_stream_copy()``). + +ALSA Topology 2 Integration +=========================== + +Equalizer modules are declared in ALSA Topology 2 files using the ``eqfir.conf`` and ``eqiir.conf`` component classes: + +* **Effect Widget**: Instantiated with widget type ``effect`` and dedicated component UUIDs: + + - **FIR Equalizer UUID**: ``e7:0c:a9:43:a5:f3:df:41:ac:06:ba:98:65:1a:e6:a3`` + - **IIR Equalizer UUID**: ``e6:c0:50:51:f9:27:c8:4e:83:51:c7:05:b6:42:d1:2f`` + +* **Static ROM Initialization**: Default speaker and microphone tuning blobs can be embedded directly into compiled topology binaries (``.bin``), ensuring optimal audio quality immediately upon system boot before userspace drivers initialize. + +.. graphviz:: + :caption: Multi-Channel Response Assignment and ALSA Topology 2 Widget Integration + + digraph multichannel_topology { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + in_stream [label="Multi-Channel Audio\n(e.g. Stereo Stream)", fillcolor="#EDF2F7", color="#CBD5E0"]; + + subgraph cluster_map { + label="Channel Response Assignment (assign_response[])"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + ch0 [label="Channel 0 (Left)\nAssign: Response 0\n(Left Speaker Profile)", fillcolor="#FAF089", color="#B7791F"]; + ch1 [label="Channel 1 (Right)\nAssign: Response 1\n(Right Speaker Profile)", fillcolor="#FAF089", color="#B7791F"]; + } + + subgraph cluster_filters { + label="Filter Engine Instances"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + f0 [label="FIR / IIR Response 0\nTuned for Left Cavity", fillcolor="#BEE3F8", color="#3182CE"]; + f1 [label="FIR / IIR Response 1\nTuned for Right Cavity", fillcolor="#BEE3F8", color="#3182CE"]; + } + + out_stream [label="Equalized Stereo Audio\nBalanced Acoustic Output", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + in_stream -> ch0; + in_stream -> ch1; + ch0 -> f0; + ch1 -> f1; + f0 -> out_stream; + f1 -> out_stream; + } + +--- + +.. _simd_eq_acceleration: + +7. SIMD Vector Acceleration Across DSP Architectures +**************************************************** + +Multi-channel equalization with dense FIR tap lines (e.g. 128 taps across 4 channels = 512 multiply-accumulates per frame) or cascaded IIR biquads (11 biquads = 55 MACs per frame per channel) demands substantial processor throughput. + +SOF provides optimized vector assembly kernels across target DSP architectures: + +* **Cadence Tensilica Xtensa HiFi 3 (``fir_hifi3.c``, ``iir_df1_hifi3.c``)**: + + - Utilizes 64-bit dual multiply-accumulate instructions (``AE_MULAA32RA``, ``AE_S32X2``). + - Processes two 32-bit audio samples concurrently with hardware saturation. + +* **Cadence Tensilica Xtensa HiFi 4 (``iir_df1_hifi4.c``)**: + + - Employs 128-bit SIMD registers executing four 32x32 multiplications per cycle. + - Leverages vector circular pointer instructions (``AE_L32X4_XC``) to advance delay line indices with zero scalar addressing overhead. + +* **Cadence Tensilica Xtensa HiFi 5 (``fir_hifi5.c``, ``iir_df1_hifi5.c``)**: + + - Octa 32-bit vector processing engine (256-bit data bus) executing eight 32-bit multiply-accumulate operations in parallel. + - Dual memory load buses allow simultaneously fetching filter coefficients and audio delay buffers in a single clock cycle. + +* **Generic Portable C Reference (``fir_generic.c``, ``iir_df1_generic.c``)**: + + - Clean, portable scalar C implementations designed for non-Xtensa platforms (e.g. ARM Cortex-M7 on Teensy 4.1, RISC-V on ESP32-P4). + +.. graphviz:: + :caption: SIMD Vector Processing and Circular Delay Line Buffering across DSP Architectures + + digraph simd_eq { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_gen { + label="Generic Scalar C (fir_generic.c / iir_df1_generic.c)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + g_core [label="Portable Scalar C Loop\n1 sample per iteration\nTarget: ARM Cortex-M, RISC-V, Simulator", fillcolor="#FFFFFF", color="#CBD5E0"]; + } + + subgraph cluster_hf3 { + label="Xtensa HiFi 3 (fir_hifi3.c / iir_df1_hifi3.c)"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + h3_core [label="Dual 32-bit Vector Engine\n2 samples processed per cycle\n64-bit dual MAC instructions", fillcolor="#BEE3F8", color="#3182CE"]; + } + + subgraph cluster_hf4 { + label="Xtensa HiFi 4 (iir_df1_hifi4.c)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + h4_core [label="Quad 32-bit Vector Engine (128-bit)\n4 samples processed per instruction cycle\nCircular delay line auto-wrapping", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + } + + subgraph cluster_hf5 { + label="Xtensa HiFi 5 (fir_hifi5.c / iir_df1_hifi5.c)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + h5_core [label="Octa 32-bit Vector Engine (256-bit bus)\n8 samples processed per cycle\nDual 128-bit memory buses for coefficients & delay line", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + g_core -> h3_core [label="2x Speedup", color="#3182CE"]; + h3_core -> h4_core [label="2x Speedup (4x Total)", color="#B7791F"]; + h4_core -> h5_core [label="2x Speedup (8x Total)", color="#38A169", style="bold"]; + } + +--- + +.. _upstream_eq_references: + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level implementation details, filter coefficient structures, and acoustic tuning scripts: + +* **Upstream Component Specifications**: + - `thesofproject/sof: src/audio/eq_fir/README.md `_ + - `thesofproject/sof: src/audio/eq_iir/README.md `_ +* **FIR Equalizer Source Files**: + - ``src/audio/eq_fir/eq_fir.c``: Component initialization, channel assignment, and buffer copying. + - ``src/audio/eq_fir/eq_fir.h``: FIR private data structures (``struct comp_data``) and format function pointers. + - ``src/include/user/fir.h``: FIR user configuration structures (``struct sof_fir_coef_data``). + - ``src/math/fir_generic.c``: Portable scalar C FIR convolution kernel. + - ``src/math/fir_hifi3.c``: Tensilica Xtensa HiFi 3 SIMD vector kernel. + - ``src/math/fir_hifi5.c``: Tensilica Xtensa HiFi 5 octa-vector kernel. +* **IIR Equalizer Source Files**: + - ``src/audio/eq_iir/eq_iir.c``: Component lifecycle, blob validation, and processing dispatch. + - ``src/audio/eq_iir/eq_iir.h``: IIR private structures and biquad state headers. + - ``src/include/user/eq.h``: IIR biquad structures (``struct sof_eq_iir_biquad``) and configuration headers (``struct sof_eq_iir_config``). + - ``src/math/iir_df1_generic.c``: Portable scalar C Direct Form I biquad cascade. + - ``src/math/iir_df1_hifi3.c``: Tensilica Xtensa HiFi 3 SIMD biquad kernel. + - ``src/math/iir_df1_hifi4.c``: Tensilica Xtensa HiFi 4 SIMD biquad kernel. + - ``src/math/iir_df1_hifi5.c``: Tensilica Xtensa HiFi 5 SIMD biquad kernel. +* **Topology Definitions**: + - ``tools/topology/topology2/include/components/eqfir.conf``: ALSA Topology 2 configuration class for FIR widgets. + - ``tools/topology/topology2/include/components/eqiir.conf``: ALSA Topology 2 configuration class for IIR widgets. +* **Acoustic Measurement & Filter Tuning**: + - :ref:`equalizers_tuning`: Comprehensive acoustic measurement runbook for tuning speaker equalizers using calibrated reference microphones, sine sweeps, and Octave/MATLAB scripts. + +Related Subsystem Architecture Guides +===================================== + +* :ref:`volume_module`: Per-channel gain scaling, smooth ramping, zero-crossing muting, and volume controls preceding/following equalizers. +* :ref:`src_asrc`: Sample rate conversion architecture handling fixed and drifting clocks across heterogeneous pipelines. +* :ref:`mixin_mixout`: Multi-stream audio mixing and distribution across post-equalizer loudspeaker and headphone buses. +* :ref:`module_framework`: The standardized module interface, Source/Sink APIs, and memory sandboxing wrapping FIR and IIR equalizers. +* :ref:`audio_buffer_management`: Lockless circular ring buffers, multi-tier DSP memory, and cache coherency. +* :ref:`pipeline_architecture`: How equalizer modules are integrated into directed acyclic audio graphs (DAGs). diff --git a/developer_guides/firmware/fw_init_boot.rst b/developer_guides/firmware/fw_init_boot.rst new file mode 100644 index 00000000..23f6b837 --- /dev/null +++ b/developer_guides/firmware/fw_init_boot.rst @@ -0,0 +1,504 @@ +.. _fw_init_boot: + +Firmware Initialization & Boot Architecture +########################################### + +The **Firmware Initialization & Boot** subsystem in Sound Open Firmware (SOF) governs the complete sequence through which the audio Digital Signal Processor (DSP) transitions from an unpowered or quiescent hardware state into a fully initialized, real-time audio computing engine. + +Operating as an embedded real-time system across diverse silicon architectures (Intel CAVS/ACE, NXP i.MX, AMD ACP, and embedded microcontrollers like ESP32 and Teensy), SOF couples low-level hardware bootstrap sequences with the **Zephyr RTOS** kernel lifecycle, multi-tier platform hardware bringup, host driver synchronization handshakes, and multi-core power restoration. + +This guide provides a comprehensive, high-level architectural walkthrough of the firmware initialization and boot framework without delving into low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +.. _fw_boot_lifecycle: + +1. End-to-End Boot & Initialization Lifecycle +********************************************* + +Bringing an audio DSP from host power-on to active audio stream processing spans multiple distinct execution domains: host operating system orchestration, DSP hardware boot ROM, Zephyr RTOS kernel initialization, SOF primary core initialization, application thread startup, and host-firmware synchronization. + +The Five Architectural Phases of Boot +===================================== + +1. **Host Driver Pre-Boot Staging**: The host operating system (e.g., Linux mainline ALSA/ASoC driver) parses the signed firmware ELF binary, inspects embedded metadata headers, allocates host DMA buffers (or Isolated Memory Regions / IMR), programs DSP base address registers (BARs), and deasserts the hardware DSP core reset latch. +2. **DSP Hardware Boot ROM Execution**: The DSP's embedded on-chip ROM begins executing on Core 0. The ROM powers up internal SRAM banks, configures early clock trees, validates cryptographic signatures and hash manifests, configures DSP memory management page tables, copies the firmware image from host memory into DSP SRAM, and vectors execution to the operating system entry point (``_start``). +3. **Zephyr RTOS Kernel Bringup**: The Zephyr operating system initializes processor registers, zeroes BSS, unpacks initialized data sections, initializes architectural exception vectors, and progresses through deterministic kernel initialization stages (``EARLY``, ``PRE_KERNEL_1``, ``PRE_KERNEL_2``, and ``POST_KERNEL``). +4. **SOF Core & Platform Subsystem Initialization**: Registered at Zephyr's ``POST_KERNEL`` stage, SOF's entry function (``sof_init()``) executes on Core 0. It sets up logging and DMA trace buffers, initializes system-wide notifiers, configures runtime power management, invokes platform-specific peripheral drivers (clocks, DMACs, IPC mailboxes, audio schedulers), and unpacks secondary core storage manifests. +5. **Application Main Handoff & Host Ready Handshake**: Zephyr transitions execution to the application main thread (``sof_app_main()``). SOF verifies library integrity (such as dynamically restored LLEXT components), writes firmware status and ABI details to the hardware mailbox, asserts the host interrupt, and transitions to the active running state, awaiting host IPC audio pipeline commands. + +.. graphviz:: + :caption: End-to-End SOF Boot Flow & System Lifecycle from Host Driver Staging to Audio Readiness + + digraph fw_boot_flow { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_host { + label="Host Operating System (Linux Kernel ALSA/ASoC SOF Driver)"; + style="filled,rounded"; + fillcolor="#F7FAFC"; + color="#CBD5E0"; + + h1 [label="Parse Firmware ELF Binary\nInspect Extended Manifest (.fw_metadata)", fillcolor="#EDF2F7", color="#CBD5E0"]; + h2 [label="Stage Firmware into Host DMA / IMR\nProgram DSP BARs & Power Registers", fillcolor="#EDF2F7", color="#CBD5E0"]; + h3 [label="Deassert DSP Hardware Reset Latch\nStart DSP Boot Timeout Monitor", fillcolor="#EDF2F7", color="#CBD5E0"]; + h4 [label="Receive Mailbox FW Ready Interrupt\nVerify ABI & Register Sound Card", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + h1 -> h2 -> h3; + } + + subgraph cluster_rom { + label="DSP Hardware Boot ROM (Core 0)"; + style="filled,rounded"; + fillcolor="#FFF5F5"; + color="#FEB2B2"; + + r1 [label="Hardware Reset Vector\nInit Early Clocks, Cache & Internal SRAM", fillcolor="#FED7D7", color="#E53E3E"]; + r2 [label="Validate Cryptographic Signature\nVerify Hash Manifest & Manifest Headers", fillcolor="#FED7D7", color="#E53E3E"]; + r3 [label="Program MMU/MPU Page Tables\nDMA Load SOF Image into DSP SRAM/TCM", fillcolor="#FED7D7", color="#E53E3E"]; + r4 [label="Branch to Operating System Entry Point\nJump to Zephyr _start Vector", fillcolor="#FEB2B2", color="#C53030"]; + + r1 -> r2 -> r3 -> r4; + } + + subgraph cluster_zephyr { + label="Zephyr RTOS Initialization Stages"; + style="filled,rounded"; + fillcolor="#EBF8FF"; + color="#BEE3F8"; + + z1 [label="Architecture Setup (crt0.S)\nClear BSS, Copy .data, Init Vectors", fillcolor="#BEE3F8", color="#3182CE"]; + z2 [label="Zephyr PRE_KERNEL Stages\nInit CPU, Interrupt Controllers & Timers", fillcolor="#BEE3F8", color="#3182CE"]; + z3 [label="Zephyr POST_KERNEL Stage\nTrigger Registered Drivers & SYS_INIT Hooks", fillcolor="#90CDF4", color="#2B6CB0"]; + + z1 -> z2 -> z3; + } + + subgraph cluster_sof { + label="Sound Open Firmware Subsystems (Core 0)"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + s1 [label="SOF Framework Hook: sof_init()\nprimary_core_init(sof)", fillcolor="#C6F6D5", color="#38A169"]; + s2 [label="Subsystem Bringup: trace_init(),\ninit_system_notify(), pm_runtime_init()", fillcolor="#E6FFFA", color="#319795"]; + s3 [label="Platform Bringup: platform_init()\nClocks, Schedulers (EDF, LL, DP), IPC, DMAC", fillcolor="#E6FFFA", color="#319795"]; + s4 [label="Component Registry & Unpack:\nsys_comp_init(), lp_sram_unpack()", fillcolor="#E6FFFA", color="#319795"]; + s5 [label="Application Entry: sof_app_main()\nstart_complete() -> boot_complete()", fillcolor="#9AE6B4", color="#2F855A", fontcolor="#1C4532"]; + s6 [label="Write Mailbox FW Ready & Status\nRaise Host Doorbell Interrupt", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + s1 -> s2 -> s3 -> s4 -> s5 -> s6; + } + + h3 -> r1 [label="Reset Deassert", color="#E53E3E", style="dashed"]; + r4 -> z1 [label="Vector Jump", color="#3182CE"]; + z3 -> s1 [label="SYS_INIT(POST_KERNEL, 99)", color="#2B6CB0"]; + s6 -> h4 [label="Mailbox Doorbell Interrupt (FW Ready)", color="#276749", style="bold"]; + } + +--- + +.. _ext_manifest_architecture: + +2. Extended Firmware Manifest & Host Pre-Boot Discovery +******************************************************* + +Before the DSP hardware is taken out of reset, the host operating system must discover firmware capabilities, ABI compatibility constraints, memory window geometries, and debugging parameters. + +SOF accomplishes this via the **Extended Firmware Manifest**, an embedded data structure placed directly into the dedicated ``.fw_metadata`` section of the compiled firmware ELF binary (implemented in ``src/init/ext_manifest.c``). + +Manifest Structure & Header Elements +==================================== + +The extended manifest consists of a contiguous sequence of self-describing structured elements. Each element begins with a standard header (``ext_man_elem_header``) containing an element type identifier and a total element payload byte length: + +* **Firmware Version (``ext_man_fw_version``)**: Exposes the major, minor, micro, build tag, and cryptographic Git commit hash of the compiled firmware binary. The host uses this to verify driver compatibility before downloading. +* **Compiler & Toolchain Version (``ext_man_cc_version``)**: Contains the compiler name, toolchain version, and build timestamp (e.g., LLVM/Clang or Cadence XCC) used to build the image. +* **Extraction Probe Support (``ext_man_probe_support``)**: Informs the host driver whether live trace probe DMA extraction points are enabled and provides buffer sizing limits for real-time telemetry streaming. +* **Debug ABI Specification (``ext_man_dbg_abi``)**: Declares the user-space debugger and probe ABI version (such as dictionary-based log extraction schemas used by ``smex`` and ``sof-logger``). +* **Configuration Dictionary (``ext_man_config_data``)**: A key-value array of hardware and build configuration constants, including maximum IPC message sizes (``SOF_IPC_MSG_MAX_SIZE``), memory window offsets, and platform capabilities. + +.. graphviz:: + :caption: Extended Manifest (`.fw_metadata`) Binary Layout and Pre-Boot Host Parsing Flow + + digraph ext_manifest_layout { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_elf { + label="Compiled SOF Firmware ELF Binary"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + elf_hdr [label="Standard ELF Header\n& Program Headers", fillcolor="#FFFFFF", color="#CBD5E0"]; + text_sec [label="Executable Code\n.text, .literal", fillcolor="#FFFFFF", color="#CBD5E0"]; + data_sec [label="Initialized Data\n.data, .rodata", fillcolor="#FFFFFF", color="#CBD5E0"]; + + subgraph cluster_meta { + label="Section: .fw_metadata"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + em_hdr [label="ext_man_header\nMagic: 0x3e456d78\nTotal Manifest Size", fillcolor="#FAF089", color="#B7791F"]; + em_ver [label="ext_man_fw_version\nMajor, Minor, Micro\nGit Commit Hash", fillcolor="#FAF089", color="#B7791F"]; + em_cc [label="ext_man_cc_version\nToolchain: Clang / XCC\nBuild Description", fillcolor="#FAF089", color="#B7791F"]; + em_prb [label="ext_man_probe_support\nProbe Extraction Limits\nTrace DMA Capabilities", fillcolor="#FAF089", color="#B7791F"]; + em_dbg [label="ext_man_dbg_abi\nDebugger ABI Version\nLog Schema Hashes", fillcolor="#FAF089", color="#B7791F"]; + em_cfg [label="ext_man_config_data\nKey-Value Configuration\nMax IPC Size, Windows", fillcolor="#FAF089", color="#B7791F"]; + + em_hdr -> em_ver -> em_cc -> em_prb -> em_dbg -> em_cfg; + } + } + + subgraph cluster_host_driver { + label="Host Linux ASoC Driver (snd-sof)"; + style="filled,rounded"; + fillcolor="#E6FFFA"; + color="#319795"; + + h_parse [label="Manifest Parser (sof_ext_man_parse)\nExtracts Metadata Before DSP Power-On", fillcolor="#B2F5EA", color="#319795"]; + h_compat [label="Version & ABI Verification\nMatch Kernel Driver Capabilities", fillcolor="#B2F5EA", color="#319795"]; + h_prep [label="Allocate Mailbox & Trace Buffers\nConfigure Stream DMA Windows", fillcolor="#B2F5EA", color="#319795"]; + + h_parse -> h_compat -> h_prep; + } + + em_hdr -> h_parse [label="Host Pre-Boot Inspection", color="#319795", style="dashed"]; + } + +Because the host driver inspects this manifest directly from the binary file prior to downloading code into the DSP, mismatched firmware builds or incompatible ABI revisions are intercepted immediately, preventing kernel panics or DSP hangs. + +--- + +.. _zephyr_boot_stages: + +3. Zephyr RTOS Multi-Stage Initialization +***************************************** + +Sound Open Firmware is natively constructed upon the **Zephyr RTOS**. Zephyr utilizes a deterministic, multi-level initialization table where drivers, core kernel primitives, and application subsystems are systematically registered and executed using the ``SYS_INIT()`` macro. + +Deterministic Initialization Levels +=================================== + +Zephyr defines five sequential initialization levels: + +1. **EARLY**: Low-level platform hardware initialization executed before any OS abstractions exist. No kernel structures or memory allocators are available. +2. **PRE_KERNEL_1**: Core CPU architecture features, basic interrupt controllers, and essential hardware console devices are brought online. No thread scheduling or kernel synchronization primitives exist. +3. **PRE_KERNEL_2**: High-resolution hardware system timers, memory management units (MMU/MPU), and hardware clock trees are initialized. +4. **POST_KERNEL**: The Zephyr kernel is fully operational. Dynamic memory allocators, thread creation, semaphores, and inter-thread messaging primitives are ready. Device drivers and middleware services initialize during this level. +5. **APPLICATION**: Executed after all kernel and device driver subsystems are ready, immediately prior to invoking the main application thread. + +.. graphviz:: + :caption: Zephyr RTOS Multi-Stage Initialization Pipeline and SOF SYS_INIT Integration + + digraph zephyr_stages { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + z_early [label="Level 1: EARLY\nLow-level SoC pinmux, early silicon clocks\n(No OS features available)", fillcolor="#EDF2F7", color="#CBD5E0"]; + z_pk1 [label="Level 2: PRE_KERNEL_1\nCPU registers, vector tables, interrupt controller\nHardware console / early UART", fillcolor="#EDF2F7", color="#CBD5E0"]; + z_pk2 [label="Level 3: PRE_KERNEL_2\nSystem tick timer (HPET/DSP timer), MMU/MPU tables\nClock domain managers", fillcolor="#EDF2F7", color="#CBD5E0"]; + z_post [label="Level 4: POST_KERNEL\nKernel Core Active: Heaps, Threads, Mutexes, Workqueues\nDevice Drivers, Audio Hardware Peripherals", fillcolor="#BEE3F8", color="#3182CE"]; + + subgraph cluster_sof_hook { + label="SOF Entry Hook: SYS_INIT(sof_init, POST_KERNEL, 99)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + sof_entry [label="sof_init() (src/init/init.c)\nExecutes at POST_KERNEL Priority 99\nGuarantees Full OS Infrastructure Ready", fillcolor="#FAF089", color="#B7791F", fontcolor="#744210"]; + } + + z_app [label="Level 5: APPLICATION\nApplication-level services, background monitors", fillcolor="#EDF2F7", color="#CBD5E0"]; + z_main [label="Application Thread: main() -> sof_app_main()\nStart Real-Time Audio Tasks & IPC Mailbox Handoff", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + z_early -> z_pk1 -> z_pk2 -> z_post; + z_post -> sof_entry [label="POST_KERNEL Execution Order"]; + sof_entry -> z_app; + z_app -> z_main; + } + +The Rationale for `POST_KERNEL, 99` +=================================== + +SOF explicitly binds its primary initialization entry point via: + +.. code-block:: c + + /* Registered in src/init/init.c */ + SYS_INIT(sof_init, POST_KERNEL, 99); + +Selecting ``POST_KERNEL`` at priority level ``99`` (the lowest priority within that stage) guarantees that: + +* All hardware buses, DMA controllers, and interrupt routing controllers registered by Zephyr drivers have finished their initialization. +* The Zephyr kernel heap allocator is fully operational, allowing SOF to dynamically allocate its global context structures and buffer descriptors. +* Zephyr thread creation and synchronization APIs (such as ``k_work_queue`` and ``k_thread``) are ready for SOF's deferred IPC handler and real-time audio schedulers. +* The SOF initialization code runs synchronously to completion on Core 0 *before* Zephyr switches execution to user application threads. + +--- + +.. _primary_core_platform_init: + +4. Primary Core Platform Initialization (`primary_core_init`) +************************************************************* + +When Zephyr invokes ``sof_init()``, control transitions immediately to ``primary_core_init()`` in ``src/init/init.c``. This function orchestrates the deterministic bringup of SOF's internal audio subsystem and invokes hardware-specific platform initializers. + +Primary Core Initialization Stages +================================== + +.. graphviz:: + :caption: Primary Core (`primary_core_init`) Execution Flow & Platform Subsystem Bringup Sequence + + digraph primary_core_flow { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + pc1 [label="1. Context Allocation\nAllocate global 'struct sof' context\nBind command arguments and runtime pointers", fillcolor="#EDF2F7", color="#CBD5E0"]; + pc2 [label="2. Logging & DMA Tracing (trace_init)\nConfigure Zephyr log timestamps (k_cycle_get_32)\nAllocate DMA trace buffer; print firmware version banner", fillcolor="#EBF8FF", color="#3182CE"]; + pc3 [label="3. System Notification & Power (pm_runtime_init)\nInitialize system-wide notification dispatch queue\nConfigure runtime power management & idle states", fillcolor="#EBF8FF", color="#3182CE"]; + pc4 [label="4. Platform Bringup (platform_init)\nPlatform clock init & dynamic KCPS budgeting\nInitialize Schedulers: EDF, LL Timer Domain, DP, TWB\nConfigure System Agent, DMACs, IPC Mailbox & Watchdog", fillcolor="#FEFCBF", color="#D69E2E"]; + pc5 [label="5. AltBootManifest Unpack (lp_sram_unpack)\nUnpack LP-SRAM text/data sections for secondary cores\nFlush data cache to memory (dcache_writeback_region)", fillcolor="#E2E8F0", color="#A0AEC0"]; + pc6 [label="6. Audio Registry & Component Setup\nRegister built-in audio components (sys_comp_init)\nInitialize pipeline position offsets (pipeline_posn_init)", fillcolor="#F0FFF4", color="#38A169"]; + pc7 [label="7. Task Loop Handoff (task_main_start)\nComplete primary core setup; enter ready state", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + pc1 -> pc2 -> pc3 -> pc4 -> pc5 -> pc6 -> pc7; + } + +1. **Global Context Setup**: Allocates and binds the singleton ``struct sof`` firmware context, which anchors pointers to memory pools, platform configurations, and audio schedulers. +2. **Logging, Timestamps, and Trace Buffering**: Configures Zephyr's logging timestamp source to the high-resolution hardware cycle counter (``k_cycle_get_32()`` or 64-bit system ticks). Initializes the circular DMA trace buffer (``trace_init()``) and prints the official firmware ABI, build hash, and version banner. +3. **System Notifiers & Runtime Power Management**: Initializes the asynchronous system notification bus (``init_system_notify()``) used for inter-component messaging (such as clock changes and audio underrun broadcasts). Brings up runtime power management (``pm_runtime_init()``) to prepare low-power idle policies. +4. **Platform Hardware Bringup (``platform_init()``)**: Calls the platform-specific hardware initialization routine (e.g., ``src/platform/intel/ace/platform.c`` or ``cavs/platform.c``): + - **Clocks & KCPS**: Configures DSP clock frequencies and initializes the kilo-cycles-per-second (KCPS) dynamic frequency scaling budget. + - **Audio Schedulers**: Instantiates the Earliest Deadline First (EDF) scheduler, the Low-Latency (LL) timer domain, the Data Processing (DP) preemptive thread scheduler, and the Thread With Budget (TWB) scheduler. + - **System Agent**: Configures periodic background health monitors (``sa_init()``) and hardware watchdog timers. + - **Audio DMACs**: Initializes host and peripheral DMA controllers (HD-Audio DMA, GPDMA). + - **Host IPC & IDC**: Allocates shared SRAM mailbox windows (Windows 0 to 3) and configures Inter-Domain Communication (IDC) for multi-core DSPs. +5. **AltBootManifest Unpacking (``lp_sram_unpack()``)**: On platforms where secondary cores lack hardware boot ROMs, the primary core parses the linker-generated ``AltBootManifest`` to copy secondary core executable code and read-only data into Low-Power SRAM (LP-SRAM), followed by data cache write-back flushing. +6. **Component Registry & Pipeline Setup**: Registers built-in processing modules (Volume, Mixer, SRC, EQ) into the component factory table (``sys_comp_init()``) and initializes stream position tracking structures. + +--- + +.. _host_fw_handshake: + +5. Host-Firmware Boot Synchronization & FW Ready Handshake +********************************************************** + +Once the primary core completes internal hardware bringup, it must formally notify the host operating system that the DSP is operational and ready to accept audio stream commands. The host and firmware synchronize through the hardware mailbox and doorbell interrupt registers. + +Protocol Generational Differences: IPC3 vs IPC4 +================================================ + +The handshake mechanism differs fundamentally between protocol generations: + +.. graphviz:: + :caption: Host-Firmware Boot Synchronization & FW Ready Handshake (IPC3 vs IPC4) + + digraph fw_ready_handshake { + graph [rankdir=LR, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + subgraph cluster_ipc3 { + label="IPC3 Boot Handshake (Static Topology)"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + i3_dsp [label="DSP Core 0 completes boot\nConstructs struct sof_ipc_fw_ready\n(Version, Flags, Window Offsets)", fillcolor="#FFFFFF", color="#CBD5E0"]; + i3_win [label="Writes payload to Mailbox Window 0\nRaises Host Doorbell Interrupt", fillcolor="#BEE3F8", color="#3182CE"]; + i3_hst [label="Host receives FW_READY interrupt\nReads Window 0 memory structure\nValidates ABI; Loads Topology", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + i3_dsp -> i3_win -> i3_hst [color="#3182CE"]; + } + + subgraph cluster_ipc4 { + label="IPC4 Boot Handshake (Dynamic Object Model)"; + style="filled,rounded"; + fillcolor="#FEFCBF"; + color="#D69E2E"; + + i4_dsp [label="DSP Core 0 completes boot\nWrites ABI version to fw_reg.abi_ver\nUpdates FW State: FW_STATUS_READY", fillcolor="#FFFFFF", color="#D69E2E"]; + i4_win [label="Sets Mailbox Window 0 Status Register\nFires Host Notification Interrupt", fillcolor="#FAF089", color="#B7791F"]; + i4_hst [label="Host detects FW_STATUS_READY\nReads Window 0 base registers\nSends IPC4 Base FW Capabilities Query", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + + i4_dsp -> i4_win -> i4_hst [color="#B7791F"]; + } + } + +* **IPC3 Handshake Protocol**: + 1. The DSP constructs a structured ``sof_ipc_fw_ready`` message containing ABI major/minor versions, build tags, and an array of memory window descriptors (defining the base offsets and lengths of Windows 0, 1, 2, and 3). + 2. The DSP writes this message directly into Mailbox Window 0 (the Outbox) and rings the host doorbell interrupt. + 3. The host driver's ISR reads Window 0, verifies ABI compatibility, records mailbox memory geometries, clears its boot watchdog timer, and proceeds to parse and download the monolithic topology binary. +* **IPC4 Handshake Protocol**: + 1. The DSP writes the ABI version of the firmware register layout into the ``abi_ver`` field of the firmware status structure within Mailbox Window 0. + 2. The DSP updates the firmware status register to ``SOF_IPC4_FW_STATUS_READY``. + 3. The host driver detects this state transition (via either an interrupt or status register polling), cancels the boot timeout, and issues an initial IPC4 ``GLB_GET_FW_VERSION`` or capabilities query to dynamically discover audio pipeline and module parameters. + +Boot Timeout Protection +======================= + +During boot, the host driver starts a hardware boot timeout monitor (typically 2 to 5 seconds). If the DSP boot ROM, cryptographic validation, or firmware initialization encounters a fatal crash: + +1. The DSP writes panic code dumps, exception vectors, and stack frames into Mailbox Window 0 before halting. +2. If the DSP hangs completely without writing to the mailbox, the host boot timer expires. +3. The host driver logs a boot failure error, captures the DSP register dump, triggers a hardware power-cycle or reset sequence, and prevents sound card registration from hanging the host operating system. + +--- + +.. _multicore_secondary_init: + +6. Multi-Core Initialization & Secondary Core Boot +************************************************** + +Modern audio DSPs (such as Intel cAVS 2.5, ACE 1.5, ACE 2.0, and ACE 3.0) feature multi-core symmetric multiprocessing (SMP) clusters (Dual-Core, Quad-Core, or Octa-Core). To conserve power, secondary cores are kept in low-power power-gated states during early boot and are powered up on demand. + +The Secondary Core Boot Flow +============================ + +When an audio pipeline requires processing capacity beyond Core 0, the host or primary core powers up secondary cores (Core 1, Core 2, Core 3): + +1. **Power Domain Activation**: Core 0 writes to the platform power management control registers to ungated clocks and energize the secondary core's power well. +2. **Zephyr SMP Core Bringup**: The secondary core vectors out of reset into Zephyr's secondary CPU startup stub. +3. **State Assessment (``check_restore()``)**: The secondary core executes ``secondary_core_init()`` in ``src/init/init.c``. It immediately evaluates whether this boot is a **Cold Boot** or a **Power Restore** (e.g., resuming from low-power D0ix retention where memory remained energized): + - If persistent structures (schedulers, notifiers, IDC contexts) are already present in shared memory, ``check_restore()`` returns true, invoking ``secondary_core_restore()``. This bypasses re-allocation, preventing memory leaks and preserving pipeline state. + - If memory was unpowered, the core proceeds with a full cold boot initialization. + +.. graphviz:: + :caption: Secondary Core Boot, Power State Assessment (`check_restore`), and Dynamic Activation Flow + + digraph secondary_core_flow { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.35, ranksep=0.4]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + sc1 [label="Core 0 Power Request\nEnergize Secondary Core Power Well & Release Reset", fillcolor="#EDF2F7", color="#CBD5E0"]; + sc2 [label="Secondary Core Starts: secondary_core_init()\nExecute Early CPU Register Initialization", fillcolor="#EBF8FF", color="#3182CE"]; + sc_check [label="check_restore() Evaluation\nAre Schedulers & IDC Contexts already allocated?", shape=diamond, fillcolor="#FEFCBF", color="#D69E2E"]; + + subgraph cluster_restore { + label="Low-Power Retention Wake"; + style="filled,rounded"; + fillcolor="#F0FFF4"; + color="#C6F6D5"; + + sc_rest [label="secondary_core_restore()\nSkip Structure Re-Allocation\nRe-enable Core Interrupts & IDC", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + } + + subgraph cluster_cold { + label="Full Cold Boot Initialization"; + style="filled,rounded"; + fillcolor="#EDF2F7"; + color="#CBD5E0"; + + sc_not [label="Initialize Core Notifiers\ninit_system_notify(sof)", fillcolor="#FFFFFF", color="#CBD5E0"]; + sc_ll [label="Initialize Core Schedulers\nLL Timer Domain & LL DMA Domain", fillcolor="#FFFFFF", color="#CBD5E0"]; + sc_dp [label="Initialize DP Scheduler\nscheduler_dp_init()", fillcolor="#FFFFFF", color="#CBD5E0"]; + sc_idc [label="Initialize IDC Communications\nidc_init() & AMS Messaging Service", fillcolor="#FFFFFF", color="#CBD5E0"]; + sc_clk [label="Adjust Core Clock Budget\ncore_kcps_adjust(cpu_id, SECONDARY_BASE)", fillcolor="#FFFFFF", color="#CBD5E0"]; + + sc_not -> sc_ll -> sc_dp -> sc_idc -> sc_clk; + } + + sc_ready [label="Secondary Core Enters Idle Loop\nReady to Accept IDC Pipeline Processing Tasks", fillcolor="#68D391", color="#276749", fontcolor="#1C4532"]; + + sc1 -> sc2 -> sc_check; + sc_check -> sc_rest [label="True (D0ix Retention Wake)"]; + sc_check -> sc_not [label="False (Cold Boot)"]; + sc_rest -> sc_ready; + sc_clk -> sc_ready; + } + +Cold Boot Subsystem Configuration +================================= + +During a cold boot, the secondary core configures its own local resources: + +* **Local Core Notifiers**: Registers local core notification queues for intra-core event handling. +* **Independent Low-Latency (LL) Domain**: Sets up dedicated per-core timer domains and DMA domains, allowing the secondary core to drive real-time audio tasks without lock contention with Core 0. +* **Local Data Processing (DP) Scheduler**: Initializes preemptive thread pools for compute-heavy audio algorithms. +* **Inter-Domain Communication (IDC)**: Binds hardware doorbell interrupts between Core 0 and the secondary core, allowing Core 0 to forward host IPC commands and synchronize audio scheduling across cores. +* **Dynamic KCPS Budget**: Adjusts core clock frequencies to match its active processing workload. + +--- + +.. _power_states_boot_lifecycles: + +7. Power State Lifecycles & Wake Transitions +******************************************** + +Firmware initialization occurs not only during system power-on, but also across runtime power state transitions. SOF coordinates with the host operating system to optimize energy efficiency through dynamic power management. + +Power States & Transition Topologies +==================================== + +The DSP transitions across three principal operational states: + +1. **D3 (Cold / Powered Off)**: The entire DSP power well is severed. All internal SRAM contents, registers, and cache lines are completely lost. Waking from D3 requires a complete cold boot: host binary download, DSP ROM cryptographic validation, Zephyr initialization, and full SOF platform bringup. +2. **D0 (Active / Operational)**: The DSP is fully powered. Core 0 and optional secondary cores actively execute audio pipelines, process DMA interrupts, and handle host IPC transactions. +3. **D0ix (Low-Power Idle / Retention)**: When no audio streams are active (or when streams enter extended pause), the DSP transitions into low-power idle. High-Performance SRAM (HP-SRAM) banks are dynamically powered down, and essential context is preserved in Low-Power SRAM (LP-SRAM) or Host DRAM. Secondary cores are powered off. Waking from D0ix bypasses full image download, executing a fast-restore path that re-enables clocks and restores execution in microseconds. + +.. graphviz:: + :caption: Power State Lifecycle Transitions, Wake Sequences, and Context Preservation + + digraph power_states { + graph [rankdir=TB, bgcolor="transparent", fontsize=10, fontname="Arial", nodesep=0.4, ranksep=0.5]; + node [shape=box, style="rounded,filled", fontname="Arial", fontsize=9, margin="0.15,0.1"]; + edge [fontname="Arial", fontsize=8, color="#4A5568", fontcolor="#2D3748"]; + + d3 [label="D3: Fully Powered Off\nPower wells severed; SRAM lost\nZero power draw", fillcolor="#FED7D7", color="#E53E3E", fontcolor="#742A2A"]; + d0 [label="D0: Fully Active Streaming\nCore 0 Active; Secondary Cores Enabled\nFull Audio Processing & DMA Streaming", fillcolor="#C6F6D5", color="#38A169", fontcolor="#22543D"]; + d0ix [label="D0ix: Low-Power Retention Idle\nSecondary cores powered down; HP-SRAM gated\nContext retained in LP-SRAM / Host DRAM", fillcolor="#FEFCBF", color="#D69E2E", fontcolor="#744210"]; + + d3 -> d0 [label="Cold Boot Sequence (Full Init)\nHost DMA download -> ROM verify -> Zephyr -> SOF\nLatency: ~50-150 ms", color="#3182CE", style="bold"]; + d0 -> d3 [label="Host Driver Unbind / System Shutdown\nFlush DMA, save persistent stats, sever power", color="#E53E3E"]; + + d0 -> d0ix [label="Stream Pause / Inactivity Timeout\nSave context to LP-SRAM/DRAM; gate HP-SRAM\nLatency: ~1 ms", color="#D69E2E"]; + d0ix -> d0 [label="Fast Restore Wake (check_restore == True)\nPower up HP-SRAM; skip memory re-allocation\nLatency: ~5-15 µs", color="#38A169", style="bold"]; + } + +LLEXT Dynamic Library Restoration +================================= + +When waking from low-power states where HP-SRAM banks were powered down, dynamically loaded Linkable Loadable Extension (LLEXT) modules must be preserved without requiring the host to re-download shared libraries over PCIe. + +SOF's LLEXT manager (``llext_manager_restore_from_dram()``) caches module text and data sections in host-backed DRAM or non-volatile LP-SRAM. During the wake sequence, the manager automatically verifies image checksums and restores the module code directly into DSP execution memory before the host ready handshake is signaled, ensuring seamless audio playback resumption. + +--- + +.. _upstream_init_references: + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level implementation details, data structure definitions, and linker scripts: + +* **Upstream DSP Initialization Specifications**: + - `thesofproject/sof: src/init/README.md `_ + - `thesofproject/sof: src/platform/intel/ace/platform.c `_ +* **Core Firmware Implementation Files**: + - ``src/init/init.c``: Primary and secondary core initialization logic, ``sof_init()`` hook, and version banners. + - ``src/init/ext_manifest.c``: Extended firmware manifest structure definitions, header parsers, and metadata tables. + - ``zephyr/wrapper.c``: Zephyr application handoff stubs, ``sof_app_main()``, and ``boot_complete()`` signaling. + - ``src/include/sof/init.h``: Global firmware context definitions and initialization function prototypes. + - ``src/include/sof/trace/trace-boot.h``: Boot-time trace point macros and debug markers. + +Related Subsystem Architecture Guides +===================================== + +* :ref:`ipc_infrastructure`: How the host and DSP exchange control messages and synchronize boot state via hardware mailboxes. +* :ref:`scheduler_architecture`: Real-time scheduling domains (LL, DP, TWB) initialized during platform bringup. +* :ref:`audio_buffer_management`: Ring buffer sizing, memory hierarchies (TCM, HP-SRAM, LP-SRAM), and cache operations. +* :ref:`pipeline_architecture`: Dynamic audio processing graph construction following boot completion. +* :ref:`module_framework`: Audio component lifecycle, module adapters, and parameter configuration. diff --git a/developer_guides/firmware/hostless_firmware.rst b/developer_guides/firmware/hostless_firmware.rst new file mode 100644 index 00000000..e053ba7f --- /dev/null +++ b/developer_guides/firmware/hostless_firmware.rst @@ -0,0 +1,564 @@ +.. _sof_hostless_firmware: + +Hostless Embedded Firmware Architecture +####################################### + +While Sound Open Firmware (SOF) is widely deployed as an audio DSP coprocessor driven by an upstream Linux host kernel driver (``snd-sof``), SOF also natively supports **hostless embedded operation**. In hostless mode, the firmware boots autonomously under the **Zephyr RTOS**, establishes audio processing graphs from compiled-in **Static Topologies**, and executes deterministic real-time audio signal processing without requiring an external host operating system or IPC mailbox connection. + +This architecture enables SOF deployment across standalone microcontrollers, dedicated USB/Bluetooth audio bridges, smart speakers, hearing augmentation devices, and embedded IoT appliances. + +.. figure:: images/hostless_system_architecture.svg + :alt: SOF Hostless Embedded Firmware System Architecture + :align: center + :width: 100% + + System-level architecture showing the autonomous Zephyr RTOS foundation, SOF Static Pipeline Loader, in-memory processing engine, endpoint ecosystem, and interactive Zephyr Shell diagnostics. + +Architectural Paradigm: Host-Driven vs. Hostless +************************************************ + +In a standard host-driven architecture, the DSP firmware operates as a subordinate subsystem: an external host operating system (Linux, ChromeOS, Android, Windows) powers on the DSP, downloads signed firmware and topology binaries over DMA, dynamically instantiates pipelines via IPC messages (IPC3 or IPC4), and continuously configures mixer gains and algorithm controls via ALSA user-space daemons. + +In contrast, **Hostless Embedded Operation** shifts full system autonomy to the DSP microcontroller: + +.. list-table:: Architectural Comparison: Host-Driven vs. Hostless Embedded SOF + :widths: 22 39 39 + :header-rows: 1 + + * - Architectural Dimension + - Host-Driven Mode (Coprocessor) + - Hostless Embedded Mode (Autonomous) + * - **System Executive** + - External Host OS (Linux ``snd-sof``) via PCIe/HDA/I2S + - Native **Zephyr RTOS** running directly on DSP / MCU + * - **Topology Source** + - Dynamic binary blobs (``.tplg``) streamed over IPC + - Compiled-in static C graph structures in flash memory + * - **Buffer Allocation** + - Dynamic heap allocation triggered by host IPC stream open + - Pre-allocated static ring buffers in SRAM / PSRAM + * - **Control & Calibration** + - ALSA mixer kcontrols, UCM profiles, and topology blobs + - Static default structs, flash calibration, or Zephyr Shell + * - **Boot Latency** + - Hundreds of milliseconds (PCIe link up, DMA handshake) + - Sub-10 millisecond autonomous cold-boot from flash + * - **Hardware Endpoints** + - Host DMA buffers, SoundWire, Intel SSP, HDA links + - USB Audio Class 2.0 (UAC2), Bluetooth LE Audio, I2S/SAI, PDM, S/PDIF + * - **Target Platforms** + - Intel cAVS/ACE, AMD ACP, NXP i.MX8 DSP cores + - PJRC Teensy 4.1 (i.MX RT1062), Espressif ESP32-P4 / ESP32-C6 + +Static Pipeline Architecture +**************************** + +Hostless platforms define their audio topology graph directly in compiled C source structures rather than parsing serialized ALSA topology binaries at runtime. The SOF Static Pipeline subsystem (``src/audio/pipeline/static_pipeline_loader.c``) interprets these structures at boot time and configures the audio processing pipeline graph. + +.. figure:: images/hostless_static_pipeline_graph.svg + :alt: SOF Hostless Static Audio Pipeline Graph DAG + :align: center + :width: 100% + + Directed Acyclic Graph (DAG) of the hostless playback and capture pipelines showing component bindings, intermediate ring buffers, static kcontrols, and binary calibration injection points. + +Static Topology Data Structures +=============================== + +The static pipeline API (``include/sof/audio/pipeline/static_pipeline.h``) defines a declarative schema for audio components, buffers, routes, and controls: + +.. code-block:: c + + #include + #include + + /* 1. Component Declaration */ + struct sof_static_comp { + uint32_t id; /* Unique component ID */ + uint32_t pipeline_id; /* Owning pipeline ID */ + const char *name; /* Human-readable component name */ + enum sof_static_comp_type type; /* Host/USB Terminal, Module, or DAI */ + const struct sof_uuid *uuid; /* Component RFC 4122 UUID */ + uint32_t direction; /* SOF_IPC_STREAM_PLAYBACK or CAPTURE */ + struct sof_static_caps caps; /* Formats, rates, channel masks */ + + /* Hardware endpoint configuration */ + enum sof_static_ep_type ep_type; + union { + struct { uint32_t terminal_id; } usb; + struct { uint32_t dai_type; uint32_t dai_index; uint32_t format; } dai; + } ep; + + /* Static configuration blob (ABI header + coefficients) */ + const void *init_blob; + size_t init_blob_size; + }; + + /* 2. Intermediate Buffer Declaration */ + struct sof_static_buffer { + uint32_t id; /* Unique buffer identifier */ + size_t size; /* Buffer capacity in bytes */ + enum sof_ipc_frame fmt; /* Frame format (S16_LE, FLOAT, S32_LE) */ + uint32_t flags; /* SOF_MEM_FLAG_DMA | SOF_MEM_FLAG_USER */ + }; + + /* 3. Pipeline Interconnect Route */ + struct sof_static_route { + uint32_t src_comp_id; /* Upstream producer component */ + uint32_t buffer_id; /* Shared circular ring buffer */ + uint32_t sink_comp_id; /* Downstream consumer component */ + }; + + /* 4. Static Kcontrol Definition */ + struct sof_static_kcontrol { + uint32_t id; /* Control identifier */ + const char *name; /* Display name (e.g. "Main Playback Volume") */ + enum sof_static_ctrl_type type; /* Volume, Switch, Enum, Binary */ + uint32_t target_comp_id; /* Attached processing component */ + int32_t min; /* Minimum control value */ + int32_t max; /* Maximum control value */ + int32_t def; /* Default initial value */ + uint8_t uac2_entity_id; /* Bound USB Audio Feature Unit ID */ + }; + + /* 5. Root Topology Container */ + struct sof_static_topology { + const char *name; + size_t num_pipelines; + const struct sof_static_pipeline_desc *pipelines; + size_t num_comps; + const struct sof_static_comp *comps; + size_t num_buffers; + const struct sof_static_buffer *buffers; + size_t num_routes; + const struct sof_static_route *routes; + size_t num_controls; + const struct sof_static_kcontrol *controls; + }; + +Static Pipeline Loader Initialization Flow +========================================== + +When the firmware boots, ``sof_static_pipelines_init()`` invokes ``sof_static_topology_init()`` to instantiate the pipeline graph: + +1. **Pipeline Creation**: For each ``sof_static_pipeline_desc``, a kernel pipeline scheduling object is created via ``pipeline_new()``, specifying priority, execution period (typically 1000 µs), and core affinity. +2. **Component Instantiation**: The loader iterates through ``comps[]``, resolving each component's driver via its UUID (``comp_driver_find()``), allocating the ``struct comp_dev`` instance, and applying any embedded ``init_blob`` coefficients (e.g., initial IIR filter taps or DRC speaker limit profiles). +3. **Circular Buffer Allocation**: Audio ring buffers declared in ``buffers[]`` are allocated in DMA-accessible memory using ``buffer_alloc()``, enforcing cache-line alignment and page constraints. +4. **Graph Routing Connection**: Each entry in ``routes[]`` binds the source component's output sink to the designated buffer and connects that buffer to the downstream sink component via ``pipeline_connect()``. +5. **Kcontrol Binding**: Default gain faders, mute switches, and bypass controls are attached to target components and mapped to external entities (such as USB Audio Class 2.0 Feature Units). +6. **Trigger Pipeline**: Pipelines configured with ``auto_start = true`` or triggered via the shell transition through ``COMP_TRIGGER_PREPARE`` and ``COMP_TRIGGER_START``, arming the Zephyr Low-Latency timer scheduler. + +Supported Hostless Hardware Platforms +************************************* + +SOF hostless firmware is ported and validated across multiple 32-bit and 64-bit embedded microcontroller architectures: + +.. list-table:: Supported Hostless Embedded Platforms Matrix + :widths: 20 22 18 20 20 + :header-rows: 1 + + * - Hardware Platform + - Core Architecture + - Clock Speed + - Digital Audio Interfaces + - Endpoint Connectivity + * - **PJRC Teensy 4.1** + - NXP i.MX RT1062 (ARM Cortex-M7) + - 600 MHz + - SAI1 (I2S / TDM), S/PDIF TX/RX + - USB High-Speed UAC2, eDMA + * - **Espressif ESP32-P4** + - Dual-Core RISC-V (HP Core) + FPU/SIMD + - 400 MHz + - I2S0, I2S1, PDM RX / TX + - USB 2.0 OTG (UAC2), GDMA + * - **Espressif ESP32-C6** + - Single-Core 32-bit RISC-V + - 160 MHz + - I2S, PDM + - Wi-Fi 6, Bluetooth 5.4 LE Audio + +PJRC Teensy 4.1 (NXP i.MX RT1062) +================================= + +The **Teensy 4.1** platform delivers high-performance audio processing on an ARM Cortex-M7 microcontroller: + +* **Audio PLL4 Clock Architecture**: Generates fractional audio root clocks (MCLK, e.g. 12.288 MHz or 24.576 MHz) with low phase noise and jitter, providing exact sample rates for 44.1 kHz and 48 kHz families. +* **Synchronous Audio Interface (SAI)**: Hardware SAI1 supports multi-channel I2S and TDM streaming up to 32-bit depth. +* **Hardware S/PDIF**: Dedicated S/PDIF transmitter and receiver peripheral with biphase mark encoding, verified in automated loopback testing with SNR exceeding 100 dB. +* **Dual-Board Loopback Test Rig**: Board A (Clock Provider / Tx) and Board B (Clock Consumer / Rx) are cross-connected to provide automated pre-commit hardware qualification. + +Espressif ESP32-P4 +================== + +The **ESP32-P4** features dual RISC-V cores with dedicated vector DSP extensions and hardware floating-point units: + +* **Dual-Core Processing Engine**: Core 0 executes real-time pipeline scheduling and DAI transfers, while Core 1 can execute computationally demanding floating-point signal processing (such as Time-Domain Fixed Beamforming or Mel-Frequency Cepstral Coefficient extraction). +* **High-Speed USB 2.0 PHY**: Integrated 480 Mbps USB High-Speed transceiver running USB Audio Class 2.0, streaming up to 192 kHz multi-channel audio with microframe asynchronous rate feedback. +* **Dual Audio Interfaces**: Two independent I2S controllers and dedicated PDM hardware decimation filters for microphone arrays. + +Espressif ESP32-C6 +================== + +The **ESP32-C6** operates as an ultra-compact, low-power wireless audio coprocessor or standalone audio beacon: + +* **Wireless Standards**: Integrated 2.4 GHz Wi-Fi 6 (802.11ax), Bluetooth 5.4 LE Audio, and IEEE 802.15.4 (Zigbee / Thread). +* **Coprocessor Link**: Connects to the primary ESP32-P4 host controller over a high-speed UART / HCI bridge and shared I2S audio bus, offloading Bluetooth Low Energy Audio broadcast processing. + +Endpoint Ecosystem: USB Audio Class 2.0 & Bluetooth +*************************************************** + +Hostless firmware seamlessly bridges external digital transports directly into the SOF audio pipeline. + +USB Audio Class 2.0 (UAC2) Bridge +================================= + +SOF integrates with Zephyr's modular USB device stack (``zephyr/usb/class/usbd_uac2.h``) via ``src/audio/usb_audio.c``: + +.. code-block:: text + + Host PC (ALSA / WASAPI / CoreAudio) + | + | High-Speed USB 2.0 (480 Mbps ISO Endpoints) + v + +-------------------------------------------------------------+ + | Zephyr USB Device Stack (USBD UAC2) | + | - Input Terminal (USB Streaming Out -> Playback Pipeline) | + | - Output Terminal (Capture Pipeline -> USB Streaming In) | + | - Feature Units (Volume & Mute Entity Descriptors) | + +-------------------------------------------------------------+ + | + v (Asynchronous Rate Feedback & PCM FIFO) + +-------------------------------------------------------------+ + | SOF Host Component (USB_PB / USB_CAP) | + | SOF_STATIC_COMP_HOST (.ep.usb.terminal_id) | + +-------------------------------------------------------------+ + +* **Asynchronous Rate Feedback**: The firmware computes fractional sample rate deviations between the local audio hardware clock and USB bus SOF (Start-of-Frame) microframe tokens, transmitting rate feedback packets to the USB host to prevent buffer overrun or underrun. +* **Entity Mapping**: Hardware volume and mute changes from the host OS are routed directly to static kcontrols bound to ``PLAYBACK_FU_ID`` and ``CAPTURE_FU_ID``. + +Bluetooth LE Audio & Wireless Streaming +======================================= + +On platforms equipped with wireless transceivers (ESP32-P4 paired with ESP32-C6), SOF incorporates a dedicated Bluetooth Audio service (``src/audio/bt_service.c`` and ``bt_audio.c``): + +* **LE Audio & Auracast**: Broadcasts and receives Low Complexity Communication Codec (LC3) compressed streams over Bluetooth Low Energy isochronous channels. +* **Classic A2DP & HFP**: Standard Advanced Audio Distribution Profile (SBC/AAC) and Hands-Free Profile with mSBC wideband speech encoding. +* **Dynamic Audio Routing**: The firmware routes audio seamlessly between USB, physical DAIs, and Bluetooth using the shell command ``sof route ``. + +Clocking Topologies & Audio Test Bridges +**************************************** + +Clock synchronization is critical in hostless operation, where the firmware may run as an autonomous clock provider or synchronize its converters as a clock consumer to an external device under test (DUT). + +.. figure:: images/hostless_clocking_audio_bridges.svg + :alt: SOF Hostless Clocking Topologies and Hardware Loopback Audio Bridges + :align: center + :width: 100% + + Clock distribution and synchronization modes: Mode 1 (Clock Provider), Mode 2 (Clock Consumer bridge to DUT), and Mode 3 (Dual-card standalone pre-commit loopback test rig). + +Clock Synchronization Modes +=========================== + +The clocking mode of physical interfaces (I2S and PDM) is governed by ``sof_static_pipeline_set_clock_mode()``: + +1. **Clock Provider Mode**: + The microcontroller's internal PLL generates Bit Clock (BCLK), Frame Sync (FSYNC / LRCK), and audio root clock (MCLK). BCLK frequency satisfies: + + .. math:: + + f_{\text{BCLK}} = 2 \times f_s \times \text{slot\_width} + + For a standard 48.0 kHz 2-channel 32-bit slot configuration: + + .. math:: + + f_{\text{BCLK}} = 2 \times 48000 \times 32 = 3.072\text{ MHz} + + The internal audio PLL drives external DACs, smart amplifiers, and codecs. + +2. **Clock Consumer Mode**: + The microcontroller disables its internal bit-clock dividers and synchronizes its DMA receiver/transmitter to external BCLK and FSYNC lines driven by a host DUT (such as Intel Tiger Lake CAVS or Panther Lake ACE). The microcontroller FIFO tracks external word clocks with zero phase slip. Note that platforms with 1.8V digital I/O (such as Intel Panther Lake) require an inline bidirectional 1.8V to 3.3V voltage level shifter when interfacing with 3.3V microcontroller GPIOs to prevent electrical overstress and ensure signal logic thresholds are met. + +3. **DMIC Injector Mode**: + Specialized clocking configuration where the host DUT drives the PDM clock line, and the hostless bridge generates a phase-aligned PDM microphone bitstream on the data pin, simulating hardware digital microphones for driver automated testing. + +Example: Dual-Card Loopback Test Rig +===================================== + +To prevent regressions in driver registers, DMA controllers, and processing components, hostless boards can be deployed in paired cross-over test configurations: + +.. list-table:: Example: Header J1 Hardware Cross-Jumper Interconnect (Clock Provider Tx to Clock Consumer Rx) + :widths: 25 25 50 + :header-rows: 1 + + * - Transmitter Pin (Provider Tx) + - Receiver Pin (Consumer Rx) + - Signal Description & Hardware Verification + * - **Pin 2 (GPIO 20)** + - **Pin 2 (GPIO 20)** + - I2S Bit Clock (BCLK, 1.536 MHz or 3.072 MHz) + * - **Pin 4 (GPIO 21)** + - **Pin 4 (GPIO 21)** + - I2S Frame Sync (FSYNC / Word Select, 48.000 kHz) + * - **Pin 6 (GPIO 22)** + - **Pin 8 (GPIO 23)** + - I2S Audio Data Out (Transmitter DOUT) to Data In (Receiver DIN) + * - **Pin 10 (GPIO 24)** + - **Pin 10 (GPIO 24)** + - PDM Microphone Clock (PDM_CLK, 3.072 MHz) + * - **Pin 12 (GPIO 25)** + - **Pin 12 (GPIO 25)** + - PDM Microphone Bitstream (PDM_DAT) + * - **Pin 14 (GND)** + - **Pin 14 (GND)** + - Common digital signal ground reference + +Interactive Zephyr Shell Diagnostics +************************************ + +Hostless firmware embeds an interactive command-line diagnostic shell (``src/debug/shell/esp32_shell.c``) accessible via UART or USB CDC ACM virtual serial ports. + +Diagnostic Command Reference +============================ + +.. list-table:: SOF Zephyr Shell Diagnostic Commands + :widths: 30 70 + :header-rows: 1 + + * - Command Syntax + - Description & Operational Behavior + * - ``sof status`` + - Dumps complete firmware telemetry: active pipelines, sample rate, MAC address, clock mode, route, volume, mute, and algorithm bypass states. + * - ``sof mode `` + - Dynamically switches clocking roles without rebooting the microcontroller. + * - ``sof vol `` + - Adjusts playback or capture volume in decibels (e.g. ``sof vol pb -6``). + * - ``sof mute `` + - Mutes or unmutes stream with soft ramping to prevent acoustic pops. + * - ``sof play `` + - Starts or stops the playback pipeline scheduler. + * - ``sof cap `` + - Controls capture pipeline; ``dump`` prints raw PCM sample buffers to console. + * - ``sof tone [freq]`` + - Generates an onboard sine wave (default 1000 Hz) for audio path verification. + * - ``sof eq `` + - Toggles parametric IIR equalizer processing on playback or capture paths. + * - ``sof drc `` + - Toggles dynamic range compressor / speaker excursion limiter. + * - ``sof tdfb `` + - Toggles Time-Domain Fixed Beamformer microphone array filter. + * - ``sof route `` + - Selects audio routing matrix between USB, serial DAI, and Bluetooth transceivers. + * - ``sof bt `` + - Manages Bluetooth LE Audio streaming, format presets, and Auracast broadcasts. + * - ``sof regs`` + - Dumps low-level peripheral hardware registers (I2S/PDM FIFOs, DMA descriptors, clock dividers). + +Example Interactive Shell Session +================================= + +.. code-block:: text + + uart:~$ sof status + === Sound Open Firmware (SOF) Status === + MAC Address: dc:54:75:e8:87:c0 + Playback Pipeline: RUNNING + Capture Pipeline: RUNNING + Active Interface: I2S0 + Clock Mode: CONSUMER (Default) + Audio Route: USB <-> DAI (Default) + BT Audio Stream: DISABLED + Sample Rate: 48000 Hz + Playback Volume: 0 dB (Mute: NO) + Capture Volume: 0 dB (Mute: NO) + Playback EQ: ENABLED + Playback DRC: ENABLED + Capture TDFB: BYPASS + Capture EQ: ENABLED + ======================================== + + uart:~$ sof mode i2s provider + Configured I2S0 clock mode to PROVIDER (BCLK: 1536 kHz, FSYNC: 48 kHz). + + uart:~$ sof tone on 1000 + Generating 1000 Hz sine wave on Playback Pipeline... + + uart:~$ sof cap dump --samples 8 + [00] 0x0000 0x0124 0x02a8 0x03fe 0x04f1 0x05a0 0x0602 0x05f8 + +Developer Tutorial: Authoring a Custom Hostless Pipeline +******************************************************** + +Follow this step-by-step workflow to implement a custom static audio processing topology on an embedded platform. + +Step 1: Define Static Topology in C +=================================== + +Create a new pipeline definition file (e.g. ``src/platform/my_mcu/my_pipeline_def.c``): + +.. code-block:: c + + #include + #include + + /* Extern module UUIDs */ + extern const struct sof_uuid usb_audio_uuid; + extern const struct sof_uuid volume_uuid; + extern const struct sof_uuid eq_iir_uuid; + extern const struct sof_uuid dai_uuid; + + /* 1. Component instances */ + static const struct sof_static_comp my_comps[] = { + SOF_STATIC_COMP_HOST( + .id = 1, .pipeline_id = 1, .name = "USB_IN", + .uuid = &usb_audio_uuid, .direction = SOF_IPC_STREAM_PLAYBACK, + .caps = SOF_STATIC_CAPS(SOF_IPC_FRAME_S16_LE, 48000, 2), + .ep.usb.terminal_id = 1 + ), + SOF_STATIC_COMP_MODULE( + .id = 2, .pipeline_id = 1, .name = "VOL_MAIN", + .uuid = &volume_uuid, .direction = SOF_IPC_STREAM_PLAYBACK, + .caps = SOF_STATIC_CAPS(SOF_IPC_FRAME_S16_LE, 48000, 2) + ), + SOF_STATIC_COMP_DAI( + .id = 3, .pipeline_id = 1, .name = "DAI_OUT", + .uuid = &dai_uuid, .direction = SOF_IPC_STREAM_PLAYBACK, + .caps = SOF_STATIC_CAPS(SOF_IPC_FRAME_S16_LE, 48000, 2), + .ep.dai.dai_type = SOF_DAI_ESP32_I2S, + .ep.dai.dai_index = 0, + .ep.dai.format = SOF_DAI_FMT_I2S + ), + }; + + /* 2. Circular ring buffers */ + static const struct sof_static_buffer my_buffers[] = { + SOF_STATIC_BUFFER(.id = 1, .size = 3072, .fmt = SOF_IPC_FRAME_S16_LE), + SOF_STATIC_BUFFER(.id = 2, .size = 3072, .fmt = SOF_IPC_FRAME_S16_LE), + }; + + /* 3. Audio routes connecting components */ + static const struct sof_static_route my_routes[] = { + SOF_STATIC_ROUTE(.src_comp_id = 1, .buffer_id = 1, .sink_comp_id = 2), + SOF_STATIC_ROUTE(.src_comp_id = 2, .buffer_id = 2, .sink_comp_id = 3), + }; + + /* 4. Playback pipeline descriptor */ + static const struct sof_static_pipeline_desc my_pipelines[] = { + { + .pipeline_id = 1, + .name = "Playback Pipeline", + .direction = SOF_IPC_STREAM_PLAYBACK, + .priority = 0, + .core = 0, + .period = 1000, + .frames_per_sched = 48, + .time_domain = SOF_TIME_DOMAIN_TIMER, + .sched_comp_id = 1, + .source_comp_id = 1, + .sink_comp_id = 3, + } + }; + + /* 5. Root topology structure */ + const struct sof_static_topology g_my_static_topology = { + .name = "My Custom Audio Topology", + .num_pipelines = ARRAY_SIZE(my_pipelines), + .pipelines = my_pipelines, + .num_comps = ARRAY_SIZE(my_comps), + .comps = my_comps, + .num_buffers = ARRAY_SIZE(my_buffers), + .buffers = my_buffers, + .num_routes = ARRAY_SIZE(my_routes), + .routes = my_routes, + }; + +Step 2: Enable Static Pipeline in Kconfig +========================================= + +Enable the static pipeline loader and selected processing modules in ``prj.conf``: + +.. code-block:: kconfig + + CONFIG_SOF_STATIC_PIPELINE=y + CONFIG_COMP_VOLUME=y + CONFIG_COMP_EQ_IIR=y + CONFIG_COMP_TONE=y + CONFIG_USB_DEVICE_STACK_NEXT=y + CONFIG_USBD_AUDIO_CLASS_2=y + CONFIG_SHELL=y + CONFIG_SOF_DEBUG_SHELL=y + +Step 3: Build, Flash & Verify +============================= + +Build the firmware using Zephyr's ``west`` tool: + +.. code-block:: bash + + # Build for ESP32-P4 + west build -b esp32p4_function_ev_board app -- -DEXTRA_CONF_FILE="prj_hostless.conf" + + # Flash board over USB serial + west flash + + # Verify audio loopback playback and capture using ALSA utilities: + # 1. Start capture stream on receiver + arecord -D hw:CARD=P4,DEV=0 -r 48000 -f S16_LE -c 2 -d 4 capture.wav & + + # 2. Play test tone through transmitter + aplay -D hw:CARD=P4,DEV=0 -r 48000 -f S16_LE -c 2 test_1000hz.wav + + # Alternatively, execute an automated loopback test harness: + # python3 /test_loopback.py --mode i2s + +Troubleshooting & Diagnostic Matrix +*********************************** + +.. list-table:: Common Hostless Firmware Issues & Diagnostic Recipes + :widths: 25 35 40 + :header-rows: 1 + + * - Error Symptom + - Root Cause + - Diagnostic & Resolution Procedure + * - **Audio Glitches / Periodic Clicks** + - Clock drift between USB SOF tokens and physical I2S word clock. + - Verify asynchronous rate feedback endpoint in ``usb_audio.c``; ensure feedback interval is 1 ms and DMA period matches ``frames_per_sched`` (48 frames at 48 kHz). + * - **Buffer Starvation (XRUN)** + - Microcontroller configured as Clock Provider while connected to an active Clock Provider DUT. + - Switch clocking role: execute ``sof mode i2s consumer`` via the shell so microcontroller FIFOs synchronize to incoming external BCLK/FSYNC. + * - **Static Noise on Floating-Point Processing** + - Bit-depth quantization mismatch between S16_LE buffers and FLOAT processing modules. + - Check ``struct sof_static_buffer`` declarations; ensure PCM converters or format flags match module capability masks (e.g. S16_LE for Volume, FLOAT for EQ/DRC). + * - **DMIC Capture Silence** + - Missing PDM clock or incorrect GPIO multiplexing. + - Inspect peripheral registers with ``sof regs``; verify PDM clock divider generates nominal 3.072 MHz and pinmux connects PDM_CLK and PDM_DAT. + * - **Teensy S/PDIF Unlock** + - Fractional divider on Audio PLL4 uncalibrated. + - Check PLL4 numerator/denominator registers in ``imx_rt_clk.c``; ensure PLL4 locks to exactly 48.000 kHz phase lock. + +Terminal Diagnostic Recipes +=========================== + +* **Test Waveform Playback from Host PC**: + + .. code-block:: bash + + # Stream 1000 Hz test sine wave into hostless USB audio bridge + aplay -D hw:CARD=P4,DEV=0 -r 48000 -f S16_LE -c 2 test_1000hz.wav + +* **Inspect USB Audio Class 2.0 Descriptors**: + + .. code-block:: bash + + # Query terminal descriptors and feature units + lsusb -d 303a: -v | grep -A 10 "AudioControl" + +* **Capture Recorded Audio from Digital Bridge**: + + .. code-block:: bash + + # Record captured audio stream for FFT and SNR calculation + arecord -D hw:CARD=P4,DEV=0 -r 48000 -f S16_LE -c 2 -d 4 capture.wav diff --git a/developer_guides/firmware/images/hostless_clocking_audio_bridges.svg b/developer_guides/firmware/images/hostless_clocking_audio_bridges.svg new file mode 100644 index 00000000..81e979bf --- /dev/null +++ b/developer_guides/firmware/images/hostless_clocking_audio_bridges.svg @@ -0,0 +1,256 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SOF Hostless Clocking Topologies & Audio Bridges + + + Clock Provider Generation, Clock Consumer Synchronization, and Dual-Card Loopback Test Rigs + + + + + + + + + + Mode 1: Provider Mode (Standalone Audio Device / Smart Node) + + + + + + + SOF Hostless MCU (Clock Provider) + ESP32-P4 / Teensy 4.1 + Internal Clock Generator: + • Audio PLL4 (i.MX RT) + • APLL (ESP32-P4) + Clock Outputs Generated: + • MCLK: 12.288 MHz + • BCLK: 1.536 MHz (32fs) + • FSYNC: 48.000 kHz + Audio Data Transceiver: + • DOUT: PCM Tx Stream + • DIN: PCM Rx Stream + + + + + + + + + External Peripherals (Clock Consumer) + Synchronized to MCU Clocks + • Audio Codec / DAC + Receives MCLK, BCLK, FSYNC + • Smart Power Amp (DSM) + I/V sense capture feedback + • S/PDIF Receiver / DAC + SPDIF coaxial/optical lock + Zero host PC required + Autonomous boots from flash + + + + + + + + + + + Mode 2: Consumer Mode (Audio Test Bridge to Host DUT) + + + + + + + Host DUT (Clock Provider) + Tiger Lake (TGL) / Panther Lake (PTL) + DSP Audio Interface: + • Intel cAVS 2.5 / ACE 3.0 + • SSP / I2S / PDM Port + Clocks Driven to Bridge: + • BCLK: Driven by DUT + • FSYNC: Driven by DUT + DMIC Clocking: + • PDM CLK driven by DUT + • Bridge injects PDM DATA + + + + + + + + + SOF Audio Bridge (Clock Consumer) + ESP32-P4 (Target Audio Bridge) + Clock Configuration: + • SOF_CLOCK_CONSUMER + • SOF_CLOCK_DMIC + FIFO Synchronization: + • GDMA follows DUT ticks + • Zero buffer slip / XRUN + USB UAC2 to Linux Host: + arecord -D hw:<card> + Streams audio to test rig + + + + + + + + + + + Mode 3: Dual-Board Standalone Hardware Loopback (Mandatory Pre-Commit Verification Rig) + + + + + + + Board 1: Transmitter (Clock Provider Tx) + Transmitter (ESP32-P4 / Teensy 4.1) + + + + 1. USB Audio Playback Stream (Host PC) + aplay -D hw:<card_tx>,0 test_1000hz.wav + + 2. Hardware Clock Generation (Provider): + • APLL / Audio PLL4 generates 48.000 kHz FSYNC + • Generates 1.536 MHz BCLK (32-bit slot) + + 3. Physical Pinout Driving (Header J1 / Pins): + • BCLK → Pin 2 (GPIO 20 / Pin 21) + • FSYNC → Pin 4 (GPIO 21 / Pin 20) + • DOUT → Pin 6 (GPIO 22 / Pin 7) + • PDM → Pin 10 (CLK) + Pin 12 (DATA) + + Autonomous Tone Option: sof tone on + + + + + + Hardware Cross-Jumper Link + Direct Board-to-Board Interconnect + + + Tx Card (J1) + Signal + Rx Card (J1) + + + + Pin 2 (GPIO20) + + Pin 2 (BCLK) + + Pin 4 (GPIO21) + + Pin 4 (FSYNC) + + Pin 6 (GPIO22) + + Pin 8 (DIN) + + Pin 10 (PDM_CLK) + + Pin 10 (PDM_CLK) + + Pin 12 (PDM_DAT) + + Pin 12 (PDM_DAT) + + Pin 14 (GND) + + Pin 14 (GND) + + + + No external breadboards or active transceivers required. + + + + + + + Board 2: Receiver (Clock Consumer Rx) + Receiver (ESP32-P4 / Teensy 4.1) + + + + 1. Clock Configuration (Consumer): + sof mode i2s consumer | sof mode pdm consumer + + 2. Hardware Capture & Recording: + arecord -D hw:<card_rx>,0 -r 48000 cap.wav + + 3. Automated Pre-Commit Acceptance Criteria: + • I2S SNR: > 80.0 dB (Nominal: 93.9 dB) + • PDM SNR: > 65.0 dB (Nominal: 83.7 dB) + • Teensy SAI1: > 80.0 dB (Nominal: 141.5 dB) + • Teensy S/PDIF: > 70.0 dB (Nominal: 109.0 dB) + • Dropped Frames / Glitches: 0 + + + + diff --git a/developer_guides/firmware/images/hostless_static_pipeline_graph.svg b/developer_guides/firmware/images/hostless_static_pipeline_graph.svg new file mode 100644 index 00000000..57c3679a --- /dev/null +++ b/developer_guides/firmware/images/hostless_static_pipeline_graph.svg @@ -0,0 +1,346 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SOF Hostless Static Audio Pipeline Graph (DAG) + + + Component Declarations, Intermediate Ring Buffers, Kcontrol Interconnects & Binary Calibration Injection + + + + + + + + + + Pipeline 1 (Playback): USB / Bluetooth Endpoint → Volume → Parametric IIR EQ → DRC Limiter → Physical DAI (I2S / SAI) + + + + + + + + USB_PB (Comp 1) + SOF_STATIC_COMP_HOST + USB Terminal ID: + PLAYBACK_TERM_ID + UAC2 ISO Out Endpoint + 48 kHz, 16-bit, 2ch + + + + + + Buffer 1 + 3072 B + S16_LE + Ring Buf + + + + + + VOL_PB (Comp 2) + SOF_STATIC_COMP_MODULE + Main Playback Volume + • Soft-ramp curve + • Linear / dB volume + 48 kHz, 16-bit, 2ch + + + + + + Buffer 2 + 3072 B + S16_LE + Ring Buf + + + + + + EQ_PB (Comp 3) + SOF_STATIC_COMP_MODULE + Parametric Biquad IIR + • 4-band Direct Form II + • Hardware FPU execution + 48 kHz, FLOAT, 2ch + + + + + + Buffer 3 + 3072 B + FLOAT + Ring Buf + + + + + + DRC_PB (Comp 4) + SOF_STATIC_COMP_MODULE + Dynamic Range Compressor + • Speaker excursion limit + • Lookahead peak detector + 48 kHz, FLOAT, 2ch + + + + + + Buffer 4 + 3072 B + S16_LE + Ring Buf + + + + + + DAI_PB (Comp 5) + SOF_STATIC_COMP_DAI + Physical Serial Out: + ESP32 I2S / SAI1 + Provider or Consumer Clk + BCLK, FSYNC, DOUT + + + + + + + + + Static Kcontrols (VOL) + • KCONTROL_VOLUME + Target: VOL_PB, 0 dB default + • KCONTROL_SWITCH + Playback Mute (PLAYBACK_FU) + + + + + + EQ Calibration & Switch + • init_blob: IIR_2ch (51 words) + Biquad coefficients in ROM + • KCONTROL_SWITCH + Bypass toggle (sof eq pb bypass) + + + + + + DRC Profile & Switch + • init_blob: DRC (35 words) + Static curve & attack/decay + • KCONTROL_SWITCH + Bypass toggle (sof drc pb bypass) + + + + + + + + + + + Pipeline 2 (Capture): Physical Ingress DAI (I2S / PDM) → TDFB Beamformer → Parametric EQ → Volume → USB / BT Host + + + + + + + + DAI_CAP (Comp 6) + SOF_STATIC_COMP_DAI + Physical Serial In: + I2S / PDM Microphone + Provider, Consumer, DMIC + BCLK, FSYNC, DIN + + + + + + Buffer 5 + 3072 B + S16_LE + Ring Buf + + + + + + TDFB_CAP (Comp 7) + SOF_STATIC_COMP_MODULE + Fixed Beamformer + • Multi-mic spatial filter + • Directional acoustic focus + 48 kHz, FLOAT, 2ch + + + + + + Buffer 6 + 3072 B + FLOAT + Ring Buf + + + + + + EQ_CAP (Comp 8) + SOF_STATIC_COMP_MODULE + Mic Calibration EQ + • High-pass DC rejection + • Transducer equalization + 48 kHz, FLOAT, 2ch + + + + + + Buffer 7 + 3072 B + FLOAT + Ring Buf + + + + + + VOL_CAP (Comp 9) + SOF_STATIC_COMP_MODULE + Capture Gain & Mute + • Mic preamp digital gain + • Soft-ramp transitions + 48 kHz, 16-bit, 2ch + + + + + + Buffer 8 + 3072 B + S16_LE + Ring Buf + + + + + + USB_CAP (Comp 10) + SOF_STATIC_COMP_HOST + USB Terminal ID: + CAPTURE_TERM_ID + UAC2 ISO In Endpoint + 48 kHz, 16-bit, 2ch + + + + + + + + + TDFB Beam Controls + • KCONTROL_SWITCH + TDFB Bypass switch + • Shell toggle: + sof tdfb enable|bypass + + + + + + Capture EQ Calibration + • init_blob: IIR_2ch + Mic high-pass biquads + • KCONTROL_SWITCH + Bypass toggle (sof eq cap) + + + + + + Static Kcontrols (VOL_CAP) + • KCONTROL_VOLUME + Target: VOL_CAP (dB fader) + • KCONTROL_SWITCH + Capture Mute (CAPTURE_FU) + + + + diff --git a/developer_guides/firmware/images/hostless_system_architecture.svg b/developer_guides/firmware/images/hostless_system_architecture.svg new file mode 100644 index 00000000..f53ca59c --- /dev/null +++ b/developer_guides/firmware/images/hostless_system_architecture.svg @@ -0,0 +1,277 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SOF Hostless Embedded Firmware: System Architecture + + + Autonomous Zephyr RTOS Foundation, Static Pipeline Loader, Processing Engine & Endpoint Ecosystem + + + + + + + + + + Zephyr RTOS Hardware Foundation & Board Bringup + + + + + + Kernel Init & Device Tree Bindings + + + + Hardware Audio PLL & Clock Tree + + + + DMA Engine (eDMA / GDMA Transfers) + + + + Hardware FPU & Vector DSP Acceleration + + + + + + + + + + + + + + SOF Static Pipeline Engine (static_pipeline_loader.c) + + + + + + + ROM-Compiled Static Topology Descriptor + struct sof_static_topology + + • Defines pipelines, components, buffers, routes, and kcontrols in flash memory. + + + • Zero IPC parsing overhead; instantiated instantly during firmware initialization. + + + + + + + In-Memory Audio Processing Pipeline + + + + + Volume / Ramp + + + Parametric EQ + + + DRC Limiter + + + TDFB Array + + + Tone Gen + + + + • Intermediate ring buffers allocated via SOF_STATIC_BUFFER() in SRAM/PSRAM. + + + • Deterministic Low-Latency scheduler executes 1 ms processing ticks on Core 0. + + + • Dynamic coefficient injection via compiled-in binary calibration blobs. + + + + + + + Static Kcontrols & Board Calibration + + Initializes volume gains (0 dB unity), soft-unmute ramps, EQ/DRC bypass toggles, + + + and channel mappings directly from flash without waiting for ALSA mixer setup. + + + + + + + + + + + + Endpoint Ecosystem: USB, Bluetooth & Physical DAIs + + + + + + + USB Audio Class 2.0 (UAC2 Bridge) + usb_audio.c / zephyr/usb/class/usbd_uac2.h + + • Asynchronous ISO endpoints with SOF microframe rate feedback. + + + • Maps UAC2 Feature Units to SOF volume and mute static controls. + + + • Provides standard USB audio card to external host PCs and test rigs. + + + + + + + Bluetooth LE Audio & Wireless Transceiver + bt_audio.c / bt_service.c (ESP32-C6 Coprocessor Link) + + • LE Audio Broadcast (Auracast) and Unicast with LC3 compression. + + + • Dual-chip coprocessor architecture (HCI over UART / Shared I2S bus). + + + • A2DP Sink/Source & HFP/mSBC voice call support. + + + + + + + Physical Serial Audio DAIs + + • I2S / TDM Transceiver: Multi-channel audio (ESP32 I2S, Teensy SAI1). + + + • PDM / DMIC Receiver: Pulse-density decimation for microphone arrays. + + + • S/PDIF Optical/Coaxial: High-fidelity 24-bit audio stream output. + + + • Dynamic Provider/Consumer clock switching (sof mode i2s provider|consumer). + + + + + + + + + + + + + + + + Interactive Zephyr Shell Diagnostics & Hardware Telemetry (esp32_shell.c) + + + + + + + Diagnostic & Control CLI (UART / USB CDC ACM) + sof status + : Print pipelines, sample rate, MAC, route, volume + sof mode <i2s|pdm> <provider|consumer|dmic> + : Switch clocking mode + sof vol <pb|cap> <dB> | sof mute <pb|cap> <on|off> + sof eq|drc|tdfb <enable|bypass> | sof tone <on|off> + + + + + + Hardware Testbench & Loopback Validation + + • sof cap dump --samples N: Real-time raw PCM buffer inspection. + + + • sof regs: Dump live hardware I2S/PDM control registers, FIFOs & dividers. + + + • sof bt status|broadcast|scan: LE Audio Auracast broadcast management. + + + • Standalone audio loopback validation (automated test scripts or ALSA tools). + + + + + diff --git a/developer_guides/firmware/images/llext_build_packaging_flow.svg b/developer_guides/firmware/images/llext_build_packaging_flow.svg new file mode 100644 index 00000000..2f1b523b --- /dev/null +++ b/developer_guides/firmware/images/llext_build_packaging_flow.svg @@ -0,0 +1,311 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SOF LLEXT Build, Link & Packaging Pipeline + + + From C Source and TOML Header to Relocatable ELF Linking, Rimage Signing (-l), and UUID Deployment Symlinks + + + + + + + + + + + 1. Source & Metadata + + + + + + + + Module Adapter Driver + src/audio/<module>/ + • Implements standard API: + module_adapter_ops + • init, process, set_config + Pure algorithm logic + SIMD + + + + + + LLEXT Helper Macros + SOF_LLEXT_BUILDINFO + Emits .mod_buildinfo ABI + SOF_LLEXT_MODULE_MANIFEST + Emits .module descriptor: + - UUID, entry, affinity + - SOF_MAN_MOD_TYPE_LLEXT + + + + + + Config Header: llext.toml.h + #include "platform.toml" + #include <module.toml> + Declares module descriptor: + • uuid = UUIDREG_STR_* + • affinity, domain, auto_start + + + + + + Kconfig Tristate Switch + CONFIG_COMP_<MOD>=m + Builds as standalone LLEXT + CONFIG_COMP_<MOD>=y + Builds built-in to basefw + + + + + + + + + + + + + + 2. CMake & Toolchain Link + + + + + + + + CMake: sof_llext_build() + sof_llext_build("${module}" + SOURCES ... + LIB openmodules + ) + Calls add_llext_target() + + + + + + Xtensa Compiler Flags + -mtext-section-literals + Places literal pools inline + directly before L32R instructions + -nostdlib -nodefaultlibs + Eliminates standard library bloat + + + + + + Relocatable vs Shared + CONFIG_LLEXT_TYPE_ELF_ + RELOCATABLE=y (-r) + Produces relocatable object file + Shared Mode (-shared) + Alternative ELF shared library + + + + + + Intermediate Output + ${module}.llext + Contains un-resolved imports + referencing basefw EXPORT_SYMBOL + + + + + + + + + + + + + + 3. TOML & Rimage Signing + + + + + + + + C Preprocessor TOML Gen + gcc -E llext.toml.h -P + -imacros autoconf.h + -imacros uuid-registry.h + Expands __COUNTER__ and + UUIDREG_STR_* to rimage.toml + + + + + + Size & Link Helpers + llext_offset_calc.py + Maintains cumulative size + llext_link_helper.py + Sets --text-addr offset based + on CONFIG_LIBRARY_BASE_ADDRESS + + + + + + Rimage Dynamic Mode (-l) + rimage -l -k key.pem + -c rimage_config.toml + -o ${module}.ri ${module}.llext + • Preserves dynamic relocations + • Computes SHA digests & CSS + + + + + + Extended Manifest Sidecar + ${module}.ri.xman + Contains build version & ABI flags; + prepended during packaging + + + + + + + + + + + + + + 4. Deployment & Symlinks + + + + + + + + xtensa-build-zephyr.py + Concatenates container: + .ri.xman + .ri -> .llext + Places into deployment tree: + /lib/firmware/intel/sof-ipc4/ + <platform>/${module}.llext + + + + + + UUID-Named Symlinks + Topology references module UUID; + driver requests file by UUID: + ${UUID}.bin -> ${module}.llext + Multi-UUID support (e.g. mixin): + 39656EB2...bin -> mixin.llext + 3C56505A...bin -> mixin.llext + + + + + + Auxiliary Libraries + aux\d_*.llext + Shared algorithmic engines + openmodules.llext + Consolidated open-source bundle + + + + + + Ready for Host DMA + Linux kernel request_firmware() + loads ${UUID}.bin on-demand + during pipeline instantiation. + + + + + + + + + Key Build Guarantee: + + + Modules compile with zero static dependencies on unexported internals; Rimage validates binary layout against platform TOML. + + + The resulting container encapsulates an Extended Manifest (.xman), CSE directory, CSS signature, module descriptor table, and relocatable ELF segments. + + + diff --git a/developer_guides/firmware/images/llext_runtime_lifecycle.svg b/developer_guides/firmware/images/llext_runtime_lifecycle.svg new file mode 100644 index 00000000..6f5f9f0d --- /dev/null +++ b/developer_guides/firmware/images/llext_runtime_lifecycle.svg @@ -0,0 +1,265 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SOF LLEXT Runtime Lifecycle & Memory Protection State Machine + + + Host IPC4 Dynamic Loading Handshake, Virtual Memory Mapping, Cache Synchronization, and Lifecycle Management + + + + + + + + + + Phase 1: Linux Host Driver Handshake (sound/soc/sof/ipc4-loader.c) + + + + + + + + 1. Topology UUID Request + + Pipeline creation requests module UUID; driver loads /lib/firmware/.../${UUID}.bin + + + + + + + 2. IPC4_GLB_LOAD_LIBRARY_PREPARE + + Host sets up DMA buffer; allocates DSP dma_id and assigns module lib_id. + + + + + + + 3. DMA Transfer & IPC4_GLB_LOAD_LIBRARY + + Host streams binary container into DSP staging buffer; sends trigger to execute load. + + + + + + + + + + + + Phase 2: Ingestion, Crypto Verify & ABI Validation (lib_manager.c) + + + + + + + + 1. Cryptographic Manifest Verification + + Validates CSS signature ($MN2) and ADSP descriptor ($AM1) against platform keys. + + + + + + + 2. ABI Version Handshake + + Verifies: buildinfo->api_version_number == SOF_MODULE_API_CURRENT_VERSION + + + + + + + 3. Library Context Registration + + Allocates struct lib_manager_mod_ctx; stores manifest and segment table. + + + + + + + + + + + + Phase 3: Virtual Memory Mapping, Section Relocation & Permission State Machine (llext_manager.c) + + + + + + + + A. Temporary RW Map + sys_mm_drv_align_map() + • Initial permission: + SYS_MM_MEM_PERM_RW + • Aligned to 4KB page + Allows loader to copy + data & patch symbols. + + + + + + B. Relocation & Copy + llext_load() engine + • Copy .text, .rodata, .data + • Zero .bss section + • Patch Xtensa opcodes + • Link EXPORT_SYMBOL + Resolves all imports. + + + + + + C. D-Cache Flush + dcache_writeback_region() + • Mandatory before + changing MMU flags + • Flushes patched code + from L1/L2 data cache + into physical SRAM. + + + + + + D. Permission Lockdown + update_region_flags() + • .text -> RX (EXEC) + • .rodata -> Read-Only + • .data/.bss -> Read-Write + Enforces W^X security: + No writable & exec pages. + + + + + + E. I-Cache Invalidate + icache_invalidate_region() + • Clears stale instruction + cache lines on all cores + • Guarantees pipeline fetches + newly written opcodes + Safe for DSP execution. + + + + + + + + + + + Phase 4: Module Allocation & Audio Processing + + + + + • llext_manager_allocate_module(): checks dependencies, increments refcount (n_dependent). + + + • Calls module entry_point() -> returns pointer to struct module_interface. + + + • Hooks into LL/DP scheduler: calls module_adapter_process() per 1 ms audio tick. + + + Audio processing runs with native performance; zero virtualization or IPC overhead. + + + + + + + + + + + Phase 5: Unloading & Cold-Store DRAM Staging + + + + + • Pipeline free: llext_manager_free_module() decrements reference count. + + + • IPC4 UNLOAD_LIBRARY: unmaps virtual memory, frees SRAM pages. + + + • D3 Power State: llext_manager_store_to_dram() backs up modules into host DRAM; + + + restores instantly on wake without requiring host driver to re-stream over DMA. + + + + diff --git a/developer_guides/firmware/images/llext_system_architecture.svg b/developer_guides/firmware/images/llext_system_architecture.svg new file mode 100644 index 00000000..edc120c9 --- /dev/null +++ b/developer_guides/firmware/images/llext_system_architecture.svg @@ -0,0 +1,268 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SOF LLEXT Dynamic Loadable Modules: System Architecture + + + Modular Run-Time Extension Subsystem Bridging Base Firmware, Zephyr LLEXT API, and Audio Pipelines + + + + + + + + + + Base Firmware Core Subsystems & Export Table + + + + + + + Audio Engine & Dispatch + • LL & DP Task Schedulers + • IPC4 Command Protocol Engine + + + + Audio Buffers & Heap + • Audio Circular Ring Buffers + • rballoc / rfree Heap Allocators + + + + EXPORT_SYMBOL() Global Linkage Table + + • Logging: tr_err, tr_warn, tr_info | Safety: memcpy_s, memset_s + + + • Notifiers: notifier_register, notifier_unregister, notifier_event + + + • Clocks: cpu_clock_manager_request, cpu_clock_manager_release + + + Base firmware exposes safe public symbols; module unresolved symbols linked dynamically + + + + + Base Firmware ABI Handshake: + + SOF_MODULE_API_CURRENT_VERSION (Enforces binary compatibility with loaded LLEXT) + + + + + + + + + + + Dynamic LLEXT Module Binary (.llext / .ri) + + + + + + + .text & .literal + Executable Code (SYS_MM_MEM_PERM_EXEC) + Xtensa literals placed via -mtext-section-literals + + + .rodata + Read-Only Data (SYS_MM_MEM_PERM_RO) + Filter taps, math LUTs, coefficient matrices + + + + .data + Initialized Writable Data (RW) + Default states, driver lookup structures + + + .bss + Zero-Initialized Data (Within .data) + Scratch memory, zero-cleared on module load + + + + .mod_buildinfo + — SOF_LLEXT_BUILDINFO macro + + Encodes struct sof_module_api_build_info with ABI version and build tag. + + + Validated during module allocation; prevents loading incompatible binaries. + + + + + .module Manifest + — SOF_LLEXT_MODULE_MANIFEST + + UUID, entry_point (module_init), affinity_mask, load_type (SOF_MAN_MOD_TYPE_LLEXT) + + + + + + + + + Dynamic + Linkage + + + + + + + + + SOF LLEXT Manager Execution & Memory Orchestration Pipeline (llext_manager.c) + + + + + + + + 1 + IPC4 Library Dispatch + + • LOAD_LIBRARY_PREPARE + • LOAD_LIBRARY over DMA + • Host transfers signed .ri + • Manifest signature verify + • Staged in memory buffer + + Library context created; module ID mapped to component UUID table. + + + + + + + + 2 + Zephyr LLEXT Loader + + • Parse ELF headers & shdr + • Relocate Xtensa opcodes + • Resolve exported symbols + • Dependency resolution + • Refcount: n_dependent + + Symbol table lookup matches imports against basefw EXPORT_SYMBOL list. + + + + + + + + 3 + MMU / MPU Security + + • sys_mm_drv_align_map + • Page size: PAGE_SZ (4 KB) + • Copy phase: Map as RW + • Exec phase: Update to RX + • Read-Only: .rodata locked + + Memory domain (k_mem_domain) isolates user-mode algorithms from kernel. + + + + + + + + 4 + Cache & Instantiation + + • dcache_writeback_region + • icache_invalidate_region + • Invoke entry_point() + • Bind module_interface + • DRAM cold store backup + + Module adapter hooks into active audio pipeline; processes real-time PCM. + + + + diff --git a/developer_guides/firmware/ipc_infrastructure.rst b/developer_guides/firmware/ipc_infrastructure.rst new file mode 100644 index 00000000..d9199ce6 --- /dev/null +++ b/developer_guides/firmware/ipc_infrastructure.rst @@ -0,0 +1,590 @@ +.. _ipc_infrastructure: + +IPC Infrastructure (IPC3 & IPC4) +################################ + +The **Inter-Processor Communication (IPC)** infrastructure in Sound Open Firmware (SOF) is the primary messaging conduit and control plane bridging the host operating system (mainline Linux ASoC drivers, Windows audio subsystems) and the Digital Signal Processor (DSP) firmware. It coordinates audio pipeline topologies, runtime module parameter updates, hardware interface configurations, stream power states, and real-time diagnostic telemetry. + +This guide provides a high-level conceptual overview of the IPC messaging framework, hardware mailbox windows, doorbell interrupt handshakes, deferred Zephyr work queues, protocol evolution from IPC3 to IPC4, dynamic module binding, asynchronous telemetry, and multi-core Inter-Domain Communication (IDC) without focusing on low-level C code. + +.. contents:: Table of Contents + :local: + :depth: 2 + +--- + +1. IPC Infrastructure & Communication Model +******************************************* + +The Dual Planes of Inter-Processor Communication +================================================ + +In modern audio systems, the DSP operates as an autonomous processor requiring tightly coordinated, bidirectional communication with the host kernel: + +1. **The Control Plane (Host to DSP)**: + * **Pipeline Topology Instantiation**: Dynamically assembling audio pipelines, allocating memory buffers, and binding processing components. + * **Parameter Configuration**: Applying volume curves, equalizer filter coefficients, dynamic range compressor profiles, and microphone calibration blobs. + * **Stream State Machine**: Transitioning audio streams through operational states (``PREPARE``, ``START``, ``PAUSE``, ``STOP``, ``RESET``). + * **Power Management**: Coordinating clock scaling, core sleep states, and host D0ix runtime power transitions. + +2. **The Telemetry & Event Plane (DSP to Host)**: + * **Stream Position Tracking**: High-frequency DMA buffer pointer updates allowing the host ALSA subsystem to maintain accurate audio-video synchronization without host polling. + * **XRUN Alerts**: Instantaneous notifications when an audio buffer underrun (starvation) or overrun (overflow) occurs. + * **Diagnostic Traces & Crash Telemetry**: Streaming real-time debug log packets and exception backtraces directly into host trace buffers. + +System-Level Architecture +========================= + +.. graphviz:: + :caption: System-Level IPC Architecture: Host Driver to DSP Firmware Dispatch + :align: center + + digraph ipc_system_arch { + rankdir=TB; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_host { + label = "Host Operating System (Linux Kernel / Windows)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1b4f72"; + + host_alsa [label="ALSA / ASoC Core\n(snd-soc-core / PCM Stream Ops)", fillcolor="#aed6f1"]; + host_drv [label="SOF Host Driver (snd-sof)\n(IPC Protocol Encoder / Decoder)", fillcolor="#aed6f1", style="filled,bold"]; + host_pci [label="PCIe / Shim Transport Layer\n(Bar Mapping & Interrupt Dispatch)", fillcolor="#aed6f1"]; + + host_alsa -> host_drv -> host_pci; + } + + subgraph cluster_hw { + label = "Hardware Mailbox & Doorbell Interconnect (PCIe BARs / SRAM)"; + style = "filled,rounded"; + color = "#7f8c8d"; + fillcolor = "#f2f4f4"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#2c3e50"; + + mbox_in [label="Mailbox Window 1: Inbox (Host -> DSP)\n(Command Payloads & Parameter Blobs)", fillcolor="#d5dbdb", shape=cylinder]; + mbox_out [label="Mailbox Window 0: Outbox (DSP -> Host)\n(Replies, Notifications & Boot Info)", fillcolor="#d5dbdb", shape=cylinder]; + doorbells [label="Hardware Doorbells\nHost Doorbell (IPC IRQ to DSP)\nDSP Doorbell (Done/Reply IRQ to Host)", fillcolor="#bdc3c7"]; + } + + subgraph cluster_dsp { + label = "DSP Firmware Architecture (SOF on Zephyr RTOS)"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1e8449"; + + dsp_isr [label="Mailbox ISR\n(Catches Doorbell IRQ & Validates)", fillcolor="#a9dfbf"]; + dsp_work [label="Zephyr Work Queue (k_work)\n(Deferred Thread Processing)", fillcolor="#a9dfbf", style="filled,bold"]; + dsp_core [label="Core IPC Framework\n(ipc-common.c: Dispatcher & State Machine)", fillcolor="#a9dfbf"]; + + subgraph cluster_protocols { + label = "Protocol-Specific Handlers"; + style = "filled,rounded"; + color = "#d35400"; + fillcolor = "#fef5e7"; + fontname = "Verdana-Bold"; + fontsize = 9; + + ipc3_hdl [label="IPC3 Handler\n(Scalar Commands: Stream, DAI, PM)", fillcolor="#fad7a0"]; + ipc4_hdl [label="IPC4 Handler\n(Dynamic Objects: Pipeline, Module, Bind)", fillcolor="#fad7a0", style="filled,bold"]; + } + + dsp_isr -> dsp_work [label="Enqueues"]; + dsp_work -> dsp_core [label="Executes"]; + dsp_core -> ipc3_hdl [label="IPC3 Msg"]; + dsp_core -> ipc4_hdl [label="IPC4 Msg"]; + } + + host_pci -> mbox_in [label="Writes Payload", color="#2980b9", penwidth=1.5]; + host_pci -> doorbells [label="Rings Host Doorbell", color="#2980b9", penwidth=1.5]; + doorbells -> dsp_isr [label="Hardware IRQ", color="#c0392b", penwidth=1.5]; + + ipc3_hdl -> mbox_out [label="Writes Reply", style=dashed, color="#27ae60"]; + ipc4_hdl -> mbox_out [label="Writes Reply", style=dashed, color="#27ae60"]; + dsp_core -> doorbells [label="Rings DSP Doorbell", color="#27ae60", penwidth=1.5]; + doorbells -> host_pci [label="Reply IRQ", color="#27ae60", penwidth=1.5]; + mbox_out -> host_pci [label="Reads Status", color="#2980b9", style=dashed]; + } + +--- + +2. Hardware Mailbox Architecture & Memory Windows +************************************************* + +Inter-processor messaging relies on dedicated **Shared SRAM Windows** mapped directly across PCIe Base Address Registers (BARs) on the host and accessible over the DSP system interconnect. + +Shared Memory Mailbox Windows +============================= + +Modern SOF platforms partition shared SRAM into distinct functional memory windows: + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Window + - Direction + - Architectural Purpose + * - **Window 0 (Outbox & Status)** + - DSP to Host + - Stores firmware reply payloads, asynchronous notifications, boot status words, and firmware version descriptors. + * - **Window 1 (Inbox)** + - Host to DSP + - Receives incoming host command headers, large parameter configuration blobs, and pipeline state commands. + * - **Window 2 (Debug & Traces)** + - DSP to Host + - Real-time debug log buffer accessed by host logging daemons (such as ``sof-logger`` or trace DMA). + * - **Window 3 (Stream Payloads)** + - Bidirectional + - Hosts large coefficient matrices (e.g. 10-band equalizer filter tables) and page-table descriptors for host DMA gateways. + +The Doorbell Interrupt Handshake Protocol +========================================= + +To coordinate memory access without race conditions, the host and DSP follow a strict **Doorbell Handshake Protocol**: + +.. graphviz:: + :caption: Bidirectional Hardware Mailbox and Doorbell Handshake Sequence + :align: center + + digraph doorbell_handshake { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_h2d { + label = "Host-to-DSP Command Transaction"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 9; + + h1 [label="1. Host writes command payload\ninto Mailbox Window 1 (Inbox)", fillcolor="#aed6f1"]; + h2 [label="2. Host asserts Host Doorbell IRQ\n(Sets Busy bit in PCIe register)", fillcolor="#aed6f1"]; + h3 [label="3. DSP ISR catches interrupt,\nclears IRQ & schedules work", fillcolor="#a9dfbf"]; + h4 [label="4. DSP processes command,\nwrites reply to Window 0 (Outbox)", fillcolor="#a9dfbf"]; + h5 [label="5. DSP asserts Done / Reply IRQ\n(Clears Busy bit; rings Host IRQ)", fillcolor="#a9dfbf", style="filled,bold"]; + h6 [label="6. Host catches reply IRQ,\nreads Window 0 & releases lock", fillcolor="#aed6f1"]; + + h1 -> h2 -> h3 -> h4 -> h5 -> h6; + } + } + +1. **Atomic Ownership**: While the Busy bit is asserted, the host is barred from overwriting the inbox. Ownership belongs exclusively to the DSP. +2. **Deterministic Acknowledgment**: The DSP signals completion by asserting the Done interrupt and writing status codes directly into Window 0, ensuring that the host driver never experiences mailbox data corruption. + +--- + +3. Core Framework & Zephyr Thread Handoff +***************************************** + +Why IPC Processing is Decoupled from Interrupts +=============================================== + +When the host triggers a mailbox doorbell interrupt, the DSP responds inside a hardware **Interrupt Service Routine (ISR)**. However, executing the entire IPC message within the ISR is strictly forbidden in real-time audio systems: + +* **Real-Time Latency Spikes**: Parsing complex pipeline topologies, allocating dynamic heaps, or configuring DAI clocks requires thousands of cycles. If executed inside an ISR, audio DMA interrupts would be delayed, causing immediate audio glitches and buffer underruns. +* **Blocking & DMA Waits**: Certain commands require waiting for DMA page table synchronization or inter-core responses. Interrupt service routines cannot sleep or block. + +Deferred Work Queue Architecture +================================ + +Sound Open Firmware solves this by delegating all command handling to the **Zephyr Work Queue subsystem** (``k_work``): + +.. graphviz:: + :caption: Mailbox ISR to Zephyr Work Queue Handoff and Message State Machine + :align: center + + digraph isr_handoff { + rankdir=TB; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_isr { + label = "Hardware Interrupt Context (Immediate, Zero Delay)"; + style = "filled,rounded"; + color = "#c0392b"; + fillcolor = "#f9ebea"; + fontname = "Verdana-Bold"; + fontsize = 9; + + irq_step1 [label="1. Hardware Mailbox IRQ Fires", fillcolor="#f5b7b1"]; + irq_step2 [label="2. Read Primary Header Word\n(Validates message boundaries)", fillcolor="#f5b7b1"]; + irq_step3 [label="3. Acknowledge Hardware Level\n(Clears interrupt latch)", fillcolor="#f5b7b1"]; + irq_step4 [label="4. Enqueue Work Item into Zephyr\nk_work_submit(&ipc->ipc_work)", fillcolor="#f5b7b1", style="filled,bold"]; + + irq_step1 -> irq_step2 -> irq_step3 -> irq_step4; + } + + subgraph cluster_thread { + label = "Thread Context (Zephyr Kernel Work Queue: ipc_work_handler)"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 9; + + th_step1 [label="5. Worker Thread Awakens\n(Runs at high cooperative priority)", fillcolor="#a9dfbf"]; + th_step2 [label="6. Decode Command & Dispatch\n(Routes to IPC3 or IPC4 handler)", fillcolor="#a9dfbf"]; + th_step3 [label="7. Execute Graph / Module Operation\n(Pipeline build, bind, or parameter update)", fillcolor="#a9dfbf", style="filled,bold"]; + th_step4 [label="8. Complete Transaction\n(Writes reply & rings Host Doorbell)", fillcolor="#a9dfbf"]; + + th_step1 -> th_step2 -> th_step3 -> th_step4; + } + + irq_step4 -> th_step1 [label="Context Switch", color="#27ae60", penwidth=1.5]; + } + +Message Lifecycle & Backpressure Handling +========================================= + +Firmware-initiated messages (such as notifications or stream position updates) are governed by an internal state machine: + +1. **State Progression**: Messages transition through ``UNREGISTERED`` $\rightarrow$ ``QUEUED`` $\rightarrow$ ``PROCESSING`` $\rightarrow$ ``ACK_PENDING`` $\rightarrow$ ``COMPLETED``. +2. **Outbox Message Queueing**: If the DSP needs to send an asynchronous notification while the hardware mailbox is already occupied by a previous pending message, the core IPC framework places the new message onto an internal transmission list (``ipc_msg_send``), preventing message loss under heavy host bus traffic. + +--- + +4. Protocol Generations: IPC3 vs. IPC4 +************************************** + +Sound Open Firmware supports two major generations of the Inter-Processor Communication protocol. While older hardware architectures use IPC3, all modern Intel platforms (Tiger Lake, Meteor Lake, Arrow Lake, Panther Lake) and contemporary designs utilize IPC4. + +Architectural Comparison +======================== + +.. graphviz:: + :caption: Structural Comparison: IPC3 Flat Scalar Model vs IPC4 Dynamic Compound Object Model + :align: center + + digraph ipc_comparison { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_ipc3 { + label = "IPC3: Static Scalar Model (Legacy)"; + style = "filled,rounded"; + color = "#7f8c8d"; + fillcolor = "#f2f4f4"; + fontname = "Verdana-Bold"; + fontsize = 9; + + ipc3_hdr [label="sof_ipc_cmd_hdr\n(Global Command Type + Size)", fillcolor="#d5dbdb"]; + ipc3_pcm [label="SOF_IPC_GLB_STREAM_MSG\n(pcm_params, trigger, position)", fillcolor="#d5dbdb"]; + ipc3_dai [label="SOF_IPC_GLB_DAI_MSG\n(dai_config, ssp/hda config)", fillcolor="#d5dbdb"]; + ipc3_topo [label="Static Graph Deployment\n(Topology loaded monolithically at probe)", fillcolor="#bdc3c7", style="filled,bold"]; + + ipc3_hdr -> ipc3_pcm; + ipc3_hdr -> ipc3_dai; + ipc3_pcm -> ipc3_topo; + } + + subgraph cluster_ipc4 { + label = "IPC4: Dynamic Compound Object Model (Modern)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 9; + + ipc4_hdr [label="64-Bit Primary Compact Header\n(Type, Rsp, Target, Status, Ext)", fillcolor="#aed6f1", style="filled,bold"]; + ipc4_ppl [label="Pipeline Management\n(new_pipeline, set_state, delete)", fillcolor="#aed6f1"]; + ipc4_mod [label="Dynamic Modules\n(init_instance, set/get_params)", fillcolor="#aed6f1"]; + ipc4_bind [label="Dynamic Pin Binding\n(ipc4_bind / ipc4_unbind)", fillcolor="#aed6f1", style="filled,bold"]; + + ipc4_hdr -> ipc4_ppl; + ipc4_hdr -> ipc4_mod; + ipc4_hdr -> ipc4_bind; + } + } + +Key Differences +=============== + +.. list-table:: + :widths: 20 40 40 + :header-rows: 1 + + * - Dimension + - IPC3 (Scalar Architecture) + - IPC4 (Compound Object Architecture) + * - **Topology Model** + - **Static**: Entire pipeline graph is compiled into a monolithic topology binary and parsed at driver probe. + - **Dynamic**: Pipelines and modules are constructed, bound, and torn down dynamically at runtime via individual IPC commands. + * - **Component Addressing** + - Global 32-bit component IDs assigned statically by the topology compiler. + - Modular 32-bit Tuple: ``module_id`` (algorithm type UUID) combined with an ``instance_id`` (unique runtime instance). + * - **Command Density** + - Scalar: Each operation requires a separate round-trip command/response handshake. + - Compound: Multiple operations (create pipeline, instantiate modules, bind pins) can be batched in a single transaction. + * - **Memory Footprint** + - Graph nodes and buffers are pre-allocated statically during system boot. + - Memory heaps are allocated and reclaimed on-demand as audio streams open and close. + +--- + +5. Pipeline Lifecycle & Dynamic Graph Control +********************************************* + +In IPC4, the host operating system dynamically constructs, connects, and controls the audio processing graph: + +Dynamic Graph Instantiation Flow +================================ + +.. graphviz:: + :caption: IPC4 Dynamic Pipeline Construction and Streaming Sequence + :align: center + + digraph ipc4_lifecycle { + rankdir=TB; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + s1 [label="1. Create Pipeline (ipc4_new_pipeline)\nHost defines pipeline ID, execution priority, and core affinity", fillcolor="#d4e6f1"]; + s2 [label="2. Instantiate Modules (ipc4_init_module_instance)\nDSP allocates component memory sandbox and initializes algorithm state", fillcolor="#d4e6f1"]; + s3 [label="3. Bind Component Pins (ipc4_bind)\nHost links Source Pin of Module A to Sink Pin of Module B via intermediate ring buffer", fillcolor="#aed6f1", style="filled,bold"]; + s4 [label="4. Configure Parameters (ipc4_set_module_params)\nHost delivers coefficient matrices, volume curves, and audio format descriptors", fillcolor="#d4e6f1"]; + s5 [label="5. Set Pipeline State (ipc4_set_pipeline_state)\nTransitions pipeline: INIT -> PAUSED -> RUNNING", fillcolor="#abebc6", style="filled,bold"]; + s6 [label="6. Audio Streaming\nScheduler domains (LL / DP) process audio frames across circular buffers", fillcolor="#abebc6"]; + s7 [label="7. Teardown (ipc4_unbind & ipc4_delete_pipeline)\nPipeline halted, memory sandbox reclaimed, and buffers deallocated", fillcolor="#fadbd8"]; + + s1 -> s2 -> s3 -> s4 -> s5 -> s6 -> s7; + } + +Core State Machine Integration +============================== + +The host controls pipeline progression by sending ``ipc4_set_pipeline_state()`` commands. The IPC framework maps these high-level host requests directly into SOF core state machine triggers: + +* **``IPC4_PIPELINE_STATE_RESET``** $\rightarrow$ Re-initializes buffers and resets filter delay lines. +* **``IPC4_PIPELINE_STATE_PAUSED``** $\rightarrow$ Halts active processing while preserving audio parameters and buffer memory. +* **``IPC4_PIPELINE_STATE_RUNNING``** $\rightarrow$ Dispatches ``COMP_TRIGGER_START``, enabling real-time timer or DMA interrupts. +* **``IPC4_PIPELINE_STATE_EOS``** $\rightarrow$ Signals End-Of-Stream, allowing remaining samples in ring buffers to drain cleanly without truncation. + +--- + +6. Firmware-Initiated Notifications & Telemetry +*********************************************** + +While commands flow from Host to DSP, the IPC infrastructure also provides a high-efficiency path for **Firmware-Initiated Asynchronous Notifications** (DSP to Host). + +Asynchronous Telemetry Flow +=========================== + +.. graphviz:: + :caption: Firmware-Initiated Asynchronous Notification Architecture + :align: center + + digraph notification_flow { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_events { + label = "DSP Event Generators"; + style = "filled,rounded"; + color = "#8e44ad"; + fillcolor = "#f4ecf7"; + fontname = "Verdana-Bold"; + fontsize = 9; + + ev_pos [label="Position Reporter\n(Stream DMA sample offset)", fillcolor="#d7bde2"]; + ev_xrun [label="XRUN Monitor\n(Buffer underrun / overrun)", fillcolor="#d7bde2"]; + ev_panic [label="Exception Handler\n(Crash dump & register state)", fillcolor="#f5b7b1"]; + } + + subgraph cluster_queue { + label = "Notification Management (notification_pool.c)"; + style = "filled,rounded"; + color = "#d35400"; + fillcolor = "#fef5e7"; + fontname = "Verdana-Bold"; + fontsize = 9; + + pool_mgr [label="Notification Pool Allocator\n(Pre-allocated descriptors)", fillcolor="#fad7a0"]; + tx_queue [label="Outbox Transmission Queue\n(Buffers notifications if mailbox busy)", fillcolor="#fad7a0", style="filled,bold"]; + pool_mgr -> tx_queue; + } + + subgraph cluster_outbox { + label = "Mailbox Outbox & Host IRQ"; + style = "filled,rounded"; + color = "#27ae60"; + fillcolor = "#eafaf1"; + fontname = "Verdana-Bold"; + fontsize = 9; + + mb_out [label="Window 0 (Outbox SRAM)\nWrites notification payload", fillcolor="#a9dfbf", shape=cylinder]; + mb_irq [label="Assert DSP Doorbell IRQ\nSignals Host PCIe interrupt", fillcolor="#a9dfbf", style="filled,bold"]; + mb_out -> mb_irq; + } + + ev_pos -> tx_queue [label="Periodic"]; + ev_xrun -> tx_queue [label="Immediate"]; + ev_panic -> tx_queue [label="Fatal"]; + + tx_queue -> mb_out [label="Dispatches to SRAM"]; + } + +Notification Types & Purpose +============================ + +1. **Stream Position Updates**: + * Sent periodically as hardware DMA transfers audio frames to/from host memory. + * Updates host ALSA ring buffer pointers, allowing user-space applications to track playback timing with microsecond accuracy. +2. **XRUN Notifications**: + * Instantly alerts the host kernel if an audio underrun or overrun occurs, enabling the host driver to log diagnostics and initiate recovery. +3. **Firmware Panic & Error Reports**: + * In the rare event of a CPU exception, watchdog timeout, or kernel assert, the exception handler formats a panic descriptor containing CPU register states, execution backtraces, and memory faults into Window 0 before resetting the DSP. + +--- + +7. Multi-Core IPC & Inter-Domain Communication (IDC) +**************************************************** + +Modern Intel and partner DSPs feature multi-core architectures (Dual-Core, Quad-Core, or Octa-Core). However, the physical PCIe mailbox hardware and doorbell interrupt registers are physically routed **only to Core 0**. + +Core 0 as the Central Host Gateway +================================== + +Core 0 acts as the central gateway for all external host communication: + +* All incoming host doorbell interrupts are caught exclusively by Core 0's mailbox ISR. +* All outgoing notifications and replies must be written to Window 0 by Core 0. + +Inter-Domain Communication (IDC) Architecture +============================================= + +When the host issues an IPC command targeting a pipeline, audio module, or power state located on a secondary core (such as Core 1, Core 2, or Core 3), SOF utilizes **Inter-Domain Communication (IDC)**: + +.. graphviz:: + :caption: Multi-Core IPC Routing Topology: Core 0 (Host Gateway) and Core 1 (Secondary Core) via IDC + :align: center + + digraph idc_topology { + rankdir=LR; + nodesep=0.3; + ranksep=0.4; + node [shape=box, style="filled,rounded", fontname="Verdana", fontsize=9, margin="0.12,0.06"]; + edge [fontname="Verdana", fontsize=8, color="#333333"]; + + subgraph cluster_c0 { + label = "DSP Core 0 (Host Gateway & Primary Dispatcher)"; + style = "filled,rounded"; + color = "#2980b9"; + fillcolor = "#ebf5fb"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#1b4f72"; + + c0_isr [label="Mailbox ISR\n(Catches Host Doorbell)", fillcolor="#aed6f1"]; + c0_dec [label="Core Target Decoder\n(Detects command targets Core 1)", fillcolor="#aed6f1"]; + c0_idc [label="IDC Sender\n(Writes IDC shared mailbox\n& rings Inter-Core Doorbell)", fillcolor="#aed6f1", style="filled,bold"]; + c0_reply [label="Host Reply Aggregator\n(Writes Window 0 & rings Host IRQ)", fillcolor="#aed6f1"]; + + c0_isr -> c0_dec -> c0_idc; + c0_reply -> c0_isr [style=invis]; + } + + subgraph cluster_shared { + label = "Inter-Core Shared Memory (HP-SRAM)"; + style = "filled,rounded"; + color = "#d35400"; + fillcolor = "#fef5e7"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#a04000"; + + idc_msg [label="IDC Message Structure\n(Shared Memory Buffer)", fillcolor="#fad7a0", shape=cylinder]; + idc_irq [label="Hardware Inter-Core Doorbell\n(DSP Architectural IRQ)", fillcolor="#fad7a0"]; + } + + subgraph cluster_c1 { + label = "DSP Core 1 (Secondary Compute Core)"; + style = "filled,rounded"; + color = "#8e44ad"; + fillcolor = "#f4ecf7"; + fontname = "Verdana-Bold"; + fontsize = 10; + fontcolor = "#512e5f"; + + c1_isr [label="IDC ISR\n(Catches Core 0 Doorbell)", fillcolor="#d7bde2"]; + c1_work [label="IDC Worker Thread\n(Executes target module operation)", fillcolor="#d7bde2", style="filled,bold"]; + c1_ack [label="IDC Reply\n(Signals completion back to Core 0)", fillcolor="#d7bde2"]; + + c1_isr -> c1_work -> c1_ack; + } + + c0_idc -> idc_msg [label="Write Payload", color="#2980b9", penwidth=1.5]; + c0_idc -> idc_irq [label="Assert IRQ", color="#2980b9", penwidth=1.5]; + idc_irq -> c1_isr [label="Hardware Interrupt", color="#c0392b", penwidth=1.5]; + idc_msg -> c1_work [label="Read Payload", color="#8e44ad", style=dashed]; + + c1_ack -> c0_reply [label="IDC Return Status", color="#27ae60", penwidth=1.5]; + } + +1. **Transparent Routing**: The host driver remains completely agnostic to core partitioning. The host targets a module by ID; Core 0's IPC framework transparently resolves which core owns the module. +2. **IDC Doorbell Interrupts**: Core 0 copies the message payload into shared inter-core SRAM and triggers a hardware inter-core interrupt to awaken Core 1. +3. **Status Aggregation**: When Core 1 finishes processing the command, it returns an acknowledgment via IDC. Core 0 aggregates the response and completes the transaction to the host. + +--- + +8. Upstream Code References & Related Guides +******************************************** + +For developers seeking low-level C implementation details, data structures, and function prototypes: + +* **Upstream IPC Specifications**: + * Core IPC framework architecture: `thesofproject/sof: src/ipc/README.md `_. + * IPC3 scalar architecture: `thesofproject/sof: src/ipc/ipc3/README.md `_. + * IPC4 dynamic object architecture: `thesofproject/sof: src/ipc/ipc4/README.md `_. + +* **Core Source Files**: + + * ``src/ipc/ipc-common.c``: Core message state machine, dispatcher, and outbox queue management. + * ``src/ipc/ipc-zephyr.c``: Zephyr work queue thread handoff (``ipc_work_handler``). + * ``src/ipc/ipc3/handler.c``: IPC3 global command dispatcher (stream, DAI, PM). + * ``src/ipc/ipc4/handler-kernel.c``: IPC4 primary header parser, global pipeline state engine, and module dispatcher. + * ``src/ipc/ipc4/ams_helpers.c``: IPC4 dynamic module instantiation and pin binding helpers. + * ``src/ipc/notification_pool.c``: Pre-allocated asynchronous notification pool allocator. + +* **Core Header Files**: + + * ``src/include/ipc/header.h``: Common IPC message header definitions and command enums. + * ``src/include/ipc/stream.h``: Stream parameter, trigger, and position payload definitions. + * ``src/include/ipc/topology.h``: Topology IPC structures and component creation payloads. + * ``src/include/sof/ipc/schedule.h``: Scheduling domain integration with IPC work queues. + +Related Guides +============== + +* :ref:`pipeline_architecture`: How IPC commands dynamically create, prepare, and trigger audio pipelines. +* :ref:`module_framework`: How IPC parameter blobs configure processing modules and runtime algorithms. +* :ref:`scheduler_architecture`: Real-time scheduling domains (LL, DP, TWB) that coordinate with IPC work queues. +* :ref:`audio_buffer_management`: Allocating and binding circular ring buffers during IPC pipeline construction. +* :ref:`fw_init_boot`: Boot flow, hardware mailbox FW Ready handshake, and Zephyr initialization. +* :ref:`topology2`: How ALSA Topology 2.0 configuration files generate IPC topology commands. diff --git a/developer_guides/firmware/kpb_wov.rst b/developer_guides/firmware/kpb_wov.rst new file mode 100644 index 00000000..edd836f0 --- /dev/null +++ b/developer_guides/firmware/kpb_wov.rst @@ -0,0 +1,808 @@ +.. _kpb_wov: + +========================================================== +Key Phrase Buffer (KPB) & Wake-on-Voice (WoV) Architecture +========================================================== + +.. contents:: + :local: + :depth: 3 + +Sound Open Firmware (SOF) provides an autonomous, low-power audio architecture designed to support **Wake-on-Voice (WoV)** and always-listening acoustic keyword activation. In modern mobile laptops, smart home hubs, automotive cockpits, and wearable devices, users expect immediate responsiveness to spoken wake phrases (such as *"Hey Computer"* or *"OK Assistant"*). However, keeping the host application processor and PCIe/USB interconnects continuously awake to analyze ambient microphone audio would consume several watts of power, draining portable batteries in a matter of hours. + +To resolve this challenge, modern acoustic architectures offload keyword spotting and voice activity detection to an ultra-low-power Digital Signal Processor (DSP) running SOF. While the host CPU remains in deep system sleep (such as ACPI S0ix / Modern Standby, S3 suspend-to-RAM, or S4 hibernation) drawing only microamperes, the audio DSP operates in an autonomous, power-optimized D0ix state. + +A critical engineering obstacle in always-listening architectures is **The Pre-Roll Dilemma**: acoustic keyword spotters—whether running neural networks via TensorFlow Lite for Microcontrollers (TFLM) or proprietary vendor models—require an integration window of 500 ms to 1500 ms of spoken phonemes before achieving statistical confidence to trigger a detection event. Furthermore, waking the host CPU, resuming platform power rails, re-initializing PCIe/SoundWire DMA controllers, and starting host user-space capture pipelines introduces an additional system resume latency of 1000 ms to 2000 ms. If microphone audio is not buffered during this multi-second interval, the opening syllables of the user's command (*"Hey Computer, what is the weather?"*) are permanently lost before host recording begins. + +The **Key Phrase Buffer (KPB)** component (``src/audio/kpb.c``, ``COMP_KPB``, UUID ``D8218443-5FF3-4A4C-B388-6CFE07B9562E``) solves this problem by maintaining a continuous circular ring buffer of incoming microphone audio. Operating as a specialized dual-sink streaming engine, KPB simultaneously provides a real-time low-latency stream to local on-DSP keyword spotters and maintains a multi-second history buffer. Upon a keyword detection event, KPB transitions into an accelerated draining engine that burst-transfers the pre-roll history to the host DMA buffer before seamlessly handing off to real-time audio capture without dropping a single acoustic frame. + +.. graphviz:: + :caption: SOF Wake-on-Voice (WoV) System Architecture: Host Sleep, DSP D0ix & Wake Sequence + :alt: Architectural block diagram showing host CPU sleep, DSP autonomous listening in D0ix, keyword detection, and pre-roll draining. + + digraph wov_system_overview { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=11, shape=box, style="filled,rounded", margin="0.15,0.08"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=10]; + + subgraph cluster_ambient_sound { + label = "Acoustic Environment"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + mic_input [label="Acoustic Speech\n'Hey Computer...'\nVoice Waveform", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + dmic_hw [label="DMIC Hardware Array\nLow-Power PDM Clock\n(16 kHz Sampling)", fillcolor="#E2E8F0", color="#4A5568", penwidth=1.5]; + } + + subgraph cluster_dsp_d0ix { + label = "DSP Autonomous Domain (D0ix Ultra-Low Power)"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + dcblock [label="DC Blocker\nIIR High-Pass\nOffset Removal", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + kpb_core [label="Key Phrase Buffer (KPB)\nDual-Sink Dispatch\nCircular History Ring\n(1.5 to 3.0 s Pre-Roll)", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + kwd_engine [label="Keyword Spotter\n(TFLM / MFCC / KD)\nContinuous Evaluation\nPin 0 (Real-Time)", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + dma_drain [label="Host Draining Sink\nBurst Transfer Task\n(Fast Mode Engine)\nPin 1 (Host Sink)", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + } + + subgraph cluster_host_domain { + label = "Host CPU System Domain"; + style = "filled,rounded"; + color = "#FED7D7"; + fillcolor = "#FFF5F5"; + + host_sleep [label="Host CPU Sleep\n(ACPI S0ix / Modern Standby)\nPCIe & DRAM Suspended", fillcolor="#FEB2B2", color="#C53030", penwidth=1.5]; + host_resume [label="Host Wake & Audio Resume\nKernel ALSA Driver\nHost DMA Capture Active", fillcolor="#FED7D7", color="#9B2C2C", penwidth=1.8]; + voice_app [label="Voice Assistant Application\nCloud / Local ASR\nReceives Intact Utterance", fillcolor="#FAF5FF", color="#6B46C1", penwidth=1.5]; + } + + mic_input -> dmic_hw [label="Sound Wave", color="#3182CE", penwidth=1.5]; + dmic_hw -> dcblock [label="PDM Frames", color="#4A5568", penwidth=1.5]; + dcblock -> kpb_core [label="16 kHz PCM", color="#2B6CB0", penwidth=1.8]; + + kpb_core -> kwd_engine [label="1. Continuous Stream\n(Real-Time Pin 0)", color="#B7791F", penwidth=1.6]; + kwd_engine -> kpb_core [label="2. Trigger Event\n(Keyword Match)", color="#C53030", style="dashed", penwidth=1.8]; + kwd_engine -> host_sleep [label="3. Wakeup IRQ\n(IPC / MSI)", color="#C53030", style="bold", penwidth=2.0]; + + host_sleep -> host_resume [label="Platform Resume\n(1000 - 2000 ms)", color="#9B2C2C", style="dashed", penwidth=1.5]; + kpb_core -> dma_drain [label="4. Burst Draining", color="#2B6CB0", penwidth=1.8]; + dma_drain -> host_resume [label="5. Pre-Roll + Live Data\n(Host DMA)", color="#2B6CB0", penwidth=2.0]; + host_resume -> voice_app [label="Uncut Audio Stream", color="#6B46C1", penwidth=1.8]; + } + +Principles of Low-Power Wake-on-Voice & The Pre-Roll Dilemma +============================================================ + +In modern computing platforms, acoustic energy efficiency is governed by the operational power consumption of different platform processing tiers: + +.. list-table:: Energy & Power Tiers in Voice-Enabled Embedded Systems + :widths: 22 18 25 35 + :header-rows: 1 + + * - Platform Power Tier + - Typical Power + - Wake Latency + - Active Audio Processing Capabilities + * - **Host Active (S0)** + - 10 W -- 45 W + - 0 ms (running) + - Full desktop OS, cloud streaming, complex large language models, high-resolution rendering. + * - **Host Modern Standby (S0ix)** + - 500 mW -- 1.5 W + - 500 ms -- 1500 ms + - Host cores in deep C-states; PCIe, DRAM controllers, and display engines clock-gated. + * - **Host Suspend-to-RAM (S3)** + - 100 mW -- 300 mW + - 1000 ms -- 2500 ms + - Host completely powered off except DRAM refresh logic; interconnects dormant. + * - **DSP Low-Power Mode (D0ix)** + - 3 mW -- 12 mW + - < 1 ms + - Primary DSP core running at reduced clock frequency (e.g. 24 MHz -- 38.4 MHz); autonomous DMIC audio capture, low-power Voice Activity Detection (VAD), and keyword spotters. + +The Pre-Roll Timing Equation +---------------------------- + +To understand the necessity of historical buffering, consider the chronological progression of a voice activation sequence: + +1. **Acoustic Speech Commencement** (:math:`t = t_0`): + The user begins uttering the activation phrase (*"Hey Computer"*). +2. **Voice Activity Detection** (:math:`t = t_0 + \Delta t_{\text{VAD}}`): + Energy-based or spectral VAD algorithms detect acoustic activity above background ambient noise (:math:`\approx 50\text{--}150\text{ ms}`). +3. **Keyword Model Inference Latency** (:math:`t = t_0 + \Delta t_{\text{KWD}}`): + The acoustic keyword classifier integrates temporal audio frames over a multi-layer neural network or acoustic model. Because phonetic recognition requires sufficient acoustic context across syllables, confident detection occurs near the end of the phrase (:math:`\Delta t_{\text{KWD}} \approx 800\text{--}1500\text{ ms}`). +4. **Host Wakeup & Platform Rail Settlement** (:math:`t = t_0 + \Delta t_{\text{KWD}} + \Delta t_{\text{wake}}`): + Upon keyword detection, the DSP asserts a platform interrupt (IPC or PCIe MSI). The host power management IC (PMIC) ramps platform voltage rails, DRAM exits self-refresh, the kernel resumes, and the ALSA audio driver invokes hardware parameters and stream prepare (:math:`\Delta t_{\text{wake}} \approx 800\text{--}2000\text{ ms}`). +5. **Host DMA Capture Activation** (:math:`t = t_0 + \Delta t_{\text{total\_latency}}`): + The host application initiates reading from the ALSA capture device (e.g. ``arecord``). + +The cumulative latency before the host application begins receiving audio data is: + +.. math:: + + T_{\text{total\_latency}} = \Delta t_{\text{KWD}} + \Delta t_{\text{wake}} + \Delta t_{\text{dma\_startup}} + +If :math:`\Delta t_{\text{KWD}} = 1200\text{ ms}` and :math:`\Delta t_{\text{wake}} = 1500\text{ ms}`, the total elapsed duration is :math:`2700\text{ ms}`. Without a circular buffer holding at least :math:`2.7\text{ seconds}` of historical microphone data, the entire wake word and the initial segment of the user command would be completely lost. + +The KPB component eliminates this data loss by continuously recording into a dedicated circular history buffer in DSP SRAM while the host is asleep. When the host resumes and initiates capture, KPB transfers this buffered historical speech into the host DMA buffer at accelerated speed before transitioning seamlessly to real-time audio. + +KPB Component State Machine & Execution Lifecycle +================================================= + +The KPB component is implemented as an audio processing module conforming to the SOF component driver interface. Internally, KPB maintains ten discrete states that govern its execution during audio streaming, buffer writing, trigger events, and draining. + +.. list-table:: KPB Component Lifecycle States (enum kpb_state) + :widths: 25 15 60 + :header-rows: 1 + + * - State Enumeration + - Value + - Functional Role & Operational Behavior + * - ``KPB_STATE_DISABLED`` + - 0 + - Initial unconfigured state prior to memory allocation and pipeline initialization. + * - ``KPB_STATE_RESET_FINISHING`` + - 1 + - Ephemeral cleanup state entered when a reset interrupt interrupts an ongoing buffering or draining operation. + * - ``KPB_STATE_CREATED`` + - 2 + - Module instance allocated, driver private data initialized, and unique identifier (UUID) assigned. + * - ``KPB_STATE_PREPARING`` + - 3 + - Validation of sampling rate (16 kHz), container width, channel count, and circular history buffer allocation during ``kpb_prepare()``. + * - ``KPB_STATE_RUN`` + - 4 + - Normal listening mode. Incoming DMIC frames are copied to the internal history buffer and simultaneously forwarded to the active real-time selector sink (pin 0). + * - ``KPB_STATE_BUFFERING`` + - 5 + - Transient state entered within ``kpb_copy()`` while writing audio frames into the active circular history ring buffer. + * - ``KPB_STATE_INIT_DRAINING`` + - 6 + - Triggered by client detection event. Locks state, calculates backward read pointer in history rings, and prepares asynchronous draining task. + * - ``KPB_STATE_DRAINING`` + - 7 + - Asynchronous draining active. The background draining task reads historical audio from the ring buffer and copies it to the host sink at accelerated speed. + * - ``KPB_STATE_HOST_COPY`` + - 8 + - Draining completed ("draining on demand"). History buffer is emptied, and incoming real-time audio is copied directly to the host capture sink without latency. + * - ``KPB_STATE_RESETTING`` + - 9 + - Teardown requested via pipeline trigger stop or reset command. Halts background tasks and frees resources. + +.. graphviz:: + :caption: KPB Component State Machine (10 Lifecycle States: Reset, Run, Buffering, Draining, and Host Copy) + :alt: Detailed finite state machine diagram showing all 10 states of the KPB component and their transitions. + + digraph kpb_state_machine { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + s_disabled [label="KPB_STATE_DISABLED\n(Uninitialized)", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + s_created [label="KPB_STATE_CREATED\n(Instance Created)", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + s_prep [label="KPB_STATE_PREPARING\n(Buffer Allocation)", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + s_run [label="KPB_STATE_RUN\n(Normal Listening / Real-Time Dispatch)", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + s_buff [label="KPB_STATE_BUFFERING\n(Writing to History Ring)", fillcolor="#C6F6D5", color="#276749", penwidth=1.5]; + s_init_drn [label="KPB_STATE_INIT_DRAINING\n(Pointer Calc & Task Setup)", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + s_draining [label="KPB_STATE_DRAINING\n(Accelerated Burst Draining Task)", fillcolor="#FEEBC8", color="#C05621", penwidth=2.0]; + s_hcopy [label="KPB_STATE_HOST_COPY\n(Real-Time Streaming to Host Sink)", fillcolor="#E9D8FD", color="#6B46C1", penwidth=2.0]; + s_resetting[label="KPB_STATE_RESETTING\n(Pipeline Stop / Reset Triggered)", fillcolor="#FED7D7", color="#C53030", penwidth=1.5]; + s_rst_fin [label="KPB_STATE_RESET_FINISHING\n(Final Resource Teardown)", fillcolor="#FEB2B2", color="#9B2C2C", penwidth=1.2]; + + s_disabled -> s_created [label="kpb_new()", color="#4A5568"]; + s_created -> s_prep [label="kpb_prepare()", color="#3182CE"]; + s_prep -> s_run [label="kpb_trigger(START)", color="#22543D", penwidth=1.5]; + + s_run -> s_buff [label="Frame Arrival (kpb_copy)", color="#276749"]; + s_buff -> s_run [label="Frame Written", color="#276749"]; + + s_run -> s_init_drn [label="Keyword Detected\n(BEGIN_DRAINING Event)", color="#C05621", penwidth=1.8]; + s_init_drn -> s_draining [label="Task Scheduled", color="#C05621", penwidth=1.5]; + + s_draining -> s_buff [label="New Audio Buffering\nDuring Draining", color="#276749", style="dashed"]; + s_buff -> s_draining [label="Resume Draining", color="#276749", style="dashed"]; + + s_draining -> s_hcopy [label="Pre-Roll Drained\n(drain_req == 0)", color="#6B46C1", penwidth=2.0]; + + s_run -> s_resetting [label="Trigger STOP / RESET", color="#C53030"]; + s_draining -> s_resetting [label="Trigger STOP / RESET", color="#C53030"]; + s_hcopy -> s_resetting [label="Trigger STOP / RESET", color="#C53030"]; + + s_resetting -> s_rst_fin [label="Task Cancelled", color="#9B2C2C"]; + s_rst_fin -> s_created [label="kpb_reset() Complete", color="#4A5568"]; + s_created -> s_disabled [label="kpb_free()", color="#4A5568"]; + } + +Lifecycle Transitions Walkthrough +--------------------------------- + +1. **Initialization & Preparation**: + When the audio pipeline is configured via topology, ``kpb_new()`` transitions the module to ``KPB_STATE_CREATED``. Upon receiving the IPC hardware parameters and prepare commands, ``kpb_prepare()`` verifies that the sampling frequency is 16 kHz and allocates the circular history buffers in DSP internal SRAM, moving to ``KPB_STATE_PREPARING``. +2. **Normal Listening (RUN & BUFFERING)**: + Upon receiving ``COMP_TRIGGER_START``, the state transitions to ``KPB_STATE_RUN``. Each time the pipeline period executes, ``kpb_copy()`` inspects the source DMIC buffer. Audio samples are copied to the active real-time selector sink (pin 0) if downstream components (the keyword spotter) are in ``COMP_STATE_ACTIVE``. Simultaneously, KPB temporarily enters ``KPB_STATE_BUFFERING`` to append the incoming PCM frames to the circular history buffer before reverting to ``KPB_STATE_RUN``. +3. **Keyword Trigger & Draining Initialization**: + When the keyword classifier identifies the activation phrase, it emits a notification event (``KPB_EVENT_BEGIN_DRAINING``). KPB locks its private spinlock/mutex and enters ``KPB_STATE_INIT_DRAINING``. The component calculates the historical read pointer offset corresponding to the requested pre-roll duration, locks available buffer headroom, pauses the real-time selector sink, and launches an asynchronous draining task. +4. **Accelerated Burst Draining**: + In ``KPB_STATE_DRAINING``, the draining task executes at an accelerated cadence (e.g. :math:`2\times` to :math:`4\times` real-time speed), reading from the historical read pointer and writing to the host sink buffer (pin 1). If new real-time microphone samples arrive during draining, they are buffered into the history ring while a running counter (``buffered_while_draining``) extends the total remaining draining requirement. +5. **Real-Time Handoff (HOST_COPY)**: + Once the historical buffer is completely drained and all accumulated audio frames have been transferred, KPB transitions to ``KPB_STATE_HOST_COPY``. In this state, the circular history buffer is bypassed, and new incoming microphone frames are copied directly to the host capture sink in real time, guaranteeing zero-latency streaming to the host voice recognition application. + +History Circular Ring Buffer Architecture & Mathematics +======================================================== + +The KPB storage engine is built around a chained linked list of circular history buffers: + +.. math:: + + \text{Ring Structure: } \mathcal{B}_0 \rightleftharpoons \mathcal{B}_1 \rightleftharpoons \dots \rightleftharpoons \mathcal{B}_{N-1} \rightleftharpoons \mathcal{B}_0 + +In standard SOF configurations, the ring comprises two distinct buffers (``KPB_NO_OF_HISTORY_BUFFERS = 2``) managed by ``struct history_buffer``: + +.. code-block:: c + + struct history_buffer { + enum buffer_state state; /* KPB_BUFFER_FREE, KPB_BUFFER_FULL, KPB_BUFFER_OFF */ + void *start_addr; /* Base memory address of buffer in DSP SRAM */ + void *end_addr; /* Upper boundary address (start_addr + size) */ + void *w_ptr; /* Current write pointer */ + void *r_ptr; /* Current read pointer for draining */ + struct history_buffer *next; /* Pointer to next ring segment */ + struct history_buffer *prev; /* Pointer to previous ring segment */ + }; + +Mathematical Buffer Sizing Equations +------------------------------------ + +The memory footprint of the KPB history buffer is determined by four platform configuration parameters: + +* Sampling frequency (:math:`f_s`, strictly 16,000 Hz for voice keyword processing). +* Audio channel count (:math:`N_{\text{ch}}`, typically 2 to 6 channels). +* Sample container width (:math:`W_{\text{container}}`, 16 bits or 32 bits). +* Target historical buffer duration (:math:`T_{\text{buff}}`, in milliseconds). + +The sample container size is defined as: + +.. math:: + + C_{\text{size}} = \begin{cases} 2 \text{ bytes} (16\text{ bits}), & \text{if } W_{\text{sample}} = 16 \\ 4 \text{ bytes} (32\text{ bits}), & \text{if } W_{\text{sample}} \in \{24, 32\} \end{cases} + +The required history buffer capacity :math:`S_{\text{buff}}` in bytes is derived as: + +.. math:: + + S_{\text{buff}} = \left(\frac{f_s}{1000}\right) \times C_{\text{size}} \times N_{\text{ch}} \times T_{\text{buff}} + +.. list-table:: KPB History Buffer Memory Allocations Across Configurations + :widths: 20 15 15 20 30 + :header-rows: 1 + + * - Platform Target + - Channels (:math:`N_{\text{ch}}`) + - Width (:math:`W_{\text{sample}}`) + - History (:math:`T_{\text{buff}}`) + - Total Allocated Memory + * - **Tiger Lake (TGL)** + - 2 (Stereo) + - 16-bit + - 3000 ms + - :math:`16 \times 2 \times 2 \times 3000 = 192{,}000\text{ bytes} \approx 187.5\text{ KB}` + * - **Tiger Lake (TGL)** + - 4 (Quad) + - 16-bit + - 3000 ms + - :math:`16 \times 2 \times 4 \times 3000 = 384{,}000\text{ bytes} \approx 375.0\text{ KB}` + * - **Generic CAVS / ACE** + - 2 (Stereo) + - 16-bit + - 2100 ms + - :math:`16 \times 2 \times 2 \times 2100 = 134{,}400\text{ bytes} \approx 131.25\text{ KB}` + * - **Generic CAVS / ACE** + - 4 (Quad) + - 32-bit + - 2100 ms + - :math:`16 \times 4 \times 4 \times 2100 = 537{,}600\text{ bytes} \approx 525.0\text{ KB}` + +.. graphviz:: + :caption: Dual-Sink Buffer Architecture: Continuous Keyword Detector Feed vs Burst Draining Host Sink + :alt: Diagram illustrating the KPB dual-sink streaming architecture connecting DMIC input, history ring buffers, real-time detector sink, and host draining sink. + + digraph kpb_dual_sink { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + subgraph cluster_input { + label = "Audio Input"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + src_dmic [label="Source Buffer\n(DMIC Capture Stream)\n16 kHz, 2-6 Channels", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + } + + subgraph cluster_kpb_internals { + label = "KPB Core Architecture"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + kpb_dispatch [label="KPB Copy Engine\n(Format Check &\nChannel Parsing)", fillcolor="#9AE6B4", color="#22543D", penwidth=1.8]; + + subgraph cluster_history { + label = "Dual Circular History Buffers"; + style = "filled,rounded"; + color = "#C6F6D5"; + fillcolor = "#F0FFF4"; + + hb0 [label="History Buffer 0\n(50% Capacity)\nstart_addr .. end_addr", fillcolor="#E2E8F0", color="#4A5568", penwidth=1.2]; + hb1 [label="History Buffer 1\n(50% Capacity)\nstart_addr .. end_addr", fillcolor="#E2E8F0", color="#4A5568", penwidth=1.2]; + + hb0 -> hb1 [label="next", color="#276749", constraint=false]; + hb1 -> hb0 [label="next", color="#276749", constraint=false]; + } + + mic_sel [label="Mic Channel Selector\n(Configurable Bitmask)\nExtracts Voice Channels", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.4]; + } + + subgraph cluster_sinks { + label = "Dual Output Sinks"; + style = "filled,rounded"; + color = "#E9D8FD"; + fillcolor = "#F7FAFC"; + + sink_rt [label="Pin 0: Real-Time Sink\n(sel_sink)\nFeeds Keyword Spotter\nZero Buffering Latency", fillcolor="#FAF5FF", color="#6B46C1", penwidth=1.6]; + sink_host [label="Pin 1: Host Sink\n(host_sink)\nFeeds Host DMA Copier\nBurst Draining & Live Stream", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.8]; + } + + src_dmic -> kpb_dispatch [label="Periodic Frames", color="#3182CE", penwidth=1.5]; + kpb_dispatch -> hb0 [label="Continuous Write\n(w_ptr update)", color="#22543D", penwidth=1.6]; + kpb_dispatch -> mic_sel [label="Voice Channels", color="#B7791F", penwidth=1.4]; + mic_sel -> sink_rt [label="Continuous Stream", color="#6B46C1", penwidth=1.6]; + + hb0 -> sink_host [label="Draining Task\n(r_ptr playback)", color="#3182CE", penwidth=1.8, style="dashed"]; + hb1 -> sink_host [label="Draining Task\n(r_ptr playback)", color="#3182CE", penwidth=1.8, style="dashed"]; + } + +Pointer Mechanics & Overwrite Protection +----------------------------------------- + +During normal listening (``KPB_STATE_RUN``), the write pointer (``w_ptr``) advances sequentially through the memory of the active buffer. When ``w_ptr`` reaches ``end_addr``, the buffer state is flagged as ``KPB_BUFFER_FULL``, the write pointer is reset to ``start_addr`` of the subsequent buffer (``buff->next``), and writing continues without disruption. + +When a keyword trigger initiates draining of :math:`B_{\text{req}}` bytes, the read pointer :math:`P_{\text{read}}` must be positioned exactly :math:`B_{\text{req}}` bytes behind the current write pointer :math:`P_{\text{write}}` across the circular buffer boundaries: + +.. math:: + + P_{\text{read}} = \begin{cases} P_{\text{write}} - B_{\text{req}}, & \text{if } (P_{\text{write}} - P_{\text{start}}) \ge B_{\text{req}} \\ P_{\text{prev\_end}} - \left(B_{\text{req}} - (P_{\text{write}} - P_{\text{start}})\right), & \text{otherwise} \end{cases} + +To prevent newly arriving microphone audio from overwriting history samples that are staged for host draining, KPB dynamically clamps its writable headroom: + +.. math:: + + \text{FreeHeadroom} = S_{\text{buff}} - B_{\text{req}} + +As the draining task reads and emits audio to the host sink, it increments ``kpb->hd.free``, restoring writable memory space in exact synchrony with host consumption. + +.. graphviz:: + :caption: History Circular Ring Buffer Pointer Mechanics: Pre-Roll Window, Wrap Safety & Overwrite Protection + :alt: Detailed memory layout and pointer mechanics showing write pointer progression, backward read pointer positioning, and boundary wrap safety. + + digraph kpb_pointer_mechanics { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + subgraph cluster_ring_layout { + label = "Circular Ring Memory Topology"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + subgraph cluster_buf0 { + label = "History Buffer Segment 0 (FULL)"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#EDF2F7"; + + b0_start [label="start_addr (0x0000)", fillcolor="#E2E8F0", color="#4A5568"]; + b0_rptr [label="r_ptr (Drain Start)\nCalculated Backward Offset", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + b0_mid [label="Staged Pre-Roll Audio Data\n(Protected from Overwrite)", fillcolor="#FEEBC8", color="#C05621"]; + b0_end [label="end_addr (0x17700)", fillcolor="#E2E8F0", color="#4A5568"]; + + b0_start -> b0_rptr -> b0_mid -> b0_end [style="invis"]; + } + + subgraph cluster_buf1 { + label = "History Buffer Segment 1 (ACTIVE / FREE)"; + style = "filled,rounded"; + color = "#C6F6D5"; + fillcolor = "#F0FFF4"; + + b1_start [label="start_addr (0x17700)", fillcolor="#E2E8F0", color="#4A5568"]; + b1_wptr [label="w_ptr (Current Write)\nTrigger Event Instant", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + b1_free [label="Available Headroom\n(free = total - drain_req)", fillcolor="#EBF8FF", color="#3182CE"]; + b1_end [label="end_addr (0x2EE00)", fillcolor="#E2E8F0", color="#4A5568"]; + + b1_start -> b1_wptr -> b1_free -> b1_end [style="invis"]; + } + } + + b0_end -> b1_start [label="Ring Boundary Link (next)", color="#276749", penwidth=1.5]; + b1_end -> b0_start [label="Wrap-Around Link (next)", color="#276749", penwidth=1.5]; + + b1_wptr -> b0_rptr [label="Reverse Offset Search: -drain_req bytes\n(Walks backward across buffer link)", color="#C05621", style="dashed", penwidth=1.8]; + } + +Dual-Sink Architecture & Microphone Channel Selection +===================================================== + +The KPB component is architected with dual output pins (``num_output_pins = 2``): + +1. **Pin 0: Real-Time Selector Sink (``sel_sink``, ``REALTIME_PIN_ID``)**: + This sink is dedicated to low-latency processing and feeds local on-DSP keyword detection engines (e.g. TFLM, MFCC feature extractors, or vendor detection algorithms). During normal system sleep, audio is delivered directly to Pin 0 on every pipeline period. +2. **Pin 1: Host Draining Sink (``host_sink``)**: + This sink connects to the host capture pipeline through downstream volume and copier components. During host sleep, Pin 1 remains inactive and paused. Upon a keyword activation event, Pin 1 receives the burst-drained pre-roll historical audio and subsequent live microphone speech. + +Microphone Channel Selection (MicSelector) +------------------------------------------ + +In modern platforms equipped with digital microphone arrays (such as 3-mic or 4-mic beamforming arrays with reference loopback channels), passing the full multi-channel stream to the keyword detector during low-power sleep would waste substantial memory bandwidth and DSP processing cycles. + +To minimize energy consumption, KPB incorporates an integrated microphone channel selector (``kpb_micselector_config``, configured via IPC4 parameter ``KP_BUF_CLIENT_MIC_SELECT``): + +.. code-block:: c + + struct kpb_micselector_config { + uint32_t mask; /* Channel selection bitmask */ + }; + +When ``kpb->num_of_sel_mic`` is configured (e.g. selecting channel 0 or channel 1 via bitmask ``0x01`` or ``0x02``), KPB automatically demultiplexes and extracts only the designated voice microphone channel when copying to the real-time sink (Pin 0). Meanwhile, the full multi-channel stream is preserved intact in the circular history buffer, ensuring that when the host wakes up, beamforming and multi-channel noise suppression algorithms have access to all physical microphone signals for high-fidelity speech recognition. + +Accelerated Burst Draining & Dynamic Pace Adjustment +==================================================== + +When a keyword trigger initiates host streaming, transferring historical data at standard real-time speed (:math:`1\times`) would be inadequate: if the host resumes 2 seconds after the trigger, draining 2 seconds of pre-roll at :math:`1\times` speed would mean the host remains perpetually 2 seconds behind real-time audio. + +To eliminate this lag, KPB executes an asynchronous **Burst Draining Task** (``kpb_draining_task``) scheduled via the SOF Earliest Deadline First (EDF) scheduler. The draining task empties the history buffer at a multiple of real-time speed before transitioning seamlessly into live streaming. + +.. graphviz:: + :caption: Accelerated Burst Draining Timeline & Dynamic Interval Adjustment (FMT vs Real-Time Hand-off) + :alt: Timing diagram comparing real-time capture progression with accelerated burst draining and seamless live hand-off. + + digraph kpb_draining_timeline { + rankdir=TB; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + subgraph cluster_timeline { + label = "WoV Audio Draining Progression"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + t0 [label="Phase 1: Ambient Listening (t < t_trig)\nHost Asleep (S0ix) | DSP D0ix\nContinuous Buffering: 16 kHz Audio -> History Ring\nReal-Time Feed -> Keyword Spotter (Pin 0)", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + + t1 [label="Phase 2: Keyword Activation (t = t_trig)\n'Hey Computer' Detected by On-DSP Classifier\nHost Wake IRQ Asserted | KPB enters INIT_DRAINING\nReverse Read Pointer Calculated (-2000 ms)", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.8]; + + t2 [label="Phase 3: Host Resume Lag (t_trig < t < t_host_ready)\nHost PMIC & Rails Settling (800 - 1500 ms)\nKPB Continues Buffering Incoming Microphone Audio\nbuffered_while_draining Counter Tracks Accumulation", fillcolor="#FEEBC8", color="#C05621", penwidth=1.5]; + + t3 [label="Phase 4: Accelerated Burst Draining (2x to 4x Pace)\nHost DMA Active | Draining Task Scheduled\nHistory Flushed Rapidly into Host Buffer\nDynamic Pace Adjustment (adjust_drain_interval)", fillcolor="#FED7D7", color="#C53030", penwidth=2.0]; + + t4 [label="Phase 5: Catch-up Convergence (drain_req == 0)\nPre-Roll Completely Transferred\nKPB Transitions to KPB_STATE_HOST_COPY\nHistory Buffer Bypassed", fillcolor="#E9D8FD", color="#6B46C1", penwidth=1.8]; + + t5 [label="Phase 6: Uncut Real-Time Streaming\nLive Microphone Audio Streamed to Host DMA at 1x Pace\nZero Lost Syllables | Zero Audio Discontinuities", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + + t0 -> t1 -> t2 -> t3 -> t4 -> t5 [color="#2B6CB0", penwidth=1.8]; + } + } + +Synchronized Draining & Dynamic Pace Adjustment +----------------------------------------------- + +SOF supports two operational draining modes: + +1. **Unsynchronized (Unlimited) Draining**: + Audio samples are copied to the host sink buffer as fast as downstream memory and DMA allow, constrained only by available sink space. +2. **Synchronized Draining (``sync_draining_mode``)**: + Draining is paced to prevent overflowing host DMA ring buffers while remaining significantly faster than real-time consumption. The target interval is governed by: + + .. math:: + + I_{\text{drain}} = \frac{T_{\text{host\_period}}}{M_{\text{drain}}} + + where :math:`M_{\text{drain}} = \text{KPB\_DRAIN\_NUM\_OF\_PPL\_PERIODS\_AT\_ONCE} = 2`. Draining operates at double the normal pipeline period rate. + +Dynamic Pace Regulation Algorithm +--------------------------------- + +Because host interrupt response and DMA scheduling exhibit jitter, KPB incorporates an adaptive pace controller (``adjust_drain_interval``) evaluated every 32 task iterations using 64-bit DSP wall-clock cycles (``sof_cycle_get_64()``): + +.. math:: + + P_{\text{actual}} = \frac{\Delta \text{DrainedBytes}}{\Delta t_{\text{elapsed}}} \times 1000 + +.. math:: + + P_{\text{optimal}} = \text{PeriodBytes} \times M_{\text{drain}} \times 1000 + +If :math:`P_{\text{actual}} < P_{\text{optimal}}` (draining is falling behind target pace), the drain interval is reduced: + +.. math:: + + I_{\text{drain}} \leftarrow I_{\text{drain}} \times \left(\frac{P_{\text{actual}}}{P_{\text{optimal}}}\right) - \frac{I_{\text{drain}}}{8} + +Conversely, if :math:`P_{\text{actual}} > P_{\text{optimal}}`, the interval is lengthened proportionally, maintaining stable DMA buffer levels without underrun or overrun. + +Fast Mode Task (FMT) Pipeline Infrastructure +-------------------------------------------- + +In complex audio graphs, intermediate components (such as Gain/Volume widgets or PCM Format Converters) may sit between KPB and the Host DMA Copier. Under standard scheduling, these intermediate modules execute only once per pipeline period (e.g. every 1 ms or 4 ms). + +To prevent these intermediate modules from throttling burst draining, SOF implements the **Fast Mode Task (FMT)** framework (``struct fast_mode_task``, configured via IPC4 parameter ``KP_BUF_CFG_FM_MODULE``). FMT registers downstream modules into an accelerated execution list, triggering their processing routines in direct synchronization with KPB burst cycles until pre-roll draining finishes. + +Event Notification Framework: IPC3 Notifiers vs IPC4 AMS +========================================================= + +Communication between keyword spotters, client pipelines, and the KPB component differs across SOF IPC architectures: + +.. graphviz:: + :caption: Event Notification Architecture: IPC3 Notifier Dispatch vs IPC4 Asynchronous Message Service (AMS) + :alt: Architectural comparison between IPC3 notifier callbacks and IPC4 Asynchronous Message Service (AMS) dispatching wake events to KPB. + + digraph kpb_event_architecture { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + subgraph cluster_ipc3 { + label = "IPC3 Notifier Framework"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#F7FAFC"; + + kwd3 [label="Keyword Detector\n(KD Module)", fillcolor="#FEFCBF", color="#B7791F"]; + notif_core [label="SOF Notifier Engine\nNOTIFIER_ID_KPB_CLIENT_EVT\nSynchronous Callbacks", fillcolor="#E2E8F0", color="#4A5568", penwidth=1.5]; + kpb_ev_hdl [label="kpb_event_handler()\nDispatches Events:\n- REGISTER_CLIENT\n- BEGIN_DRAINING", fillcolor="#9AE6B4", color="#22543D", penwidth=1.8]; + + kwd3 -> notif_core [label="notifier_event()", color="#B7791F"]; + notif_core -> kpb_ev_hdl [label="Direct Callback", color="#22543D", penwidth=1.5]; + } + + subgraph cluster_ipc4 { + label = "IPC4 Asynchronous Message Service (AMS)"; + style = "filled,rounded"; + color = "#FED7D7"; + fillcolor = "#FFF5F5"; + + kwd4 [label="Keyword Spotter\n(IPC4 KPD Module)", fillcolor="#FEFCBF", color="#B7791F"]; + ams_core [label="AMS Message Router\nCONFIG_AMS Enabled\nAsynchronous Mailbox", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.5]; + kpb_ams [label="kpb_set_large_config()\nKP_BUF_CFG_FM_MODULE\nKP_BUF_CLIENT_MIC_SELECT", fillcolor="#FEB2B2", color="#C53030", penwidth=1.8]; + + kwd4 -> ams_core [label="ams_send_message()", color="#B7791F"]; + ams_core -> kpb_ams [label="Large Config IPC", color="#C53030", penwidth=1.5]; + } + } + +IPC3 Notifier Implementation +---------------------------- + +In IPC3 topologies, communication between the detection module and KPB relies on the internal core notifier system: + +.. code-block:: c + + enum kpb_event { + KPB_EVENT_REGISTER_CLIENT = 0, + KPB_EVENT_UPDATE_PARAMS, + KPB_EVENT_BEGIN_DRAINING, + KPB_EVENT_STOP_DRAINING, + KPB_EVENT_UNREGISTER_CLIENT, + }; + +Clients (such as ``detect_test``) register with KPB by passing ``KPB_EVENT_REGISTER_CLIENT`` along with their requested history draining window (``drain_req``, up to ``KPB_MAX_DRAINING_REQ`` = 2000 ms to 3000 ms). When the keyword model confirms an utterance match, it fires ``KPB_EVENT_BEGIN_DRAINING``, causing KPB to calculate the historical read pointer and start the draining task. + +IPC4 Asynchronous Message Service (AMS) +--------------------------------------- + +Under IPC4, inter-module signaling leverages the **Asynchronous Message Service (AMS)** (``CONFIG_AMS``). Modules communicate via standardized large configuration parameters: + +* ``KP_BUF_CFG_FM_MODULE`` (Parameter ID 1): Configures the list of downstream modules participating in the Fast Mode Task during accelerated pre-roll draining. +* ``KP_BUF_CLIENT_MIC_SELECT`` (Parameter ID 11): Updates the real-time microphone channel selection mask without tearing down active audio pipelines. + +Linux Driver & DAPM Control Sequencing +-------------------------------------- + +On the Linux host, keyword detection pipelines are managed through ALSA Dynamic Audio Power Management (DAPM). Two intertwined pipelines are constructed: + +1. **Pipeline 8 (Host Capture Pipeline)**: DMIC :math:`\to` Volume :math:`\to` KPB :math:`\to` Host Copier :math:`\to` ALSA PCM capture device. +2. **Pipeline 9 (Keyword Detect Pipeline)**: KPB Pin 0 :math:`\to` Selector :math:`\to` Detector Module :math:`\to` Virtual Detector Sink. + +.. list-table:: ALSA DAPM Control Sequence for Keyword Detection + :widths: 20 25 25 30 + :header-rows: 1 + + * - Stream Control Action + - Host Pipeline (Pipe 8) + - Detector Pipeline (Pipe 9) + - Operational Hardware State + * - **1. HW Parameters** + - ``snd_pcm_hw_params()`` + - ``DAPM_PRE_PMU`` Event + - DSP sets 16 kHz sampling, validates minimum host buffer (:math:`\ge 67200\text{ frames}`). + * - **2. Trigger Start** + - Host suspended + - Pipeline 9 Started + - DSP enters D0ix; KPB buffers incoming audio; Detector continuously scans. + * - **3. Keyword Detected** + - Host resumes via IRQ + - Draining triggered + - KPB empties pre-roll history to host DMA; transitions to live copy. + * - **4. Capture Stop** + - ``snd_pcm_drain()`` + - ``DAPM_POST_PMD`` Event + - Host application finishes reading speech command; pipeline resets to listening state. + +End-to-End WoV System Pipeline & Topology 2 Wiring +================================================== + +The integration of KPB within an end-to-end Sound Open Firmware audio graph is illustrated in Figure 201: + +.. graphviz:: + :caption: End-to-End WoV Audio Graph: DMIC Array, DC Blocker, KPB, Keyword Spotter & Host DMA Copier + :alt: Complete end-to-end audio processing pipeline connecting physical DMIC inputs to DC Blocker, KPB, Keyword Spotter, and Host DMA Copier. + + digraph wov_complete_graph { + rankdir=LR; + bgcolor="transparent"; + node [fontname="Helvetica,Arial,sans-serif", fontsize=10, shape=box, style="filled,rounded", margin="0.12,0.06"]; + edge [fontname="Helvetica,Arial,sans-serif", fontsize=9]; + + subgraph cluster_dmic_be { + label = "DAI Back-End Pipeline (Pipe 1)"; + style = "filled,rounded"; + color = "#CBD5E0"; + fillcolor = "#F7FAFC"; + + hw_dmic [label="DMIC Hardware\nArray (16 kHz)\n4-Channel PDM", fillcolor="#E2E8F0", color="#4A5568", penwidth=1.5]; + dai_copier [label="DAI Copier\n(dai-copier.1)\nMulti-Channel DMA", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.5]; + dcblock [label="DC Blocker\n(dcblock.1)\nRemoves ADC DC", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + pga_kwd [label="Capture Volume\n(pga.1)\nGain Adjustment", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + } + + subgraph cluster_kpb_hub { + label = "KPB Core Hub (Pipe 2)"; + style = "filled,rounded"; + color = "#BEE3F8"; + fillcolor = "#FFFFFF"; + + kpb_widget [label="Key Phrase Buffer\n(kpb.1)\nUUID: D8218443...\nDual-Output Widget", fillcolor="#9AE6B4", color="#22543D", penwidth=2.0]; + } + + subgraph cluster_detect_fe { + label = "Detection Pipeline (Pipe 9)"; + style = "filled,rounded"; + color = "#FEFCBF"; + fillcolor = "#FFFFF0"; + + selector [label="Channel Selector\n(selector.1)\nSelects Voice Mic", fillcolor="#FEFCBF", color="#B7791F", penwidth=1.4]; + detector [label="Keyword Detector\n(TFLM / MFCC / KD)\nEvaluates Wake Phrase", fillcolor="#FEEBC8", color="#C05621", penwidth=1.8]; + det_sink [label="Virtual Detector Sink\n(DAPM Control Node)", fillcolor="#EDF2F7", color="#4A5568", penwidth=1.2]; + } + + subgraph cluster_host_fe { + label = "Host Capture Pipeline (Pipe 8)"; + style = "filled,rounded"; + color = "#E9D8FD"; + fillcolor = "#FAF5FF"; + + host_copier [label="Host Copier\n(copier.host.1)\nFast Mode Capable", fillcolor="#EBF8FF", color="#3182CE", penwidth=1.8]; + host_dma [label="Host ALSA Capture\n(hw:0,8)\narecord / Voice AI", fillcolor="#FAF5FF", color="#6B46C1", penwidth=2.0]; + } + + hw_dmic -> dai_copier [label="PDM Pins", color="#4A5568"]; + dai_copier -> dcblock [label="Raw PCM", color="#3182CE"]; + dcblock -> pga_kwd [label="HPF PCM", color="#3182CE"]; + pga_kwd -> kpb_widget [label="4-Ch 16 kHz Stream", color="#22543D", penwidth=1.8]; + + kpb_widget -> selector [label="Pin 0: Real-Time Stream", color="#B7791F", penwidth=1.6]; + selector -> detector [label="1-Ch Voice Stream", color="#B7791F", penwidth=1.5]; + detector -> det_sink [label="Detection Events", color="#4A5568"]; + + kpb_widget -> host_copier [label="Pin 1: Draining & Live Stream", color="#3182CE", penwidth=2.0]; + host_copier -> host_dma [label="PCIe / Memory DMA", color="#6B46C1", penwidth=2.0]; + + detector -> kpb_widget [label="Trigger Event (BEGIN_DRAINING)", color="#C53030", style="dashed", penwidth=1.8]; + } + +Topology 2 Widget Declaration +----------------------------- + +In ALSA Topology 2 (``tools/topology/topology2/include/components/kpb.conf``), the KPB widget is declared as an effect class with one input pin and two output pins: + +.. code-block:: text + + Class.Widget."kpb" { + DefineAttribute."index" {} + DefineAttribute."instance" {} + DefineAttribute."cpc" { + token_ref "comp.word" + } + + + + attributes { + !constructor [ + "index" + "instance" + ] + !mandatory [ + "no_pm" + "uuid" + ] + !immutable [ + "uuid" + ] + unique "instance" + } + + type "effect" + num_input_audio_formats 1 + num_output_audio_formats 1 + + # UUID: D8218443-5FF3-4A4C-B388-6CFE07B9562E + uuid "43:84:21:d8:f3:5f:4c:4a:b3:88:6c:fe:07:b9:56:2e" + no_pm "true" + cpc 720000 + num_input_pins 1 + num_output_pins 2 + } + +Backend Pipeline Integration +---------------------------- + +In ``tools/topology/topology2/include/pipelines/cavs/dai-kpb-be.conf``, the KPB widget is instantiated downstream of the DAI copier: + +.. code-block:: text + + Object.Widget.kpb."1" { + index $DRAINING_PIPELINE_ID + num_input_audio_formats 2 + num_output_audio_formats 2 + + Object.Base.input_audio_format [ + { + in_rate 16000 + in_bit_depth 32 + in_valid_bit_depth 32 + } + { + in_rate 16000 + in_channels 4 + in_bit_depth 32 + in_valid_bit_depth 32 + in_ch_cfg $CHANNEL_CONFIG_3_POINT_1 + } + ] + } + +Host Buffer Sizing Requirements & Best Practices +------------------------------------------------ + +.. important:: + **Host DMA Buffer Sizing**: + Platform resume from ACPI S0ix / Modern Standby requires between 1000 ms and 2000 ms under typical operating conditions. To ensure that pre-roll historical audio is not overwritten before the host application begins consuming samples, the ALSA capture buffer must be dimensioned adequately: + + * The host ``buffer-size`` must be configured to at least **67,200 frames** (:math:`\approx 4.2\text{ seconds}` at 16 kHz). + * Host capture should be invoked with memory-mapped non-blocking I/O: + + .. code-block:: bash + + arecord -Dhw:0,8 -M -N -c 2 -f S16_LE -r 16000 --buffer-size=68000 capture.wav -vvv + + * Smaller buffer allocations will be rejected by the SOF firmware during the ``hw_params`` validation stage with an ``-EINVAL`` error to prevent buffer overrun corruption. diff --git a/developer_guides/firmware/level_multiplier.rst b/developer_guides/firmware/level_multiplier.rst new file mode 100644 index 00000000..e2f31fd2 --- /dev/null +++ b/developer_guides/firmware/level_multiplier.rst @@ -0,0 +1,902 @@ +.. _level_multiplier: + +Level Multiplier Architecture +============================= + +The **Level Multiplier** subsystem in Sound Open Firmware (SOF) is an ultra-low-latency, zero-overhead digital linear gain and attenuation component. Operating strictly on fixed-point **Q9.23** arithmetic, the Level Multiplier scales digital audio signals across a vast dynamic range from :math:`-138.47\text{ dB}` to :math:`+48.17\text{ dB}`. Unlike full-featured software volume controls that implement multi-channel curves, logarithmic lookups, and multi-millisecond smoothing ramps, the Level Multiplier applies a direct scalar factor across all channels without state ramping overhead or algorithmic delay. + +The Level Multiplier is extensively deployed in voice capture front-ends (such as Automatic Speech Recognition and far-field voice trigger pipelines) to calibrate microphone sensitivity independently from user-facing media volume controls. Furthermore, the component integrates an automated **zero-overhead fast-path bypass**: whenever the configured gain equals unity (:math:`0\text{ dB}`, `LEVEL_MULTIPLIER_GAIN_ONE`), the component completely bypasses arithmetic multiplication loops and executes a direct memory copy, minimizing processor cycles and active power consumption. + +.. contents:: Table of Contents + :local: + :depth: 3 + +------------------------------------------------------------------------------- + +Architectural Overview & Functional Role +---------------------------------------- + +Audio processing pipelines frequently require precise level adjustments that are independent of user-controlled volume sliders. Typical examples include microphone pre-amplification calibration, transducer sensitivity matching across multi-microphone arrays, inter-stage digital headroom management, and platform-specific acoustic tuning. + +Conventional SOF components address level adjustment with different design trade-offs: + +- **Volume Control Subsystem** (:ref:`volume_module`): + Designed for user-facing listening controls. Features logarithmic-to-linear curve translation, per-channel independent attenuation sliders (:math:`-\infty` to :math:`0\text{ dB}`), mute state machines, and smooth multi-millisecond linear ramping to prevent audible zipper noise when the user interacts with an ALSA mixer slider. This functionality requires stateful ramp management and per-sample interpolation overhead. +- **Aria Subsystem** (:ref:`aria`): + Designed for dynamic lookahead peak limiting and transient back-off. It enforces a target pre-amplification boost (:math:`0`, :math:`+6`, :math:`+12`, :math:`+18\text{ dB}`) while dynamically ducking gain during loud bursts, introducing an exact :math:`1\text{ ms}` algorithmic lookahead latency via an internal circular delay buffer. +- **Level Multiplier Subsystem**: + Designed for ultra-fast, deterministic, zero-latency scalar multiplication. It applies a uniform fixed-point multiplier across all channels without ramp overhead, introducing **identically 0 ms of algorithmic delay**. When set to unity gain (:math:`0\text{ dB}`), it completely bypasses arithmetic execution via a direct fast-path. + +.. list-table:: Architectural Comparison: Level Multiplier vs Volume vs Aria + :widths: 20 25 25 30 + :header-rows: 1 + + * - Parameter + - Level Multiplier + - Volume Control + - Aria (Automatic Regressive) + * - **Gain Representation** + - Linear Q9.23 fixed-point + - Logarithmic dB / Linear Q1.31 + - Discrete modes (:math:`0, 6, 12, 18\text{ dB}`) + * - **Gain Range** + - :math:`-138.47\text{ dB}` to :math:`+48.17\text{ dB}` + - :math:`-\infty\text{ dB}` to :math:`0\text{ dB}` (attenuation only) + - :math:`0\text{ dB}` to :math:`+18\text{ dB}` (with regressive ducking) + * - **Algorithmic Latency** + - **0 ms** (instantaneous sample processing) + - **0 ms** (instantaneous sample processing) + - **1 ms** (lookahead circular ring buffer) + * - **Ramp Smoothing** + - None (direct scalar application) + - Smooth per-sample linear ramp (16 to 500 ms) + - Per-sample lookahead linear interpolation + * - **Fast-Path Bypass** + - Automated direct memory copy at unity gain (:math:`0\text{ dB}`) + - Arithmetic bypass at 0 dB if unmuted + - Invariant 1 ms circular delay buffer routing + * - **Primary Use Cases** + - Voice capture sensitivity calibration, ASR tuning + - Main playback volume, application streams + - Far-field mic boost with anti-clipping protection + +.. _figure_223: + +.. graphviz:: + :align: center + :caption: SOF Level Multiplier Architecture: Ingress, Fast-Path Bypass & Fixed-Point Gain Scaling Core + + digraph level_multiplier_architecture { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_ingress { + label = "Audio Egress / Producer"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + source [label="Source Stream Buffer\n(S16_LE / S24_4LE / S32_LE)\nsource_get_data_*()", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + subgraph cluster_module { + label = "Level Multiplier Module (UUID: 30397456-4661...)"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + decision [label="Unity Gain Check\ncd->gain == 0x00800000?", fillcolor="#d97706", fontcolor="#ffffff", color="#fbbf24", shape="diamond"]; + fastpath [label="Zero-Overhead Fast-Path\nsource_to_sink_copy()\n(Direct Memory Copy)", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + + subgraph cluster_dsp_core { + label = "Fixed-Point Q9.23 Scaling Core"; + style = "solid"; + color = "#0369a1"; + bgcolor = "#0369a111"; + + s16_proc [label="S16 Engine\nq_multsr_sat_32x32_16\n(Shift = 23)", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + s24_proc [label="S24 Engine\nq_multsr_sat_32x32_24\n(Shift = 23)", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + s32_proc [label="S32 Engine\nq_multsr_sat_32x32\n(Shift = 23)", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + } + } + + subgraph cluster_egress { + label = "Audio Ingress / Consumer"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + sink [label="Sink Stream Buffer\nsink_commit_buffer()", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + source -> decision [label="Ingress frames"]; + decision -> fastpath [label="True (0 dB)"]; + decision -> s16_proc [label="False (S16)"]; + decision -> s24_proc [label="False (S24)"]; + decision -> s32_proc [label="False (S32)"]; + + fastpath -> sink [label="Copied samples"]; + s16_proc -> sink [label="Scaled S16"]; + s24_proc -> sink [label="Scaled S24"]; + s32_proc -> sink [label="Scaled S32"]; + } + +------------------------------------------------------------------------------- + +Fixed-Point Q9.23 Number System & Gain Range +-------------------------------------------- + +The Level Multiplier represents linear gain as a 32-bit signed integer using the **Q9.23** fixed-point numeric format, defined in :file:`level_multiplier.h`: + +.. code-block:: c + + #define LEVEL_MULTIPLIER_QXY_X 9 + #define LEVEL_MULTIPLIER_QXY_Y 23 + #define LEVEL_MULTIPLIER_GAIN_ONE (1 << LEVEL_MULTIPLIER_QXY_Y) + +Bitfield Structure +~~~~~~~~~~~~~~~~~~ + +A 32-bit word in Q9.23 allocates bits as follows: + +.. math:: + + \underbrace{b_{31}}_{\text{Sign}} \quad \underbrace{b_{30} \quad b_{29} \quad b_{28} \quad b_{27} \quad b_{26} \quad b_{25} \quad b_{24} \quad b_{23}}_{8 \text{ Integer Bits}} \quad \underbrace{b_{22} \quad b_{21} \quad \dots \quad b_1 \quad b_0}_{23 \text{ Fractional Bits}} + +- **Sign Bit** (:math:`b_{31}`): Supports both non-inverting (:math:`+`) and phase-inverting (:math:`-`) multipliers. +- **Integer Bits** (:math:`b_{30} \dots b_{23}`): 8 bits of integer magnitude, providing a maximum positive integer value of :math:`2^8 - 1 = 255`. +- **Fractional Bits** (:math:`b_{22} \dots b_0`): 23 bits of fractional precision, yielding an elemental quantization resolution of: + + .. math:: + + \Delta = 2^{-23} \approx 1.1920928955 \times 10^{-7} + +Unity Gain Definition +~~~~~~~~~~~~~~~~~~~~~ + +Unity gain (:math:`1.0\times`, corresponding to :math:`0.00\text{ dB}`) is represented when the fractional component is zero and the integer component is :math:`1`: + +.. math:: + + \text{LEVEL\_MULTIPLIER\_GAIN\_ONE} = 1 \cdot 2^{23} = 8,388,608 = \text{0x00800000} + +Dynamic Range & Extremes +~~~~~~~~~~~~~~~~~~~~~~~~ + +The Q9.23 format enables an exceptionally wide dynamic range: + +1. **Maximum Positive Amplification**: + The largest representable positive gain word is: + + .. math:: + + \text{gain}_{\max} = 2^{31} - 1 = \text{0x7FFFFFFF} = 256.0 - 2^{-23} \approx 255.99999988 + + In decibels: + + .. math:: + + G_{\max} = 20 \log_{10}(256) \approx +48.1648\text{ dB} \approx +48.17\text{ dB} + +2. **Minimum Positive Non-Zero Resolution**: + The smallest positive increment above zero is a single LSB: + + .. math:: + + \text{gain}_{\min} = 1 = \text{0x00000001} \implies 2^{-23} + + In decibels: + + .. math:: + + G_{\min} = 20 \log_{10}(2^{-23}) \approx -138.4739\text{ dB} \approx -138.47\text{ dB} + +3. **Total Dynamic Span**: + The span from maximum boost to minimum non-zero resolution encompasses: + + .. math:: + + \text{Span} = 48.17\text{ dB} - (-138.47\text{ dB}) = 186.64\text{ dB} + + well exceeding the 144 dB theoretical dynamic range of 24-bit audio converters. +4. **Complete Silence**: + Setting :math:`\text{gain} = 0` (:math:`\text{0x00000000}`) completely mutes the signal (:math:`-\infty\text{ dB}`). + +Decibel to Q9.23 Linear Translation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To calculate the 32-bit Q9.23 integer word for a desired gain in decibels (:math:`G_{dB}`): + +.. math:: + + \text{gain}_{\text{Q9.23}} = \left\lfloor 10^{\frac{G_{dB}}{20}} \cdot 2^{23} + 0.5 \right\rfloor + +.. list-table:: Standard Decibel to Q9.23 Conversion Matrix + :widths: 20 25 25 30 + :header-rows: 1 + + * - Desired Gain (dB) + - Linear Multiplier + - Hexadecimal Value + - Decimal Q9.23 Integer + * - **+40.0 dB** + - :math:`100.0000\times` + - ``0x32000000`` + - 838,860,800 + * - **+30.0 dB** + - :math:`31.6228\times` + - ``0x0FD0A499`` + - 265,331,865 + * - **+20.0 dB** + - :math:`10.0000\times` + - ``0x05000000`` + - 83,886,080 + * - **+10.0 dB** + - :math:`3.1623\times` + - ``0x01948332`` + - 26,510,130 + * - **0.0 dB (Unity)** + - :math:`1.0000\times` + - ``0x00800000`` + - 8,388,608 + * - **-10.0 dB** + - :math:`0.3162\times` + - ``0x00287A26`` + - 2,652,710 + * - **-20.0 dB** + - :math:`0.1000\times` + - ``0x000CCCCD`` + - 838,861 + * - **-30.0 dB** + - :math:`0.0316\times` + - ``0x00040C37`` + - 265,271 + * - **-40.0 dB** + - :math:`0.0100\times` + - ``0x000147AE`` + - 83,886 + +.. _figure_224: + +.. graphviz:: + :align: center + :caption: Fixed-Point Q9.23 Number System: Dynamic Range (-138.47 dB to +48.17 dB) & Bit Allocation + + digraph level_multiplier_q9_23 { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_bitfield { + label = "32-Bit Q9.23 Word Memory Organization"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + sign_bit [label="Bit 31\nSign Bit (s)\n0: Positive\n1: Negative", fillcolor="#dc2626", fontcolor="#ffffff", color="#f87171"]; + int_bits [label="Bits 30 .. 23\n8 Integer Bits (Integer Magnitude)\nMax Integer = 255", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + frac_bits [label="Bits 22 .. 0\n23 Fractional Bits (Fractional Precision)\nResolution LSB = 2^-23 (~1.19e-7)", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + subgraph cluster_range { + label = "Dynamic Range Scale"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + r_max [label="Maximum Amplification: +48.17 dB\nGain = 0x7FFFFFFF (~256.0x)", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + r_one [label="Unity Gain (Fast-Path): 0.00 dB\nGain = 0x00800000 (1.0x)", fillcolor="#d97706", fontcolor="#ffffff", color="#fbbf24"]; + r_min [label="Minimum Resolution: -138.47 dB\nGain = 0x00000001 (2^-23)", fillcolor="#334155", fontcolor="#94a3b8", color="#475569"]; + r_mute [label="Digital Silence: -Infinity dB\nGain = 0x00000000 (0.0x)", fillcolor="#1e293b", fontcolor="#94a3b8", color="#475569"]; + } + + sign_bit -> int_bits [style="invis"]; + int_bits -> frac_bits [style="invis"]; + + r_max -> r_one [label="Attenuation"]; + r_one -> r_min [label="Extreme Attenuation"]; + r_min -> r_mute [label="Mute"]; + } + +------------------------------------------------------------------------------- + +Universal PCM Frame Format Processing Engines +--------------------------------------------- + +To support the full range of audio endpoints across the SOF ecosystem, the Level Multiplier implements dedicated processing kernels for three standard PCM frame formats: + +- **16-bit PCM** (:c:macro:`SOF_IPC_FRAME_S16_LE`) +- **24-bit PCM** (:c:macro:`SOF_IPC_FRAME_S24_4LE`) +- **32-bit PCM** (:c:macro:`SOF_IPC_FRAME_S32_LE`) + +Shift Constant Derivation +~~~~~~~~~~~~~~~~~~~~~~~~~ + +During fixed-point multiplication, the product of an :math:`N`-bit sample and the 23-bit fractional component must be shifted right to align the output back to the original container format with saturation. The shift constants are declared in :file:`level_multiplier-generic.c`: + +.. code-block:: c + + #define LEVEL_MULTIPLIER_S16_SHIFT Q_SHIFT_BITS_32(15, LEVEL_MULTIPLIER_QXY_Y, 15) + #define LEVEL_MULTIPLIER_S24_SHIFT Q_SHIFT_BITS_64(23, LEVEL_MULTIPLIER_QXY_Y, 23) + #define LEVEL_MULTIPLIER_S32_SHIFT Q_SHIFT_BITS_64(31, LEVEL_MULTIPLIER_QXY_Y, 31) + +Using the SOF fixed-point shift macro :math:`Q\_SHIFT\_BITS(X, Y, Z) = X + Y - Z`: + +.. math:: + + \text{Shift}_{S16} = 15 + 23 - 15 = 23 + +.. math:: + + \text{Shift}_{S24} = 23 + 23 - 23 = 23 + +.. math:: + + \text{Shift}_{S32} = 31 + 23 - 31 = 23 + +In all three format domains, the required right-shift is identically **23 bits**, perfectly canceling the :math:`2^{23}` scale factor of Q9.23 unity gain. + +Format Processing Loops +~~~~~~~~~~~~~~~~~~~~~~~ + +1. **16-bit Processing Loop** (:c:func:`level_multiplier_s16`): + Operates on 16-bit signed audio samples. Each sample is multiplied by the 32-bit Q9.23 gain using the standard helper :c:func:`q_multsr_sat_32x32_16`, which handles intermediate 48-bit multiplication, 23-bit right-shifting, and saturation clamping to :math:`[-32768, 32767]`: + + .. code-block:: c + + for (i = 0; i < samples_without_wrap; i++) { + *y = q_multsr_sat_32x32_16(*x, gain, LEVEL_MULTIPLIER_S16_SHIFT); + x++; + y++; + } + +2. **24-bit Processing Loop** (:c:func:`level_multiplier_s24`): + Audio is stored in 32-bit containers with 24-bit valid audio. The sample is sign-extended using :c:func:`sign_extend_s24` to ensure correct two's complement sign propagation before multiplication. The result is clamped to the 24-bit dynamic range :math:`[-8388608, 8388607]`: + + .. code-block:: c + + for (i = 0; i < samples_without_wrap; i++) { + *y = q_multsr_sat_32x32_24(sign_extend_s24(*x), gain, + LEVEL_MULTIPLIER_S24_SHIFT); + x++; + y++; + } + +3. **32-bit Processing Loop** (:c:func:`level_multiplier_s32`): + Operates on full 32-bit samples. The multiplication produces a 64-bit product, right-shifted by 23 bits and clamped with 32-bit symmetric saturation: + + .. code-block:: c + + for (i = 0; i < samples_without_wrap; i++) { + *y = q_multsr_sat_32x32(*x, gain, LEVEL_MULTIPLIER_S32_SHIFT); + x++; + y++; + } + +Buffer Wrap Segmentation +~~~~~~~~~~~~~~~~~~~~~~~~ + +To prevent memory faults when reading from and writing to ring buffers, the processing loop computes the largest contiguous block of samples that can be processed before either the source or sink buffer wraps: + +.. code-block:: c + + source_samples_without_wrap = x_end - x; + samples_without_wrap = y_end - y; + samples_without_wrap = MIN(samples_without_wrap, source_samples_without_wrap); + samples_without_wrap = MIN(samples_without_wrap, remaining_samples); + +The inner loop executes across this contiguous segment without branching. Once completed, pointers wrap around via pointer arithmetic: + +.. code-block:: c + + x = (x >= x_end) ? x - x_size : x; + y = (y >= y_end) ? y - y_size : y; + +.. _figure_225: + +.. graphviz:: + :align: center + :caption: Multi-Format Arithmetic Engine: S16_LE, S24_4LE, and S32_LE Multiply-Shift Pipelines + + digraph level_multiplier_formats { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_s16 { + label = "S16_LE Pipeline (16-Bit Container)"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + s16_in [label="Input: x[n] (Q1.15)\n[-32768, 32767]", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + s16_mult [label="Multiply: x * gain\nQ1.15 * Q9.23 -> Q10.38", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + s16_shift [label="Right Shift & Saturation\n>> 23 (LEVEL_MULTIPLIER_S16_SHIFT)\nClamp to [-32768, 32767]", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + s16_out [label="Output: y[n] (Q1.15)", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + + s16_in -> s16_mult -> s16_shift -> s16_out; + } + + subgraph cluster_s24 { + label = "S24_4LE Pipeline (32-Bit Container)"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b22"; + + s24_in [label="Input: x[n] (24-bit in 32-bit)\nsign_extend_s24(*x)", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + s24_mult [label="Multiply: x * gain\nQ1.23 * Q9.23 -> Q10.46", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + s24_shift [label="Right Shift & Saturation\n>> 23 (LEVEL_MULTIPLIER_S24_SHIFT)\nClamp to [-8388608, 8388607]", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + s24_out [label="Output: y[n] (Q1.23)", fillcolor="#10b981", fontcolor="#ffffff", color="#6ee7b7"]; + + s24_in -> s24_mult -> s24_shift -> s24_out; + } + + subgraph cluster_s32 { + label = "S32_LE Pipeline (32-Bit Full Scale)"; + style = "solid"; + color = "#d97706"; + bgcolor = "#78350f22"; + + s32_in [label="Input: x[n] (Q1.31)\n[-2^31, 2^31 - 1]", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + s32_mult [label="64-Bit Multiply: x * gain\nQ1.31 * Q9.23 -> Q10.54", fillcolor="#b45309", fontcolor="#ffffff", color="#fbbf24"]; + s32_shift [label="64-Bit Shift & Saturation\n>> 23 (LEVEL_MULTIPLIER_S32_SHIFT)\nClamp to [-2^31, 2^31 - 1]", fillcolor="#d97706", fontcolor="#ffffff", color="#fbbf24"]; + s32_out [label="Output: y[n] (Q1.31)", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + + s32_in -> s32_mult -> s32_shift -> s32_out; + } + } + +------------------------------------------------------------------------------- + +Zero-Overhead Fast-Path Bypass Architecture +------------------------------------------- + +A primary design requirement for SOF signal chains is energy efficiency. In many topologies, a Level Multiplier is instantiated statically in a pipeline to allow dynamic calibration during manufacturing or runtime mode changes, but remains at unity gain (:math:`0.00\text{ dB}`) during standard operation. + +Fast-Path Implementation +~~~~~~~~~~~~~~~~~~~~~~~~ + +In :c:func:`level_multiplier_process`, the component inspects the active gain variable before initiating any processing loops: + +.. code-block:: c + + if (cd->gain != LEVEL_MULTIPLIER_GAIN_ONE) + /* Process the data with the requested gain. */ + return cd->level_multiplier_func(mod, source, sink, frames); + + /* Just copy from source to sink. */ + source_to_sink_copy(source, sink, true, frames * cd->frame_bytes); + return 0; + +When ``cd->gain`` equals :c:macro:`LEVEL_MULTIPLIER_GAIN_ONE` (:math:`\text{0x00800000}`): + +1. **Elimination of Math Loops**: + The component completely skips the function pointer call to ``cd->level_multiplier_func``. No arithmetic multiplication, bit-shifting, sign extension, or saturation logic is executed. +2. **Direct Block Copy**: + The function :c:func:`source_to_sink_copy` is invoked directly. This executes optimized memory copy primitives (e.g. 64-bit or 128-bit wide word block transfers) or hardware DMA transfers between circular buffers. +3. **Power and Cycle Minimization**: + CPU cycles are reduced to the absolute physical memory transfer minimum, significantly lowering active DSP power consumption during standard passthrough. + +.. _figure_226: + +.. graphviz:: + :align: center + :caption: Zero-Overhead Fast-Path Bypass vs Active Processing Decision Crossbar + + digraph level_multiplier_fastpath { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_dispatch { + label = "Runtime Process Dispatch in level_multiplier_process()"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + chk [label="Inspect Active Gain Value\nIs cd->gain == LEVEL_MULTIPLIER_GAIN_ONE (0x00800000)?", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569", shape="diamond"]; + } + + subgraph cluster_paths { + label = "Execution Pathways"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + path_fast [label="FAST-PATH BYPASS\nsource_to_sink_copy()\n- Zero arithmetic instructions\n- Minimal CPU cycle footprint\n- Maximal memory bandwidth", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + path_active [label="ACTIVE SCALING PATH\ncd->level_multiplier_func()\n- HiFi SIMD / Scalar vector loops\n- Format-specific shift and saturation\n- Linear level amplification / attenuation", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + } + + subgraph cluster_ret { + label = "Sink Egress"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + ret [label="Return Status (0 = Success)\nFrames Committed to Downstream Sink", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + chk -> path_fast [label="YES (0 dB)"]; + chk -> path_active [label="NO (Gain != 0 dB)"]; + + path_fast -> ret; + path_active -> ret; + } + +------------------------------------------------------------------------------- + +Tensilica HiFi SIMD Vector Acceleration +--------------------------------------- + +To achieve peak computational efficiency on Intel audio DSP platforms, the Level Multiplier includes highly optimized assembly kernels tailored for **Tensilica HiFi3 / HiFi4** and **Tensilica HiFi5** processor architectures. + +HiFi3 / HiFi4 Dual-Lane Vectorization (:file:`level_multiplier-hifi3.c`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On HiFi3 and HiFi4 architectures, the DSP utilizes 64-bit vector registers (:c:type:`ae_f32x2`, :c:type:`ae_f16x4`): + +1. **16-Bit Processing** (:c:func:`level_multiplier_s16`): + Loads 4 samples simultaneously using :c:macro:`AE_LA16X4_IP`. The 16-bit samples are multiplied by the 32-bit Q9.23 gain using dual fractional multipliers: + + .. code-block:: c + + samples0 = AE_MULFP32X16X2RS_H(gain, samples); + samples1 = AE_MULFP32X16X2RS_L(gain, samples); + + The intermediate products are shifted left by 8 bits with saturation to convert from Q9.23 to Q1.31: + + .. code-block:: c + + samples0 = AE_SLAI32S(samples0, 8); + samples1 = AE_SLAI32S(samples1, 8); + + Finally, the 32-bit values are symmetrically rounded back to 16-bit representation using :c:macro:`AE_ROUND16X4F32SSYM` and stored via :c:macro:`AE_SA16X4_IP`. +2. **24-Bit Processing** (:c:func:`level_multiplier_s24`): + Processes two 32-bit containers per vector operation. Samples are shifted left by 8 bits to align 24-bit audio to the most significant bits: + + .. code-block:: c + + AE_LA32X2_IP(samples, x_align, x); + samples = AE_MULFP32X2RS(gain, AE_SLAI32(samples, 8)); + samples = AE_SLAI32S(samples, 8); + samples = AE_SRAI32(samples, 8); + AE_SA32X2_IP(samples, y_align, y); + +3. **32-Bit Processing** (:c:func:`level_multiplier_s32`): + Multiplies two 32-bit samples by the 32-bit gain, producing 64-bit accumulators: + + .. code-block:: c + + mult0 = AE_MULF32R_HH(gain, samples); + mult1 = AE_MULF32R_LL(gain, samples); + mult0 = AE_SLAI64(mult0, LEVEL_MULTIPLIER_S32_SHIFT); + mult1 = AE_SLAI64(mult1, LEVEL_MULTIPLIER_S32_SHIFT); + samples = AE_ROUND32X2F48SSYM(mult0, mult1); + AE_SA32X2_IP(samples, y_align, y); + +HiFi5 Quad/Octal 128-Bit Vectorization (:file:`level_multiplier-hifi5.c`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On HiFi5 cores (featured in Intel Lunar Lake, Panther Lake, and newer architectures), vector execution is doubled via **128-bit vector pipelines**: + +1. **Octal 16-Bit Processing**: + Loads 8 16-bit samples per instruction cycle (:c:macro:`AE_LA16X4X2_IP`) and computes 8 parallel multiply-accumulate operations simultaneously using :c:macro:`AE_MULF2P32X16X4RS`. +2. **Quad 32-Bit Processing (S24 & S32)**: + Loads 4 32-bit samples per cycle (:c:macro:`AE_LA32X2X2_IP`) and evaluates 4 lanes simultaneously with quad-vector instruction :c:macro:`AE_MULF2P32X4RS`. + +This achieves double the vector throughput of HiFi3/4, reducing processor clock cycle requirements by up to 50%. + +.. _figure_227: + +.. graphviz:: + :align: center + :caption: HiFi3/HiFi4 Dual-MAC vs HiFi5 Quad-MAC 128-bit Vector Processing Pipelines + + digraph level_multiplier_simd { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_hifi3 { + label = "Tensilica HiFi3 / HiFi4 (64-Bit Vector Architecture)"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + h3_load [label="64-Bit Vector Load: AE_LA32X2_IP\nLoads 2 x 32-bit (or 4 x 16-bit) samples", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + h3_mult [label="Dual 32x32 MAC: AE_MULF32R_HH & LL\nParallel dual-lane multiplication", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + h3_round [label="Symmetric Round: AE_ROUND32X2F48SSYM\nConverts 64-bit products to 32-bit output", fillcolor="#0d9488", fontcolor="#ffffff", color="#2dd4bf"]; + h3_store [label="64-Bit Vector Store: AE_SA32X2_IP\nWrites 2 samples to sink ring", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + + h3_load -> h3_mult -> h3_round -> h3_store; + } + + subgraph cluster_hifi5 { + label = "Tensilica HiFi5 (128-Bit Vector Architecture)"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b22"; + + h5_load [label="128-Bit Vector Load: AE_LA32X2X2_IP\nLoads 4 x 32-bit (or 8 x 16-bit) samples", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + h5_mult [label="Quad 32x32 MAC: AE_MULF2P32X4RS\nParallel 4-lane simultaneous multiplication", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + h5_round [label="Quad Symmetric Round & Slew\nVectorized saturation and bit alignment", fillcolor="#10b981", fontcolor="#ffffff", color="#6ee7b7"]; + h5_store [label="128-Bit Vector Store: AE_SA32X2X2_IP\nWrites 4 samples to sink ring in 1 cycle", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + + h5_load -> h5_mult -> h5_round -> h5_store; + } + } + +------------------------------------------------------------------------------- + +IPC4 Modular Interface, LLEXT Packaging & Topology 2 Graph +---------------------------------------------------------- + +The Level Multiplier component conforms to the Intel IPC4 modular interface and can be built statically into firmware or packaged as a dynamic Loadable Linkable Extension (LLEXT). + +IPC4 Control Configuration Handler (:file:`level_multiplier-ipc4.c`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Runtime parameter updates are processed by :c:func:`level_multiplier_set_config`: + +.. code-block:: c + + switch (param_id) { + case SOF_IPC4_SWITCH_CONTROL_PARAM_ID: + case SOF_IPC4_ENUM_CONTROL_PARAM_ID: + comp_err(dev, "Illegal control param_id %d.", param_id); + return -EINVAL; + } + + if (fragment_size != sizeof(int32_t)) { + comp_err(dev, "Illegal fragment size %d.", fragment_size); + return -EINVAL; + } + + memcpy_s(&cd->gain, sizeof(int32_t), fragment, sizeof(int32_t)); + +- The component validates that the incoming payload size exactly matches 4 bytes (`sizeof(int32_t)`). +- The 32-bit Q9.23 gain value is copied directly into ``cd->gain``. +- The update takes effect on the very next processing tick without pipeline re-initialization. + +Modular LLEXT Packaging +~~~~~~~~~~~~~~~~~~~~~~~ + +When modular compilation is enabled (``CONFIG_COMP_LEVEL_MULTIPLIER = "m"``), the component is linked into :file:`level_multiplier.llext`: + +.. code-block:: c + + SOF_LLEXT_MOD_ENTRY(level_multiplier, &level_multiplier_interface); + + static const struct sof_man_module_manifest mod_manifest __section(".module") __used = + SOF_LLEXT_MODULE_MANIFEST("LEVEL_MULTIPLIER", level_multiplier_llext_entry, 1, + SOF_REG_UUID(level_multiplier), 40); + +- **Module Name**: ``"LEVEL_MULTIPLIER"`` +- **Component UUID**: ``30397456-4661-4644-97e5-39a9e5ab1778`` (Topology GUID: ``56:74:39:30:61:46:44:46:97:e5:39:a9:e5:ab:17:78``). +- **Max Instances**: 40 concurrent instances. +- **Stack Size**: 40 bytes minimum stack overhead. + +Performance Profile (:file:`level_multiplier.toml`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +From :file:`src/audio/level_multiplier/level_multiplier.toml`: + +- **Cycles Per Chunk (CPC)**: 1,000,000 CPS nominal budget. +- **Input/Output Buffer Size**: 128 samples. +- **Memory Footprint**: Only 32 bytes of instance private data (:c:struct:`level_multiplier_comp_data`). + +ALSA Topology 2 Widget Definition +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In :file:`tools/topology/topology2/include/components/level_multiplier.conf`: + +.. code-block:: text + + Class.Widget."level_multiplier" { + DefineAttribute."index" { + type "integer" + } + DefineAttribute."instance" { + type "integer" + } + + + attributes { + !constructor [ "index" "instance" ] + !mandatory [ "num_input_pins" "num_output_pins" + "num_input_audio_formats" "num_output_audio_formats" ] + !immutable [ "uuid" "type" ] + unique "instance" + } + + uuid "56:74:39:30:61:46:44:46:97:e5:39:a9:e5:ab:17:78" + type "effect" + no_pm "true" + num_input_pins 1 + num_output_pins 1 + } + +Octave / MATLAB Tuning Script (:file:`sof_level_multiplier_blobs.m`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +SOF provides an Octave script to generate pre-computed binary blobs across the standard tuning sweep from :math:`-40\text{ dB}` to :math:`+40\text{ dB}` in :math:`10\text{ dB}` steps: + +.. code-block:: octave + + for param = -40:10:40 + gain_value = sof_level_multiplier_db2lin(param); + blob8 = sof_level_multiplier_build_blob(gain_value); + tplg2_fn = sprintf("%s/gain_%d_db.conf", sof_tplg_level_multiplier, param); + sof_tplg2_write(tplg2_fn, blob8, "level_multiplier_config", ...); + end + +.. _figure_228: + +.. graphviz:: + :align: center + :caption: IPC4 Runtime Configuration Delivery, Tuning Blobs & LLEXT Dynamic Module Binding + + digraph level_multiplier_ipc4_flow { + graph [rankdir=TB, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_host { + label = "Host Driver & Userspace ALSA Plane"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + octave [label="Octave Tuning Tool\n(sof_level_multiplier_blobs.m)\nExports gain_-40_db..+40_db.conf", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + alsatplg [label="ALSA Topology Compiler\n(alsatplg)\nBuilds level_multiplier.conf widget", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + amixer [label="ALSA Mixer / ctl Control\nSends 32-bit Q9.23 gain payload", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + } + + subgraph cluster_dsp { + label = "SOF Audio DSP Pipeline"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + ipc4 [label="IPC4 Configuration Dispatcher\nChecks param_id & fragment_size == 4", fillcolor="#0d9488", fontcolor="#ffffff", color="#2dd4bf"]; + llext [label="Zephyr LLEXT Dynamic Linker\nLoads level_multiplier.llext", fillcolor="#047857", fontcolor="#ffffff", color="#34d399"]; + core [label="Level Multiplier Private Data\nAtomically updates cd->gain", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + octave -> alsatplg [label="Tuning Blobs"]; + alsatplg -> ipc4 [label="Pipeline Binding"]; + amixer -> ipc4 [label="Runtime Gain Update"]; + + ipc4 -> llext [label="Module Init"]; + ipc4 -> core [label="Gain Update"]; + } + +.. _figure_229: + +.. graphviz:: + :align: center + :caption: ALSA Topology 2 Voice Capture Sensitivity Pipeline Graph + + digraph level_multiplier_pipeline { + graph [rankdir=LR, bgcolor="#0f172a", fontname="Helvetica, Arial, sans-serif", fontsize=11, compound=true, pad=0.4, nodesep=0.5, ranksep=0.6]; + node [shape=rect, style="filled,rounded", fontname="Helvetica, Arial, sans-serif", fontsize=10, penwidth=1.5]; + edge [fontname="Helvetica, Arial, sans-serif", fontsize=9, color="#94a3b8", fontcolor="#cbd5e1", penwidth=1.2]; + + subgraph cluster_hw { + label = "Physical Audio Ingress"; + style = "solid"; + color = "#059669"; + bgcolor = "#064e3b22"; + + dmic [label="DMIC / SoundWire Gateway\n(dai-copier.1)\nDigital Microphone Ingress", fillcolor="#059669", fontcolor="#ffffff", color="#34d399"]; + } + + subgraph cluster_pipe { + label = "Voice Capture Pre-Processing Pipeline (Pipeline 1)"; + style = "solid"; + color = "#0284c7"; + bgcolor = "#082f4922"; + + dcblock [label="DC Blocker\n(dcblock.1)\nRemoves Hardware DC Bias", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + lvmult [label="Level Multiplier\n(level_multiplier.1)\nSensitivity Boost (+10 dB to +30 dB)\nUUID: 56:74:39:30...", fillcolor="#0284c7", fontcolor="#ffffff", color="#38bdf8"]; + tdfb [label="Beamformer (TDFB)\n(tdfb.1)\nDirectional Array Focus", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + rtnr [label="Noise Reduction (RTNR)\n(rtnr.1)\nSuppresses Ambient Noise", fillcolor="#0369a1", fontcolor="#ffffff", color="#38bdf8"]; + } + + subgraph cluster_host { + label = "Host Delivery"; + style = "solid"; + color = "#334155"; + bgcolor = "#1e293b55"; + + host_copier [label="Host Copier Gateway\n(host-copier.1)\nDMA to Speech Recognition (ASR)", fillcolor="#1e293b", fontcolor="#e2e8f0", color="#475569"]; + } + + dmic -> dcblock [label="Raw Digital Audio"]; + dcblock -> lvmult [label="DC-Free Stream"]; + lvmult -> tdfb [label="Sensitivity Boosted"]; + tdfb -> rtnr [label="Beamformed Focus"]; + rtnr -> host_copier [label="Clean Speech Stream"]; + } + +------------------------------------------------------------------------------- + +Factory Bringup, Acoustic Quality & Verification Runbook +-------------------------------------------------------- + +This runbook provides step-by-step instructions to compile, deploy, and verify the Level Multiplier component on physical development platforms (e.g. Panther Lake, Arrow Lake, or Tiger Lake). + +1. Topology Compilation & Deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Compile an ALSA Topology 2 configuration incorporating the Level Multiplier: + +.. code-block:: bash + + # Step 1: Generate tuning blobs across -40 dB to +40 dB + cd tools/tune/level_multiplier + octave --no-gui sof_level_multiplier_blobs.m + + # Step 2: Compile Topology 2 binary + cd ../../topology/topology2 + alsatplg -c development/sof-hda-benchmark-level_multiplier24.conf \ + -o sof-hda-benchmark-level_multiplier24.tplg + + # Step 3: Deploy topology binary to target DUT + scp sof-hda-benchmark-level_multiplier24.tplg root@:/lib/firmware/intel/sof-ipc4/ + +2. Driver Initialization & Module Verification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Reload the SOF kernel driver and check kernel logs for clean module creation: + +.. code-block:: bash + + # Reload kernel audio driver + ssh root@ 'modprobe -r snd_sof_pci_intel_mtl && modprobe snd_sof_pci_intel_mtl' + + # Confirm module instantiation and UUID registration + ssh root@ 'dmesg | grep -i level_multiplier' + +Expected kernel trace: + +.. code-block:: text + + sof-audio-pci-intel-mtl: module LEVEL_MULTIPLIER [30397456-4661-4644-97e5-39a9e5ab1778] loaded + sof-audio-pci-intel-mtl: level_multiplier.1.1: initialized with default unity gain (0x00800000) + +3. Precision Linearity & Gain Accuracy Test +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Verify output signal amplitude against input signal across gain settings: + +.. code-block:: bash + + # Generate reference sine tone at -30 dBFS (1 kHz, 24-bit, 48 kHz) + sox -n -r 48000 -c 2 -b 24 ref_tone_minus30dBFS.wav synth 5 sine 1000 vol -30dB + + # Play reference tone through pipeline + ssh root@ 'aplay -D hw:0,0 ref_tone_minus30dBFS.wav' + + # 1. Test Unity Gain (0 dB, 0x00800000) -> Output must measure exactly -30.0 dBFS + # 2. Set Gain to +10 dB (0x01948332): + ssh root@ 'sof-ctl -D hw:0 -n "level_multiplier.1.1.extctl" -s /lib/firmware/intel/sof-ipc4/gain_10_db.txt' + # -> Measured Output must equal -20.0 dBFS (+/- 0.05 dB) + + # 3. Set Gain to -10 dB (0x00287A26): + ssh root@ 'sof-ctl -D hw:0 -n "level_multiplier.1.1.extctl" -s /lib/firmware/intel/sof-ipc4/gain_-10_db.txt' + # -> Measured Output must equal -40.0 dBFS (+/- 0.05 dB) + +4. Fast-Path Bypass Verification & Power Profiling +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Confirm that unity gain engages the fast-path memory copy and reduces DSP cycle consumption: + +.. code-block:: bash + + # Benchmark DSP Cycles Per Chunk (CPC) with dut-monitor + dut-monitor --telemetry --interval 1000 + + # Active Gain (+10 dB): Observe active DSP cycles + # Unity Gain (0 dB): Cycles drop sharply as source_to_sink_copy() bypasses multiplication diff --git a/developer_guides/firmware/llext_modules.rst b/developer_guides/firmware/llext_modules.rst index 1385bc21..325efb90 100644 --- a/developer_guides/firmware/llext_modules.rst +++ b/developer_guides/firmware/llext_modules.rst @@ -1,106 +1,509 @@ .. _llext_modules: -LLEXT Modules -############# +LLEXT Dynamic Loadable Modules Architecture +########################################### -|SOF| support for loadable modules, using Zephyr LLEXT API. - -Zephyr LLEXT API -**************** - -Please refer to https://docs.zephyrproject.org/latest/services/llext/index.html -for detailed documentation. In short, the Zephyr Linkable Loadable Extensions -(LLEXT) API implements support for run-time loading and unloading of ELF-format -executable code and data. - -SOF use of the LLEXT API -************************ - -SOF has multiple ways to implement loadable modules. LLEXT is one of them. -With it modules are built as shared or relocatable ELF objects with an addition -of a cryptographic signature, using any user-supplied key, and a manifest. When -loaded and instantiated, Zephyr LLEXT functionality is used to dynamically -resolve module internal as well as SOF and Zephyr external code and data -references. In the future support for inter-module linking will be added. - -Accessing the base firmware from LLEXT modules -********************************************** - -LLEXT modules can access all code and data from the base firmware exported, -using the ``EXPORT_SYMBOL()`` macro. Therefore writing LLEXT modules isn't very -different from built-in ones. - -Implementing LLEXT modules -************************** - -At the moment only modules implementing the `Module Adapter API `_ are supported. - -.. _multiple-adapter-modules: - -It is possible to implement multiple Module Adapter modules with a common code -base, i.e. sharing a set of source files and functions. Then a single LLEXT -object would be created, implementing multiple Module Adapter interfaces. In -that case an array of ``struct sof_module_api_build_info`` objects is needed and -the TOML file should contain those multiple module entries too. -src/audio/mixin_mixout/mixin_mixout.c is an example of such a case. - -As explained above, LLEXT modules in general look very similar to native SOF -code, with the only restriction of having no access to not-exported symbols. - -LLEXT modules should also contain a ``.buildinfo`` section, containing a -``struct sof_module_api_build_info`` object and a ``.module`` section, -containing a ``struct sof_man_module_manifest`` object. The latter should also -contain a pointer to a module entry point function, returning a pointer to the -module's ``struct module_interface`` instance. All these additions can be -performed, using ``SOF_LLEXT_MOD_ENTRY()``, ``SOF_LLEXT_MODULE_MANIFEST()`` and -``SOF_LLEXT_BUILDINFO`` helper macros. See src/audio/eq_iir/eq_iir.c for an -example. - -A TOML configuration file is needed for building of LLEXT modules too. It is -generated by the C preprocessor at build time from the same components, as would -be used for a monolithic build. For this preprocessor run a small header file is -added. It mostly just includes ``platform.toml`` and ``${module}.toml``, similar -to src/samples/audio/smart_amp_test_llext/llext.toml.h. - -Finally an additional CMakeLists.txt is needed similar to -src/samples/audio/smart_amp_test_llext/CMakeLists.txt. It contains a single call -to ``sof_llext_build()``, which is an SOF helper function, using Zephyr LLEXT -cmake support by calling ``add_llext_target()`` and ``add_llext_command()``. - -With that in place, it is also possible to switch between monolithic and modular -builds by specifying the module as "tristate" in its Kconfig and selecting "m" -for modular builds. Note, that it is possible to implement third party Module -Adapter drivers, that would be built exclusively as loadable modules. Such -modules don't have to use "tristate" in their Kconfig entries. - -Installation -************ - -The |SOF| Linux kernel driver loads SOF dynamic modules by their UUIDs -(staging from ``/lib/firmware/intel/sof-ipc4/`` or platform module directories), -as specified in the topology. For SOF in-tree modules the process of creation and -installation of modules in a deployment tree is automated by the -xtensa-build-zephyr.py script. It copies modules to the deployment tree as -files with a "llext" extension and creates symbolic links to them named as -``${UUID}.bin``. E.g. - -.. code-block:: cfg - - B36EE4DA-006F-47F9-A06D-FECBE2D8B6CE.bin -> drc.llext - -Note, that as described :ref:`above ` multiple UUIDs -can be associated with a single module, in such cases multiple symbolic links -will be created, e.g. - -.. code-block:: cfg - - 39656EB2-3B71-4049-8D3F-F92CD5C43C09.bin -> mixin_mixout.llext - 3C56505A-24D7-418F-BDDC-C1F5A3AC2AE0.bin -> mixin_mixout.llext - -See :ref:`uuid` for more information on UUID use by SOF -component and module adapter drivers. - -It is also possible to avoid using the script by running ``west build`` to build -an SOF image and any modules, then using the cross-compiler to preprocess TOML -files and finally by running rimage to sign them. This would generate the same -result but figuring out all the command-line arguments would be rather difficult. +Sound Open Firmware (SOF) incorporates dynamic runtime loading of audio processing components using the **Zephyr Linkable Loadable Extensions (LLEXT)** subsystem. Rather than compiling every audio filter, codec, algorithm, and vendor processing library into a single monolithic firmware executable, LLEXT enables components to be built as standalone, relocatable Executable and Linkable Format (ELF) objects (``.llext`` files). + +These modular objects are signed using **Rimage** in dynamic library mode (``rimage -l``), staged on the host filesystem under ``/lib/firmware/intel/sof-ipc4/``, and loaded dynamically into audio DSP memory on demand by the Linux kernel driver (``snd-sof``) via the Intel IPC4 protocol when audio pipelines are created. + +.. figure:: images/llext_system_architecture.svg + :alt: SOF LLEXT Dynamic Module System Architecture + :align: center + :width: 100% + + System-level architecture showing relationships between Base Firmware, Zephyr LLEXT API, SOF LLEXT Manager, relocatable modules, and memory protection boundaries. + +Architectural Motivation & Design Goals +*************************************** + +The transition from monolithic firmware builds to dynamically loadable LLEXT modules addresses several critical architectural challenges in modern audio DSP platforms: + +1. **SRAM Footprint Optimization**: + Embedded DSP High-Performance SRAM (HP-SRAM) is constrained (often 2 MB to 4 MB). A monolithic image containing dozens of audio processing algorithms (reverberation, beamforming, active noise reduction, multi-band dynamic range compression, keyword spotting, neural network models) quickly exhausts available SRAM. LLEXT allows the DSP to keep only the base operating system and currently active stream modules in memory, freeing SRAM when pipelines are stopped. +2. **Post-Silicon Extensibility & Rapid Delivery**: + New audio processing algorithms or bug fixes can be packaged, cryptographically signed, and distributed to end-user systems as standalone module files without updating or rebooting the base firmware. +3. **Vendor IP & Proprietary Algorithm Isolation**: + Third-party acoustic processing algorithms (e.g., proprietary speaker protection, spatial audio synthesizers, licensed decoders) can be compiled against the SOF Module Adapter API and distributed as pre-compiled, relocatable binaries without exposing vendor source code or linking against the full GPL/BSD base firmware source tree. +4. **Fine-Grained Memory Protection**: + Dynamic modules run within dedicated Zephyr memory domains (``struct k_mem_domain``) with hardware MPU/MMU enforcement (:math:`W \oplus X` security policy), isolating algorithmic processing from critical RTOS data structures and interrupt handlers. + +Module Binary Anatomy & Section Descriptors +******************************************* + +An LLEXT module is an ELF32 relocatable object file (or shared library) containing standard code/data sections along with specialized SOF metadata sections required for runtime ABI validation and manifest registration. + +.. list-table:: LLEXT Module ELF Section Hierarchy + :widths: 18 20 22 40 + :header-rows: 1 + + * - Section Name + - Section Type + - Memory Permissions + - Description & Contents + * - ``.text`` + - ``SHT_PROGBITS`` + - ``SYS_MM_MEM_PERM_EXEC`` (RX) + - Executable machine instructions. On Xtensa, literal pools are colocated via ``-mtext-section-literals``. + * - ``.rodata`` + - ``SHT_PROGBITS`` + - Read-Only (RO) + - Constant data, coefficient tables, filter tap matrices, and math lookup tables. + * - ``.data`` + - ``SHT_PROGBITS`` + - Read-Write (RW) + - Initialized global and static variables. + * - ``.bss`` + - ``SHT_NOBITS`` + - Read-Write (RW) + - Zero-initialized variables. Enforced to reside contiguously within or adjacent to ``.data`` memory boundaries. + * - ``.mod_buildinfo`` + - ``SHT_PROGBITS`` + - Read-Only (RO) + - Contains ``struct sof_module_api_build_info``. Defines the module API version, format tag, and build hash. + * - ``.module`` + - ``SHT_PROGBITS`` + - Read-Only (RO) + - Contains ``struct sof_man_module_manifest``. Specifies module UUID, entry point, affinity mask, and load type. + +LLEXT Integration Macros +======================== + +SOF provides standardized macros in ``include/module/module/llext.h`` to simplify module authoring: + +ABI Compatibility Check (``SOF_LLEXT_BUILDINFO``) +------------------------------------------------- + +.. code-block:: c + + #define SOF_LLEXT_BUILDINFO \ + static const struct sof_module_api_build_info buildinfo \ + __section(".mod_buildinfo") __used = { \ + .format = SOF_MODULE_API_BUILD_INFO_FORMAT, \ + .api_version_number.full = SOF_MODULE_API_CURRENT_VERSION, \ + } + +When the module is loaded, ``llext_manager_allocate_module()`` inspects the ``.mod_buildinfo`` section. If ``buildinfo->api_version_number.full`` does not match ``SOF_MODULE_API_CURRENT_VERSION`` in the running base firmware, the load request is rejected with ``-EINVAL``, preventing runtime panics caused by ABI drift. + +Module Manifest Registration (``SOF_LLEXT_MODULE_MANIFEST``) +------------------------------------------------------------ + +.. code-block:: c + + #define SOF_LLEXT_MODULE_MANIFEST(manifest_name, entry, affinity, mod_uuid, instances, ...) \ + { \ + .module = { \ + .name = manifest_name, \ + .uuid = mod_uuid, \ + .entry_point = (uint32_t)(entry), \ + .instance_max_count = instances, \ + .type = { \ + .load_type = SOF_MAN_MOD_TYPE_LLEXT, \ + .domain_ll = 1, \ + }, \ + .affinity_mask = (affinity), \ + } \ + } + +This macro registers: + +* **UUID**: The 128-bit RFC 4122 component identifier matched against ALSA Topology widget UUIDs. +* **Entry Point**: Function pointer (e.g. ``module_init``) called upon instantiation, returning the driver's ``struct module_interface *``. +* **Affinity Mask**: Bitmask of DSP cores permitted to run the module (e.g. ``0x1`` for Core 0, ``0x3`` for Cores 0 and 1). +* **Load Type**: Set to ``SOF_MAN_MOD_TYPE_LLEXT`` (``2``) for standard processing modules, or ``SOF_MAN_MOD_TYPE_LLEXT_AUX`` (``3``) for auxiliary helper libraries. + +Symbol Export Linkage (``EXPORT_SYMBOL``) +========================================= + +LLEXT modules do not link against a copy of the RTOS or C library. Instead, unresolved external symbols are resolved at load time against symbols explicitly exported by the base firmware using the ``EXPORT_SYMBOL()`` macro in ``zephyr/include/zephyr/llext/symbol.h``: + +.. code-block:: c + + /* Example base firmware symbol exports in SOF core */ + EXPORT_SYMBOL(tr_err); + EXPORT_SYMBOL(tr_warn); + EXPORT_SYMBOL(tr_info); + EXPORT_SYMBOL(memcpy_s); + EXPORT_SYMBOL(memset_s); + EXPORT_SYMBOL(rballoc); + EXPORT_SYMBOL(rfree); + EXPORT_SYMBOL(notifier_register); + EXPORT_SYMBOL(notifier_unregister); + EXPORT_SYMBOL(notifier_event); + EXPORT_SYMBOL(cpu_clock_manager_request); + EXPORT_SYMBOL(cpu_clock_manager_release); + +Any attempt by an LLEXT module to call a function not marked with ``EXPORT_SYMBOL()`` in the base firmware will fail during runtime relocation linking, preventing unauthorized access to private kernel internals. + +Multi-Module Packaging (Shared Codebases) +***************************************** + +In many audio processing pipelines, multiple distinct component drivers share a single common codebase. A primary example is ``src/audio/mixin_mixout/``, where both the **MIXIN** (audio stream multiplexer) and **MIXOUT** (audio stream fanout) component drivers reside in the same source files. + +LLEXT natively supports packaging multiple component drivers into a single ``.llext`` binary container: + +1. **Manifest Array**: The source file declares an array of ``struct sof_man_module_manifest`` structures, with each entry binding a distinct UUID, entry point, and component name: + + .. code-block:: c + + /* In mixin_mixout.c */ + static const struct sof_man_module_manifest mixin_mixout_manifest[] __section(".module") = { + SOF_LLEXT_MODULE_MANIFEST("MIXIN", mixin_init, 0x3, UUIDREG_STR_MIXIN, 8), + SOF_LLEXT_MODULE_MANIFEST("MIXOUT", mixout_init, 0x3, UUIDREG_STR_MIXOUT, 8), + }; + +2. **TOML Preprocessor Descriptor**: The platform TOML preprocessor template (``llext.toml.h``) declares multiple ``[[module.entry]]`` blocks corresponding to each UUID. +3. **Symlink Generation**: When ``xtensa-build-zephyr.py`` packages the build, it reads all UUIDs associated with the target from ``llext.uuid`` and generates individual deployment symlinks pointing to the single shared container: + + .. code-block:: text + + 39656EB2-3B71-4049-8D3F-F92CD5C43C09.bin -> mixin_mixout.llext (MIXIN) + 3C56505A-24D7-418F-BDDC-C1F5A3AC2AE0.bin -> mixin_mixout.llext (MIXOUT) + +Auxiliary Libraries & Shared Engines +==================================== + +Components that rely on large shared math algorithms (such as FIR filter convolution or IIR biquad matrix engines) can be factored into **Auxiliary Libraries** (``SOF_MAN_MOD_TYPE_LLEXT_AUX``). Auxiliary libraries are loaded once and linked against dependent LLEXT modules using refcounted tracking (``LLEXT_MAX_DEPENDENCIES``). + +Build System & Toolchain Pipeline +********************************* + +LLEXT modules are built using Zephyr's CMake extensions and signed using Rimage. + +.. figure:: images/llext_build_packaging_flow.svg + :alt: SOF LLEXT Build and Packaging Pipeline + :align: center + :width: 100% + + End-to-end LLEXT compilation, relocatable linking, C-preprocessor TOML generation, Rimage dynamic signing, and deployment symlink assembly. + +Kconfig Tristate Integration +============================ + +Audio modules in SOF support tristate Kconfig definitions (``n``, ``m``, ``y``): + +.. code-block:: kconfig + + config COMP_VOLUME + tristate "Volume control component" + default y + help + Select 'y' to link volume statically into base firmware. + Select 'm' to compile volume as an LLEXT loadable module. + Select 'n' to disable the component. + +When ``CONFIG_LLEXT_FORCE_ALL_MODULAR=y`` is enabled, all processing components configured as tristate are automatically built as modular LLEXT packages, minimizing base firmware size. + +The ``sof_llext_build()`` CMake Function +======================================== + +In each module's ``llext/CMakeLists.txt``, the build is defined using SOF's high-level helper function: + +.. code-block:: cmake + + # Example: src/audio/volume/llext/CMakeLists.txt + sof_llext_build("volume" + SOURCES + ../volume_generic.c + ../volume_hifi3.c + ../volume_hifi4.c + ../volume_hifi5.c + ../volume_generic_with_peakvol.c + ../volume_hifi3_with_peakvol.c + ../volume_hifi4_with_peakvol.c + ../volume_hifi5_with_peakvol.c + ../volume.c + ../volume_ipc4.c + LIB openmodules + ) + +Compiler & Linker Directives +---------------------------- + +Under the hood, ``sof_llext_build()`` executes the following critical build steps: + +1. **Xtensa Literal Placement**: + Injects ``-mtext-section-literals``. Because LLEXT modules are linked without a full linker script, literal pools must be emitted inline directly preceding the ``L32R`` instructions that reference them, preventing out-of-range PC-relative displacement faults. +2. **Library Stripping**: + Applies ``-nostdlib -nodefaultlibs`` to eliminate duplicate C runtime dependencies. +3. **Relocatable Linking**: + When ``CONFIG_LLEXT_TYPE_ELF_RELOCATABLE=y``, the linker produces an incremental relocatable object (``-r``), preserving symbol relocation tables for the Zephyr runtime loader. +4. **Preprocessed TOML Configuration**: + Invokes the C preprocessor on ``llext.toml.h`` with autoconf macros to generate ``rimage_config.toml``. +5. **Rimage Dynamic Signing** (``-l``): + Executes Rimage with the ``-l`` flag: + + .. code-block:: bash + + rimage -l -k keys/otc_private.pem \ + -c rimage_config.toml \ + -o build/volume_llext/volume.ri \ + build/volume_llext/volume.llext + + The ``-l`` flag instructs Rimage that the input ELF is a dynamic module rather than a bootloader executable, calculating module segment digests, appending the module table entry (``$AME``), and generating an Extended Manifest sidecar (``volume.ri.xman``). + +Helper Utilities +================ + +The build pipeline leverages specialized Python utilities in ``scripts/``: + +* ``llext_link_helper.py``: Calculates section VMA placements according to ``CONFIG_LIBRARY_BASE_ADDRESS``. +* ``llext_offset_calc.py``: Maintains a cumulative persistent module size counter, guaranteeing non-overlapping memory regions. +* ``llext_write_uuids.cmake``: Inspects module headers and writes ``llext.uuid`` containing all component UUIDs for deployment packaging. + +Runtime Lifecycle & Memory Management +************************************* + +The runtime lifecycle of an LLEXT module is managed jointly by the Linux host driver (``sound/soc/sof/ipc4-loader.c``), the SOF Library Manager (``src/library_manager/lib_manager.c``), and the SOF LLEXT Manager (``src/library_manager/llext_manager.c``). + +.. figure:: images/llext_runtime_lifecycle.svg + :alt: SOF LLEXT Runtime Lifecycle and Memory Protection State Machine + :align: center + :width: 100% + + Detailed runtime execution flow: Host IPC4 loading handshake, virtual memory allocation, permission transitions, cache maintenance, and teardown. + +Phase 1: Host IPC4 Loading Protocol +=================================== + +When an audio use case is triggered (e.g., playback stream opening), the ALSA topology parser determines which component modules are required by the pipeline. If a module is not currently resident in DSP memory: + +1. **Firmware File Resolution**: The host driver requests the firmware binary from the filesystem by UUID: ``/lib/firmware/intel/sof-ipc4//.bin``. +2. **Library Prepare** (``SOF_IPC4_GLB_LOAD_LIBRARY_PREPARE``): + The host sends an IPC message allocating a host-to-DSP DMA stream buffer (``dma_id``) and assigning a numeric library identifier (``lib_id``, typically 1 to 15). +3. **DMA Payload Transfer**: + The host streams the signed LLEXT container (Extended Manifest + CPD + CSS + ELF payload) into the pre-allocated DSP memory window. +4. **Library Trigger** (``SOF_IPC4_GLB_LOAD_LIBRARY``): + The host signals the DSP to initiate image parsing and dynamic linking. + +Phase 2: Authentication & ABI Handshake +======================================= + +On the DSP, the IPC4 message is received by ``ipc4_load_library()`` and dispatched to ``lib_manager_load_library()``: + +1. **Cryptographic Validation**: The CSS signature (``$MN2``) and ADSP descriptor (``$AM1``) are authenticated against platform verification keys. +2. **ABI Verification**: The LLEXT manager locates the ``.mod_buildinfo`` section and verifies that ``buildinfo->api_version_number.full == SOF_MODULE_API_CURRENT_VERSION``. +3. **Context Allocation**: A ``struct lib_manager_mod_ctx`` is allocated in DSP heap, binding the ``lib_id`` to the module's manifest table. + +Phase 3: Virtual Memory Mapping & Permissions State Machine +=========================================================== + +Memory mapping is performed by ``llext_manager_load_data_from_storage()`` using Zephyr's system memory management driver (``sys_mm_drv``): + +.. list-table:: LLEXT Memory Protection State Machine + :widths: 15 25 30 30 + :header-rows: 1 + + * - Step + - Function Invoked + - Memory Permission + - Operational Objective + * - **1. Staging** + - ``sys_mm_drv_align_map()`` + - ``SYS_MM_MEM_PERM_RW`` + - Maps virtual SRAM pages aligned to ``PAGE_SZ`` (4 KB) with full Read-Write permissions. + * - **2. Copy & Link** + - ``llext_load()`` / ``memcpy_s()`` + - ``SYS_MM_MEM_PERM_RW`` + - Copies ``.text``, ``.rodata``, and ``.data`` into place; clears ``.bss``; resolves external symbols via ``EXPORT_SYMBOL`` table. + * - **3. Cache Flush** + - ``dcache_writeback_region()`` + - ``SYS_MM_MEM_PERM_RW`` + - Flushes patched executable instructions and data from L1/L2 data cache lines to physical SRAM. + * - **4. Lockdown** + - ``sys_mm_drv_update_region_flags()`` + - ``SYS_MM_MEM_PERM_EXEC`` / Read-Only / Read-Write + - Enforces :math:`W \oplus X` security: ``.text`` is locked to RX (executable, no write); ``.rodata`` is locked to Read-Only; ``.data``/``.bss`` remains RW. + * - **5. Invalidate** + - ``icache_invalidate_region()`` + - ``SYS_MM_MEM_PERM_EXEC`` + - Flushes instruction cache lines across all active DSP cores, ensuring instruction pipelines fetch freshly relocated opcodes. + +Phase 4: Component Instantiation & Real-Time Processing +======================================================= + +When an audio pipeline creates an instance of the component: + +1. ``llext_manager_allocate_module()`` checks all declared dependencies (``LLEXT_MAX_DEPENDENCIES``) and increments their reference counters (``dep->n_dependent++``). +2. The module entry point function (``entry_point()``) is invoked, returning a pointer to the driver's ``struct module_interface``. +3. The component binds to the SOF Module Adapter framework and is registered with the Low-Latency (LL) or Data Processing (DP) task scheduler. +4. During streaming, the scheduler calls ``module_adapter_process()`` periodically (e.g. every 1 ms), processing PCM audio buffers with native DSP performance and zero virtualization overhead. + +Phase 5: Teardown & Cold-Store DRAM Staging +=========================================== + +* **Instance Teardown**: When an audio stream closes, ``llext_manager_free_module()`` releases instance memory and decrements dependency refcounts. +* **Library Unloading**: When the host sends ``SOF_IPC4_GLB_UNLOAD_LIBRARY``, the LLEXT manager unmaps virtual memory regions via ``sys_mm_drv_unmap_region()``, freeing SRAM pages back to the global pool. +* **Low-Power D3 Staging** (``llext_manager_dram.c``): + When the system transitions into low-power suspend (D3), ``llext_manager_store_to_dram()`` backs up loaded module images into host DRAM carveouts. Upon system wake, ``llext_manager_restore_from_dram()`` rapidly restores the modules without requiring the Linux host driver to re-stream multi-megabyte binaries over DMA, slashing wake latency. + +Developer Tutorial: Authoring a New LLEXT Module +************************************************ + +To create a new loadable audio processing component (e.g., ``my_filter``), follow this step-by-step workflow: + +Step 1: Implement the Module Adapter Driver +=========================================== + +In ``src/audio/my_filter/my_filter.c``, implement standard processing hooks: + +.. code-block:: c + + #include + #include + + /* 1. Declare ABI build info */ + SOF_LLEXT_BUILDINFO; + + static int my_filter_init(struct processing_module *mod) + { + /* Initialize component state */ + return 0; + } + + static int my_filter_process(struct processing_module *mod, + struct input_stream_buffer *bsource, + struct output_stream_buffer *bsink) + { + /* Execute audio processing */ + return 0; + } + + static struct module_interface my_filter_interface = { + .init = my_filter_init, + .process = my_filter_process, + }; + + static struct module_interface *my_filter_entry(void) + { + return &my_filter_interface; + } + + /* 2. Declare hardware manifest */ + static const struct sof_man_module_manifest my_filter_manifest + __section(".module") = SOF_LLEXT_MODULE_MANIFEST( + "MY_FILTER", + my_filter_entry, + 0x1, /* Affinity: Core 0 */ + UUIDREG_STR_MY_FILTER, /* Component UUID */ + 4 /* Max 4 instances */ + ); + +Step 2: Create the CMake LLEXT Definition +========================================= + +Create ``src/audio/my_filter/llext/CMakeLists.txt``: + +.. code-block:: cmake + + # Copyright (c) 2026 Sound Open Firmware + # SPDX-License-Identifier: Apache-2.0 + + sof_llext_build("my_filter" + SOURCES + ../my_filter.c + LIB openmodules + ) + +Step 3: Create the TOML Header Template +======================================= + +Create ``src/audio/my_filter/llext/llext.toml.h``: + +.. code-block:: c + + #include "platform.toml" + #include