From 7e1336e8731159d86082d5947ba3fe1a421062d2 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Tue, 16 Jun 2026 19:44:57 +0100 Subject: [PATCH 01/35] scripts: llext: support relocatable link bypass When CONFIG_LLEXT_TYPE_ELF_RELOCATABLE is active, bypass appending static address flags (-Ttext, --section-start, -Tdata) in the linker helper script. This keeps section base addresses at 0. Also adjust the offset calculator to avoid integer parsing errors when all section addresses are set to 0. Signed-off-by: Liam Girdwood --- scripts/llext_offset_calc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/llext_offset_calc.py b/scripts/llext_offset_calc.py index 0f302a8cbe12..2a07984b725f 100755 --- a/scripts/llext_offset_calc.py +++ b/scripts/llext_offset_calc.py @@ -47,6 +47,9 @@ def get_elf_size(elf_name): if section.header['sh_addr'] + section.header['sh_size'] > end: end = section.header['sh_addr'] + section.header['sh_size'] + if start == 0xffffffff: + return 0 + size = end - start return size From e30adc4cdf25b07315906f2f4bc271b2da01404c Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Tue, 16 Jun 2026 19:44:59 +0100 Subject: [PATCH 02/35] library_manager: llext: implement page-level VMA allocator for relocatable modules Implement page-level virtual memory mapping using Zephyr's sys_bitarray utility over the library region. Compile section layout at load-time to allocate virtual addresses and rewrite section sh_addr headers in-place. This enables Zephyr LLEXT to naturally relocate references. Signed-off-by: Liam Girdwood --- app/llext_relocatable.conf | 1 + src/include/sof/lib_manager.h | 3 + src/library_manager/llext_manager.c | 509 ++++++++++++++++++++++- src/library_manager/llext_manager_dram.c | 5 + 4 files changed, 515 insertions(+), 3 deletions(-) diff --git a/app/llext_relocatable.conf b/app/llext_relocatable.conf index 76b5339e1bb0..ce8dafe3a6aa 100644 --- a/app/llext_relocatable.conf +++ b/app/llext_relocatable.conf @@ -1 +1,2 @@ CONFIG_LLEXT_TYPE_ELF_RELOCATABLE=y +CONFIG_LLEXT_EXPORT_BUILTINS_BY_SLID=y diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h index ceb47e6abe6c..d1994f76233e 100644 --- a/src/include/sof/lib_manager.h +++ b/src/include/sof/lib_manager.h @@ -94,6 +94,7 @@ enum { LIB_MANAGER_DATA, LIB_MANAGER_RODATA, LIB_MANAGER_BSS, + LIB_MANAGER_EXPORT, LIB_MANAGER_COLD, LIB_MANAGER_COLDRODATA, LIB_MANAGER_N_SEGMENTS, @@ -118,6 +119,8 @@ struct lib_manager_module { bool mapped; bool domain_dp; struct lib_manager_segment_desc segment[LIB_MANAGER_N_SEGMENTS]; + uintptr_t vma_base; + size_t vma_size; }; struct lib_manager_mod_ctx { diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 241e7e45fa6c..f23b01e6c249 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -52,6 +52,368 @@ extern struct tr_ctx lib_manager_tr; #define PAGE_SZ CONFIG_MM_DRV_PAGE_SIZE +#include +#include + +#define LLEXT_LIB_PAGES (CONFIG_LIBRARY_REGION_SIZE / PAGE_SZ) +SYS_BITARRAY_DEFINE_STATIC(lib_vma_bitarray, LLEXT_LIB_PAGES); + +static uintptr_t llext_manager_alloc_vma(size_t size) +{ + size_t num_pages = ALIGN_UP(size, PAGE_SZ) / PAGE_SZ; + size_t offset; + int ret; + + ret = sys_bitarray_alloc(&lib_vma_bitarray, num_pages, &offset); + if (ret < 0) { + tr_err(&lib_manager_tr, "llext_manager_alloc_vma: failed to allocate %zu pages", num_pages); + return 0; + } + + return CONFIG_LIBRARY_BASE_ADDRESS + offset * PAGE_SZ; +} + +void llext_manager_free_vma(uintptr_t vma, size_t size) +{ + if (!vma || !size) + return; + + size_t num_pages = ALIGN_UP(size, PAGE_SZ) / PAGE_SZ; + size_t offset = (vma - CONFIG_LIBRARY_BASE_ADDRESS) / PAGE_SZ; + + sys_bitarray_free(&lib_vma_bitarray, num_pages, offset); +} + +static enum llext_mem llext_manager_get_sec_mem_idx(const char *name, const elf_shdr_t *shdr) +{ + if (strcmp(name, ".exported_sym") == 0) + return LLEXT_MEM_EXPORT; + + switch (shdr->sh_type) { + case SHT_NOBITS: + return LLEXT_MEM_BSS; + case SHT_PROGBITS: + if (shdr->sh_flags & SHF_EXECINSTR) + return LLEXT_MEM_TEXT; + else if (shdr->sh_flags & SHF_WRITE) + return LLEXT_MEM_DATA; + else + return LLEXT_MEM_RODATA; + case SHT_PREINIT_ARRAY: + return LLEXT_MEM_PREINIT; + case SHT_INIT_ARRAY: + return LLEXT_MEM_INIT; + case SHT_FINI_ARRAY: + return LLEXT_MEM_FINI; + default: + return LLEXT_MEM_COUNT; + } +} + +/* + * llext_manager_layout_sections() rebases the addresses of recognized + * sections in place, directly in the raw ELF buffer, before llext_load() + * ever parses the file. Two classes of data in the file still reference the + * OLD (pre-rebase) addresses after that mutation, and need fixing up + * ourselves: + * + * 1. .rela.dyn/.rela.plt r_offset fields (byte-for-byte copies from the + * build-time ELF) -- otherwise llext_link_plt()'s llext_file_offset() + * lookup fails ("Offset not found") for every relocation whose target + * section moved. + * + * 2. R_XTENSA_RELATIVE relocations are a complete no-op in Zephyr's llext + * core whenever ldr_parm->pre_located is set (see + * arch_elf_relocate_local() in arch/xtensa/core/elf.c), under the + * assumption that a pre-located relative relocation's stored pointer + * value is already correct. SOF's rebase step violates that assumption, + * so we must apply the same delta to the pointer VALUE stored at each + * R_XTENSA_RELATIVE's target ourselves. + * + * 3. .symtab/.dynsym st_value fields: llext_copy_symbols() takes st_value + * as the final absolute address, unmodified, whenever pre_located is + * set (same assumption as above, same violation) -- this feeds both + * llext_find_sym() lookups (R_XTENSA_GLOB_DAT resolution) and exported + * symbol addresses, so stale st_value silently propagates stale + * addresses through both. + * + * Track the (old_addr, size, delta, section index) of every section + * actually rebased during the real (vma_base != 0) pass, then walk the ELF + * a second time applying all three fixups above. + */ +struct llext_manager_sec_rebase { + uintptr_t old_addr; + size_t size; + ptrdiff_t delta; + int shdr_idx; +}; + +#define LLEXT_MANAGER_MAX_REBASED_SECTIONS 32 + +/* Keep in sync with arch/xtensa/core/elf.c -- not exposed via a shared header. */ +#define SOF_LLEXT_R_XTENSA_RELATIVE 5 + +static ptrdiff_t llext_manager_delta_for_old_addr(const struct llext_manager_sec_rebase *rebase, + int rebase_cnt, uintptr_t old_addr) +{ + for (int k = 0; k < rebase_cnt; k++) { + if (old_addr >= rebase[k].old_addr && old_addr < rebase[k].old_addr + rebase[k].size) + return rebase[k].delta; + } + + return 0; +} + +static bool llext_manager_addr_to_file_off(uint8_t *elf_buf, uintptr_t addr, size_t *file_off) +{ + elf_ehdr_t *hdr = (elf_ehdr_t *)elf_buf; + elf_shdr_t *shdrs = (elf_shdr_t *)(elf_buf + hdr->e_shoff); + + for (int i = 0; i < hdr->e_shnum; i++) { + elf_shdr_t *shdr = shdrs + i; + + if (!(shdr->sh_flags & SHF_ALLOC) || !shdr->sh_size) + continue; + + if (addr >= shdr->sh_addr && addr < shdr->sh_addr + shdr->sh_size) { + *file_off = shdr->sh_offset + (addr - shdr->sh_addr); + return true; + } + } + + return false; +} + +static void llext_manager_fixup_rela(uint8_t *elf_buf, + const struct llext_manager_sec_rebase *rebase, + int rebase_cnt) +{ + elf_ehdr_t *hdr = (elf_ehdr_t *)elf_buf; + elf_shdr_t *shdrs = (elf_shdr_t *)(elf_buf + hdr->e_shoff); + + for (int i = 0; i < hdr->e_shnum; i++) { + elf_shdr_t *shdr = shdrs + i; + + if (shdr->sh_type != SHT_RELA) + continue; + + int cnt = shdr->sh_size / shdr->sh_entsize; + elf_rela_t *relas = (elf_rela_t *)(elf_buf + shdr->sh_offset); + + for (int j = 0; j < cnt; j++) { + elf_rela_t *rela = relas + j; + ptrdiff_t r_delta = llext_manager_delta_for_old_addr(rebase, rebase_cnt, + rela->r_offset); + + if (!r_delta) + continue; + + rela->r_offset += r_delta; + + if (ELF_R_TYPE(rela->r_info) == SOF_LLEXT_R_XTENSA_RELATIVE) { + size_t file_off; + + if (llext_manager_addr_to_file_off(elf_buf, rela->r_offset, + &file_off)) { + uint32_t *val = (uint32_t *)(elf_buf + file_off); + ptrdiff_t v_delta = llext_manager_delta_for_old_addr( + rebase, rebase_cnt, *val); + + *val += v_delta; + } + } + } + } +} + +static void llext_manager_fixup_symtab(uint8_t *elf_buf, + const struct llext_manager_sec_rebase *rebase, + int rebase_cnt) +{ + elf_ehdr_t *hdr = (elf_ehdr_t *)elf_buf; + elf_shdr_t *shdrs = (elf_shdr_t *)(elf_buf + hdr->e_shoff); + + for (int i = 0; i < hdr->e_shnum; i++) { + elf_shdr_t *shdr = shdrs + i; + + if (shdr->sh_type != SHT_SYMTAB && shdr->sh_type != SHT_DYNSYM) + continue; + + int cnt = shdr->sh_size / shdr->sh_entsize; + elf_sym_t *syms = (elf_sym_t *)(elf_buf + shdr->sh_offset); + + for (int j = 0; j < cnt; j++) { + elf_sym_t *sym = syms + j; + int k; + + for (k = 0; k < rebase_cnt; k++) { + if (sym->st_shndx == rebase[k].shdr_idx) { + sym->st_value += rebase[k].delta; + break; + } + } + } + } +} + +static size_t llext_manager_layout_sections(uint8_t *elf_buf, uintptr_t vma_base) +{ + elf_ehdr_t *hdr = (elf_ehdr_t *)elf_buf; + elf_shdr_t *shdrs = (elf_shdr_t *)(elf_buf + hdr->e_shoff); + elf_shdr_t *shstr_shdr = shdrs + hdr->e_shstrndx; + const char *shstrtab = (const char *)(elf_buf + shstr_shdr->sh_offset); + + uintptr_t current_vma = vma_base; + enum llext_mem last_region = LLEXT_MEM_COUNT; + struct llext_manager_sec_rebase rebase[LLEXT_MANAGER_MAX_REBASED_SECTIONS]; + int rebase_cnt = 0; + bool region_opened[LLEXT_MEM_COUNT] = { false }; + ptrdiff_t region_delta[LLEXT_MEM_COUNT] = { 0 }; + + for (int i = 0; i < hdr->e_shnum; i++) { + elf_shdr_t *shdr = shdrs + i; + + if (!(shdr->sh_flags & SHF_ALLOC) || shdr->sh_size == 0) + continue; + + const char *name = shstrtab + shdr->sh_name; + enum llext_mem s_region = llext_manager_get_sec_mem_idx(name, shdr); + + if (s_region == LLEXT_MEM_BSS) { + /* + * For layout purposes only, treat .bss as part of the DATA + * region rather than its own distinct region. llext_manager_ + * load_module() requires .bss to be immediately contiguous + * with writable DATA (they share a single VMA mapping, since + * .bss has no file backing of its own and just extends + * DATA's mapped-and-zeroed tail). If .bss were laid out as + * its own region, any OTHER region (e.g. .exported_sym) + * appearing between DATA's last member and .bss in file order + * would consume the address range .bss needs to be adjacent + * to, forcing .bss onto a spurious extra page beyond where + * DATA actually ends. Aliasing to LLEXT_MEM_DATA here reuses + * the exact same delta/contiguous-packing path already used + * for DATA members that reappear after such an interruption. + * This only affects the rebased sh_addr written below; it + * does not change which segment .bss is reported under + * elsewhere (that comes from Zephyr's own section type-based + * classification, independent of this local variable). + */ + s_region = LLEXT_MEM_DATA; + } + + if (s_region == LLEXT_MEM_COUNT) { + /* + * Not part of any llext region (e.g. .dynamic) and its own + * sh_addr is left untouched below. But it still occupies + * real file space, possibly between two sections of a + * region that IS being repacked here -- account for its + * slot so later same-region sections keep the same + * relative spacing that llext_map_sections()'s ET_DYN + * consistency check (sh_addr delta == sh_offset delta) + * requires. + */ + current_vma = ALIGN_UP(current_vma, shdr->sh_addralign); + current_vma += shdr->sh_size; + continue; + } + + if (region_opened[s_region]) { + /* + * This region already had a section rebased earlier and is + * reappearing now, with a genuinely different region's + * section(s) interleaved in between in file order (e.g. + * .exported_sym sitting between two .data-region members). + * Reuse the region's already-established rebase delta so + * this member keeps the exact same (sh_addr - sh_offset) as + * the region's first member -- that constant delta is what + * llext_map_sections()'s ET_DYN consistency check actually + * requires. Re-packing it back into the forward current_vma + * layout instead (as if it were a fresh region) would insert + * a spurious page-aligned gap that doesn't exist in the + * original file. + */ + if (vma_base) { + uintptr_t old_addr = shdr->sh_addr; + uintptr_t new_addr = (uintptr_t)((ptrdiff_t)old_addr + + region_delta[s_region]); + + if (new_addr != old_addr && rebase_cnt < ARRAY_SIZE(rebase)) { + rebase[rebase_cnt].old_addr = old_addr; + rebase[rebase_cnt].size = shdr->sh_size; + rebase[rebase_cnt].delta = region_delta[s_region]; + rebase[rebase_cnt].shdr_idx = i; + rebase_cnt++; + } + shdr->sh_addr = new_addr; + + /* + * A reopened region's member keeps the region's original + * constant address delta (required above for the ET_DYN + * sh_addr/sh_offset consistency check), which is NOT + * necessarily contiguous with current_vma's independently + * tracked cursor -- any other region's section(s) placed + * (and possibly page-aligned) since this region was last + * open, e.g. .exported_sym, can leave current_vma short of + * where this delta-preserving placement actually ends. + * Re-anchor current_vma to this section's real end so the + * next freshly-opened region's page-alignment starts after + * this region's true extent, not a stale, too-small cursor + * value -- otherwise the next region can be placed + * overlapping this region's tail. + */ + current_vma = new_addr + shdr->sh_size; + } + + /* + * Keep last_region tracking the region of the section that + * ACTUALLY precedes the next one in file order, not just "the + * last freshly-opened region". Without this, a fresh region + * immediately following a reused-region section (e.g. .bss + * right after a re-visited .data member, with .exported_sym + * sandwiched earlier) sees a stale last_region from whatever + * region was last freshly opened (e.g. EXPORT) instead of this + * section's own region (DATA), and wrongly concludes it needs + * yet another PAGE_SZ-aligned transition on top of the one + * already inserted for EXPORT -- stacking two page gaps where + * only one belongs, which desyncs .bss from the writable-data + * region it must remain contiguous with. + */ + last_region = s_region; + continue; + } + + if (last_region != LLEXT_MEM_COUNT && last_region != s_region) { + current_vma = ALIGN_UP(current_vma, PAGE_SZ); + } + last_region = s_region; + + current_vma = ALIGN_UP(current_vma, shdr->sh_addralign); + if (vma_base) { + uintptr_t old_addr = shdr->sh_addr; + + if (old_addr != current_vma && rebase_cnt < ARRAY_SIZE(rebase)) { + rebase[rebase_cnt].old_addr = old_addr; + rebase[rebase_cnt].size = shdr->sh_size; + rebase[rebase_cnt].delta = (ptrdiff_t)current_vma - (ptrdiff_t)old_addr; + rebase[rebase_cnt].shdr_idx = i; + rebase_cnt++; + } + region_delta[s_region] = (ptrdiff_t)current_vma - (ptrdiff_t)old_addr; + region_opened[s_region] = true; + shdr->sh_addr = current_vma; + } + current_vma += shdr->sh_size; + } + + if (vma_base && rebase_cnt) { + llext_manager_fixup_rela(elf_buf, rebase, rebase_cnt); + llext_manager_fixup_symtab(elf_buf, rebase, rebase_cnt); + } + + return current_vma - vma_base; +} + static int llext_manager_update_flags(void __sparse_cache *vma, size_t size, uint32_t flags) { size_t pre_pad_size = (uintptr_t)vma & (PAGE_SZ - 1); @@ -252,6 +614,11 @@ static int llext_manager_load_module(struct lib_manager_module *mctx) mctx->segment[LIB_MANAGER_RODATA].addr; size_t rodata_size = mctx->segment[LIB_MANAGER_RODATA].size; + /* Exported symbol table (.exported_sym), read-only like .rodata */ + void __sparse_cache *va_base_export = (void __sparse_cache *) + mctx->segment[LIB_MANAGER_EXPORT].addr; + size_t export_size = mctx->segment[LIB_MANAGER_EXPORT].size; + /* Writable data (.data, .bss and others) */ void __sparse_cache *va_base_data = (void __sparse_cache *) mctx->segment[LIB_MANAGER_DATA].addr; @@ -319,6 +686,12 @@ static int llext_manager_load_module(struct lib_manager_module *mctx) if (ret < 0) goto e_text; + /* Copy exported symbol table */ + ret = llext_manager_load_data_from_storage(virtual_region, ldr, ext, LLEXT_MEM_EXPORT, + va_base_export, export_size, 0); + if (ret < 0) + goto e_rodata; + /* Copy writable data */ /* * NOTE: va_base_data and data_size refer to an address range that @@ -371,6 +744,11 @@ static int llext_manager_unload_module(struct lib_manager_module *mctx) mctx->segment[LIB_MANAGER_RODATA].addr; size_t rodata_size = mctx->segment[LIB_MANAGER_RODATA].size; + /* Exported symbol table (.exported_sym), read-only like .rodata */ + void __sparse_cache *va_base_export = (void __sparse_cache *) + mctx->segment[LIB_MANAGER_EXPORT].addr; + size_t export_size = mctx->segment[LIB_MANAGER_EXPORT].size; + /* Writable data (.data, .bss, etc.) */ void __sparse_cache *va_base_data = (void __sparse_cache *) mctx->segment[LIB_MANAGER_DATA].addr; @@ -417,6 +795,12 @@ static int llext_manager_unload_module(struct lib_manager_module *mctx) if (ret < 0 && !err) err = ret; + llext_manager_unmap_detached_sections(ldr, ext, LLEXT_MEM_EXPORT, + va_base_export, export_size); + ret = llext_manager_align_unmap(va_base_export, export_size); + if (ret < 0 && !err) + err = ret; + #ifdef CONFIG_SOF_USERSPACE_LL llext_manager_rm_partition(zephyr_ll_mem_domain(), (uintptr_t)shdr, total, K_MEM_PARTITION_P_RW_U_NA | XTENSA_MMU_CACHED_WB); @@ -435,6 +819,73 @@ static bool llext_manager_section_detached(const elf_shdr_t *shdr) return shdr->sh_addr < SOF_MODULE_DRAM_LINK_END; } +/* + * Zephyr's llext_load() reads several sections' actual byte content + * synchronously, during the load itself -- e.g. llext_export_symbols() + * (called unconditionally to build the extension's exported-symbol table + * for future dependency resolution) reads .exported_sym, and other parts + * of the eager load path read .rodata and friends. This is unlike the + * later, on-demand copy SOF performs when a module is actually + * instantiated (llext_manager_load_module()), which only runs well after + * llext_manager_link() has already called llext_load(). So every + * "attached" (non-detached, i.e. real VMA, see + * llext_manager_section_detached()) allocated section with real file + * content (skipping .bss/SHT_NOBITS, which has none) has to be populated + * here, directly from the raw ELF buffer, before llext_load() is invoked + * below. + */ +static int llext_manager_load_sections_early(uint8_t *elf_buf) +{ + elf_ehdr_t *hdr = (elf_ehdr_t *)elf_buf; + elf_shdr_t *shdrs = (elf_shdr_t *)(elf_buf + hdr->e_shoff); + const struct sys_mm_drv_region *virtual_memory_regions; + const struct sys_mm_drv_region *virtual_region; + int i; + + virtual_memory_regions = sys_mm_drv_query_memory_regions(); + if (!virtual_memory_regions) + return -EFAULT; + + SYS_MM_DRV_MEMORY_REGION_FOREACH(virtual_memory_regions, virtual_region) { + if (virtual_region->attr == VIRTUAL_REGION_LLEXT_LIBRARIES_ATTR) + break; + } + + if (!virtual_region->size) + return -EFAULT; + + for (i = 0; i < hdr->e_shnum; i++) { + elf_shdr_t *shdr = shdrs + i; + void __sparse_cache *vma; + int ret; + + if (!(shdr->sh_flags & SHF_ALLOC) || shdr->sh_size == 0) + continue; + + if (shdr->sh_type == SHT_NOBITS) + continue; + + if (llext_manager_section_detached(shdr)) + continue; + + vma = (void __sparse_cache *)(uintptr_t)shdr->sh_addr; + + ret = llext_manager_align_map(virtual_region, vma, shdr->sh_size, + SYS_MM_MEM_PERM_RW); + if (ret < 0) + return ret; + + ret = memcpy_s((__sparse_force void *)(uintptr_t)shdr->sh_addr, shdr->sh_size, + elf_buf + shdr->sh_offset, shdr->sh_size); + if (ret < 0) + return ret; + + dcache_writeback_region(vma, shdr->sh_size); + } + + return 0; +} + static int llext_manager_link(const char *name, struct lib_manager_module *mctx, const void **buildinfo, const struct sof_man_module_manifest **mod_manifest) @@ -457,6 +908,36 @@ static int llext_manager_link(const char *name, } if (!*llext || mctx->mapped) { + if (!*llext) { + uint8_t *elf_buf = (uint8_t *)mctx->ebl->buf; + size_t total_size = llext_manager_layout_sections(elf_buf, 0); + if (total_size == 0) { + tr_err(&lib_manager_tr, "llext_manager_link: layout sections failed"); + return -EINVAL; + } + + uintptr_t vma_base = llext_manager_alloc_vma(total_size); + if (!vma_base) { + tr_err(&lib_manager_tr, "llext_manager_link: VMA allocation failed"); + return -ENOMEM; + } + + mctx->vma_base = vma_base; + mctx->vma_size = total_size; + + llext_manager_layout_sections(elf_buf, vma_base); + + ret = llext_manager_load_sections_early(elf_buf); + if (ret < 0) { + tr_err(&lib_manager_tr, + "llext_manager_link: early section copy failed: %d", ret); + llext_manager_free_vma(mctx->vma_base, mctx->vma_size); + mctx->vma_base = 0; + mctx->vma_size = 0; + return ret; + } + } + /* * Either the very first time loading this module, or the module * is already mapped, we just call llext_load() to refcount it @@ -469,8 +950,15 @@ static int llext_manager_link(const char *name, }; ret = llext_load(ldr, name, llext, &ldr_parm); - if (ret) + if (ret) { + tr_err(&lib_manager_tr, "llext_load failed: ret=%d", ret); + if (mctx->vma_base) { + llext_manager_free_vma(mctx->vma_base, mctx->vma_size); + mctx->vma_base = 0; + mctx->vma_size = 0; + } return ret; + } } /* All code sections */ @@ -509,6 +997,20 @@ static int llext_manager_link(const char *name, mctx->segment[LIB_MANAGER_BSS].addr, mctx->segment[LIB_MANAGER_BSS].size); + /* + * Exported symbol table (.exported_sym). Some toolchains (e.g. GNU ld, + * unlike the Clang LLEXT overlay which merges it into .rodata) keep this + * as its own distinct allocatable section, so it needs its own tracked + * segment and its own copy-from-storage pass, same as .rodata. + */ + llext_get_region_info(ldr, *llext, LLEXT_MEM_EXPORT, &hdr, NULL, NULL); + mctx->segment[LIB_MANAGER_EXPORT].addr = hdr->sh_addr; + mctx->segment[LIB_MANAGER_EXPORT].size = hdr->sh_size; + + tr_dbg(&lib_manager_tr, ".exported_sym: start: %#lx size %#x", + mctx->segment[LIB_MANAGER_EXPORT].addr, + mctx->segment[LIB_MANAGER_EXPORT].size); + *buildinfo = NULL; ret = llext_section_shndx(ldr, *llext, ".mod_buildinfo"); if (ret >= 0) { @@ -1203,13 +1705,14 @@ int llext_manager_add_library(uint32_t module_id) } for (i = 0; i < ctx->n_mod; i++) { - const struct sof_man_module *mod = lib_manager_get_module_manifest(module_id + i); + unsigned int idx = ctx->mod[i].start_idx; + const struct sof_man_module *mod = lib_manager_get_module_manifest(module_id + idx); if (mod->type.load_type == SOF_MAN_MOD_TYPE_LLEXT_AUX) { const struct sof_man_module_manifest *mod_manifest; const struct sof_module_api_build_info *buildinfo; - ret = llext_manager_link_single(module_id + i, desc, ctx, + ret = llext_manager_link_single(module_id + idx, desc, ctx, (const void **)&buildinfo, &mod_manifest); if (ret < 0) return ret; diff --git a/src/library_manager/llext_manager_dram.c b/src/library_manager/llext_manager_dram.c index 2f6cff2b3501..1cc8fad942f8 100644 --- a/src/library_manager/llext_manager_dram.c +++ b/src/library_manager/llext_manager_dram.c @@ -14,6 +14,8 @@ LOG_MODULE_DECLARE(lib_manager, CONFIG_SOF_LOG_LEVEL); +void llext_manager_free_vma(uintptr_t vma, size_t size); + struct lib_manager_dram_storage { struct ext_library ext_lib; struct lib_manager_mod_ctx *ctx; @@ -332,6 +334,9 @@ int llext_manager_restore_from_dram(void) if (mod[k].llext) llext_unload(&mod[k].llext); + if (mod[k].vma_base) + llext_manager_free_vma(mod[k].vma_base, mod[k].vma_size); + if (mod[k].ebl) rfree(mod[k].ebl); } From 9eba8914a649d6d125f2646a1965e63c604c0ef5 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 16 Aug 2026 18:15:39 +0100 Subject: [PATCH 03/35] zephyr: llext: link non-relocatable modules -Bsymbolic-functions Without this flag, calls between GLOBAL-visibility functions defined in different translation units of the SAME llext module are emitted as PLT calls. Zephyr's llext_link_plt() only resolves PLT symbols against the base image's export table, this module's own .exported_sym table, or other already-loaded extensions -- never against symbols merely defined locally in this module's own .dynsym. Multi-TU C++ libraries (e.g. TensorFlow Lite Micro) call plenty of non-exported internal helpers across .cc files, so without this flag those calls fail to link at load time. Signed-off-by: Liam Girdwood --- zephyr/CMakeLists.txt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 1c27e1c06670..e7be53dfb0cc 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -102,7 +102,16 @@ function(sof_llext_build module) if(CONFIG_LLEXT_TYPE_ELF_RELOCATABLE) set(EXTRA_LINKER_PARAMS -nostdlib -nodefaultlibs -r) else() - set(EXTRA_LINKER_PARAMS -nostdlib -nodefaultlibs -shared) + # -Bsymbolic-functions: without this, calls between GLOBAL-visibility + # functions defined in different translation units of the SAME llext + # module are emitted as PLT calls. Zephyr's llext_link_plt() resolves + # PLT symbols only against the base image's export table, this + # module's own .exported_sym table, or other already-loaded + # extensions -- never against symbols merely defined locally in this + # module's own .dynsym. Multi-TU C++ libraries (e.g. TFLite Micro) + # call plenty of non-exported internal helpers across .cc files, so + # without this flag those calls fail to link at load time. + set(EXTRA_LINKER_PARAMS -nostdlib -nodefaultlibs -shared -Wl,-Bsymbolic-functions) endif() foreach(path ${SOF_LLEXT_LIBS_PATH}) From 3ae612ca307fc091ebe7732f491caa24a62659e3 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 16 Aug 2026 18:15:27 +0100 Subject: [PATCH 04/35] lib: cpp: export operator new/delete and __cxa_pure_virtual for LLEXT LLEXT modules linking C++ code (e.g. TensorFlow Lite Micro) can end up with undefined references to global operator new/delete and __cxa_pure_virtual even when built with -fno-exceptions -- some support code (e.g. TFLM's arena allocators) still emits calls to the sized deallocation form in generated destructors, and any TU referencing an abstract class's vtable needs __cxa_pure_virtual resolvable for its pure-virtual slots. zephyr/lib/cpp/minimal/cpp_new.cpp and cpp_virtual.c already define these, but neither is referenced anywhere in the base image build, so their definitions are never pulled into the link or exported. Add thin wrappers in src/lib/cpp_new_export.cpp and export them under the mangled names so LLEXT modules can resolve against the base image. Signed-off-by: Liam Girdwood --- src/lib/CMakeLists.txt | 4 +++ src/lib/cpp_new_export.cpp | 55 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 src/lib/cpp_new_export.cpp diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt index 707753635f69..d9213ef4473d 100644 --- a/src/lib/CMakeLists.txt +++ b/src/lib/CMakeLists.txt @@ -23,6 +23,10 @@ if(CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL) add_local_sources(sof cpu-clk-manager.c) endif() +if(CONFIG_CPP) + add_local_sources(sof cpp_new_export.cpp) +endif() + is_zephyr(zephyr) if(zephyr) ### Zephyr ### diff --git a/src/lib/cpp_new_export.cpp b/src/lib/cpp_new_export.cpp new file mode 100644 index 000000000000..9b226a9f80c3 --- /dev/null +++ b/src/lib/cpp_new_export.cpp @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. All rights reserved. + +#include +#include +#include + +/* + * LLEXT modules linking C++ code (e.g. TFLite Micro) can end up with + * undefined references to global operator new/delete even when built with + * -fno-exceptions, because some support code (e.g. TFLM's arena allocators) + * still emits a call to the sized deallocation form in its generated + * destructors. Export wrappers here so such extensions can resolve them + * against the base image, mirroring zephyr/lib/cpp/minimal/cpp_new.cpp + * (which isn't itself referenced anywhere in the base image build, so its + * definitions are never pulled into the link or exported). + */ + +namespace { + +[[maybe_unused]] void *sof_operator_new(size_t size) +{ + return malloc(size); +} + +[[maybe_unused]] void sof_operator_delete(void *ptr, size_t size) +{ + (void)size; + free(ptr); +} + +/* + * Same problem as operator new/delete above, one level lower: an abstract + * class's vtable (e.g. TFLM's MicroOpResolver base) fills the slots for its + * pure virtual methods with the address of __cxa_pure_virtual, so any + * translation unit that references such a vtable needs this symbol + * resolvable even though it should never actually be called at runtime. + * zephyr/lib/cpp/minimal/cpp_virtual.c already defines a real + * __cxa_pure_virtual, but -- exactly like cpp_new.cpp above -- nothing in + * the base image build references it directly, so it is never pulled into + * the link or exported for LLEXT modules to resolve against. Provide our + * own and export it under the same name. + */ +[[maybe_unused]] void sof_cxa_pure_virtual(void) +{ + while (1) { + } +} + +} /* namespace */ + +EXPORT_SYMBOL_NAMED(sof_operator_new, _Znwj); +EXPORT_SYMBOL_NAMED(sof_operator_delete, _ZdlPvj); +EXPORT_SYMBOL_NAMED(sof_cxa_pure_virtual, __cxa_pure_virtual); From d6e41f322b24d420d2ee37f09d960e6ddea2e862 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 16 Aug 2026 18:17:12 +0100 Subject: [PATCH 05/35] lib: ams: export producer API for LLEXT modules Export ams_send(), ams_helper_register_producer(), ams_helper_unregister_producer(), and ams_helper_prepare_payload() so an LLEXT module can act as an AMS message producer (e.g. a keyword-spotting component signaling KPB directly) without needing these calls statically linked into the base image. Signed-off-by: Liam Girdwood --- src/ipc/ipc4/ams_helpers.c | 5 +++++ src/lib/ams.c | 39 +++++++++++++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/ipc/ipc4/ams_helpers.c b/src/ipc/ipc4/ams_helpers.c index 6f21ee3e3623..b95acf176e88 100644 --- a/src/ipc/ipc4/ams_helpers.c +++ b/src/ipc/ipc4/ams_helpers.c @@ -7,6 +7,7 @@ #include #include +#include #if CONFIG_AMS @@ -76,4 +77,8 @@ void ams_helper_prepare_payload(const struct comp_dev *dev, payload->message = message; } +EXPORT_SYMBOL(ams_helper_register_producer); +EXPORT_SYMBOL(ams_helper_unregister_producer); +EXPORT_SYMBOL(ams_helper_prepare_payload); + #endif /* CONFIG_AMS */ diff --git a/src/lib/ams.c b/src/lib/ams.c index 7f4e6ebb8664..1809763acef7 100644 --- a/src/lib/ams.c +++ b/src/lib/ams.c @@ -22,6 +22,7 @@ #include #include #include +#include LOG_MODULE_REGISTER(ams, CONFIG_SOF_LOG_LEVEL); @@ -284,9 +285,8 @@ static uint32_t ams_push_slot(struct ams_shared_context __sparse_cache *ctx_shar for (uint32_t i = 0; i < ARRAY_SIZE(ctx_shared->slots); ++i) { if (ctx_shared->slot_uses[i] == 0) { - /* the slot only carries the payload struct (read back - * via u.msg); message points to a caller-owned buffer - * rather than inline data, so copy exactly the struct + /* the slot carries the payload struct (read back + * via u.msg) and inline message data */ err = memcpy_s((__sparse_force void *)ctx_shared->slots[i].u.msg_raw, sizeof(ctx_shared->slots[i].u.msg_raw), @@ -295,6 +295,23 @@ static uint32_t ams_push_slot(struct ams_shared_context __sparse_cache *ctx_shar if (err != 0) return AMS_INVALID_SLOT; + if (msg->message && msg->message_length > 0) { + size_t max_data = sizeof(ctx_shared->slots[i].u.msg_raw) - + sizeof(*msg); + + if (msg->message_length > max_data) { + tr_err(&ams_tr, "Message too large: %u > %zu", + msg->message_length, max_data); + return AMS_INVALID_SLOT; + } + + err = memcpy_s((__sparse_force void *)(ctx_shared->slots[i].u.msg_raw + sizeof(*msg)), + max_data, + msg->message, msg->message_length); + if (err != 0) + return AMS_INVALID_SLOT; + } + ctx_shared->slots[i].module_id = module_id; ctx_shared->slots[i].instance_id = instance_id; ctx_shared->slot_done[i] = 0; @@ -468,6 +485,8 @@ int ams_send(const struct ams_message_payload *const ams_message_payload) AMS_INVALID_SLOT); } +EXPORT_SYMBOL(ams_send); + int ams_message_send_mi(struct async_message_service *ams, const struct ams_message_payload *const ams_message_payload, uint16_t target_module, uint16_t target_instance) @@ -488,12 +507,26 @@ static int ams_process_slot(struct async_message_service *ams, uint32_t slot) { struct ams_shared_context __sparse_cache *shared_c; struct ams_message_payload msg; + uint8_t msg_buf[256]; uint16_t module_id; uint16_t instance_id; shared_c = ams_acquire(ams->ams_context->shared); msg = shared_c->slots[slot].u.msg; + if (msg.message && msg.message_length > 0) { + if (msg.message_length <= sizeof(msg_buf)) { + if (memcpy_s(msg_buf, sizeof(msg_buf), + (__sparse_force void *)(shared_c->slots[slot].u.msg_raw + sizeof(msg)), + msg.message_length) != 0) { + ams_release(shared_c); + return -EINVAL; + } + msg.message = msg_buf; + } else { + msg.message = (__sparse_force uint8_t *)(shared_c->slots[slot].u.msg_raw + sizeof(msg)); + } + } module_id = shared_c->slots[slot].module_id; instance_id = shared_c->slots[slot].instance_id; From e00eb80189ab696842247180664c8e56c03b332b Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 16 Aug 2026 18:19:25 +0100 Subject: [PATCH 06/35] scripts: xtensa-build-zephyr: create per-UUID symlinks in install_lib() Consume the lib_uuids dict already populated earlier in the script: when a library's final .bin file did not yet exist at UUID-collection time, its UUIDs were deferred into lib_uuids instead of being symlinked immediately. install_lib() now walks lib_uuids[key] and creates the deferred .bin symlink/copy once the library is actually installed. Signed-off-by: Liam Girdwood --- scripts/tensorflow-clone.sh | 67 ++++++++++++++++++++++++---------- scripts/xtensa-build-zephyr.py | 4 ++ 2 files changed, 51 insertions(+), 20 deletions(-) diff --git a/scripts/tensorflow-clone.sh b/scripts/tensorflow-clone.sh index d3f67d0a835e..caa0f58577ea 100755 --- a/scripts/tensorflow-clone.sh +++ b/scripts/tensorflow-clone.sh @@ -1,6 +1,6 @@ #!/bin/bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2025 Intel Corporation. All rights reserved. +# Copyright(c) 2025-2026 Intel Corporation. All rights reserved. # fail immediately on any errors set -e @@ -14,8 +14,7 @@ declare -a REPOS=( "https://github.com/google/ruy" ) -# Commit ID to check for (optional). If specified, the script will update -# the repository if this commit ID is not found. Leave empty to skip. +# Commit ID to check out. If specified, the script will checkout this commit. declare -a COMMIT_ID=( "cdedfb1a1044eb774915de21b63a1b6aa93276f6" "e86d97b6237f88ab5925c0b41e3e3589a1560d86" @@ -25,7 +24,10 @@ declare -a COMMIT_ID=( ) # Directory where repositories will be cloned/updated. -BASE_DIR="$HOME/work/sof" # Or any other desired location +# Default to the parent directory containing the SOF workspace. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SOF_PARENT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" +BASE_DIR="${1:-"$SOF_PARENT_DIR"}" # Function to check if a commit ID exists in a repository check_commit() { @@ -36,40 +38,65 @@ check_commit() { return 0 # Skip check if no commit ID is provided fi - if ! git -C "$repo_dir" rev-parse --quiet --verify "$commit_id" >/dev/null 2>&1; then - return 1 # Commit ID not found + if ! git -C "$repo_dir" rev-parse --quiet --verify "$commit_id^{commit}" >/dev/null 2>&1; then + return 1 # Commit ID not found locally else - return 0 # Commit ID found + return 0 # Commit ID found locally fi } - # Function to update the repository update_repo() { local repo_dir="$1" echo "Updating repository: $repo_dir" - git -C "$repo_dir" fetch --all - git -C "$repo_dir" pull + git -C "$repo_dir" fetch --all --tags --prune +} + +# Function to checkout the required commit ID +checkout_commit() { + local repo_dir="$1" + local commit_id="$2" + local repo_name="$3" + + if [ -z "$commit_id" ]; then + return 0 + fi + + local current_commit + current_commit=$(git -C "$repo_dir" rev-parse HEAD) + + local target_commit + target_commit=$(git -C "$repo_dir" rev-parse "$commit_id^{commit}") + + if [ "$current_commit" != "$target_commit" ]; then + echo "Checking out $commit_id in $repo_name..." + git -C "$repo_dir" checkout -q "$commit_id" + else + echo "Repository $repo_name is already at commit $commit_id." + fi } # Main script logic mkdir -p "$BASE_DIR" for ((i = 0; i < ${#REPOS[@]}; i++)); do - echo "Counter: $i, Value: ${REPOS[i]}" - repo_url=${REPOS[i]} - - repo_name=$(basename "$repo_url" .git) # Extract repo name + repo_url="${REPOS[i]}" + commit_id="${COMMIT_ID[i]}" + repo_name=$(basename "$repo_url" .git) repo_dir="$BASE_DIR/$repo_name" - if [ ! -d "$repo_dir" ]; then - echo "Cloning repository: $repo_url" + echo "=== [$((i + 1))/${#REPOS[@]}] $repo_name ===" + + if [ ! -d "$repo_dir/.git" ]; then + echo "Cloning repository: $repo_url -> $repo_dir" git clone "$repo_url" "$repo_dir" || { echo "git clone failed for $repo_url"; exit 1; } - elif ! check_commit "$repo_dir" "${COMMIT_ID[i]}"; then + fi + + if ! check_commit "$repo_dir" "$commit_id"; then update_repo "$repo_dir" - else - echo "Repository $repo_name is up to date." fi + + checkout_commit "$repo_dir" "$commit_id" "$repo_name" done -echo "All repositories processed." +echo "All repositories processed and checked out to required commits." diff --git a/scripts/xtensa-build-zephyr.py b/scripts/xtensa-build-zephyr.py index f20c95f7262a..735842e0ba54 100755 --- a/scripts/xtensa-build-zephyr.py +++ b/scripts/xtensa-build-zephyr.py @@ -1174,6 +1174,10 @@ def install_lib(platform, sof_output_dir, abs_build_dir, platform_wconfig): symlink_or_copy(lib_install_dir, lib_name, lib_dir, alias_libname) + for uuid in lib_uuids.get(key, []): + linkname = uuid + '.bin' + symlink_or_copy(lib_install_dir, lib_name, sof_lib_dir, linkname) + def install_platform(platform, sof_output_dir, platf_build_environ, platform_wconfig): # Keep in sync with caller From e08ccf917468f49c09fc963476c98f5dbe9852a1 Mon Sep 17 00:00:00 2001 From: Seppo Ingalsuo Date: Mon, 24 Aug 2026 17:11:00 +0300 Subject: [PATCH 07/35] platform: cavs: initialize DP scheduler during platform_init Call scheduler_dp_init() in platform_init() on cAVS platforms when CONFIG_ZEPHYR_DP_SCHEDULER is enabled so that DP tasks (such as MFCC and MWW in the Data Processing domain) can bind to the DP scheduler without failing with -ENODEV (-19). Signed-off-by: Seppo Ingalsuo --- app/boards/intel_adsp/Kconfig.defconfig | 3 +++ app/boards/intel_adsp_cavs25.conf | 2 ++ src/platform/intel/cavs/platform.c | 7 +++++++ zephyr/Kconfig | 8 ++++---- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/app/boards/intel_adsp/Kconfig.defconfig b/app/boards/intel_adsp/Kconfig.defconfig index 0fd7126d58d1..ac8c6fb6f190 100644 --- a/app/boards/intel_adsp/Kconfig.defconfig +++ b/app/boards/intel_adsp/Kconfig.defconfig @@ -87,6 +87,9 @@ config L3_HEAP config ZEPHYR_DP_SCHEDULER default y +config DP_TO_DP_BIND + default y + config ZEPHYR_NATIVE_DRIVERS default y diff --git a/app/boards/intel_adsp_cavs25.conf b/app/boards/intel_adsp_cavs25.conf index 7b2a260db89d..0a89f1a669a7 100644 --- a/app/boards/intel_adsp_cavs25.conf +++ b/app/boards/intel_adsp_cavs25.conf @@ -19,6 +19,8 @@ CONFIG_PCM_CONVERTER_FORMAT_S24_3LE=y # SOF / infrastructure CONFIG_AMS=y +CONFIG_IDC_TIMEOUT_US=100000 +CONFIG_P4WQ_INIT_STAGE_EARLY=y CONFIG_LP_MEMORY_BANKS=1 CONFIG_HP_MEMORY_BANKS=30 diff --git a/src/platform/intel/cavs/platform.c b/src/platform/intel/cavs/platform.c index 8a1a7c59c3b5..4752916a85b5 100644 --- a/src/platform/intel/cavs/platform.c +++ b/src/platform/intel/cavs/platform.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -122,6 +123,12 @@ int platform_init(struct sof *sof) sof->platform_timer_domain = timer_domain_init(sof->platform_timer, 0); scheduler_init_ll(sof->platform_timer_domain); +#if CONFIG_ZEPHYR_DP_SCHEDULER + ret = scheduler_dp_init(); + if (ret < 0) + return ret; +#endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ + /* init the system agent */ trace_point(TRACE_BOOT_PLATFORM_AGENT); sa_init(sof, CONFIG_SYSTICK_PERIOD); diff --git a/zephyr/Kconfig b/zephyr/Kconfig index 867100623b42..6c0035dc55c4 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -241,19 +241,19 @@ config DMA_DOMAIN_SEM_LIMIT config PIPELINE_2_0 bool "Enable pipeline 2.0 changes" depends on IPC_MAJOR_4 - default y if ACE + default y if (ACE || CAVS) help This flag enables changes to new pipeline structure, known as pipeline2_0 It is required for certain new features, like DP_SCHEDULER. config ZEPHYR_DP_SCHEDULER bool "use Zephyr thread based DP scheduler" - default y if ACE + default y if (ACE || CAVS) depends on IPC_MAJOR_4 depends on ZEPHYR_SOF_MODULE - depends on ACE + depends on (ACE || CAVS) depends on PIPELINE_2_0 - imply XTENSA_HIFI_SHARING if XTENSA_HIFI + imply XTENSA_HIFI_SHARING if (XTENSA_HIFI && ACE) help Enable Data Processing preemptive scheduler based on Zephyr preemptive threads. From 40e41ca2f244c3d4279ba253a6154d841b96c84b Mon Sep 17 00:00:00 2001 From: Seppo Ingalsuo Date: Thu, 27 Aug 2026 15:24:07 +0300 Subject: [PATCH 08/35] audio: mfcc: add mel40 and mel40_10ms profiles for microWakeWord frontend Add 40-bin mel filterbank configurations (mel40, mel40_compress, mel40_10ms, and mel40_10ms_compress) to setup_mfcc.m and generate the corresponding topology blobs for microWakeWord streaming frontends. Signed-off-by: Seppo Ingalsuo --- src/audio/mfcc/mfcc_common.c | 2 +- src/audio/mfcc/tune/setup_mfcc.m | 44 +++++++++++++++++++ src/include/sof/audio/mfcc/mfcc_vad.h | 8 ++-- .../topology2/cavs-benchmark-hda.conf | 10 +++++ .../development/tplg-targets-bench.cmake | 2 + .../include/bench/mfcc_controls_capture.conf | 1 + .../include/bench/mfcc_controls_playback.conf | 1 + .../include/bench/mfccmel40_10ms_s16.conf | 13 ++++++ .../include/bench/mfccmel40_10ms_s24.conf | 13 ++++++ .../include/bench/mfccmel40_10ms_s32.conf | 13 ++++++ .../components/mfcc/ceps13_compress_dtx.conf | 2 +- .../include/components/mfcc/default.conf | 2 +- .../include/components/mfcc/mel40.conf | 24 ++++++++++ .../include/components/mfcc/mel40_10ms.conf | 24 ++++++++++ .../components/mfcc/mel40_10ms_compress.conf | 24 ++++++++++ .../components/mfcc/mel40_compress.conf | 24 ++++++++++ .../include/components/mfcc/mel80.conf | 2 +- .../components/mfcc/mel80_compress.conf | 2 +- .../components/mfcc/mel80_compress_dtx.conf | 2 +- 19 files changed, 203 insertions(+), 10 deletions(-) create mode 100644 tools/topology/topology2/include/bench/mfccmel40_10ms_s16.conf create mode 100644 tools/topology/topology2/include/bench/mfccmel40_10ms_s24.conf create mode 100644 tools/topology/topology2/include/bench/mfccmel40_10ms_s32.conf create mode 100644 tools/topology/topology2/include/components/mfcc/mel40.conf create mode 100644 tools/topology/topology2/include/components/mfcc/mel40_10ms.conf create mode 100644 tools/topology/topology2/include/components/mfcc/mel40_10ms_compress.conf create mode 100644 tools/topology/topology2/include/components/mfcc/mel40_compress.conf diff --git a/src/audio/mfcc/mfcc_common.c b/src/audio/mfcc/mfcc_common.c index 3890671165fd..2460493ed34d 100644 --- a/src/audio/mfcc/mfcc_common.c +++ b/src/audio/mfcc/mfcc_common.c @@ -419,7 +419,7 @@ int mfcc_stft_process(struct processing_module *mod, struct mfcc_comp_data *cd) /* Use hop counter for frame numbering (independent of VAD enable) */ state->header.frame_number = state->hop_count; - /* Run VAD on the mel log spectrum (available in both modes) */ + /* Run VAD on the scaled mel log spectrum (available in both modes) */ if (config->enable_vad) { mfcc_vad_update(&cd->vad, state->mel_log_32); diff --git a/src/audio/mfcc/tune/setup_mfcc.m b/src/audio/mfcc/tune/setup_mfcc.m index dbf69587a74f..13ffea4955ec 100644 --- a/src/audio/mfcc/tune/setup_mfcc.m +++ b/src/audio/mfcc/tune/setup_mfcc.m @@ -31,6 +31,50 @@ function setup_mfcc() setup.tplg_fn = 'mel80_compress.conf'; export_mfcc_setup(gen_cfg, setup); + % Blob for 40-bin/20ms-hop mel spectrogram, matching TFLM micro_speech's + % front-end shape (TFLM_FEATURE_SIZE=40, TFLM_FEATURE_STRIDE_MS=20, + % TFLM_FEATURE_DURATION_MS=30) for interim wake-word sanity-checking. + setup = get_mel_spectrogram_config(); + setup.frame_length = 30.0; % 480 samples at 16 kHz + setup.frame_shift = 20.0; % 320 samples at 16 kHz + setup.num_mel_bins = 40; + setup.tplg_fn = 'mel40.conf'; + export_mfcc_setup(gen_cfg, setup); + + % Same 40-bin/20ms-hop mel spectrogram with compress PCM output for the + % on-device TFLM wake-word path (KPB -> SRC -> MFCC -> tflmcly). + setup = get_mel_spectrogram_config(); + setup.frame_length = 30.0; + setup.frame_shift = 20.0; + setup.num_mel_bins = 40; + setup.compress_output = true; + setup.tplg_fn = 'mel40_compress.conf'; + export_mfcc_setup(gen_cfg, setup); + + % Blob for 40-bin/10ms-hop mel spectrogram, matching microWakeWord's + % MixConv front-end shape (40 features per 10ms stride over a 30ms + % window, 125 Hz to 7500 Hz bandwidth) -- see src/audio/microwakeword. + setup = get_mel_spectrogram_config(); + setup.frame_length = 30.0; % 480 samples at 16 kHz + setup.frame_shift = 10.0; % 160 samples at 16 kHz + setup.num_mel_bins = 40; + setup.low_freq = 125; + setup.high_freq = 7500; + setup.tplg_fn = 'mel40_10ms.conf'; + export_mfcc_setup(gen_cfg, setup); + + % Same 40-bin/10ms-hop mel spectrogram with compress PCM output for + % microWakeWord compress stream capture. + setup = get_mel_spectrogram_config(); + setup.frame_length = 30.0; + setup.frame_shift = 10.0; + setup.num_mel_bins = 40; + setup.low_freq = 125; + setup.high_freq = 7500; + setup.compress_output = true; + setup.tplg_fn = 'mel40_10ms_compress.conf'; + export_mfcc_setup(gen_cfg, setup); + % Blob for mel spectrogram with compress PCM output and DTX setup = get_mel_spectrogram_config(); setup.compress_output = true; diff --git a/src/include/sof/audio/mfcc/mfcc_vad.h b/src/include/sof/audio/mfcc/mfcc_vad.h index 6873343d334e..048ea475eba1 100644 --- a/src/include/sof/audio/mfcc/mfcc_vad.h +++ b/src/include/sof/audio/mfcc/mfcc_vad.h @@ -29,9 +29,9 @@ struct processing_module; #define MFCC_VAD_NOISE_INIT_FRAMES 100 /** - * \brief Slow noise floor rise coefficient in Q1.15 (0.003 * 2^15). + * \brief Slow noise floor rise coefficient in Q1.15 (0.001 * 2^15, tau ~10 s). */ -#define MFCC_VAD_NOISE_RISE_ALPHA 98 +#define MFCC_VAD_NOISE_RISE_ALPHA 33 /** * \brief Fast noise floor rise coefficient in Q1.15 (0.020 * 2^15). @@ -39,9 +39,9 @@ struct processing_module; #define MFCC_VAD_NOISE_RISE_ALPHA_FAST 655 /** - * \brief Energy threshold for speech detection in Q9.23 (0.30 * 2^23). + * \brief Energy threshold for speech detection in Q9.23 (0.20 * 2^23). */ -#define MFCC_VAD_ENERGY_THRESHOLD 2516582 +#define MFCC_VAD_ENERGY_THRESHOLD 1677722 /** * \brief Hangover frame count to keep VAD active after last speech detection. diff --git a/tools/topology/topology2/cavs-benchmark-hda.conf b/tools/topology/topology2/cavs-benchmark-hda.conf index d1357caa20ab..182ccb403a44 100644 --- a/tools/topology/topology2/cavs-benchmark-hda.conf +++ b/tools/topology/topology2/cavs-benchmark-hda.conf @@ -845,6 +845,16 @@ IncludeByKey.BENCH_CONFIG { } + "mfccmel40_10ms16" { + + } + "mfccmel40_10ms24" { + + } + "mfccmel40_10ms32" { + + } + # # Micsel component # diff --git a/tools/topology/topology2/development/tplg-targets-bench.cmake b/tools/topology/topology2/development/tplg-targets-bench.cmake index 2dd4f28bc07c..416448fd181e 100644 --- a/tools/topology/topology2/development/tplg-targets-bench.cmake +++ b/tools/topology/topology2/development/tplg-targets-bench.cmake @@ -20,6 +20,7 @@ set(components "level_multiplier" "mfcc" "mfccmel" + "mfccmel40_10ms" "micsel" "phase_vocoder" "rtnr" @@ -48,6 +49,7 @@ set(component_parameters "BENCH_LEVEL_MULTIPLIER_PARAMS=default" "BENCH_MFCC_PARAMS=default" "BENCH_MFCC_PARAMS=mel80" + "BENCH_MFCC_PARAMS=mel40_10ms" "BENCH_MICSEL_PARAMS=passthrough" "BENCH_PHASE_VOCODER_PARAMS=default" "BENCH_RTNR_PARAMS=default" diff --git a/tools/topology/topology2/include/bench/mfcc_controls_capture.conf b/tools/topology/topology2/include/bench/mfcc_controls_capture.conf index 8788387ec8c7..a185cdec852f 100644 --- a/tools/topology/topology2/include/bench/mfcc_controls_capture.conf +++ b/tools/topology/topology2/include/bench/mfcc_controls_capture.conf @@ -7,6 +7,7 @@ IncludeByKey.BENCH_MFCC_PARAMS { "default" "include/components/mfcc/default.conf" "mel80" "include/components/mfcc/mel80.conf" + "mel40_10ms" "include/components/mfcc/mel40_10ms.conf" } } mixer."1" { diff --git a/tools/topology/topology2/include/bench/mfcc_controls_playback.conf b/tools/topology/topology2/include/bench/mfcc_controls_playback.conf index 007dbb91cd4f..6d7ae6585663 100644 --- a/tools/topology/topology2/include/bench/mfcc_controls_playback.conf +++ b/tools/topology/topology2/include/bench/mfcc_controls_playback.conf @@ -7,6 +7,7 @@ IncludeByKey.BENCH_MFCC_PARAMS { "default" "include/components/mfcc/default.conf" "mel80" "include/components/mfcc/mel80.conf" + "mel40_10ms" "include/components/mfcc/mel40_10ms.conf" } } mixer."1" { diff --git a/tools/topology/topology2/include/bench/mfccmel40_10ms_s16.conf b/tools/topology/topology2/include/bench/mfccmel40_10ms_s16.conf new file mode 100644 index 000000000000..ec89bffb90a1 --- /dev/null +++ b/tools/topology/topology2/include/bench/mfccmel40_10ms_s16.conf @@ -0,0 +1,13 @@ + # Created with script "./bench_comp_generate.sh mfcc" + Object.Widget.mfcc.1 { + index $BENCH_PLAYBACK_HOST_PIPELINE + + + } + Object.Widget.mfcc.2 { + index $BENCH_CAPTURE_HOST_PIPELINE + + + } + + diff --git a/tools/topology/topology2/include/bench/mfccmel40_10ms_s24.conf b/tools/topology/topology2/include/bench/mfccmel40_10ms_s24.conf new file mode 100644 index 000000000000..73571fabe5f2 --- /dev/null +++ b/tools/topology/topology2/include/bench/mfccmel40_10ms_s24.conf @@ -0,0 +1,13 @@ + # Created with script "./bench_comp_generate.sh mfcc" + Object.Widget.mfcc.1 { + index $BENCH_PLAYBACK_HOST_PIPELINE + + + } + Object.Widget.mfcc.2 { + index $BENCH_CAPTURE_HOST_PIPELINE + + + } + + diff --git a/tools/topology/topology2/include/bench/mfccmel40_10ms_s32.conf b/tools/topology/topology2/include/bench/mfccmel40_10ms_s32.conf new file mode 100644 index 000000000000..75c01eaf4a43 --- /dev/null +++ b/tools/topology/topology2/include/bench/mfccmel40_10ms_s32.conf @@ -0,0 +1,13 @@ + # Created with script "./bench_comp_generate.sh mfcc" + Object.Widget.mfcc.1 { + index $BENCH_PLAYBACK_HOST_PIPELINE + + + } + Object.Widget.mfcc.2 { + index $BENCH_CAPTURE_HOST_PIPELINE + + + } + + diff --git a/tools/topology/topology2/include/components/mfcc/ceps13_compress_dtx.conf b/tools/topology/topology2/include/components/mfcc/ceps13_compress_dtx.conf index 7056b9e7cb4b..6c0c52ba09e5 100644 --- a/tools/topology/topology2/include/components/mfcc/ceps13_compress_dtx.conf +++ b/tools/topology/topology2/include/components/mfcc/ceps13_compress_dtx.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 26-May-2026 +# Exported MFCC configuration 03-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " diff --git a/tools/topology/topology2/include/components/mfcc/default.conf b/tools/topology/topology2/include/components/mfcc/default.conf index 0ac19fa71d04..7284f8209454 100644 --- a/tools/topology/topology2/include/components/mfcc/default.conf +++ b/tools/topology/topology2/include/components/mfcc/default.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 26-May-2026 +# Exported MFCC configuration 03-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " diff --git a/tools/topology/topology2/include/components/mfcc/mel40.conf b/tools/topology/topology2/include/components/mfcc/mel40.conf new file mode 100644 index 000000000000..79ecd2b74e62 --- /dev/null +++ b/tools/topology/topology2/include/components/mfcc/mel40.conf @@ -0,0 +1,24 @@ +# Exported MFCC configuration 03-Sep-2026 +# cd src/audio/mfcc/tune; octave setup_mfcc.m +Object.Base.data."mfcc_config" { + bytes " + 0x53,0x4f,0x46,0x34,0x00,0x00,0x00,0x00, + 0x74,0x00,0x00,0x00,0x01,0xd0,0x01,0x03, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x74,0x00,0x00,0x00,0x00,0x02,0x00,0x04, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x80,0x3e,0x00,0x00, + 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, + 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x03,0x00,0x00,0x00,0x01,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xe0,0x01,0x40,0x01,0x40,0x1f,0x00,0x00, + 0x00,0x00,0x28,0x00,0x00,0x00,0x00,0x04, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01, + 0x01,0x00,0x00,0x01,0x01,0x00,0x01,0x00, + 0x00,0x00,0x00,0x00" +} diff --git a/tools/topology/topology2/include/components/mfcc/mel40_10ms.conf b/tools/topology/topology2/include/components/mfcc/mel40_10ms.conf new file mode 100644 index 000000000000..cca9bba95d1b --- /dev/null +++ b/tools/topology/topology2/include/components/mfcc/mel40_10ms.conf @@ -0,0 +1,24 @@ +# Exported MFCC configuration 03-Sep-2026 +# cd src/audio/mfcc/tune; octave setup_mfcc.m +Object.Base.data."mfcc_config" { + bytes " + 0x53,0x4f,0x46,0x34,0x00,0x00,0x00,0x00, + 0x74,0x00,0x00,0x00,0x01,0xd0,0x01,0x03, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x74,0x00,0x00,0x00,0x00,0x02,0x00,0x04, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x80,0x3e,0x00,0x00, + 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, + 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x03,0x00,0x00,0x00,0x01,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xe0,0x01,0xa0,0x00,0x4c,0x1d,0x7d,0x00, + 0x00,0x00,0x28,0x00,0x00,0x00,0x00,0x04, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01, + 0x01,0x00,0x00,0x01,0x01,0x00,0x01,0x00, + 0x00,0x00,0x00,0x00" +} diff --git a/tools/topology/topology2/include/components/mfcc/mel40_10ms_compress.conf b/tools/topology/topology2/include/components/mfcc/mel40_10ms_compress.conf new file mode 100644 index 000000000000..26769e6538fb --- /dev/null +++ b/tools/topology/topology2/include/components/mfcc/mel40_10ms_compress.conf @@ -0,0 +1,24 @@ +# Exported MFCC configuration 03-Sep-2026 +# cd src/audio/mfcc/tune; octave setup_mfcc.m +Object.Base.data."mfcc_config" { + bytes " + 0x53,0x4f,0x46,0x34,0x00,0x00,0x00,0x00, + 0x74,0x00,0x00,0x00,0x01,0xd0,0x01,0x03, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x74,0x00,0x00,0x00,0x00,0x02,0x00,0x04, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x80,0x3e,0x00,0x00, + 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, + 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x03,0x00,0x00,0x00,0x01,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xe0,0x01,0xa0,0x00,0x4c,0x1d,0x7d,0x00, + 0x00,0x00,0x28,0x00,0x00,0x00,0x00,0x04, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01, + 0x01,0x00,0x00,0x01,0x01,0x00,0x01,0x01, + 0x00,0x00,0x00,0x00" +} diff --git a/tools/topology/topology2/include/components/mfcc/mel40_compress.conf b/tools/topology/topology2/include/components/mfcc/mel40_compress.conf new file mode 100644 index 000000000000..cb9f76105869 --- /dev/null +++ b/tools/topology/topology2/include/components/mfcc/mel40_compress.conf @@ -0,0 +1,24 @@ +# Exported MFCC configuration 03-Sep-2026 +# cd src/audio/mfcc/tune; octave setup_mfcc.m +Object.Base.data."mfcc_config" { + bytes " + 0x53,0x4f,0x46,0x34,0x00,0x00,0x00,0x00, + 0x74,0x00,0x00,0x00,0x01,0xd0,0x01,0x03, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x74,0x00,0x00,0x00,0x00,0x02,0x00,0x04, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x80,0x3e,0x00,0x00, + 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, + 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x03,0x00,0x00,0x00,0x01,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xe0,0x01,0x40,0x01,0x40,0x1f,0x00,0x00, + 0x00,0x00,0x28,0x00,0x00,0x00,0x00,0x04, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01, + 0x01,0x00,0x00,0x01,0x01,0x00,0x01,0x01, + 0x00,0x00,0x00,0x00" +} diff --git a/tools/topology/topology2/include/components/mfcc/mel80.conf b/tools/topology/topology2/include/components/mfcc/mel80.conf index b18baadd459b..fd2023c7f408 100644 --- a/tools/topology/topology2/include/components/mfcc/mel80.conf +++ b/tools/topology/topology2/include/components/mfcc/mel80.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 26-May-2026 +# Exported MFCC configuration 03-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " diff --git a/tools/topology/topology2/include/components/mfcc/mel80_compress.conf b/tools/topology/topology2/include/components/mfcc/mel80_compress.conf index f26f2af6980c..c219b406f70c 100644 --- a/tools/topology/topology2/include/components/mfcc/mel80_compress.conf +++ b/tools/topology/topology2/include/components/mfcc/mel80_compress.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 26-May-2026 +# Exported MFCC configuration 03-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " diff --git a/tools/topology/topology2/include/components/mfcc/mel80_compress_dtx.conf b/tools/topology/topology2/include/components/mfcc/mel80_compress_dtx.conf index d225811ca4d1..14b2aaaac2fd 100644 --- a/tools/topology/topology2/include/components/mfcc/mel80_compress_dtx.conf +++ b/tools/topology/topology2/include/components/mfcc/mel80_compress_dtx.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 26-May-2026 +# Exported MFCC configuration 03-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " From 66bb61c9c8fb6e400f8f6c5d1eced9cc171ca7ae Mon Sep 17 00:00:00 2001 From: Seppo Ingalsuo Date: Fri, 11 Sep 2026 15:11:06 +0300 Subject: [PATCH 09/35] audio: mfcc: set DP schedule period from FFT hop cadence The module_adapter DP period helper derives its scheduling period from the sink's rate and free space. For MFCC the sink is a phrase-detect / feature stream whose rate is not yet propagated at prepare time, so the derived period would be bogus (or zero) and the DP thread would be scheduled at the wrong cadence. Compute dev->period from the FFT hop size and source rate directly in mfcc_prepare(). This gives module_adapter a valid override before it inspects the sinks, and matches the natural cadence at which the MFCC component produces feature frames. Signed-off-by: Seppo Ingalsuo --- src/audio/mfcc/mfcc.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/audio/mfcc/mfcc.c b/src/audio/mfcc/mfcc.c index 88d5d4863d31..bc57d2e8ace4 100644 --- a/src/audio/mfcc/mfcc.c +++ b/src/audio/mfcc/mfcc.c @@ -241,12 +241,23 @@ static int mfcc_prepare(struct processing_module *mod, /* Initialize MFCC, max_frames is set to dev->frames + 4 */ if (cd->config && data_size > 0) { - ret = mfcc_setup(mod, dev->frames + 4, audio_stream_get_rate(&sourceb->stream), + uint32_t src_rate = audio_stream_get_rate(&sourceb->stream); + + ret = mfcc_setup(mod, dev->frames + 4, src_rate, audio_stream_get_channels(&sourceb->stream)); if (ret < 0) { comp_err(dev, "setup failed."); return ret; } + + /* Set DP scheduling period from FFT hop cadence; sink rate is + * not yet propagated at prepare time so module_adapter cannot + * derive it from mod->sinks[]. + */ + if (src_rate && cd->state.fft.fft_hop_size) + dev->period = (uint32_t)(1000000ULL * + cd->state.fft.fft_hop_size / + src_rate); } else { comp_err(dev, "configuration is missing."); return -EINVAL; From ba903bfd1bc3c595e9ae3ba04d6074abc6a750aa Mon Sep 17 00:00:00 2001 From: Seppo Ingalsuo Date: Fri, 11 Sep 2026 15:11:15 +0300 Subject: [PATCH 10/35] audio: module_adapter: skip zero-rate sinks in DP period calc module_adapter_calculate_dp_period() divides by sink_get_frame_bytes() * sink_get_rate() for every sink of the module. Phrase-detect / event modules such as microWakeWord expose sinks that carry no audio data: their rate and frame size are zero, which crashes the DP scheduler with a divide-by-zero. Skip any sink whose frame_bytes or rate is zero. Modules with such sinks are expected to set dev->period themselves (as MFCC now does based on its FFT hop cadence). Signed-off-by: Seppo Ingalsuo --- src/audio/module_adapter/module_adapter.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index 98234daa082c..b890e04aa9a7 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -388,12 +388,21 @@ static void module_adapter_calculate_dp_period(struct comp_dev *dev) unsigned int period = UINT32_MAX; for (int i = 0; i < mod->num_of_sinks; i++) { + unsigned int frame_bytes = sink_get_frame_bytes(mod->sinks[i]); + unsigned int rate = sink_get_rate(mod->sinks[i]); + + /* Skip sinks that don't produce audio data (e.g. phrase detect + * modules emit events, no rate/frame bytes) to avoid divide by + * zero. The module is expected to set dev->period itself. + */ + if (!frame_bytes || !rate) + continue; + /* calculate time required the module to provide OBS data portion - a period * use 64bit integers to avoid overflows */ unsigned int sink_period = 1000000ULL * sink_get_min_free_space(mod->sinks[i]) / - (sink_get_frame_bytes(mod->sinks[i]) * - sink_get_rate(mod->sinks[i])); + (frame_bytes * rate); /* note the minimal period for the module */ if (period > sink_period) period = sink_period; From 4e1c2630e25e819dd2c604644aa5ba6fd94c1865 Mon Sep 17 00:00:00 2001 From: Seppo Ingalsuo Date: Fri, 11 Sep 2026 15:11:24 +0300 Subject: [PATCH 11/35] sink: guard sink_get_free_frames against zero frame size sink_get_free_frames() unconditionally divided by sink_get_frame_bytes(), assuming the format had been fully propagated by the time a module queried the sink. That does not hold for component-to-component sinks whose format is set only after the upstream component finishes its own prepare (e.g. SRC->KPB before KPB publishes its input buffer format), leading to a divide-by-zero. Return 0 when frame_bytes is zero, mirroring the guard that already exists in source_get_data_frames_available(). Signed-off-by: Seppo Ingalsuo --- src/module/audio/sink_api.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/module/audio/sink_api.c b/src/module/audio/sink_api.c index 969b37f5a987..d3b856e7dcc2 100644 --- a/src/module/audio/sink_api.c +++ b/src/module/audio/sink_api.c @@ -109,12 +109,16 @@ EXPORT_SYMBOL(sink_get_frame_bytes); size_t sink_get_free_frames(struct sof_sink *sink) { - /* The frame size is a valid divisor: a host channel count of zero is - * rejected at module init (module_adapter_ipc4.c) before it reaches - * the stream, and the sample size is fixed by a valid frame format, so - * this is not re-checked on the hot path. + uint32_t frame_bytes = sink_get_frame_bytes(sink); + + /* frame_bytes can be 0 on a component-to-component sink whose format + * hasn't been propagated yet (e.g. SRC->KPB before the KPB side sets + * its input buffer format). Mirror source_get_data_frames_available(). */ - return sink_get_free_size(sink) / sink_get_frame_bytes(sink); + if (frame_bytes > 0) + return sink_get_free_size(sink) / frame_bytes; + else + return 0; } EXPORT_SYMBOL(sink_get_free_frames); From 28ebb96a8f46429749bfdc6e5e9c365fb6ed2e10 Mon Sep 17 00:00:00 2001 From: Seppo Ingalsuo Date: Fri, 11 Sep 2026 15:11:49 +0300 Subject: [PATCH 12/35] vregion: include llext/symbol.h for EXPORT_SYMBOL vregion.c uses EXPORT_SYMBOL() but only picked the macro up indirectly through other headers. When those headers stop pulling llext/symbol.h in (e.g. depending on Kconfig knobs) the build fails with an implicit declaration. Include the header directly. Signed-off-by: Seppo Ingalsuo --- zephyr/lib/vregion.c | 1 + 1 file changed, 1 insertion(+) diff --git a/zephyr/lib/vregion.c b/zephyr/lib/vregion.c index 1653d48a9a66..41831ed7ef01 100644 --- a/zephyr/lib/vregion.c +++ b/zephyr/lib/vregion.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include From 5ef344baa021125c07147ececa0d5f43ca9e810e Mon Sep 17 00:00:00 2001 From: Seppo Ingalsuo Date: Fri, 11 Sep 2026 15:12:01 +0300 Subject: [PATCH 13/35] audio: mfcc: tune: disable VAD control updates for mel40_10ms_compress The mel40_10ms_compress profile is used by the microWakeWord capture pipelines, where the MFCC widget lives inside a compress/encoder stream. The kernel currently cannot map an encoder-hosted MFCC back to a topology widget for SOF_IPC4_MODULE_NOTIFICATION events, so every VAD state change floods the mailbox with unmatched notifications and eventually times out MOD_SET_DX at teardown. Set update_controls = false for this profile so the firmware does not emit those notifications at all. Regenerate the exported blob (byte 142 flips from 0x01 to 0x00). Other profiles that ship the same date comment are refreshed by the same octave run and are otherwise unchanged. Signed-off-by: Seppo Ingalsuo --- src/audio/mfcc/tune/setup_mfcc.m | 1 + .../include/components/mfcc/ceps13_compress_dtx.conf | 2 +- tools/topology/topology2/include/components/mfcc/default.conf | 2 +- tools/topology/topology2/include/components/mfcc/mel40.conf | 2 +- .../topology2/include/components/mfcc/mel40_10ms.conf | 2 +- .../include/components/mfcc/mel40_10ms_compress.conf | 4 ++-- .../topology2/include/components/mfcc/mel40_compress.conf | 2 +- tools/topology/topology2/include/components/mfcc/mel80.conf | 2 +- .../topology2/include/components/mfcc/mel80_compress.conf | 2 +- .../topology2/include/components/mfcc/mel80_compress_dtx.conf | 2 +- 10 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/audio/mfcc/tune/setup_mfcc.m b/src/audio/mfcc/tune/setup_mfcc.m index 13ffea4955ec..e8578a264826 100644 --- a/src/audio/mfcc/tune/setup_mfcc.m +++ b/src/audio/mfcc/tune/setup_mfcc.m @@ -72,6 +72,7 @@ function setup_mfcc() setup.low_freq = 125; setup.high_freq = 7500; setup.compress_output = true; + setup.update_controls = false; setup.tplg_fn = 'mel40_10ms_compress.conf'; export_mfcc_setup(gen_cfg, setup); diff --git a/tools/topology/topology2/include/components/mfcc/ceps13_compress_dtx.conf b/tools/topology/topology2/include/components/mfcc/ceps13_compress_dtx.conf index 6c0c52ba09e5..ce4b7b65c1e5 100644 --- a/tools/topology/topology2/include/components/mfcc/ceps13_compress_dtx.conf +++ b/tools/topology/topology2/include/components/mfcc/ceps13_compress_dtx.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 03-Sep-2026 +# Exported MFCC configuration 11-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " diff --git a/tools/topology/topology2/include/components/mfcc/default.conf b/tools/topology/topology2/include/components/mfcc/default.conf index 7284f8209454..eaf74a2b7390 100644 --- a/tools/topology/topology2/include/components/mfcc/default.conf +++ b/tools/topology/topology2/include/components/mfcc/default.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 03-Sep-2026 +# Exported MFCC configuration 11-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " diff --git a/tools/topology/topology2/include/components/mfcc/mel40.conf b/tools/topology/topology2/include/components/mfcc/mel40.conf index 79ecd2b74e62..f18ad76f31b9 100644 --- a/tools/topology/topology2/include/components/mfcc/mel40.conf +++ b/tools/topology/topology2/include/components/mfcc/mel40.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 03-Sep-2026 +# Exported MFCC configuration 11-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " diff --git a/tools/topology/topology2/include/components/mfcc/mel40_10ms.conf b/tools/topology/topology2/include/components/mfcc/mel40_10ms.conf index cca9bba95d1b..bf2bdd39bb5f 100644 --- a/tools/topology/topology2/include/components/mfcc/mel40_10ms.conf +++ b/tools/topology/topology2/include/components/mfcc/mel40_10ms.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 03-Sep-2026 +# Exported MFCC configuration 11-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " diff --git a/tools/topology/topology2/include/components/mfcc/mel40_10ms_compress.conf b/tools/topology/topology2/include/components/mfcc/mel40_10ms_compress.conf index 26769e6538fb..232e655df15c 100644 --- a/tools/topology/topology2/include/components/mfcc/mel40_10ms_compress.conf +++ b/tools/topology/topology2/include/components/mfcc/mel40_10ms_compress.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 03-Sep-2026 +# Exported MFCC configuration 11-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " @@ -19,6 +19,6 @@ Object.Base.data."mfcc_config" { 0x00,0x00,0x28,0x00,0x00,0x00,0x00,0x04, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01, - 0x01,0x00,0x00,0x01,0x01,0x00,0x01,0x01, + 0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x01, 0x00,0x00,0x00,0x00" } diff --git a/tools/topology/topology2/include/components/mfcc/mel40_compress.conf b/tools/topology/topology2/include/components/mfcc/mel40_compress.conf index cb9f76105869..ac3627d2ae49 100644 --- a/tools/topology/topology2/include/components/mfcc/mel40_compress.conf +++ b/tools/topology/topology2/include/components/mfcc/mel40_compress.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 03-Sep-2026 +# Exported MFCC configuration 11-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " diff --git a/tools/topology/topology2/include/components/mfcc/mel80.conf b/tools/topology/topology2/include/components/mfcc/mel80.conf index fd2023c7f408..f1d98ea5c002 100644 --- a/tools/topology/topology2/include/components/mfcc/mel80.conf +++ b/tools/topology/topology2/include/components/mfcc/mel80.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 03-Sep-2026 +# Exported MFCC configuration 11-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " diff --git a/tools/topology/topology2/include/components/mfcc/mel80_compress.conf b/tools/topology/topology2/include/components/mfcc/mel80_compress.conf index c219b406f70c..a56c1b60e335 100644 --- a/tools/topology/topology2/include/components/mfcc/mel80_compress.conf +++ b/tools/topology/topology2/include/components/mfcc/mel80_compress.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 03-Sep-2026 +# Exported MFCC configuration 11-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " diff --git a/tools/topology/topology2/include/components/mfcc/mel80_compress_dtx.conf b/tools/topology/topology2/include/components/mfcc/mel80_compress_dtx.conf index 14b2aaaac2fd..2264afbd1086 100644 --- a/tools/topology/topology2/include/components/mfcc/mel80_compress_dtx.conf +++ b/tools/topology/topology2/include/components/mfcc/mel80_compress_dtx.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 03-Sep-2026 +# Exported MFCC configuration 11-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " From 3c1d07ce3f97527298df7fa8bdc7f7a27e11ecb8 Mon Sep 17 00:00:00 2001 From: Seppo Ingalsuo Date: Wed, 9 Sep 2026 14:16:50 +0300 Subject: [PATCH 14/35] audio: pipeline: guard against NULL source or sink in pipeline_copy() When a pipeline is being stopped or unbound, the component unbind handler may clear pipeline->source_comp before the low-latency copy task finishes its final execution tick. Attempting to access p->source_comp->direction without checking for NULL triggers a synchronous PIF exception (DSP panic). Add NULL checks for p->source_comp and start before proceeding with the pipeline graph copy walk, returning 0 cleanly if the pipeline endpoints are no longer valid. Signed-off-by: Seppo Ingalsuo --- src/audio/pipeline/pipeline-stream.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/audio/pipeline/pipeline-stream.c b/src/audio/pipeline/pipeline-stream.c index 5fd795e818db..d3cb59026c3f 100644 --- a/src/audio/pipeline/pipeline-stream.c +++ b/src/audio/pipeline/pipeline-stream.c @@ -179,6 +179,11 @@ int pipeline_copy(struct pipeline *p) PPL_LOCK(p->core); + if (!p->source_comp) { + PPL_UNLOCK(); + return 0; + } + if (p->source_comp->direction == SOF_IPC_STREAM_PLAYBACK) { dir = PPL_DIR_UPSTREAM; start = p->sink_comp; @@ -187,6 +192,11 @@ int pipeline_copy(struct pipeline *p) start = p->source_comp; } + if (!start) { + PPL_UNLOCK(); + return 0; + } + data.start = start; data.p = p; From 3cca033245aaff8052d469c3aa4cb087b2d7bfe2 Mon Sep 17 00:00:00 2001 From: Seppo Ingalsuo Date: Wed, 23 Sep 2026 18:19:12 +0300 Subject: [PATCH 15/35] math: audio: add PCAN (Per-Channel AGC Normalization) support Add PCAN (Per-Channel AGC Normalization) fixed-point math library for normalizing mel filterbank energies. PCAN estimates per-channel noise background, applies non-linear wide-dynamic-range AGC compression with fixed-point LUT gain lookup, and delivers scaled features for streaming keyword-spotting networks. Include ztest test suite with MATLAB reference models covering gain lookup, noise estimation, dynamic shrinking, stream processing, and corner cases. Also add tflite-micro dependency to west.yml. Signed-off-by: Seppo Ingalsuo --- src/include/sof/math/auditory.h | 4 + src/include/sof/math/pcan.h | 135 ++++++++++ src/math/CMakeLists.txt | 4 + src/math/Kconfig | 11 + src/math/auditory/mel_filterbank_32.c | 77 +++++- src/math/pcan/CMakeLists.txt | 37 +++ src/math/pcan/README.md | 105 ++++++++ src/math/pcan/pcan.c | 246 ++++++++++++++++++ test/cmocka/src/math/auditory/CMakeLists.txt | 1 + .../unit/math/advanced/pcan/CMakeLists.txt | 61 +++++ .../math/advanced/pcan/pcan_gain_lookup.m | 77 ++++++ .../math/advanced/pcan/pcan_noise_estimate.m | 37 +++ .../unit/math/advanced/pcan/pcan_process.m | 55 ++++ .../unit/math/advanced/pcan/pcan_shrink.m | 41 +++ .../ztest/unit/math/advanced/pcan/pcan_test.c | 173 ++++++++++++ .../advanced/pcan/pcan_wide_dynamic_func.m | 56 ++++ test/ztest/unit/math/advanced/pcan/prj.conf | 2 + test/ztest/unit/math/advanced/pcan/ref_pcan.m | 212 +++++++++++++++ .../math/advanced/pcan/ref_pcan_corners.h | 28 ++ .../unit/math/advanced/pcan/ref_pcan_func.h | 47 ++++ .../unit/math/advanced/pcan/ref_pcan_lut.h | 45 ++++ .../unit/math/advanced/pcan/ref_pcan_stream.h | 132 ++++++++++ .../unit/math/advanced/pcan/testcase.yaml | 14 + west.yml | 8 + 24 files changed, 1594 insertions(+), 14 deletions(-) create mode 100644 src/include/sof/math/pcan.h create mode 100644 src/math/pcan/CMakeLists.txt create mode 100644 src/math/pcan/README.md create mode 100644 src/math/pcan/pcan.c create mode 100644 test/ztest/unit/math/advanced/pcan/CMakeLists.txt create mode 100644 test/ztest/unit/math/advanced/pcan/pcan_gain_lookup.m create mode 100644 test/ztest/unit/math/advanced/pcan/pcan_noise_estimate.m create mode 100644 test/ztest/unit/math/advanced/pcan/pcan_process.m create mode 100644 test/ztest/unit/math/advanced/pcan/pcan_shrink.m create mode 100644 test/ztest/unit/math/advanced/pcan/pcan_test.c create mode 100644 test/ztest/unit/math/advanced/pcan/pcan_wide_dynamic_func.m create mode 100644 test/ztest/unit/math/advanced/pcan/prj.conf create mode 100644 test/ztest/unit/math/advanced/pcan/ref_pcan.m create mode 100644 test/ztest/unit/math/advanced/pcan/ref_pcan_corners.h create mode 100644 test/ztest/unit/math/advanced/pcan/ref_pcan_func.h create mode 100644 test/ztest/unit/math/advanced/pcan/ref_pcan_lut.h create mode 100644 test/ztest/unit/math/advanced/pcan/ref_pcan_stream.h create mode 100644 test/ztest/unit/math/advanced/pcan/testcase.yaml diff --git a/src/include/sof/math/auditory.h b/src/include/sof/math/auditory.h index a68f0468d591..32c07803c613 100644 --- a/src/include/sof/math/auditory.h +++ b/src/include/sof/math/auditory.h @@ -111,4 +111,8 @@ void psy_apply_mel_filterbank_16(struct psy_mel_filterbank *mel_fb, struct icomp void psy_apply_mel_filterbank_32(struct psy_mel_filterbank *mel_fb, struct icomplex32 *fft_out, int32_t *power_spectra, int32_t *mel_log, int bitshift); +void psy_apply_mel_filterbank_with_linear_32(struct psy_mel_filterbank *mel_fb, struct icomplex32 *fft_out, + int32_t *power_spectra, int32_t *mel_log, + uint32_t *mel_linear, int bitshift); + #endif /* __SOF_MATH_AUDITORY_H__ */ diff --git a/src/include/sof/math/pcan.h b/src/include/sof/math/pcan.h new file mode 100644 index 000000000000..494e193026b4 --- /dev/null +++ b/src/include/sof/math/pcan.h @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. All rights reserved. + * + * Author: Antigravity AI & SOF Team + */ + +/** + * \file include/sof/math/pcan.h + * \brief Per-Channel AGC Normalization (PCAN) library interface + * + * Wraps and integrates Google's upstream TFLite Micro microfrontend + * library (under Apache-2.0 with patent grant protection). + */ + +#ifndef __SOF_MATH_PCAN_H__ +#define __SOF_MATH_PCAN_H__ + +#include +#include +#include + +#include "tensorflow/lite/experimental/microfrontend/lib/bits.h" +#include "tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control.h" +#include "tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control_util.h" +#include "tensorflow/lite/experimental/microfrontend/lib/noise_reduction.h" +#include "tensorflow/lite/experimental/microfrontend/lib/noise_reduction_util.h" +#include "tensorflow/lite/experimental/microfrontend/lib/log_scale.h" + +#define PCAN_SNR_BITS kPcanSnrBits /* 12 */ +#define PCAN_OUTPUT_BITS kPcanOutputBits /* 6 */ +#define PCAN_SMOOTHING_COEF_BITS kNoiseReductionBits /* 14 */ +#define PCAN_WIDE_DYNAMIC_BITS kWideDynamicFunctionBits /* 32 */ +#define PCAN_LUT_SIZE kWideDynamicFunctionLUTSize /* 125 */ + +/** + * \brief SOF PCAN configuration structure. + */ +struct pcan_config { + float strength; /**< Exponent alpha (e.g. 0.95) */ + float offset; /**< Additive offset delta (e.g. 80.0) */ + int32_t gain_bits; /**< Gain bit shift (e.g. 21) */ + uint16_t smoothing_coef; /**< IIR smoothing coefficient in Q14 (e.g. 819 for 0.05) */ + uint16_t smoothing_bits; /**< Bit shift for noise estimate (e.g. 10) */ + int32_t input_correction_bits; /**< Input correction bits shift (e.g. 0) */ + bool enable_pcan; /**< Enable flag */ +}; + +/** + * \brief SOF PCAN runtime state wrapper. + */ +struct pcan_state { + struct PcanGainControlState g_pcan; /**< Google upstream PCAN state */ + struct NoiseReductionState g_noise; /**< Google upstream Noise Reduction state */ + struct LogScaleState g_log_scale; /**< Google upstream LogScale state */ + uint32_t *noise_estimate; /**< Pointer to noise estimate buffer */ + int16_t *gain_lut; /**< Pointer to gain LUT */ + int num_channels; /**< Number of channels */ + int32_t snr_shift; /**< SNR bit shift */ + uint16_t smoothing_bits; /**< Smoothing bits */ + bool enable_pcan; /**< PCAN enabled */ + bool allocated_noise; /**< Internally allocated noise buffer */ + bool allocated_lut; /**< Internally allocated gain LUT buffer */ +}; + +/** + * \brief Evaluate Google's WideDynamicFunction. + */ +static inline int16_t pcan_wide_dynamic_function(uint32_t x, const int16_t *lut) +{ + return WideDynamicFunction(x, lut); +} + +/** + * \brief Evaluate Google's PcanShrink. + */ +static inline uint32_t pcan_shrink(uint32_t x) +{ + return PcanShrink(x); +} + +/** + * \brief Compute single point in continuous PCAN gain curve. + */ +int16_t pcan_gain_lookup_function(float strength, float offset, int32_t gain_bits, + int32_t input_bits, uint32_t x); + +/** + * \brief Fill 125-entry gain lookup table for WideDynamicFunction. + */ +int pcan_compute_lut(float strength, float offset, int32_t gain_bits, + int32_t input_bits, int16_t *gain_lut); + +/** + * \brief Initialize PCAN state with supplied buffers or allocate them. + */ +int pcan_populate_state(const struct pcan_config *config, struct pcan_state *state, + uint32_t *noise_estimate_buffer, int16_t *gain_lut_buffer, + int num_channels, uint16_t smoothing_bits, + int32_t input_correction_bits); + +/** + * \brief Free allocated resources in PCAN state. + */ +void pcan_free_state(struct pcan_state *state); + +/** + * \brief Reset PCAN temporal state. + */ +void pcan_reset(struct pcan_state *state); + +/** + * \brief Apply noise reduction and update noise estimate using Google's NoiseReductionApply. + */ +static inline void pcan_noise_reduction(struct pcan_state *state, uint32_t *signal) +{ + if (state && state->enable_pcan) + NoiseReductionApply(&state->g_noise, signal); +} + +/** + * \brief Apply PCAN gain control and compression using Google's PcanGainControlApply. + */ +static inline void pcan_apply(struct pcan_state *state, uint32_t *signal) +{ + if (state && state->enable_pcan) + PcanGainControlApply(&state->g_pcan, signal); +} + +/** + * \brief Apply Google's fixed-point logarithm and scale to signal in-place. + */ +void pcan_log_scale(struct pcan_state *state, uint32_t *signal); + +#endif /* __SOF_MATH_PCAN_H__ */ diff --git a/src/math/CMakeLists.txt b/src/math/CMakeLists.txt index a52263006295..4f950725e3d5 100644 --- a/src/math/CMakeLists.txt +++ b/src/math/CMakeLists.txt @@ -83,6 +83,10 @@ if(CONFIG_MATH_AUDITORY) add_subdirectory(auditory) endif() +if(CONFIG_MATH_PCAN) + add_subdirectory(pcan) +endif() + if(CONFIG_MATH_DCT) list(APPEND base_files dct.c) endif() diff --git a/src/math/Kconfig b/src/math/Kconfig index 1feaab8f03d1..934a6a9776e6 100644 --- a/src/math/Kconfig +++ b/src/math/Kconfig @@ -287,6 +287,17 @@ config MATH_32BIT_MEL_FILTERBANK endmenu +config MATH_PCAN + bool "PCAN (Per-Channel AGC Normalization) library" + default n + select BINARY_LOGARITHM_FIXED + select MATH_EXP + select NATURAL_LOGARITHM_FIXED + help + Select this to build PCAN (Per-Channel Automatic Gain Control Normalization) + library. PCAN performs dynamic per-channel gain control and piecewise polynomial + root compression across time to suppress stationary noise and enhance acoustic transients. + config MATH_DCT bool "DCT transform library" default n diff --git a/src/math/auditory/mel_filterbank_32.c b/src/math/auditory/mel_filterbank_32.c index 414ddf482f93..18244cf64990 100644 --- a/src/math/auditory/mel_filterbank_32.c +++ b/src/math/auditory/mel_filterbank_32.c @@ -9,10 +9,23 @@ #include #include #include +#include #include -void psy_apply_mel_filterbank_32(struct psy_mel_filterbank *fb, struct icomplex32 *fft_out, - int32_t *power_spectra, int32_t *mel_log, int bitshift) +static inline uint32_t mel_sqrt32(uint32_t num) +{ + if (num == 0) + return 0; + + /* sofm_sqrt_int32 treats input as Q2.30, returning sqrt(n)*2^15. + * Scale down by 2^15 with rounding to obtain integer sqrt(num). + */ + return (uint32_t)((sofm_sqrt_int32((int32_t)num) + (1 << 14)) >> 15); +} + +void psy_apply_mel_filterbank_with_linear_32(struct psy_mel_filterbank *fb, struct icomplex32 *fft_out, + int32_t *power_spectra, int32_t *mel_log, + uint32_t *mel_linear, int bitshift) { int64_t pmax; int64_t p; @@ -67,20 +80,56 @@ void psy_apply_mel_filterbank_32(struct psy_mel_filterbank *fb, struct icomplex3 */ log_arg = sat_int32(Q_SHIFT_RND(p, 45, 25)); log_arg = MAX(log_arg, AUDITORY_EPS_Q31); - log = base2_logarithm((uint32_t)log_arg); - log -= AUDITORY_LOG2_2P25_Q16; - /* Compensate Mel triangles scale */ - log += fb->scale_log2; + if (mel_linear) { + /* Compensate dynamic lshift and FFT bitshift so mel_linear reflects + * true acoustic magnitude calibrated to Google microfrontend range. + */ + uint32_t s = mel_sqrt32((uint32_t)log_arg); + /* Total power shift applied was: lshift + 2 * bitshift */ + int neg_shift = -((int32_t)lshift + 2 * bitshift); + int int_shift = neg_shift >> 1; + int frac_shift = neg_shift & 1; - /* Subtract the applied lshift for power spectra - * log2(x * 2^(-n)) = log2(x) - n. Note that the bitshift need to be subtracted - * as doubled because it was applied in linear domain, from log(x * 2^(-2 * n)) - */ - log -= ((int32_t)lshift + 2 * bitshift) << 16; + uint64_t s_comp = s; + if (frac_shift) + s_comp = (s_comp * 46341U) >> 15; /* 46341 / 32768 ~= sqrt(2) */ + + if (int_shift > 0) + s_comp <<= int_shift; + else if (int_shift < 0) + s_comp >>= -int_shift; + + /* Scale to Google microfrontend range: 25826 / 32768 ~= 0.788 */ + s_comp = (s_comp * 25826U) >> 15; + if (s_comp > 65535U) + s_comp = 65535U; - /* Scale for desired log, output as Q9.23 */ - log = Q_MULTSR_32X32((int64_t)log, fb->log_mult, 16, 29, 23); - mel_log[i] = log; /* Q9.23 */ + mel_linear[i] = (uint32_t)s_comp; + } + + if (mel_log) { + log = base2_logarithm((uint32_t)log_arg); + log -= AUDITORY_LOG2_2P25_Q16; + + /* Compensate Mel triangles scale */ + log += fb->scale_log2; + + /* Subtract the applied lshift for power spectra + * log2(x * 2^(-n)) = log2(x) - n. Note that the bitshift need to be subtracted + * as doubled because it was applied in linear domain, from log(x * 2^(-2 * n)) + */ + log -= ((int32_t)lshift + 2 * bitshift) << 16; + + /* Scale for desired log, output as Q9.23 */ + log = Q_MULTSR_32X32((int64_t)log, fb->log_mult, 16, 29, 23); + mel_log[i] = log; /* Q9.23 */ + } } } + +void psy_apply_mel_filterbank_32(struct psy_mel_filterbank *fb, struct icomplex32 *fft_out, + int32_t *power_spectra, int32_t *mel_log, int bitshift) +{ + psy_apply_mel_filterbank_with_linear_32(fb, fft_out, power_spectra, mel_log, NULL, bitshift); +} diff --git a/src/math/pcan/CMakeLists.txt b/src/math/pcan/CMakeLists.txt new file mode 100644 index 000000000000..0d94a9d79075 --- /dev/null +++ b/src/math/pcan/CMakeLists.txt @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: BSD-3-Clause + +if(NOT DEFINED TFLM_PATH) + if(EXISTS "${sof_top_dir}/../tflite-micro") + set(TFLM_PATH "${sof_top_dir}/../tflite-micro") + elseif(EXISTS "${PROJECT_SOURCE_DIR}/../tflite-micro") + set(TFLM_PATH "${PROJECT_SOURCE_DIR}/../tflite-micro") + elseif(EXISTS "${CMAKE_CURRENT_LIST_DIR}/../../../../tflite-micro") + set(TFLM_PATH "${CMAKE_CURRENT_LIST_DIR}/../../../../tflite-micro") + endif() +endif() + +set(MICROFRONTEND_DIR ${TFLM_PATH}/tensorflow/lite/experimental/microfrontend/lib) + +set(base_files + pcan.c + ${MICROFRONTEND_DIR}/pcan_gain_control.c + ${MICROFRONTEND_DIR}/noise_reduction.c + ${MICROFRONTEND_DIR}/noise_reduction_util.c + ${MICROFRONTEND_DIR}/log_scale.c + ${MICROFRONTEND_DIR}/log_lut.c +) + +is_zephyr(zephyr) +if(zephyr) ### Zephyr ### + + zephyr_include_directories(${TFLM_PATH}) + zephyr_library_sources( + ${base_files} + ) + +else() ### library, e.g. testbench or plugin ### + + include_directories(${TFLM_PATH}) + add_local_sources(sof ${base_files}) + +endif() diff --git a/src/math/pcan/README.md b/src/math/pcan/README.md new file mode 100644 index 000000000000..d038b733fa54 --- /dev/null +++ b/src/math/pcan/README.md @@ -0,0 +1,105 @@ +# Per-Channel AGC Normalization (PCAN) Math Library + +This directory contains the **PCAN (Per-Channel Automatic Gain Control Normalization)** library for Sound Open Firmware (SOF), integrating Google's upstream `microfrontend` library from [TFLite Micro](https://github.com/tensorflow/tflite-micro/tree/main/tensorflow/lite/experimental/microfrontend/lib) under the **Apache 2.0 license** with express patent grant protection. + +--- + +## 1. Overview & Mathematical Formulation + +PCAN applies per-channel adaptive dynamic gain control and root dynamic range compression across time to linear filterbank energies (e.g. Mel spectrogram bins), suppressing stationary background noise while enhancing transient acoustic events (speech onsets, keywords). + +``` + +-------------------------------------------------------------+ + | Input Filterbank Energy Matrix | + | E_in[channel, frame] | + +-------------------------------------------------------------+ + | + v ++---------------------------------------------------------------------------------------------+ +| PCAN Processing Stages | +| | +| 1. Temporal Noise Estimation (IIR Smoothing): | +| E_est[k] = ((E_in[k] << smoothing_bits)*alpha_s + E_est[k]*(1 - alpha_s)) >> 14 | +| | +| 2. Piecewise Quadratic Octave Gain Lookup: | +| gain[k] = WideDynamicFunction(E_est[k], gain_lut) | +| - Fast single-cycle CLZ/MSB (AE_NSAU on Tensilica HiFi) | +| - 10-bit fractional index quadratic interpolation | +| | +| 3. Dynamic Gain & Root Polynomial Compression: | +| SNR[k] = ((uint64_t)E_in[k] * gain[k]) >> snr_shift | +| E_out[k] = PcanShrink(SNR[k]) | +| - If SNR[k] < 8192 (2 << 12): SNR[k]^2 >> 20 | +| - If SNR[k] >= 8192: (SNR[k] >> 6) - 64 | ++---------------------------------------------------------------------------------------------+ + | + v + +-------------------------------------------------------------+ + | PCAN-Normalized Feature Tensor | + | E_out[channel, frame] | + +-------------------------------------------------------------+ +``` + +--- + +## 2. Dependencies + +The PCAN math library requires the following tools and repositories: + +| Dependency | Required Version | Description | +|---|---|---| +| **West Manifest (`west.yml`)** | $\ge 0.13$ | Pulls `tflite-micro` workspace dependency via `west update` | +| **TFLite-Micro** | Git revision `e86d97b6` | Upstream Google microfrontend C source files (`pcan_gain_control.c`, `noise_reduction.c`, `bits.h`) | +| **CMake** | $\ge 3.13$ | SOF build and configuration system | +| **Host Toolchain** | GCC $\ge 9.0$ / Clang $\ge 11.0$ | For building host testbench and CMocka unit tests | +| **DSP Toolchains** | Zephyr SDK / Xtensa XCC / Clang | For building target firmware with HiFi3 / HiFi4 / HiFi5 intrinsics | +| **CMocka** | Built with SOF | Unit testing framework for bit-exact validation | +| **GNU Octave** | $\ge 5.0$ | Reference modeling and golden test vector generation | + +--- + +## 3. Build & Test Instructions + +### 3.1 Synchronize Workspace via West +Ensure `tflite-micro` is pulled and synchronized in your workspace: +```bash +cd /path/to/sof-workspace +west update +``` + +### 3.2 Build and Run CMocka Unit Tests (Host) +Configure and run the 5-phase PCAN unit test suite on the host: +```bash +# Configure unit test build +cmake -S sof -B build_ut -DBUILD_UNIT_TESTS=ON -DBUILD_UNIT_TESTS_HOST=ON -DINIT_CONFIG=unit_test_defconfig + +# Build PCAN unit test binary +cmake --build build_ut --target pcan + +# Run PCAN test executable directly +./build_ut/test/cmocka/src/math/pcan/pcan + +# Or run via CTest with full math regression +ctest --test-dir build_ut -R "pcan|auditory|dct|matrix|window|fft" --output-on-failure +``` + +### 3.3 Regenerate Octave Golden Reference Vectors +To re-generate or verify golden test headers from the Octave reference suite: +```bash +cd test/cmocka/src/math/pcan +octave-cli --eval "ref_pcan" +``` +This updates `ref_pcan_lut.h`, `ref_pcan_func.h`, `ref_pcan_stream.h`, and `ref_pcan_corners.h`. + +### 3.4 Build Target Firmware (Zephyr / SOF) +When building SOF for a target DSP platform (e.g. Intel ACE15 / Meteor Lake / Arrow Lake): +```bash +west build -b intel_adsp_ace15_mtpm app +``` + +--- + +## 4. Kconfig & CMake Options + +- `CONFIG_MATH_PCAN`: Enables building the PCAN math library and links Google's microfrontend C source files. +- `CONFIG_COMP_MFCC`: Automatically selects `CONFIG_MATH_PCAN` when MFCC feature extraction is enabled. diff --git a/src/math/pcan/pcan.c b/src/math/pcan/pcan.c new file mode 100644 index 000000000000..eb2fccb6b1c5 --- /dev/null +++ b/src/math/pcan/pcan.c @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. All rights reserved. +// +// Author: Antigravity AI & SOF Team + +#include +#include +#include +#include +#include +#include + +/* ln(2) in Q5.27: round(ln(2) * 2^27) */ +#define PCAN_LN2_Q27 93032640 + +/* Convert PCAN config floats to fixed-point (init-time only). */ +static inline int32_t pcan_strength_q15(float strength) +{ + return (int32_t)(strength * 32768.0f + 0.5f); +} + +static inline int32_t pcan_offset_q7(float offset) +{ + return (int32_t)(offset * 128.0f + 0.5f); +} + +int16_t pcan_gain_lookup_function(float strength, float offset, int32_t gain_bits, + int32_t input_bits, uint32_t x) +{ + const int32_t strength_q15 = pcan_strength_q15(strength); + const int32_t offset_q7 = pcan_offset_q7(offset); + uint64_t u64; + uint32_t u32; + uint32_t ln_u_q27; + int32_t ln_actual_q27; + int64_t s_ln; + int32_t exp_arg_q27; + int32_t exp_arg_ceiling; + int32_t gain_q20; + int32_t gain; + + /* u = x + offset * 2^input_bits, matches upstream's (x_as_float + offset) in fixed-point. */ + u64 = (uint64_t)x + (((uint64_t)offset_q7 << input_bits) >> 7); + if (u64 == 0) + u64 = 1; + u32 = (u64 > 0xFFFFFFFFu) ? 0xFFFFFFFFu : (uint32_t)u64; + + /* ln(u) as UQ5.27. */ + ln_u_q27 = ln_int32(u32); + + /* ln(x_as_float + offset) = ln(u) - input_bits * ln(2), in Q5.27. */ + ln_actual_q27 = (int32_t)ln_u_q27 - input_bits * PCAN_LN2_Q27; + + /* strength * ln(actual): Q1.15 * Q5.27 -> Q6.42, shift down 15 -> Q5.27. */ + s_ln = ((int64_t)strength_q15 * (int64_t)ln_actual_q27) >> 15; + + /* exp_arg = gain_bits * ln(2) - strength * ln(actual), Q5.27. */ + exp_arg_q27 = gain_bits * PCAN_LN2_Q27 - (int32_t)s_ln; + + /* exp(10.3972) = 32767.5, above this the int16 clamp always fires. + * Q5.27: round(10.3972 * 2^27) = 1395385005. + */ + exp_arg_ceiling = 1395385005; + if (exp_arg_q27 >= exp_arg_ceiling) + return INT16_MAX; + if (exp_arg_q27 <= -SOFM_EXP_FIXED_INPUT_MAX) + return 0; + + /* sofm_exp_fixed returns Q12.20 for input Q5.27 in [-16, +7.6246]. Above the + * upper input bound the output saturates. Handle the intermediate range + * [7.6246, 10.3972] with an explicit split so we still get an accurate value. + */ + if (exp_arg_q27 > SOFM_EXP_FIXED_INPUT_MAX) { + /* exp(a + b) = exp(a) * exp(b); pick b = 5.5 in Q5.27. */ + const int32_t half_q27 = 738197504; /* round(5.5 * 2^27) */ + const int32_t half_exp_q20 = 256590991; /* round(exp(5.5) * 2^20) */ + int64_t full_q40; + + gain_q20 = sofm_exp_fixed(exp_arg_q27 - half_q27); + /* Q12.20 * Q12.20 -> Q24.40. Do the >> 20 to Q(?)20 as int64. */ + full_q40 = (int64_t)gain_q20 * half_exp_q20; + /* Round to integer directly to avoid intermediate int32 overflow. */ + gain = (int32_t)((full_q40 + ((int64_t)1 << 39)) >> 40); + if (gain > INT16_MAX) + return INT16_MAX; + if (gain < 0) + return 0; + return (int16_t)gain; + } + + gain_q20 = sofm_exp_fixed(exp_arg_q27); + + /* Round to nearest integer (matching upstream: gain_as_float + 0.5f, then cast). */ + gain = (gain_q20 + (1 << 19)) >> 20; + if (gain > INT16_MAX) + return INT16_MAX; + if (gain < 0) + return 0; + return (int16_t)gain; +} + +int pcan_compute_lut(float strength, float offset, int32_t gain_bits, + int32_t input_bits, int16_t *gain_lut) +{ + int interval; + + if (!gain_lut) + return -EINVAL; + + memset(gain_lut, 0, PCAN_LUT_SIZE * sizeof(int16_t)); + + /* Layout matches upstream PcanGainControlPopulateState(): entries 0 and 1 + * hold the low-x samples, then for intervals 2..kWideDynamicFunctionBits + * the triplet (y0, a1, a2) is stored at (4 * interval - 6, -5, -4). + */ + gain_lut[0] = pcan_gain_lookup_function(strength, offset, gain_bits, input_bits, 0); + gain_lut[1] = pcan_gain_lookup_function(strength, offset, gain_bits, input_bits, 1); + + for (interval = 2; interval <= kWideDynamicFunctionBits; ++interval) { + const uint32_t x0 = (uint32_t)1 << (interval - 1); + const uint32_t x1 = x0 + (x0 >> 1); + const uint32_t x2 = (interval == kWideDynamicFunctionBits) ? + x0 + (x0 - 1) : 2 * x0; + const int16_t y0 = pcan_gain_lookup_function(strength, offset, gain_bits, + input_bits, x0); + const int16_t y1 = pcan_gain_lookup_function(strength, offset, gain_bits, + input_bits, x1); + const int16_t y2 = pcan_gain_lookup_function(strength, offset, gain_bits, + input_bits, x2); + const int32_t diff1 = (int32_t)y1 - y0; + const int32_t diff2 = (int32_t)y2 - y0; + const int32_t a1 = 4 * diff1 - diff2; + const int32_t a2 = diff2 - a1; + + gain_lut[4 * interval - 6] = y0; + gain_lut[4 * interval - 5] = (int16_t)a1; + gain_lut[4 * interval - 4] = (int16_t)a2; + } + + return 0; +} + +int pcan_populate_state(const struct pcan_config *config, struct pcan_state *state, + uint32_t *noise_estimate_buffer, int16_t *gain_lut_buffer, + int num_channels, uint16_t smoothing_bits, + int32_t input_correction_bits) +{ + int ret; + + if (!config || !state || num_channels <= 0) + return -EINVAL; + + memset(state, 0, sizeof(*state)); + state->enable_pcan = config->enable_pcan; + if (!state->enable_pcan) + return 0; + + state->num_channels = num_channels; + state->smoothing_bits = smoothing_bits; + state->snr_shift = config->gain_bits - input_correction_bits - PCAN_SNR_BITS; + if (state->snr_shift < 0) + return -EINVAL; + + if (noise_estimate_buffer) { + state->noise_estimate = noise_estimate_buffer; + } else { + state->noise_estimate = malloc(num_channels * sizeof(uint32_t)); + if (!state->noise_estimate) + return -ENOMEM; + state->allocated_noise = true; + } + memset(state->noise_estimate, 0, num_channels * sizeof(uint32_t)); + + if (gain_lut_buffer) { + state->gain_lut = gain_lut_buffer; + } else { + state->gain_lut = malloc(PCAN_LUT_SIZE * sizeof(int16_t)); + if (!state->gain_lut) { + pcan_free_state(state); + return -ENOMEM; + } + state->allocated_lut = true; + } + + ret = pcan_compute_lut(config->strength, config->offset, config->gain_bits, + (int32_t)smoothing_bits - input_correction_bits, + state->gain_lut); + if (ret < 0) { + pcan_free_state(state); + return ret; + } + + /* Mirror the SOF state into the upstream struct consumed by PcanGainControlApply(). */ + state->g_pcan.enable_pcan = 1; + state->g_pcan.noise_estimate = state->noise_estimate; + state->g_pcan.num_channels = num_channels; + state->g_pcan.gain_lut = state->gain_lut; + state->g_pcan.snr_shift = state->snr_shift; + + state->g_noise.estimate = state->noise_estimate; + state->g_noise.num_channels = num_channels; + state->g_noise.smoothing_bits = smoothing_bits; + state->g_noise.even_smoothing = 410; /* 0.025 * (1 << 14) */ + state->g_noise.odd_smoothing = 983; /* 0.06 * (1 << 14) */ + state->g_noise.min_signal_remaining = 819; /* 0.05 * (1 << 14) */ + + state->g_log_scale.enable_log = 1; + state->g_log_scale.scale_shift = 6; + + return 0; +} + +void pcan_free_state(struct pcan_state *state) +{ + if (!state) + return; + + if (state->allocated_noise) + free(state->noise_estimate); + if (state->allocated_lut) + free(state->gain_lut); + + memset(state, 0, sizeof(*state)); +} + +void pcan_reset(struct pcan_state *state) +{ + if (!state || !state->noise_estimate) + return; + + memset(state->noise_estimate, 0, state->num_channels * sizeof(uint32_t)); +} + +void pcan_log_scale(struct pcan_state *state, uint32_t *signal) +{ + int i; + uint16_t *scaled; + + if (!state || !state->enable_pcan) + return; + + scaled = LogScaleApply(&state->g_log_scale, signal, state->num_channels, 3); + for (i = state->num_channels - 1; i >= 0; --i) + signal[i] = scaled[i]; +} diff --git a/test/cmocka/src/math/auditory/CMakeLists.txt b/test/cmocka/src/math/auditory/CMakeLists.txt index fe200e6b15eb..04b5feec01d2 100644 --- a/test/cmocka/src/math/auditory/CMakeLists.txt +++ b/test/cmocka/src/math/auditory/CMakeLists.txt @@ -9,4 +9,5 @@ cmocka_test(auditory ${PROJECT_SOURCE_DIR}/src/math/base2log.c ${PROJECT_SOURCE_DIR}/src/math/decibels.c ${PROJECT_SOURCE_DIR}/src/math/numbers.c + ${PROJECT_SOURCE_DIR}/src/math/sqrt_int32.c ) diff --git a/test/ztest/unit/math/advanced/pcan/CMakeLists.txt b/test/ztest/unit/math/advanced/pcan/CMakeLists.txt new file mode 100644 index 000000000000..f3bcb5af4910 --- /dev/null +++ b/test/ztest/unit/math/advanced/pcan/CMakeLists.txt @@ -0,0 +1,61 @@ +cmake_minimum_required(VERSION 3.20.0) + +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(test_math_advanced_pcan) + +set(SOF_ROOT "${PROJECT_SOURCE_DIR}/../../../../../..") + +# Set SOF top directory for UUID registry generation +set(sof_top_dir ${SOF_ROOT}) + +# Include SOF CMake utilities for proper SOF source compilation +include(${SOF_ROOT}/scripts/cmake/misc.cmake) +include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) + +# Locate the tflite-micro checkout (west-managed in the SOF workspace) +if(NOT DEFINED TFLM_PATH) + if(EXISTS "${SOF_ROOT}/../tflite-micro") + set(TFLM_PATH "${SOF_ROOT}/../tflite-micro") + elseif(EXISTS "${SOF_ROOT}/../../tflite-micro") + set(TFLM_PATH "${SOF_ROOT}/../../tflite-micro") + endif() +endif() + +if(NOT DEFINED TFLM_PATH OR NOT EXISTS "${TFLM_PATH}") + message(FATAL_ERROR + "tflite-micro not found; expected next to the SOF workspace. " + "Set TFLM_PATH manually or run 'west update' to fetch it.") +endif() + +set(MICROFRONTEND_DIR ${TFLM_PATH}/tensorflow/lite/experimental/microfrontend/lib) + +target_include_directories(app PRIVATE + ${SOF_ROOT}/zephyr/include + ${SOF_ROOT}/src/include + ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/test/cmocka/include + ${TFLM_PATH} + ${PROJECT_BINARY_DIR}/include/generated +) + +target_compile_definitions(app PRIVATE + -DCONFIG_ZEPHYR_POSIX=1 + -DCONFIG_LIBRARY=1 + -DUNIT_TEST=1 +) + +target_sources(app PRIVATE + pcan_test.c + ${SOF_ROOT}/src/math/pcan/pcan.c + ${SOF_ROOT}/src/math/log_e.c + ${SOF_ROOT}/src/math/base2log.c + ${SOF_ROOT}/src/math/exp_fcn.c + ${SOF_ROOT}/src/math/exp_fcn_hifi.c + ${MICROFRONTEND_DIR}/pcan_gain_control.c + ${MICROFRONTEND_DIR}/noise_reduction.c + ${MICROFRONTEND_DIR}/noise_reduction_util.c + ${MICROFRONTEND_DIR}/log_scale.c + ${MICROFRONTEND_DIR}/log_lut.c +) + +sof_append_relative_path_definitions(app) diff --git a/test/ztest/unit/math/advanced/pcan/pcan_gain_lookup.m b/test/ztest/unit/math/advanced/pcan/pcan_gain_lookup.m new file mode 100644 index 000000000000..68fc83f246b6 --- /dev/null +++ b/test/ztest/unit/math/advanced/pcan/pcan_gain_lookup.m @@ -0,0 +1,77 @@ +% pcan_gain_lookup - Compute PCAN continuous gain and generate LUT +% +% SPDX-License-Identifier: BSD-3-Clause +% Copyright(c) 2026 Intel Corporation. All rights reserved. + +function [gain_lut, y_lut] = pcan_gain_lookup(config, input_bits) +% Inputs: +% config.strength - Exponent alpha (e.g., 0.95) +% config.offset - Additive constant delta (e.g., 80.0) +% config.gain_bits - Scale factor exponent (e.g., 21) +% input_bits - smoothing_bits - input_correction_bits +% +% Outputs: +% gain_lut - 125-entry int16 lookup table for WideDynamicFunction +% y_lut - Raw function values at octave evaluation points + + if nargin < 2 + input_bits = 10; + end + + strength = config.strength; + offset = config.offset; + gain_bits = config.gain_bits; + + kWideDynamicFunctionBits = 32; + kWideDynamicFunctionLUTSize = 4 * kWideDynamicFunctionBits - 3; % 125 + + gain_lut = zeros(kWideDynamicFunctionLUTSize, 1, 'int16'); + + % Evaluate point x in gain function + function y = eval_gain(x_val) + x_float = double(x_val) / double(bitshift(uint64(1), input_bits)); + g_float = double(bitshift(uint64(1), gain_bits)) * ((x_float + offset) ^ (-strength)); + if g_float > 32767 + y = int16(32767); + else + y = int16(round(g_float)); + end + end + + gain_lut(1) = eval_gain(0); + gain_lut(2) = eval_gain(1); + + % Intervals 2 through 32 + % In C: lut is offset by -6 so that interval 2 writes to lut[4*2]=lut[8] -> offset 2 in array + for interval = 2:kWideDynamicFunctionBits + x0 = bitshift(uint64(1), interval - 1); + x1 = x0 + bitshift(x0, -1); + if interval == kWideDynamicFunctionBits + x2 = x0 + (x0 - 1); + else + x2 = 2 * x0; + end + + y0 = int32(eval_gain(x0)); + y1 = int32(eval_gain(x1)); + y2 = int32(eval_gain(x2)); + + diff1 = y1 - y0; + diff2 = y2 - y0; + a1 = 4 * diff1 - diff2; + a2 = diff2 - a1; + + % Map to 1-based index in gain_lut: + % In C: index is 4 * interval - 6 (0-based) -> +1 for 1-based + idx = 4 * interval - 5; + + gain_lut(idx) = int16(y0); + gain_lut(idx + 1) = int16(a1); + gain_lut(idx + 2) = int16(a2); + if interval < kWideDynamicFunctionBits + gain_lut(idx + 3) = int16(0); + end + end + + y_lut = gain_lut; +end diff --git a/test/ztest/unit/math/advanced/pcan/pcan_noise_estimate.m b/test/ztest/unit/math/advanced/pcan/pcan_noise_estimate.m new file mode 100644 index 000000000000..b9f340de52df --- /dev/null +++ b/test/ztest/unit/math/advanced/pcan/pcan_noise_estimate.m @@ -0,0 +1,37 @@ +% pcan_noise_estimate - Octave fixed-point model of PCAN temporal IIR noise smoothing +% +% SPDX-License-Identifier: BSD-3-Clause +% Copyright(c) 2026 Intel Corporation. All rights reserved. + +function [estimate_out] = pcan_noise_estimate(estimate_in, signal_in, smoothing_coef, smoothing_bits, coef_bits) +% Inputs: +% estimate_in - Previous noise estimate vector (uint32) +% signal_in - Current frame input energy vector (uint32) +% smoothing_coef - IIR smoothing factor in Q(coef_bits) (default 819 for 0.05 in Q14) +% smoothing_bits - Scale shift for input energy (default 10) +% coef_bits - Number of fractional bits in smoothing_coef (default 14) +% +% Output: +% estimate_out - Updated noise estimate vector (uint32) + + if nargin < 3 + smoothing_coef = 819; % ~0.05 in Q14 (16384 * 0.05 = 819.2) + end + if nargin < 4 + smoothing_bits = 10; + end + if nargin < 5 + coef_bits = 14; + end + + one_minus_coef = bitshift(1, coef_bits) - smoothing_coef; + num_channels = length(signal_in); + estimate_out = zeros(num_channels, 1, 'uint32'); + + for i = 1:num_channels + sig_scaled = bitshift(uint64(signal_in(i)), smoothing_bits); + est_prev = uint64(estimate_in(i)); + est_new = bitshift((sig_scaled * uint64(smoothing_coef)) + (est_prev * uint64(one_minus_coef)), -coef_bits); + estimate_out(i) = uint32(est_new); + end +end diff --git a/test/ztest/unit/math/advanced/pcan/pcan_process.m b/test/ztest/unit/math/advanced/pcan/pcan_process.m new file mode 100644 index 000000000000..d853c117da36 --- /dev/null +++ b/test/ztest/unit/math/advanced/pcan/pcan_process.m @@ -0,0 +1,55 @@ +% pcan_process - Octave model of full PCAN frame processing +% +% SPDX-License-Identifier: BSD-3-Clause +% Copyright(c) 2026 Intel Corporation. All rights reserved. + +function [output_frames, final_noise_estimate] = pcan_process(input_frames, config, smoothing_bits, input_correction_bits) +% Inputs: +% input_frames - [num_channels x num_frames] matrix of uint32 filterbank energies +% config.strength - Exponent alpha (e.g. 0.95) +% config.offset - Additive constant delta (e.g. 80.0) +% config.gain_bits - Gain bit shift (e.g. 21) +% config.smoothing_coef - Smoothing factor in Q14 (e.g. 819) +% smoothing_bits - Smoothing bit shift (e.g. 10) +% input_correction_bits - Input correction shift (e.g. 0) +% +% Outputs: +% output_frames - [num_channels x num_frames] matrix of uint32 PCAN normalized outputs +% final_noise_estimate - [num_channels x 1] final noise estimate state + + if nargin < 3 + smoothing_bits = 10; + end + if nargin < 4 + input_correction_bits = 0; + end + + [num_channels, num_frames] = size(input_frames); + input_bits = smoothing_bits - input_correction_bits; + kPcanSnrBits = 12; + snr_shift = config.gain_bits - input_correction_bits - kPcanSnrBits; + + [gain_lut, ~] = pcan_gain_lookup(config, input_bits); + + noise_estimate = zeros(num_channels, 1, 'uint32'); + output_frames = zeros(num_channels, num_frames, 'uint32'); + + for f = 1:num_frames + sig_in = input_frames(:, f); + + % 1. Update temporal noise estimate + noise_estimate = pcan_noise_estimate(noise_estimate, sig_in, config.smoothing_coef, smoothing_bits, 14); + + % 2. Apply PCAN gain control per channel + frame_out = zeros(num_channels, 1, 'uint32'); + for c = 1:num_channels + gain = pcan_wide_dynamic_func(noise_estimate(c), gain_lut); + snr = bitshift(uint64(sig_in(c)) * uint64(uint16(gain)), -snr_shift); + frame_out(c) = pcan_shrink(uint32(snr), 12, 6); + end + + output_frames(:, f) = frame_out; + end + + final_noise_estimate = noise_estimate; +end diff --git a/test/ztest/unit/math/advanced/pcan/pcan_shrink.m b/test/ztest/unit/math/advanced/pcan/pcan_shrink.m new file mode 100644 index 000000000000..2ebd98736cc9 --- /dev/null +++ b/test/ztest/unit/math/advanced/pcan/pcan_shrink.m @@ -0,0 +1,41 @@ +% pcan_shrink - Octave fixed-point model of PcanShrink piecewise compression +% +% SPDX-License-Identifier: BSD-3-Clause +% Copyright(c) 2026 Intel Corporation. All rights reserved. + +function y = pcan_shrink(x, snr_bits, output_bits) +% Inputs: +% x - 32-bit unsigned SNR input value (scalar or vector) +% snr_bits - Number of fractional bits in SNR (default 12) +% output_bits - Number of fractional bits in output (default 6) +% +% Output: +% y - 32-bit unsigned compressed output value + + if nargin < 2 + snr_bits = 12; + end + if nargin < 3 + output_bits = 6; + end + + threshold = bitshift(uint64(2), snr_bits); % 2 << 12 = 8192 + quadratic_shift = -(2 + 2 * snr_bits - output_bits); % -20 + linear_shift = -(snr_bits - output_bits); % -6 + linear_offset = uint64(bitshift(1, output_bits)); % 64 + + y = zeros(size(x), 'uint32'); + + for k = 1:numel(x) + val = uint64(x(k)); + if val < threshold + % Quadratic compression: x^2 / 4 + prod_val = val * val; + res = bitshift(prod_val, quadratic_shift); + else + % Linear compression: x - 1 + res = bitshift(val, linear_shift) - linear_offset; + end + y(k) = uint32(res); + end +end diff --git a/test/ztest/unit/math/advanced/pcan/pcan_test.c b/test/ztest/unit/math/advanced/pcan/pcan_test.c new file mode 100644 index 000000000000..0dfc6d189acc --- /dev/null +++ b/test/ztest/unit/math/advanced/pcan/pcan_test.c @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. All rights reserved. +// +// These contents may have been developed with support from one or more Intel-operated +// generative artificial intelligence solutions. +// +// Converted from cmocka to Ztest +// Original: test/cmocka/src/math/pcan/pcan_test.c + +#include +#include +#include +#include + +#include + +#include "ref_pcan_lut.h" +#include "ref_pcan_func.h" +#include "ref_pcan_stream.h" +#include "ref_pcan_corners.h" + +/* The SOF PCAN core computes the gain LUT in fixed-point (ln_int32 + + * sofm_exp_fixed) while the Octave reference uses double-precision powf. + * The two agree to within a couple of LSBs on the sampled y-values, but the + * quadratic slope terms a1 = 4*(y1-y0) - (y2-y0) and a2 = -a1 + (y2-y0) + * amplify per-sample rounding by up to ~6x, so allow a bigger LUT tolerance. + */ +#define PCAN_TEST_LSB_TOL 2 +#define PCAN_TEST_LUT_TOL 6 + +#define zassert_int_near(actual, expected, tol) \ + do { \ + long long _a = (long long)(actual); \ + long long _e = (long long)(expected); \ + long long _d = _a - _e; \ + \ + if (_d < 0) \ + _d = -_d; \ + zassert_true(_d <= (long long)(tol), \ + "value %lld deviates from reference %lld by %lld (tol %lld)", \ + _a, _e, _d, (long long)(tol)); \ + } while (0) + +ZTEST(pcan_suite, test_pcan_lut_generation) +{ + int16_t lut[PCAN_LUT_SIZE]; + int ret; + int i; + + ret = pcan_compute_lut(0.95f, 80.0f, 21, 10, lut); + zassert_equal(ret, 0); + + for (i = 0; i < PCAN_TEST_LUT1_SIZE; i++) + zassert_int_near(lut[i], ref_pcan_lut1[i], PCAN_TEST_LUT_TOL); + + ret = pcan_compute_lut(0.8f, 50.0f, 18, 12, lut); + zassert_equal(ret, 0); + + for (i = 0; i < PCAN_TEST_LUT2_SIZE; i++) + zassert_int_near(lut[i], ref_pcan_lut2[i], PCAN_TEST_LUT_TOL); +} + +ZTEST(pcan_suite, test_pcan_wide_dynamic_function) +{ + int16_t lut[PCAN_LUT_SIZE]; + int16_t out; + int ret; + int i; + + ret = pcan_compute_lut(0.95f, 80.0f, 21, 10, lut); + zassert_equal(ret, 0); + + for (i = 0; i < PCAN_TEST_WDF_NUM_POINTS; i++) { + out = pcan_wide_dynamic_function(ref_pcan_wdf_inputs[i], lut); + zassert_int_near(out, ref_pcan_wdf_outputs[i], PCAN_TEST_LSB_TOL); + } +} + +ZTEST(pcan_suite, test_pcan_shrink) +{ + uint32_t out; + int i; + + for (i = 0; i < PCAN_TEST_SHRINK_NUM_POINTS; i++) { + out = pcan_shrink(ref_pcan_shrink_inputs[i]); + zassert_equal(out, ref_pcan_shrink_outputs[i]); + } +} + +ZTEST(pcan_suite, test_pcan_streaming) +{ + struct pcan_config cfg; + struct pcan_state pstate; + uint32_t channel_data[PCAN_STREAM_NUM_CHANNELS]; + int ret; + int f; + int c; + + cfg.strength = 0.95f; + cfg.offset = 80.0f; + cfg.gain_bits = 21; + cfg.smoothing_coef = PCAN_STREAM_SMOOTHING_COEF; + cfg.smoothing_bits = PCAN_STREAM_SMOOTHING_BITS; + cfg.input_correction_bits = PCAN_STREAM_INPUT_CORRECTION_BITS; + cfg.enable_pcan = true; + + ret = pcan_populate_state(&cfg, &pstate, NULL, NULL, + PCAN_STREAM_NUM_CHANNELS, + PCAN_STREAM_SMOOTHING_BITS, + PCAN_STREAM_INPUT_CORRECTION_BITS); + zassert_equal(ret, 0); + + for (f = 0; f < PCAN_STREAM_NUM_FRAMES; f++) { + for (c = 0; c < PCAN_STREAM_NUM_CHANNELS; c++) + channel_data[c] = ref_pcan_stream_inputs[f * PCAN_STREAM_NUM_CHANNELS + c]; + + pcan_noise_reduction(&pstate, channel_data); + pcan_apply(&pstate, channel_data); + pcan_log_scale(&pstate, channel_data); + } + + for (c = 0; c < PCAN_STREAM_NUM_CHANNELS; c++) + zassert_true(pstate.noise_estimate[c] > 0, "noise estimate should be positive"); + + pcan_reset(&pstate); + for (c = 0; c < PCAN_STREAM_NUM_CHANNELS; c++) + zassert_equal(pstate.noise_estimate[c], 0); + + pcan_free_state(&pstate); +} + +ZTEST(pcan_suite, test_pcan_corner_cases) +{ + int16_t lut[PCAN_LUT_SIZE]; + struct pcan_config bad_cfg; + struct pcan_state bad_state; + int16_t wdf_out; + uint32_t shrink_out; + int ret; + int i; + + ret = pcan_compute_lut(0.95f, 80.0f, 21, 10, lut); + zassert_equal(ret, 0); + + for (i = 0; i < PCAN_CORNERS_NUM_POINTS; i++) { + wdf_out = pcan_wide_dynamic_function(ref_pcan_corner_inputs[i], lut); + zassert_int_near(wdf_out, ref_pcan_corner_wdf_outputs[i], PCAN_TEST_LSB_TOL); + + shrink_out = pcan_shrink(ref_pcan_corner_inputs[i]); + zassert_equal(shrink_out, ref_pcan_corner_shrink_outputs[i]); + } + + /* gain_bits too small -> negative snr_shift */ + bad_cfg.strength = 0.95f; + bad_cfg.offset = 80.0f; + bad_cfg.gain_bits = 5; + bad_cfg.smoothing_coef = 819; + bad_cfg.smoothing_bits = 10; + bad_cfg.input_correction_bits = 0; + bad_cfg.enable_pcan = true; + + ret = pcan_populate_state(&bad_cfg, &bad_state, NULL, NULL, 16, 10, 0); + zassert_true(ret < 0); + + ret = pcan_populate_state(NULL, &bad_state, NULL, NULL, 16, 10, 0); + zassert_true(ret < 0); + + ret = pcan_populate_state(&bad_cfg, &bad_state, NULL, NULL, 0, 10, 0); + zassert_true(ret < 0); +} + +ZTEST_SUITE(pcan_suite, NULL, NULL, NULL, NULL, NULL); diff --git a/test/ztest/unit/math/advanced/pcan/pcan_wide_dynamic_func.m b/test/ztest/unit/math/advanced/pcan/pcan_wide_dynamic_func.m new file mode 100644 index 000000000000..1ecf32041efb --- /dev/null +++ b/test/ztest/unit/math/advanced/pcan/pcan_wide_dynamic_func.m @@ -0,0 +1,56 @@ +% pcan_wide_dynamic_func - Octave fixed-point model of WideDynamicFunction +% +% SPDX-License-Identifier: BSD-3-Clause +% Copyright(c) 2026 Intel Corporation. All rights reserved. + +function y = pcan_wide_dynamic_func(x, gain_lut) +% Inputs: +% x - 32-bit unsigned input value (scalar or vector) +% gain_lut - 125-entry int16 lookup table from pcan_gain_lookup +% +% Output: +% y - 16-bit signed interpolated gain factor (same shape as x) + + y = zeros(size(x), 'int16'); + + for k = 1:numel(x) + val = uint32(x(k)); + if val <= 2 + % Directly return lut[0], lut[1], or lut[2] + y(k) = gain_lut(val + 1); + else + % Compute MSB (1 to 32) + clz_val = 0; + tmp = val; + for b = 31:-1:0 + if bitand(tmp, bitshift(uint32(1), b)) ~= 0 + break; + end + clz_val = clz_val + 1; + end + interval = 32 - clz_val; + + % In C: lut pointer is offset by (4 * interval - 6) + % 1-based index for lut[0]: + idx = 4 * interval - 5; + l0 = int32(gain_lut(idx)); + l1 = int32(gain_lut(idx + 1)); + l2 = int32(gain_lut(idx + 2)); + + if interval < 11 + frac = bitand(bitshift(val, 11 - interval), uint32(1023)); % 0x3FF + else + frac = bitand(bitshift(val, -(interval - 11)), uint32(1023)); + end + frac_i32 = int32(frac); + + res = bitshift(l2 * frac_i32, -5); + res = res + bitshift(l1, 5); + res = res * frac_i32; + res = bitshift(res + 16384, -15); % (1 << 14) = 16384 + res = res + l0; + + y(k) = int16(res); + end + end +end diff --git a/test/ztest/unit/math/advanced/pcan/prj.conf b/test/ztest/unit/math/advanced/pcan/prj.conf new file mode 100644 index 000000000000..d34c7781cd0a --- /dev/null +++ b/test/ztest/unit/math/advanced/pcan/prj.conf @@ -0,0 +1,2 @@ +CONFIG_ZTEST=y +CONFIG_SOF_FULL_ZEPHYR_APPLICATION=n diff --git a/test/ztest/unit/math/advanced/pcan/ref_pcan.m b/test/ztest/unit/math/advanced/pcan/ref_pcan.m new file mode 100644 index 000000000000..865d52757840 --- /dev/null +++ b/test/ztest/unit/math/advanced/pcan/ref_pcan.m @@ -0,0 +1,212 @@ +% ref_pcan - Generate C header files for PCAN library unit tests +% +% SPDX-License-Identifier: BSD-3-Clause +% Copyright(c) 2026 Intel Corporation. All rights reserved. + +function ref_pcan() + % Add path to cmocka helper export functions + path(path(), '../../../m'); + + try + opt.describe = export_get_git_describe(); + catch + opt.describe = 'SOF PCAN Reference Generator'; + end + + %% Phase 1: Export Golden LUTs for default and custom configurations + export_ref_lut(opt); + + %% Phase 2 & 3: Export Golden values for WideDynamicFunction and PcanShrink + export_ref_functions(opt); + + %% Phase 4: Export Streaming Multi-Channel Multi-Frame Test Vectors + export_ref_streaming(opt); + + %% Phase 5: Export Corner & Edge Cases + export_ref_corner_cases(opt); + + fprintf(1, 'All PCAN test headers exported successfully.\n'); +end + +function export_ref_lut(opt) + header_fn = 'ref_pcan_lut.h'; + fh = export_headerfile_open(header_fn); + export_comment(fh, sprintf('Generated by ref_pcan.m (%s)', opt.describe)); + + % Config 1: Default TFLM configuration + cfg1.strength = 0.95; + cfg1.offset = 80.0; + cfg1.gain_bits = 21; + input_bits1 = 10; + [lut1, ~] = pcan_gain_lookup(cfg1, input_bits1); + + export_ndefine(fh, 'PCAN_TEST_LUT1_SIZE', length(lut1)); + export_ndefine(fh, 'PCAN_TEST_LUT1_GAIN_BITS', cfg1.gain_bits); + export_ndefine(fh, 'PCAN_TEST_LUT1_INPUT_BITS', input_bits1); + export_vector(fh, 16, 'ref_pcan_lut1', lut1); + + % Config 2: Custom configuration (strength=0.8, offset=50.0, gain_bits=18) + cfg2.strength = 0.8; + cfg2.offset = 50.0; + cfg2.gain_bits = 18; + input_bits2 = 12; + [lut2, ~] = pcan_gain_lookup(cfg2, input_bits2); + + export_ndefine(fh, 'PCAN_TEST_LUT2_SIZE', length(lut2)); + export_ndefine(fh, 'PCAN_TEST_LUT2_GAIN_BITS', cfg2.gain_bits); + export_ndefine(fh, 'PCAN_TEST_LUT2_INPUT_BITS', input_bits2); + export_vector(fh, 16, 'ref_pcan_lut2', lut2); + + fclose(fh); + fprintf(1, 'Exported %s.\n', header_fn); +end + +function export_ref_functions(opt) + header_fn = 'ref_pcan_func.h'; + fh = export_headerfile_open(header_fn); + export_comment(fh, sprintf('Generated by ref_pcan.m (%s)', opt.describe)); + + cfg.strength = 0.95; + cfg.offset = 80.0; + cfg.gain_bits = 21; + input_bits = 10; + [lut, ~] = pcan_gain_lookup(cfg, input_bits); + + % 1. WideDynamicFunction Test Vector: Test various points across all 32 intervals + wdf_inputs = uint32([ ... + 0, 1, 2, 3, 4, 5, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, ... + 255, 256, 511, 512, 1023, 1024, 2047, 2048, 4095, 4096, 8191, 8192, ... + 16383, 16384, 32767, 32768, 65535, 65536, 131071, 131072, ... + 262143, 262144, 524287, 524288, 1048575, 1048576, 2097151, 2097152, ... + 4194303, 4194304, 8388607, 8388608, 16777215, 16777216, ... + 33554431, 33554432, 67108863, 67108864, 134217727, 134217728, ... + 268435455, 268435456, 536870911, 536870912, 1073741823, 1073741824, ... + 2147483647, 2147483648, 4294967295 ... + ]); + wdf_outputs = pcan_wide_dynamic_func(wdf_inputs, lut); + + export_ndefine(fh, 'PCAN_TEST_WDF_NUM_POINTS', length(wdf_inputs)); + export_uint32_vector(fh, 'ref_pcan_wdf_inputs', wdf_inputs); + export_vector(fh, 16, 'ref_pcan_wdf_outputs', wdf_outputs); + + % 2. PcanShrink Test Vector: Sweep below, around (8192), and above threshold + shrink_inputs = uint32([ ... + 0, 1, 10, 100, 500, 1000, 2000, 4000, 8000, 8190, 8191, 8192, 8193, 8194, ... + 10000, 16384, 32768, 65536, 100000, 500000, 1000000, 10000000 ... + ]); + shrink_outputs = pcan_shrink(shrink_inputs, 12, 6); + + export_ndefine(fh, 'PCAN_TEST_SHRINK_NUM_POINTS', length(shrink_inputs)); + export_uint32_vector(fh, 'ref_pcan_shrink_inputs', shrink_inputs); + export_uint32_vector(fh, 'ref_pcan_shrink_outputs', shrink_outputs); + + fclose(fh); + fprintf(1, 'Exported %s.\n', header_fn); +end + +function export_ref_streaming(opt) + header_fn = 'ref_pcan_stream.h'; + fh = export_headerfile_open(header_fn); + export_comment(fh, sprintf('Generated by ref_pcan.m (%s)', opt.describe)); + + cfg.strength = 0.95; + cfg.offset = 80.0; + cfg.gain_bits = 21; + cfg.smoothing_coef = 819; % 0.05 in Q14 + smoothing_bits = 10; + input_correction_bits = 0; + + num_channels = 16; + num_frames = 20; + + % Create deterministic synthetic test signal: + % Baseline background noise + mid-stream bursts + rng(42); + input_matrix = zeros(num_channels, num_frames, 'uint32'); + for f = 1:num_frames + for c = 1:num_channels + noise = 50 + mod(c * 17 + f * 23, 100); + if f >= 5 && f <= 12 && c >= 4 && c <= 10 + speech_burst = 2000 + (c * 150); + else + speech_burst = 0; + end + input_matrix(c, f) = uint32(noise + speech_burst); + end + end + + [output_matrix, final_noise] = pcan_process(input_matrix, cfg, smoothing_bits, input_correction_bits); + + export_ndefine(fh, 'PCAN_STREAM_NUM_CHANNELS', num_channels); + export_ndefine(fh, 'PCAN_STREAM_NUM_FRAMES', num_frames); + export_ndefine(fh, 'PCAN_STREAM_SMOOTHING_COEF', cfg.smoothing_coef); + export_ndefine(fh, 'PCAN_STREAM_SMOOTHING_BITS', smoothing_bits); + export_ndefine(fh, 'PCAN_STREAM_INPUT_CORRECTION_BITS', input_correction_bits); + + input_linear = reshape(input_matrix, [], 1); + output_linear = reshape(output_matrix, [], 1); + + export_uint32_vector(fh, 'ref_pcan_stream_inputs', input_linear); + export_uint32_vector(fh, 'ref_pcan_stream_outputs', output_linear); + export_uint32_vector(fh, 'ref_pcan_stream_final_noise', final_noise); + + fclose(fh); + fprintf(1, 'Exported %s.\n', header_fn); +end + +function export_ref_corner_cases(opt) + header_fn = 'ref_pcan_corners.h'; + fh = export_headerfile_open(header_fn); + export_comment(fh, sprintf('Generated by ref_pcan.m (%s)', opt.describe)); + + cfg.strength = 0.95; + cfg.offset = 80.0; + cfg.gain_bits = 21; + input_bits = 10; + [lut, ~] = pcan_gain_lookup(cfg, input_bits); + + % Corner case points: zeros, boundaries, transitions, overflows + corner_inputs = uint32([ ... + 0, ... + 1, ... + 2, ... + 3, ... + 4, ... + 2047, 2048, 2049, ... + 8191, 8192, 8193, ... + 65535, 65536, 65537, ... + 1073741823, 1073741824, 1073741825, ... + 2147483647, 2147483648, 2147483649, ... + 4294967294, 4294967295 ... + ]); + + wdf_corner_outs = pcan_wide_dynamic_func(corner_inputs, lut); + shrink_corner_outs = pcan_shrink(corner_inputs, 12, 6); + + export_ndefine(fh, 'PCAN_CORNERS_NUM_POINTS', length(corner_inputs)); + export_uint32_vector(fh, 'ref_pcan_corner_inputs', corner_inputs); + export_vector(fh, 16, 'ref_pcan_corner_wdf_outputs', wdf_corner_outs); + export_uint32_vector(fh, 'ref_pcan_corner_shrink_outputs', shrink_corner_outs); + + fclose(fh); + fprintf(1, 'Exported %s.\n', header_fn); +end + +function export_uint32_vector(fh, vname, data) + columns = 6; + rows = ceil(length(data) / columns); + fprintf(fh, '\nstatic const uint32_t %s[%d] = {\n', vname, length(data)); + i = 1; + for j = 1:rows + fprintf(fh, '\t%11uU,', data(i)); + i = i + 1; + for k = 2:columns + if i <= length(data) + fprintf(fh, ' %11uU,', data(i)); + i = i + 1; + end + end + fprintf(fh, '\n'); + end + fprintf(fh, '};\n'); +end diff --git a/test/ztest/unit/math/advanced/pcan/ref_pcan_corners.h b/test/ztest/unit/math/advanced/pcan/ref_pcan_corners.h new file mode 100644 index 000000000000..8fa1852b4222 --- /dev/null +++ b/test/ztest/unit/math/advanced/pcan/ref_pcan_corners.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. + */ + +/* Generated by ref_pcan.m (v1.9-rc1-7901-g9500fa321) */ + +#define PCAN_CORNERS_NUM_POINTS 22 + +static const uint32_t ref_pcan_corner_inputs[22] = { + 0U, 1U, 2U, 3U, 4U, 2047U, + 2048U, 2049U, 8191U, 8192U, 8193U, 65535U, + 65536U, 65537U, 1073741823U, 1073741824U, 1073741825U, 2147483647U, + 2147483648U, 2147483649U, 4294967294U, 4294967295U, +}; + +static const int16_t ref_pcan_corner_wdf_outputs[22] = { + 32636, 32635, 32635, 32635, 32634, 31879, 31879, 31879, 29812, 29811, + 29811, 18676, 18672, 18672, 4, 4, 4, 2, 2, 2, + 1, 1, +}; + +static const uint32_t ref_pcan_corner_shrink_outputs[22] = { + 0U, 0U, 0U, 0U, 0U, 3U, + 4U, 4U, 63U, 64U, 64U, 959U, + 960U, 960U, 16777151U, 16777152U, 16777152U, 33554367U, + 33554368U, 33554368U, 67108799U, 67108799U, +}; diff --git a/test/ztest/unit/math/advanced/pcan/ref_pcan_func.h b/test/ztest/unit/math/advanced/pcan/ref_pcan_func.h new file mode 100644 index 000000000000..f86dfb415d6f --- /dev/null +++ b/test/ztest/unit/math/advanced/pcan/ref_pcan_func.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. + */ + +/* Generated by ref_pcan.m (v1.9-rc1-7901-g9500fa321) */ + +#define PCAN_TEST_WDF_NUM_POINTS 65 + +static const uint32_t ref_pcan_wdf_inputs[65] = { + 0U, 1U, 2U, 3U, 4U, 5U, + 7U, 8U, 15U, 16U, 31U, 32U, + 63U, 64U, 127U, 128U, 255U, 256U, + 511U, 512U, 1023U, 1024U, 2047U, 2048U, + 4095U, 4096U, 8191U, 8192U, 16383U, 16384U, + 32767U, 32768U, 65535U, 65536U, 131071U, 131072U, + 262143U, 262144U, 524287U, 524288U, 1048575U, 1048576U, + 2097151U, 2097152U, 4194303U, 4194304U, 8388607U, 8388608U, + 16777215U, 16777216U, 33554431U, 33554432U, 67108863U, 67108864U, + 134217727U, 134217728U, 268435455U, 268435456U, 536870911U, 536870912U, + 1073741823U, 1073741824U, 2147483647U, 2147483648U, 4294967295U, +}; + +static const int16_t ref_pcan_wdf_outputs[65] = { + 32636, 32635, 32635, 32635, 32634, 32634, 32634, 32633, 32630, 32630, + 32624, 32624, 32612, 32612, 32587, 32587, 32539, 32539, 32443, 32443, + 32253, 32253, 31879, 31879, 31159, 31158, 29812, 29811, 27448, 27446, + 23710, 23707, 18676, 18672, 13169, 13166, 8351, 8348, 4876, 4874, + 2698, 2697, 1446, 1446, 762, 762, 398, 398, 207, 207, + 107, 107, 56, 56, 29, 29, 15, 15, 8, 8, + 4, 4, 2, 2, 1, +}; +#define PCAN_TEST_SHRINK_NUM_POINTS 22 + +static const uint32_t ref_pcan_shrink_inputs[22] = { + 0U, 1U, 10U, 100U, 500U, 1000U, + 2000U, 4000U, 8000U, 8190U, 8191U, 8192U, + 8193U, 8194U, 10000U, 16384U, 32768U, 65536U, + 100000U, 500000U, 1000000U, 10000000U, +}; + +static const uint32_t ref_pcan_shrink_outputs[22] = { + 0U, 0U, 0U, 0U, 0U, 0U, + 3U, 15U, 61U, 63U, 63U, 64U, + 64U, 64U, 92U, 192U, 448U, 960U, + 1498U, 7748U, 15561U, 156186U, +}; diff --git a/test/ztest/unit/math/advanced/pcan/ref_pcan_lut.h b/test/ztest/unit/math/advanced/pcan/ref_pcan_lut.h new file mode 100644 index 000000000000..6736041146cc --- /dev/null +++ b/test/ztest/unit/math/advanced/pcan/ref_pcan_lut.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. + */ + +/* Generated by ref_pcan.m (v1.9-rc1-7901-g9500fa321) */ + +#define PCAN_TEST_LUT1_SIZE 125 +#define PCAN_TEST_LUT1_GAIN_BITS 21 +#define PCAN_TEST_LUT1_INPUT_BITS 10 + +static const int16_t ref_pcan_lut1[125] = { + 32636, 32635, 32635, 1, -2, 0, 32634, 1, -2, 0, + 32633, -5, 2, 0, 32630, -6, 0, 0, 32624, -12, + 0, 0, 32612, -23, -2, 0, 32587, -48, 0, 0, + 32539, -96, 0, 0, 32443, -190, 0, 0, 32253, -378, + 4, 0, 31879, -739, 18, 0, 31158, -1409, 62, 0, + 29811, -2567, 202, 0, 27446, -4301, 562, 0, 23707, -6265, + 1230, 0, 18672, -7458, 1952, 0, 13166, -7030, 2212, 0, + 8348, -5342, 1868, 0, 4874, -3459, 1282, 0, 2697, -2025, + 774, 0, 1446, -1120, 436, 0, 762, -596, 232, 0, + 398, -313, 122, 0, 207, -164, 64, 0, 107, -85, + 34, 0, 56, -45, 18, 0, 29, -22, 8, 0, + 15, -13, 6, 0, 8, -8, 4, 0, 4, -2, + 0, 0, 2, -3, 2, +}; +#define PCAN_TEST_LUT2_SIZE 125 +#define PCAN_TEST_LUT2_GAIN_BITS 18 +#define PCAN_TEST_LUT2_INPUT_BITS 12 + +static const int16_t ref_pcan_lut2[125] = { + 11465, 11465, 11465, 0, 0, 0, 11465, -3, 2, 0, + 11464, 0, 0, 0, 11464, 1, -2, 0, 11463, 1, + -2, 0, 11462, -5, 2, 0, 11459, -6, 0, 0, + 11453, -9, -2, 0, 11442, -25, 2, 0, 11419, -47, + 2, 0, 11374, -91, 2, 0, 11285, -178, 4, 0, + 11111, -341, 10, 0, 10780, -637, 38, 0, 10181, -1116, + 116, 0, 9181, -1749, 286, 0, 7718, -2315, 526, 0, + 5929, -2478, 700, 0, 4151, -2156, 696, 0, 2691, -1588, + 552, 0, 1655, -1047, 378, 0, 986, -647, 238, 0, + 577, -386, 144, 0, 335, -226, 84, 0, 193, -130, + 48, 0, 111, -77, 30, 0, 64, -45, 18, 0, + 37, -24, 8, 0, 21, -15, 6, 0, 12, -7, + 2, 0, 7, -5, 2, +}; diff --git a/test/ztest/unit/math/advanced/pcan/ref_pcan_stream.h b/test/ztest/unit/math/advanced/pcan/ref_pcan_stream.h new file mode 100644 index 000000000000..1960909baa6f --- /dev/null +++ b/test/ztest/unit/math/advanced/pcan/ref_pcan_stream.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. + */ + +/* Generated by ref_pcan.m (v1.9-rc1-7901-g9500fa321) */ + +#define PCAN_STREAM_NUM_CHANNELS 16 +#define PCAN_STREAM_NUM_FRAMES 20 +#define PCAN_STREAM_SMOOTHING_COEF 819 +#define PCAN_STREAM_SMOOTHING_BITS 10 +#define PCAN_STREAM_INPUT_CORRECTION_BITS 0 + +static const uint32_t ref_pcan_stream_inputs[320] = { + 90U, 107U, 124U, 141U, 58U, 75U, + 92U, 109U, 126U, 143U, 60U, 77U, + 94U, 111U, 128U, 145U, 113U, 130U, + 147U, 64U, 81U, 98U, 115U, 132U, + 149U, 66U, 83U, 100U, 117U, 134U, + 51U, 68U, 136U, 53U, 70U, 87U, + 104U, 121U, 138U, 55U, 72U, 89U, + 106U, 123U, 140U, 57U, 74U, 91U, + 59U, 76U, 93U, 110U, 127U, 144U, + 61U, 78U, 95U, 112U, 129U, 146U, + 63U, 80U, 97U, 114U, 82U, 99U, + 116U, 2733U, 2800U, 2967U, 3134U, 3301U, + 3468U, 3635U, 52U, 69U, 86U, 103U, + 120U, 137U, 105U, 122U, 139U, 2656U, + 2823U, 2990U, 3157U, 3324U, 3491U, 3558U, + 75U, 92U, 109U, 126U, 143U, 60U, + 128U, 145U, 62U, 2679U, 2846U, 3013U, + 3180U, 3347U, 3414U, 3581U, 98U, 115U, + 132U, 149U, 66U, 83U, 51U, 68U, + 85U, 2702U, 2869U, 3036U, 3103U, 3270U, + 3437U, 3604U, 121U, 138U, 55U, 72U, + 89U, 106U, 74U, 91U, 108U, 2725U, + 2892U, 2959U, 3126U, 3293U, 3460U, 3627U, + 144U, 61U, 78U, 95U, 112U, 129U, + 97U, 114U, 131U, 2748U, 2815U, 2982U, + 3149U, 3316U, 3483U, 3550U, 67U, 84U, + 101U, 118U, 135U, 52U, 120U, 137U, + 54U, 2671U, 2838U, 3005U, 3172U, 3339U, + 3406U, 3573U, 90U, 107U, 124U, 141U, + 58U, 75U, 143U, 60U, 77U, 2694U, + 2861U, 3028U, 3195U, 3262U, 3429U, 3596U, + 113U, 130U, 147U, 64U, 81U, 98U, + 66U, 83U, 100U, 117U, 134U, 51U, + 68U, 85U, 102U, 119U, 136U, 53U, + 70U, 87U, 104U, 121U, 89U, 106U, + 123U, 140U, 57U, 74U, 91U, 108U, + 125U, 142U, 59U, 76U, 93U, 110U, + 127U, 144U, 112U, 129U, 146U, 63U, + 80U, 97U, 114U, 131U, 148U, 65U, + 82U, 99U, 116U, 133U, 50U, 67U, + 135U, 52U, 69U, 86U, 103U, 120U, + 137U, 54U, 71U, 88U, 105U, 122U, + 139U, 56U, 73U, 90U, 58U, 75U, + 92U, 109U, 126U, 143U, 60U, 77U, + 94U, 111U, 128U, 145U, 62U, 79U, + 96U, 113U, 81U, 98U, 115U, 132U, + 149U, 66U, 83U, 100U, 117U, 134U, + 51U, 68U, 85U, 102U, 119U, 136U, + 104U, 121U, 138U, 55U, 72U, 89U, + 106U, 123U, 140U, 57U, 74U, 91U, + 108U, 125U, 142U, 59U, 127U, 144U, + 61U, 78U, 95U, 112U, 129U, 146U, + 63U, 80U, 97U, 114U, 131U, 148U, + 65U, 82U, +}; + +static const uint32_t ref_pcan_stream_outputs[320] = { + 28U, 39U, 51U, 65U, 12U, 19U, + 29U, 40U, 53U, 67U, 13U, 21U, + 30U, 42U, 54U, 68U, 39U, 50U, + 62U, 12U, 21U, 30U, 40U, 52U, + 64U, 13U, 22U, 31U, 42U, 53U, + 8U, 14U, 50U, 8U, 13U, 21U, + 32U, 41U, 51U, 8U, 14U, 22U, + 33U, 42U, 52U, 9U, 16U, 23U, + 9U, 15U, 21U, 31U, 42U, 52U, + 9U, 16U, 22U, 32U, 44U, 53U, + 10U, 17U, 25U, 33U, 16U, 24U, + 31U, 922U, 936U, 949U, 979U, 1006U, + 1013U, 1036U, 6U, 11U, 17U, 26U, + 35U, 43U, 25U, 33U, 40U, 574U, + 597U, 604U, 617U, 628U, 631U, 628U, + 13U, 18U, 26U, 35U, 45U, 8U, + 34U, 43U, 7U, 421U, 432U, 435U, + 442U, 450U, 444U, 456U, 21U, 27U, + 35U, 44U, 9U, 14U, 5U, 9U, + 14U, 336U, 346U, 350U, 345U, 351U, + 355U, 361U, 30U, 36U, 6U, 10U, + 16U, 22U, 10U, 15U, 21U, 283U, + 288U, 280U, 285U, 287U, 289U, 292U, + 39U, 6U, 11U, 16U, 23U, 30U, + 17U, 23U, 29U, 240U, 234U, 236U, + 240U, 242U, 244U, 241U, 8U, 12U, + 18U, 24U, 32U, 4U, 25U, 31U, + 4U, 199U, 203U, 205U, 209U, 213U, + 210U, 215U, 14U, 19U, 26U, 32U, + 5U, 9U, 33U, 5U, 9U, 176U, + 181U, 185U, 189U, 185U, 189U, 192U, + 21U, 26U, 34U, 6U, 11U, 16U, + 6U, 10U, 15U, 0U, 0U, 0U, + 0U, 0U, 0U, 0U, 28U, 4U, + 7U, 11U, 17U, 23U, 12U, 17U, + 22U, 0U, 0U, 0U, 0U, 0U, + 0U, 0U, 5U, 8U, 13U, 17U, + 24U, 30U, 18U, 23U, 29U, 0U, + 0U, 0U, 0U, 0U, 0U, 0U, + 10U, 14U, 19U, 24U, 3U, 6U, + 25U, 3U, 6U, 0U, 0U, 0U, + 0U, 0U, 0U, 0U, 16U, 21U, + 26U, 4U, 7U, 11U, 4U, 7U, + 11U, 0U, 0U, 0U, 0U, 0U, + 0U, 0U, 22U, 28U, 5U, 8U, + 13U, 17U, 9U, 13U, 17U, 0U, + 1U, 0U, 0U, 0U, 0U, 0U, + 3U, 6U, 9U, 13U, 19U, 24U, + 14U, 19U, 23U, 0U, 0U, 0U, + 0U, 0U, 0U, 0U, 7U, 10U, + 15U, 19U, 26U, 4U, 20U, 25U, + 4U, 0U, 0U, 0U, 0U, 0U, + 0U, 0U, 12U, 16U, 21U, 26U, + 5U, 8U, +}; + +static const uint32_t ref_pcan_stream_final_noise[16] = { + 65089U, 66549U, 66741U, 658990U, 693317U, 727673U, + 763723U, 799477U, 833963U, 863004U, 61280U, 65959U, + 67716U, 69176U, 63373U, 64115U, +}; diff --git a/test/ztest/unit/math/advanced/pcan/testcase.yaml b/test/ztest/unit/math/advanced/pcan/testcase.yaml new file mode 100644 index 000000000000..064bb7debe9c --- /dev/null +++ b/test/ztest/unit/math/advanced/pcan/testcase.yaml @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: BSD-3-Clause +# +# Copyright(c) 2026 Intel Corporation. All rights reserved. +# +# PCAN (Per-Channel AGC Normalization) unit tests for Ztest framework +# + +tests: + sof.unit.math.advanced.pcan: + tags: math advanced pcan mfcc microwakeword + platform_allow: native_sim + integration_platforms: + - native_sim + build_only: false diff --git a/west.yml b/west.yml index b66c3a7deac1..c5e5ffab04ef 100644 --- a/west.yml +++ b/west.yml @@ -11,6 +11,8 @@ manifest: url-base: https://github.com/thesofproject - name: zephyrproject url-base: https://github.com/zephyrproject-rtos + - name: tensorflow + url-base: https://github.com/tensorflow # When upgrading projects here please run git log --oneline in the # project and if not too long then include the output in your commit @@ -41,6 +43,12 @@ manifest: path: sof/tools/rimage/tomlc99 revision: e3a03f5ec7d8d33be705c5ce8a632d998ce9b4d1 + - name: tflite-micro + repo-path: tflite-micro + path: tflite-micro + revision: e86d97b6237f88ab5925c0b41e3e3589a1560d86 + remote: tensorflow + - name: zephyr repo-path: zephyr revision: beeac475263ea55a74a7b61274cd4867c53c973d From b6b103106df0ee27e8a710ca10cf5fa180652230 Mon Sep 17 00:00:00 2001 From: Seppo Ingalsuo Date: Wed, 23 Sep 2026 18:19:28 +0300 Subject: [PATCH 16/35] audio: mfcc: add 8-bit PCAN feature extraction support Add 8-bit PCAN-normalized Mel feature extraction to the MFCC module. When CONFIG_COMP_MFCC_PCAN is enabled, linear Mel filterbank energies are processed through the PCAN AGC pipeline, quantized to 8-bit int8 values, and prepended with the standard mfcc_data_header. Ensure dev->frames is at least frame_shift in DP domain, calibrate linear Mel magnitudes against microfrontend scaling, and guard PCAN mode under dedicated Kconfig. Signed-off-by: Seppo Ingalsuo --- CMakeLists.txt | 8 ++- src/arch/host/configs/library_defconfig | 1 + src/audio/mfcc/Kconfig | 23 +++++++ src/audio/mfcc/README.md | 89 ++++++++++++++++++++----- src/audio/mfcc/mfcc.c | 61 +++++++++++++++-- src/audio/mfcc/mfcc_common.c | 75 ++++++++++++++++++--- src/audio/mfcc/mfcc_setup.c | 71 ++++++++++++++++++++ src/include/sof/audio/mfcc/mfcc_comp.h | 11 ++- src/include/user/mfcc.h | 10 ++- 9 files changed, 312 insertions(+), 37 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 65747ffe1a8f..4a65d394519c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,6 +2,9 @@ cmake_minimum_required(VERSION 3.13) +if(CMAKE_C_COMPILER_ID MATCHES "Clang") + add_compile_options(-mllvm --text-section-literals=false) +endif() if("${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}") message(FATAL_ERROR " In-source builds are not supported.\n" @@ -58,7 +61,10 @@ set(CMAKE_ASM_FLAGS -DASSEMBLY) # that may be exported / installed add_library(sof_public_headers INTERFACE) -target_include_directories(sof_public_headers INTERFACE ${PROJECT_SOURCE_DIR}/src/include) +target_include_directories(sof_public_headers INTERFACE + ${PROJECT_SOURCE_DIR}/src/include + ${PROJECT_SOURCE_DIR}/../tflite-micro +) # interface library that is used only as container for sof binary options # other targets can use it to build with the same options diff --git a/src/arch/host/configs/library_defconfig b/src/arch/host/configs/library_defconfig index 1279dd26c924..e6a6042302a9 100644 --- a/src/arch/host/configs/library_defconfig +++ b/src/arch/host/configs/library_defconfig @@ -11,6 +11,7 @@ CONFIG_COMP_IIR=y CONFIG_COMP_IGO_NR=y CONFIG_COMP_LEVEL_MULTIPLIER=y CONFIG_COMP_MFCC=y +CONFIG_COMP_MFCC_PCAN=y CONFIG_COMP_MODULE_ADAPTER=y CONFIG_COMP_MULTIBAND_DRC=y CONFIG_COMP_MUX=y diff --git a/src/audio/mfcc/Kconfig b/src/audio/mfcc/Kconfig index f56cadb40de2..1faa74998e72 100644 --- a/src/audio/mfcc/Kconfig +++ b/src/audio/mfcc/Kconfig @@ -24,3 +24,26 @@ config COMP_MFCC The characteristic of the audio features are defined in the binary control blob. Directory tools/tune/mfcc contains a tool to create the configurations. + +if COMP_MFCC + +config COMP_MFCC_PCAN + bool "Enable PCAN (Per-Channel AGC Normalization) in MFCC" + default n + select MATH_PCAN + help + Select to build MFCC with PCAN (Per-Channel Automatic Gain Control + Normalization) support on the Mel filterbank energies. PCAN performs + dynamic per-channel gain control and piecewise polynomial root + compression to suppress stationary noise and enhance transients, + producing normalized features suitable for direct int8 quantization + downstream (e.g. microWakeWord). The topology PCAN configuration + fields are ignored when this option is disabled. + +config COMP_MFCC_DEBUG_TRACE + bool "Verbose per-process debug traces in MFCC" + default n + help + Enable MFCC per-process debug prints in the hot path. + +endif # COMP_MFCC diff --git a/src/audio/mfcc/README.md b/src/audio/mfcc/README.md index 31f9c331e545..e2f348231f24 100644 --- a/src/audio/mfcc/README.md +++ b/src/audio/mfcc/README.md @@ -1,25 +1,84 @@ -# MFCC Feature Extraction Architecture +# MFCC & PCAN Feature Extraction Architecture -This directory contains the Mel-Frequency Cepstral Coefficients (MFCC) feature extractor. +This directory contains the **Mel-Frequency Cepstral Coefficients (MFCC)** feature extractor and integrated **PCAN (Per-Channel AGC Normalization)** audio pre-processing component for Sound Open Firmware (SOF). -## Overview +--- -MFCC extracts audio features commonly used as inputs for machine learning models, such as wake-word detection or speech recognition. +## 1. Overview -## Architecture Diagram +The MFCC module converts raw streaming time-domain PCM audio into compact acoustic feature representations (e.g. 13-bin cepstral coefficients, 40-bin or 80-bin Mel spectrograms) suitable for on-device machine learning models such as wake-word classifiers (TFLM, microWakeWord) and speech recognition engines (Whisper). + +When enabled, **PCAN** applies per-channel adaptive dynamic gain control and piecewise root dynamic range compression across time to suppress stationary background noise and normalize channel energy before feature quantization. + +--- + +## 2. Architecture & Data Flow ```mermaid graph LR - In[Audio Frame] --> Win[Windowing] - Win --> FFT[FFT] - FFT --> Mel[Mel Filterbank] - Mel --> DCT[DCT] - DCT --> Out[MFCC Output Features] + In[Audio PCM Frame] --> Win[Windowing: Hamming/Hann] + Win --> FFT[FFT: 512-pt] + FFT --> Mel[Mel Filterbank: 40/80 Bins] + Mel --> PCAN[PCAN Gain Normalization] + PCAN --> Log[Log Scaling / dB] + Log --> VAD[VAD / DTX Silence Gating] + Log --> DCT[DCT: Cepstral Matrix] + DCT --> Lifter[Cepstral Lifter] + Lifter --> Out[MFCC Output Features] +``` + +--- + +## 3. Dependencies + +| Dependency | Purpose | +|---|---| +| **West Manifest (`west.yml`)** | Manages workspace dependencies including `tflite-micro` | +| **`tflite-micro`** | Provides Google's upstream Apache-2.0 `microfrontend` PCAN library (`pcan_gain_control.c`, `noise_reduction.c`) | +| **GNU Octave ($\ge 5.0$)** | Required for topology configuration generation (`tune/setup_mfcc.m`) and reference vector export | +| **CMake ($\ge 3.13$)** | SOF build system | +| **Zephyr SDK / Xtensa XCC** | DSP cross-compilation toolchains for Cadence HiFi3 / HiFi4 SIMD kernels | +| **CMocka** | Host unit testing framework | + +--- + +## 4. Build & Tuning Instructions + +### 4.1 Synchronize Dependencies via West +```bash +west update ``` -## Configuration and Scripts +### 4.2 Build Host Unit Tests +```bash +# Configure unit tests +cmake -S sof -B build_ut -DBUILD_UNIT_TESTS=ON -DBUILD_UNIT_TESTS_HOST=ON -DINIT_CONFIG=unit_test_defconfig + +# Build all math and MFCC tests +cmake --build build_ut --target pcan +ctest --test-dir build_ut -R "pcan|auditory|dct|matrix|window|fft" --output-on-failure +``` + +### 4.3 Build Target DSP Firmware +```bash +west build -b intel_adsp_ace15_mtpm app +``` + +### 4.4 Generate Topology Configurations (Octave) +The tuning script generates binary configuration blobs and ALSA topology configurations: +```bash +cd src/audio/mfcc/tune +octave-cli --eval "setup_mfcc" +``` +Available presets generated: +- `ceps13_compress_dtx.conf`: 13-bin cepstral features with VAD and DTX silence suppression. +- `mel80_compress.conf`: 80-bin linear Mel spectrogram features. +- `mel80_compress_dtx.conf`: 80-bin Mel spectrogram with DTX gating. +- `mel80_pcan_compress.conf`: 80-bin Mel spectrogram with Google PCAN dynamic AGC normalization. + +--- + +## 5. Configuration Options -- **Kconfig**: Enables the MFCC component (`COMP_MFCC`) which selects necessary math libraries (`MATH_FFT`, `MATH_DCT`, `MATH_16BIT_MEL_FILTERBANK`, etc.). Depends on `COMP_MODULE_ADAPTER`. -- **CMakeLists.txt**: Compiles generic, common, and HIFI implementations (`mfcc_hifi3.c`, `mfcc_hifi4.c`). Provides support for Zephyr loadable extensions (`llext`). -- **mfcc.toml**: Specifies the topology configuration for the MFCC module (UUID, affinity, memory parameters, and pin formats). -- **Topology (.conf)**: Derived from `tools/topology/topology2/include/components/mfcc.conf`, configuring a `mfcc` widget object of type `effect` with UUID `73:a7:10:db:a4:1a:ea:4c:a2:1f:2d:57:a5:c9:82:eb`. +- **`CONFIG_COMP_MFCC`**: Enables the MFCC component and automatically selects required math libraries (`CONFIG_MATH_FFT`, `CONFIG_MATH_AUDITORY`, `CONFIG_MATH_DCT`, `CONFIG_MATH_PCAN`, `CONFIG_MATH_WINDOW`). +- **`CONFIG_COMP_MODULE_ADAPTER`**: Module adapter infrastructure for SOF processing components. diff --git a/src/audio/mfcc/mfcc.c b/src/audio/mfcc/mfcc.c index bc57d2e8ace4..a71948e8d8b3 100644 --- a/src/audio/mfcc/mfcc.c +++ b/src/audio/mfcc/mfcc.c @@ -33,11 +33,19 @@ #include #include #include +#if CONFIG_COMP_MFCC_DEBUG_TRACE +#include +#endif LOG_MODULE_REGISTER(mfcc, CONFIG_SOF_LOG_LEVEL); SOF_DEFINE_REG_UUID(mfcc); +#if CONFIG_COMP_MFCC_DEBUG_TRACE +static uint32_t last_mfcc_cycle; +static uint32_t mfcc_call_count; +#endif + /** \brief Source/sink API based source copy function map. */ struct mfcc_source_func_map { uint8_t source_fmt; @@ -162,6 +170,13 @@ static int mfcc_process(struct processing_module *mod, size_t source_avail; int frames; int num_ceps; +#if CONFIG_COMP_MFCC_DEBUG_TRACE + uint32_t now = k_cycle_get_32(); + uint32_t delta_cycles = now - last_mfcc_cycle; + + last_mfcc_cycle = now; + mfcc_call_count++; +#endif comp_dbg(dev, "start"); @@ -172,23 +187,47 @@ static int mfcc_process(struct processing_module *mod, * can continue while the previous period is drained. */ pending = state->header_pending || state->out_remain > 0; - if (cd->config->compress_output && pending) - return mfcc_process_output(mod, cd, sources, sinks, 0, 0); + if (cd->config->compress_output && pending) { + int ret = mfcc_process_output(mod, cd, sources, sinks, 0, 0); + +#if CONFIG_COMP_MFCC_DEBUG_TRACE + comp_info(dev, "[MFCC proc %u] delta=%u us (pending retry, ret=%d)", + mfcc_call_count, k_cyc_to_us_near32(delta_cycles), ret); +#endif + return ret; + } source_avail = source_get_data_frames_available(sources[0]); frames = MIN(source_avail, cd->max_frames); - if (!frames) + if (!frames) { +#if CONFIG_COMP_MFCC_DEBUG_TRACE + comp_info(dev, "[MFCC proc %u] delta=%u us (no frames, avail=%zu)", + mfcc_call_count, k_cyc_to_us_near32(delta_cycles), source_avail); +#endif return 0; + } /* Copy input audio from source to MFCC internal circular buffer */ cd->source_func(sources[0], &state->buf, &state->emph, frames, state->source_channel); /* Run STFT and Mel/DCT processing */ num_ceps = mfcc_stft_process(mod, cd); - if (num_ceps < 0) + if (num_ceps < 0) { +#if CONFIG_COMP_MFCC_DEBUG_TRACE + comp_info(dev, "[MFCC proc %u] delta=%u us stft error %d", + mfcc_call_count, k_cyc_to_us_near32(delta_cycles), num_ceps); +#endif return num_ceps; + } + + int ret = mfcc_process_output(mod, cd, sources, sinks, num_ceps, frames); - return mfcc_process_output(mod, cd, sources, sinks, num_ceps, frames); +#if CONFIG_COMP_MFCC_DEBUG_TRACE + comp_info(dev, "[MFCC proc %u] delta=%u us avail=%zu frames=%d ceps=%d ret=%d", + mfcc_call_count, k_cyc_to_us_near32(delta_cycles), + source_avail, frames, num_ceps, ret); +#endif + return ret; } /** @@ -242,8 +281,14 @@ static int mfcc_prepare(struct processing_module *mod, /* Initialize MFCC, max_frames is set to dev->frames + 4 */ if (cd->config && data_size > 0) { uint32_t src_rate = audio_stream_get_rate(&sourceb->stream); + int max_frames; + + if (dev->frames < cd->config->frame_shift) + dev->frames = cd->config->frame_shift; - ret = mfcc_setup(mod, dev->frames + 4, src_rate, + max_frames = dev->frames + 4; + + ret = mfcc_setup(mod, max_frames, src_rate, audio_stream_get_channels(&sourceb->stream)); if (ret < 0) { comp_err(dev, "setup failed."); @@ -317,6 +362,10 @@ static int mfcc_reset(struct processing_module *mod) /* Reset to similar state as init() */ cd->source_func = NULL; +#if CONFIG_COMP_MFCC_DEBUG_TRACE + last_mfcc_cycle = 0; + mfcc_call_count = 0; +#endif return 0; } diff --git a/src/audio/mfcc/mfcc_common.c b/src/audio/mfcc/mfcc_common.c index 2460493ed34d..ee2e2a25fe6a 100644 --- a/src/audio/mfcc/mfcc_common.c +++ b/src/audio/mfcc/mfcc_common.c @@ -13,6 +13,9 @@ #include #include #include +#if CONFIG_COMP_MFCC_PCAN +#include +#endif #include #include #include @@ -352,8 +355,23 @@ int mfcc_stft_process(struct processing_module *mod, struct mfcc_comp_data *cd) * to add the missing "gain". */ mel_scale_shift = input_shift - fft->fft_plan->len; + +#if CONFIG_COMP_MFCC_PCAN + if (state->pcan.enable_pcan) { + psy_apply_mel_filterbank_with_linear_32(&state->melfb, fft->fft_out, + state->power_spectra, state->mel_log_32, + state->mel_linear, mel_scale_shift); + pcan_noise_reduction(&state->pcan, state->mel_linear); + pcan_apply(&state->pcan, state->mel_linear); + pcan_log_scale(&state->pcan, state->mel_linear); + } else { + psy_apply_mel_filterbank_32(&state->melfb, fft->fft_out, state->power_spectra, + state->mel_log_32, mel_scale_shift); + } +#else psy_apply_mel_filterbank_32(&state->melfb, fft->fft_out, state->power_spectra, state->mel_log_32, mel_scale_shift); +#endif if (state->mel_only) { /* In Mel-only mode output Mel log spectra directly */ @@ -497,6 +515,30 @@ static void mfcc_prepare_output(struct mfcc_state *state, int num_ceps) if (num_ceps <= 0) return; +#if CONFIG_COMP_MFCC_PCAN + if (state->pcan.enable_pcan) { + int8_t *out8 = (int8_t *)state->out_stage; + + for (k = 0; k < num_ceps; k++) { + /* Map PCAN log-scaled output (~0..666) to int8 [-128..127] */ + int32_t val = (int32_t)state->mel_linear[k]; + + val = ((val * 256) + 333) / 666 - 128; + if (val > 127) + val = 127; + else if (val < -128) + val = -128; + + out8[k] = (int8_t)val; + } + + state->out_data_ptr = state->out_stage; + state->out_remain = num_ceps; + state->header_pending = true; + return; + } +#endif + /* Copy into out_stage so the next STFT hop may freely reuse * mel_log_32 / cepstral_coef while this frame is still pending. */ @@ -585,8 +627,14 @@ static int mfcc_output_compress(struct processing_module *mod, struct mfcc_comp_ state->dtx_silence_counter = 0; } +#if CONFIG_COMP_MFCC_PCAN + size_t sample_size = state->pcan.enable_pcan ? sizeof(int8_t) : sizeof(int32_t); +#else + size_t sample_size = sizeof(int32_t); +#endif + out_bytes = (state->header_pending ? sizeof(state->header) : 0) + - state->out_remain * sizeof(int32_t); + state->out_remain * sample_size; if (out_bytes == 0) return 0; @@ -609,7 +657,7 @@ static int mfcc_output_compress(struct processing_module *mod, struct mfcc_comp_ if (state->out_remain > 0) { mfcc_sink_write_bytes(&dst, sink_start, sink_buf_size, (uint8_t *)state->out_data_ptr, - state->out_remain * sizeof(int32_t)); + state->out_remain * sample_size); } ret = sink_commit_buffer(sinks[0], commit_bytes); @@ -653,11 +701,10 @@ static int mfcc_output_legacy(struct processing_module *mod, struct mfcc_comp_da void *sink_start; size_t sink_buf_size; uint8_t *dst; - int n32; int ret; /* The MFCC sink is treated as an opaque byte container: the period - * carries an MFCC blob (header + int32 features), not PCM audio. + * carries an MFCC blob (header + features), not PCM audio. * Sizing the commit as sink_frame_bytes * frames keeps the period * size matched to whatever the sink advertises (S16_LE / S24_4LE / * S32_LE), so no format-specific conversion is needed. Any payload @@ -702,17 +749,25 @@ static int mfcc_output_legacy(struct processing_module *mod, struct mfcc_comp_da } } - /* Write pending feature data (always int32) */ + /* Write pending feature data (int8 in PCAN mode, int32 otherwise) */ if (state->out_remain > 0 && avail > 0) { - data_bytes = state->out_remain * sizeof(int32_t); - to_write = MIN(data_bytes, avail) & ~(size_t)3; +#if CONFIG_COMP_MFCC_PCAN + size_t sample_size = state->pcan.enable_pcan ? sizeof(int8_t) : sizeof(int32_t); +#else + size_t sample_size = sizeof(int32_t); +#endif + data_bytes = state->out_remain * sample_size; + to_write = MIN(data_bytes, avail); + if (sample_size > 1) + to_write &= ~(sample_size - 1); if (to_write > 0) { mfcc_sink_write_bytes(&dst, sink_start, sink_buf_size, (uint8_t *)state->out_data_ptr, to_write); - n32 = to_write / sizeof(int32_t); - state->out_data_ptr += n32; - state->out_remain -= n32; + int n = to_write / sample_size; + + state->out_data_ptr = (void *)((uint8_t *)state->out_data_ptr + to_write); + state->out_remain -= n; } } diff --git a/src/audio/mfcc/mfcc_setup.c b/src/audio/mfcc/mfcc_setup.c index 1d062b2a5206..c380aa7da3cb 100644 --- a/src/audio/mfcc/mfcc_setup.c +++ b/src/audio/mfcc/mfcc_setup.c @@ -10,6 +10,9 @@ #include #include #include +#if CONFIG_COMP_MFCC_PCAN +#include +#endif #include #include #include @@ -383,6 +386,9 @@ int mfcc_setup(struct processing_module *mod, int max_frames, int sample_rate, i /* Use FFT buffer as scratch for later computed data */ state->power_spectra = (int32_t *)&fft->fft_buf[0]; state->mel_log_32 = &state->power_spectra[fft->half_fft_size]; +#if CONFIG_COMP_MFCC_PCAN + state->mel_linear = (uint32_t *)&state->mel_log_32[config->num_mel_bins]; +#endif /* Check that mel_log_32 fits in the remaining fft_buf scratch space */ mel_log_32_space = (int)(fft->fft_buffer_size / sizeof(int32_t)) - fft->half_fft_size; @@ -459,9 +465,70 @@ int mfcc_setup(struct processing_module *mod, int max_frames, int sample_rate, i } } + if (config->enable_pcan) { +#if CONFIG_COMP_MFCC_PCAN + struct pcan_config pcfg; + uint32_t *pcan_noise; + int16_t *pcan_lut; + + pcfg.strength = (config->pcan_strength > 0) ? + ((float)config->pcan_strength / 32768.0f) : 0.95f; + pcfg.offset = (config->pcan_offset > 0) ? + ((float)config->pcan_offset / 128.0f) : 80.0f; + pcfg.gain_bits = (config->pcan_gain_bits > 0) ? + config->pcan_gain_bits : 21; + pcfg.smoothing_coef = (config->pcan_smoothing_coef > 0) ? + (uint16_t)config->pcan_smoothing_coef : 819; + pcfg.smoothing_bits = 10; + pcfg.input_correction_bits = 0; + pcfg.enable_pcan = true; + + pcan_noise = mod_zalloc(mod, config->num_mel_bins * sizeof(uint32_t)); + if (!pcan_noise) { + comp_err(dev, "Failed PCAN noise estimate alloc"); + ret = -ENOMEM; + goto free_vad; + } + + pcan_lut = mod_zalloc(mod, PCAN_LUT_SIZE * sizeof(int16_t)); + if (!pcan_lut) { + comp_err(dev, "Failed PCAN gain LUT alloc"); + mod_free(mod, pcan_noise); + ret = -ENOMEM; + goto free_vad; + } + + ret = pcan_populate_state(&pcfg, &state->pcan, pcan_noise, pcan_lut, + config->num_mel_bins, 10, 0); + if (ret < 0) { + comp_err(dev, "Failed PCAN state init"); + mod_free(mod, pcan_noise); + mod_free(mod, pcan_lut); + goto free_vad; + } + } else { + state->pcan.enable_pcan = false; + state->pcan.noise_estimate = NULL; + state->pcan.gain_lut = NULL; + } +#else + comp_err(dev, "enable_pcan set but CONFIG_COMP_MFCC_PCAN is not selected"); + ret = -EINVAL; + goto free_vad; + } +#endif + comp_dbg(dev, "done"); return 0; +free_vad: + if (config->enable_vad) { + mod_free(mod, cd->vad.noise_floor); + mod_free(mod, cd->vad.weights); + cd->vad.noise_floor = NULL; + cd->vad.weights = NULL; + } + free_out_stage: mod_free(mod, state->out_stage); @@ -528,6 +595,10 @@ void mfcc_free_buffers(struct processing_module *mod) mfcc_free_and_null(mod, (void **)&cd->state.dct.matrix); mfcc_free_and_null(mod, (void **)&cd->state.lifter.matrix); mfcc_free_and_null(mod, (void **)&cd->state.out_stage); +#if CONFIG_COMP_MFCC_PCAN + mfcc_free_and_null(mod, (void **)&cd->state.pcan.noise_estimate); + mfcc_free_and_null(mod, (void **)&cd->state.pcan.gain_lut); +#endif mfcc_free_and_null(mod, (void **)&cd->vad.noise_floor); mfcc_free_and_null(mod, (void **)&cd->vad.weights); } diff --git a/src/include/sof/audio/mfcc/mfcc_comp.h b/src/include/sof/audio/mfcc/mfcc_comp.h index 885339004fc0..ad4e2afdf33c 100644 --- a/src/include/sof/audio/mfcc/mfcc_comp.h +++ b/src/include/sof/audio/mfcc/mfcc_comp.h @@ -13,6 +13,9 @@ #include #include #include +#if CONFIG_COMP_MFCC_PCAN +#include +#endif #include #include #include @@ -99,6 +102,10 @@ struct mfcc_state { struct mfcc_fft fft; /**< FFT related */ struct dct_plan_16 dct; /**< DCT related */ struct psy_mel_filterbank melfb; /**< Mel filter bank */ +#if CONFIG_COMP_MFCC_PCAN + struct pcan_state pcan; /**< PCAN state */ + uint32_t *mel_linear; /**< Linear Mel band magnitudes for PCAN */ +#endif struct mfcc_cepstral_lifter lifter; /**< Cepstral lifter coefficients */ struct mat_matrix_16b *mel_spectra; /**< Pointer to scratch */ struct mat_matrix_16b *cepstral_coef; /**< Pointer to scratch */ @@ -122,8 +129,8 @@ struct mfcc_state { bool header_pending; /**< True when data header not yet written for current output */ struct mfcc_data_header header; /**< Data header for current output frame */ size_t sample_buffers_size; /**< bytes */ - int32_t *out_data_ptr; /**< Read pointer into staging data for multi-period output */ - int out_remain; /**< Remaining int32_t samples to write to sink from staging */ + void *out_data_ptr; /**< Read pointer into staging data for multi-period output */ + int out_remain; /**< Remaining samples to write to sink from staging */ int32_t *out_stage; /**< Dedicated staging buffer for pending output, decoupled from STFT scratch */ int out_stage_size; /**< Capacity of out_stage in int32_t samples */ uint32_t hop_count; /**< FFT hop counter, increments every processed hop */ diff --git a/src/include/user/mfcc.h b/src/include/user/mfcc.h index 286ee4f5e985..e8793811509b 100644 --- a/src/include/user/mfcc.h +++ b/src/include/user/mfcc.h @@ -56,7 +56,8 @@ struct sof_mfcc_config { int16_t mmax_coef; /**< Q1.15 decay coefficient for dynamic mmax, a small value for slow */ uint16_t dtx_trailing_silence_hops; /**< DTX: number of silence hops to send after speech, 0 = send first only */ uint16_t dtx_silence_hops_interval; /**< DTX: send silence frame every Nth hop during VAD=0, 0 = disable */ - uint32_t reserved[5]; + uint32_t pcan_smoothing_coef; /**< Q14 smoothing coef for PCAN, e.g. 819 for 0.05 */ + uint32_t reserved[4]; int32_t sample_frequency; /**< Hz. e.g. 16000 */ int32_t pmin; /**< Q1.31 linear power, limit minimum Mel energy, e.g. 1e-9 */ enum sof_mfcc_mel_log_type mel_log; /**< Use MEL_LOG_IS_LOG, LOG10 or DB*/ @@ -79,7 +80,9 @@ struct sof_mfcc_config { int16_t vtln_high; /**< Reserved, no support */ int16_t vtln_low; /**< Reserved, no support */ int16_t vtln_warp; /**< Reserved, no support */ - int16_t reserved16[3]; /**< Reserved for future 16-bit fields, set to 0 */ + int16_t pcan_strength; /**< Q1.15 strength alpha, e.g. 31130 for 0.95 */ + int16_t pcan_offset; /**< Q8.7 offset delta, e.g. 10240 for 80.0 */ + int16_t pcan_gain_bits; /**< Gain bits scale, e.g. 21 */ bool htk_compat; /**< Must be false */ bool raw_energy; /**< Reserved, no support */ bool remove_dc_offset; /**< Reserved, no support */ @@ -92,7 +95,8 @@ struct sof_mfcc_config { bool enable_dtx; /**< Discontinuous transmission: suppress silence after trailing frames */ bool update_controls; /**< Update controls with VAD decision */ bool compress_output; /**< Use compress PCM output: variable size, no zero padding */ - bool reserved_bool[4]; /* Reserved for future boolean flags, set to false (0) */ + bool enable_pcan; /**< Enable PCAN normalization on Mel filterbank energies */ + bool reserved_bool[3]; /* Reserved for future boolean flags, set to false (0) */ } __attribute__((packed)); #endif /* __USER_MFCC_H__ */ From 7e79b823225d171c597af5d1d0bc06b76c8b25f7 Mon Sep 17 00:00:00 2001 From: Seppo Ingalsuo Date: Wed, 23 Sep 2026 18:19:42 +0300 Subject: [PATCH 17/35] audio: mfcc: tune: add PCAN profiles and export configuration Add PCAN export configuration to setup_mfcc.m and generate mel40_10ms_pcan_compress.conf and mel80_pcan_compress.conf topology blobs with enable_pcan set. Signed-off-by: Seppo Ingalsuo --- src/audio/mfcc/tune/setup_mfcc.m | 74 ++++++++++++++----- .../components/mfcc/ceps13_compress_dtx.conf | 8 +- .../include/components/mfcc/default.conf | 8 +- .../include/components/mfcc/mel40.conf | 16 ++-- .../components/mfcc/mel40_10ms_compress.conf | 16 ++-- .../mfcc/mel40_10ms_pcan_compress.conf | 24 ++++++ .../components/mfcc/mel40_compress.conf | 16 ++-- .../include/components/mfcc/mel80.conf | 16 ++-- .../components/mfcc/mel80_compress.conf | 16 ++-- .../components/mfcc/mel80_compress_dtx.conf | 16 ++-- .../components/mfcc/mel80_pcan_compress.conf | 24 ++++++ 11 files changed, 161 insertions(+), 73 deletions(-) create mode 100644 tools/topology/topology2/include/components/mfcc/mel40_10ms_pcan_compress.conf create mode 100644 tools/topology/topology2/include/components/mfcc/mel80_pcan_compress.conf diff --git a/src/audio/mfcc/tune/setup_mfcc.m b/src/audio/mfcc/tune/setup_mfcc.m index e8578a264826..b72855cdcf26 100644 --- a/src/audio/mfcc/tune/setup_mfcc.m +++ b/src/audio/mfcc/tune/setup_mfcc.m @@ -96,6 +96,35 @@ function setup_mfcc() setup.tplg_fn = 'ceps13_compress_dtx.conf'; export_mfcc_setup(gen_cfg, setup); + % Mel spectrogram with PCAN normalization and compress PCM output + setup = get_mel_spectrogram_config(); + setup.enable_pcan = true; + setup.compress_output = true; + setup.tplg_fn = 'mel80_pcan_compress.conf'; + export_mfcc_setup(gen_cfg, setup); + + % 40 mel bins, 10 ms hop PCAN for microWakeWord (no Slaney norm, Google-matched) + setup = get_mel_spectrogram_config(); + setup.frame_length = 30.0; + setup.frame_shift = 10.0; + setup.num_mel_bins = 40; + setup.low_freq = 125; + setup.high_freq = 7500; + setup.norm = 'none'; + setup.mel_offset = 0; + setup.mel_scale = 1.0; + setup.top_db = 0; + setup.dynamic_mmax = false; + setup.enable_pcan = true; + setup.pcan_strength = 0.95; + setup.pcan_offset = 80.0; + setup.pcan_gain_bits = 21; + setup.pcan_smoothing_coef = 819; + setup.compress_output = true; + setup.update_controls = false; + setup.tplg_fn = 'mel40_10ms_pcan_compress.conf'; + export_mfcc_setup(gen_cfg, setup); + end function cfg = get_mfcc_default_config() @@ -139,6 +168,11 @@ function setup_mfcc() cfg.dtx_silence_hops_interval = 0; cfg.update_controls = false; cfg.compress_output = false; + cfg.enable_pcan = false; + cfg.pcan_strength = 0.95; + cfg.pcan_offset = 80.0; + cfg.pcan_gain_bits = 21; + cfg.pcan_smoothing_coef = 819; end function cfg = get_mel_spectrogram_config() @@ -158,7 +192,7 @@ function setup_mfcc() cfg.num_mel_bins = 80; cfg.preemphasis_coefficient = 0; cfg.raw_energy = false; - cfg.remove_dc_offset = false; + cfg.remove_dc_offset = true; cfg.round_to_power_of_two = true; cfg.sample_frequency = 16000; cfg.snip_edges = true; @@ -168,20 +202,25 @@ function setup_mfcc() cfg.vtln_low = 0; cfg.vtln_warp = 1.0; cfg.window_type = 'hann'; - cfg.mel_log = 'log10'; + cfg.mel_log = 'log'; % Set to 'db' for librosa, set to 'log10' for matlab cfg.pmin = 1e-10; - cfg.top_db = 8; % applied for log10, would be 80 dB clamp for decibels as 10*log10() - cfg.mel_offset = 4.0; % For whisper like Mel scale and normalize - cfg.mel_scale = 0.25; % For whisper like Mel scale and normalize - cfg.mmax_init = 0; % Initial value max Mel value, data clamp is mmax - top_db - cfg.mmax_coef = 0; % Dynamic max Mel value decay coefficient (zero lock to found max) - cfg.dynamic_mmax = true; - cfg.enable_vad = true; + cfg.top_db = 80.0; + cfg.mel_offset = 4.0; % Whisper: (mel + 4.0) * 0.25 + cfg.mel_scale = 0.25; + cfg.mmax_init = 0; + cfg.mmax_coef = 0.005; % Whisper mmax tracking: slow decay + cfg.dynamic_mmax = false; + cfg.enable_vad = false; cfg.enable_dtx = false; cfg.dtx_trailing_silence_hops = 0; cfg.dtx_silence_hops_interval = 0; - cfg.update_controls = true; + cfg.update_controls = false; cfg.compress_output = false; + cfg.enable_pcan = false; + cfg.pcan_strength = 0.95; + cfg.pcan_offset = 80.0; + cfg.pcan_gain_bits = 21; + cfg.pcan_smoothing_coef = 819; end function export_mfcc_setup(gen_cfg, cfg) @@ -218,8 +257,9 @@ function export_mfcc_setup(gen_cfg, cfg) v = cfg.dtx_trailing_silence_hops; [b8, j] = add_w16b(v, b8, j); % DTX trailing silence hops v = cfg.dtx_silence_hops_interval; [b8, j] = add_w16b(v, b8, j); % DTX silence frame interval +v = cfg.pcan_smoothing_coef; [b8, j] = add_w32b(v, b8, j); % PCAN smoothing coef in Q14 % Reserved -for i = 1:5 +for i = 1:4 [b8, j] = add_w32b(0, b8, j); end @@ -245,10 +285,9 @@ function export_mfcc_setup(gen_cfg, cfg) v = 0; [b8, j] = add_w16b(v, b8, j); % vtln_high Qx.y TBD v = 0; [b8, j] = add_w16b(v, b8, j); % vtln_low Qx.y TBD v = 0; [b8, j] = add_w16b(v, b8, j); % vtln_warp Qx.y TBD -% reserved16[3] -for i = 1:3 - [b8, j] = add_w16b(0, b8, j); -end +v = q_convert(cfg.pcan_strength, 15); [b8, j] = add_w16b(v, b8, j); % PCAN strength in Q1.15 +v = q_convert(cfg.pcan_offset, 7); [b8, j] = add_w16b(v, b8, j); % PCAN offset in Q8.7 +v = cfg.pcan_gain_bits; [b8, j] = add_w16b(v, b8, j); % PCAN gain bits v = cfg.htk_compat; [b8, j] = add_w8b(v, b8, j); % bool v = cfg.raw_energy; [b8, j] = add_w8b(v, b8, j); % bool v = cfg.remove_dc_offset; [b8, j] = add_w8b(v, b8, j); % bool @@ -261,8 +300,9 @@ function export_mfcc_setup(gen_cfg, cfg) v = cfg.enable_dtx; [b8, j] = add_w8b(v, b8, j); % bool v = cfg.update_controls; [b8, j] = add_w8b(v, b8, j); % bool v = cfg.compress_output; [b8, j] = add_w8b(v, b8, j); % bool -% reserved_bool[4] -for i = 1:4 +v = cfg.enable_pcan; [b8, j] = add_w8b(v, b8, j); % bool +% reserved_bool[3] +for i = 1:3 [b8, j] = add_w8b(0, b8, j); end diff --git a/tools/topology/topology2/include/components/mfcc/ceps13_compress_dtx.conf b/tools/topology/topology2/include/components/mfcc/ceps13_compress_dtx.conf index ce4b7b65c1e5..306d0bca002a 100644 --- a/tools/topology/topology2/include/components/mfcc/ceps13_compress_dtx.conf +++ b/tools/topology/topology2/include/components/mfcc/ceps13_compress_dtx.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 11-Sep-2026 +# Exported MFCC configuration 16-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " @@ -8,7 +8,7 @@ Object.Base.data."mfcc_config" { 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x74,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x14,0x00,0xf4,0x01, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x33,0x03,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x80,0x3e,0x00,0x00, 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, @@ -17,8 +17,8 @@ Object.Base.data."mfcc_config" { 0xc3,0x35,0x00,0x2c,0x00,0x00,0x00,0x00, 0x90,0x01,0xa0,0x00,0x00,0x00,0x14,0x00, 0x0d,0x00,0x17,0x00,0x00,0x00,0x00,0x64, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x01,0x01,0x01, + 0x00,0x00,0x00,0x00,0x00,0x00,0x9a,0x79, + 0x00,0x28,0x15,0x00,0x00,0x01,0x01,0x01, 0x01,0x00,0x00,0x00,0x01,0x01,0x01,0x01, 0x00,0x00,0x00,0x00" } diff --git a/tools/topology/topology2/include/components/mfcc/default.conf b/tools/topology/topology2/include/components/mfcc/default.conf index eaf74a2b7390..297c4ec13581 100644 --- a/tools/topology/topology2/include/components/mfcc/default.conf +++ b/tools/topology/topology2/include/components/mfcc/default.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 11-Sep-2026 +# Exported MFCC configuration 16-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " @@ -8,7 +8,7 @@ Object.Base.data."mfcc_config" { 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x74,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x33,0x03,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x80,0x3e,0x00,0x00, 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, @@ -17,8 +17,8 @@ Object.Base.data."mfcc_config" { 0xc3,0x35,0x00,0x2c,0x00,0x00,0x00,0x00, 0x90,0x01,0xa0,0x00,0x00,0x00,0x14,0x00, 0x0d,0x00,0x17,0x00,0x00,0x00,0x00,0x64, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x01,0x01,0x01, + 0x00,0x00,0x00,0x00,0x00,0x00,0x9a,0x79, + 0x00,0x28,0x15,0x00,0x00,0x01,0x01,0x01, 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00" } diff --git a/tools/topology/topology2/include/components/mfcc/mel40.conf b/tools/topology/topology2/include/components/mfcc/mel40.conf index f18ad76f31b9..abede8b58913 100644 --- a/tools/topology/topology2/include/components/mfcc/mel40.conf +++ b/tools/topology/topology2/include/components/mfcc/mel40.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 11-Sep-2026 +# Exported MFCC configuration 16-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " @@ -7,18 +7,18 @@ Object.Base.data."mfcc_config" { 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x74,0x00,0x00,0x00,0x00,0x02,0x00,0x04, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0xa4,0x00,0x00,0x00,0x00,0x00, + 0x33,0x03,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x80,0x3e,0x00,0x00, - 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0xe0,0x01,0x40,0x01,0x40,0x1f,0x00,0x00, - 0x00,0x00,0x28,0x00,0x00,0x00,0x00,0x04, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01, - 0x01,0x00,0x00,0x01,0x01,0x00,0x01,0x00, + 0x00,0x00,0x28,0x00,0x00,0x00,0x00,0x28, + 0x00,0x00,0x00,0x00,0x00,0x00,0x9a,0x79, + 0x00,0x28,0x15,0x00,0x00,0x00,0x01,0x01, + 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00" } diff --git a/tools/topology/topology2/include/components/mfcc/mel40_10ms_compress.conf b/tools/topology/topology2/include/components/mfcc/mel40_10ms_compress.conf index 232e655df15c..f01a1442475f 100644 --- a/tools/topology/topology2/include/components/mfcc/mel40_10ms_compress.conf +++ b/tools/topology/topology2/include/components/mfcc/mel40_10ms_compress.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 11-Sep-2026 +# Exported MFCC configuration 16-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " @@ -7,18 +7,18 @@ Object.Base.data."mfcc_config" { 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x74,0x00,0x00,0x00,0x00,0x02,0x00,0x04, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0xa4,0x00,0x00,0x00,0x00,0x00, + 0x33,0x03,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x80,0x3e,0x00,0x00, - 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0xe0,0x01,0xa0,0x00,0x4c,0x1d,0x7d,0x00, - 0x00,0x00,0x28,0x00,0x00,0x00,0x00,0x04, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01, - 0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x01, + 0x00,0x00,0x28,0x00,0x00,0x00,0x00,0x28, + 0x00,0x00,0x00,0x00,0x00,0x00,0x9a,0x79, + 0x00,0x28,0x15,0x00,0x00,0x00,0x01,0x01, + 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x01, 0x00,0x00,0x00,0x00" } diff --git a/tools/topology/topology2/include/components/mfcc/mel40_10ms_pcan_compress.conf b/tools/topology/topology2/include/components/mfcc/mel40_10ms_pcan_compress.conf new file mode 100644 index 000000000000..dcbf5ea5164c --- /dev/null +++ b/tools/topology/topology2/include/components/mfcc/mel40_10ms_pcan_compress.conf @@ -0,0 +1,24 @@ +# Exported MFCC configuration 16-Sep-2026 +# cd src/audio/mfcc/tune; octave setup_mfcc.m +Object.Base.data."mfcc_config" { + bytes " + 0x53,0x4f,0x46,0x34,0x00,0x00,0x00,0x00, + 0x74,0x00,0x00,0x00,0x01,0xd0,0x01,0x03, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x74,0x00,0x00,0x00,0x00,0x00,0x00,0x10, + 0x00,0x00,0xa4,0x00,0x00,0x00,0x00,0x00, + 0x33,0x03,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x80,0x3e,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x03,0x00,0x00,0x00,0x01,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xe0,0x01,0xa0,0x00,0x4c,0x1d,0x7d,0x00, + 0x00,0x00,0x28,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x9a,0x79, + 0x00,0x28,0x15,0x00,0x00,0x00,0x01,0x01, + 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x01, + 0x01,0x00,0x00,0x00" +} diff --git a/tools/topology/topology2/include/components/mfcc/mel40_compress.conf b/tools/topology/topology2/include/components/mfcc/mel40_compress.conf index ac3627d2ae49..c4881ac0060f 100644 --- a/tools/topology/topology2/include/components/mfcc/mel40_compress.conf +++ b/tools/topology/topology2/include/components/mfcc/mel40_compress.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 11-Sep-2026 +# Exported MFCC configuration 16-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " @@ -7,18 +7,18 @@ Object.Base.data."mfcc_config" { 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x74,0x00,0x00,0x00,0x00,0x02,0x00,0x04, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0xa4,0x00,0x00,0x00,0x00,0x00, + 0x33,0x03,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x80,0x3e,0x00,0x00, - 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0xe0,0x01,0x40,0x01,0x40,0x1f,0x00,0x00, - 0x00,0x00,0x28,0x00,0x00,0x00,0x00,0x04, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01, - 0x01,0x00,0x00,0x01,0x01,0x00,0x01,0x01, + 0x00,0x00,0x28,0x00,0x00,0x00,0x00,0x28, + 0x00,0x00,0x00,0x00,0x00,0x00,0x9a,0x79, + 0x00,0x28,0x15,0x00,0x00,0x00,0x01,0x01, + 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x01, 0x00,0x00,0x00,0x00" } diff --git a/tools/topology/topology2/include/components/mfcc/mel80.conf b/tools/topology/topology2/include/components/mfcc/mel80.conf index f1d98ea5c002..16b9d12fabed 100644 --- a/tools/topology/topology2/include/components/mfcc/mel80.conf +++ b/tools/topology/topology2/include/components/mfcc/mel80.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 11-Sep-2026 +# Exported MFCC configuration 16-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " @@ -7,18 +7,18 @@ Object.Base.data."mfcc_config" { 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x74,0x00,0x00,0x00,0x00,0x02,0x00,0x04, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0xa4,0x00,0x00,0x00,0x00,0x00, + 0x33,0x03,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x80,0x3e,0x00,0x00, - 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x90,0x01,0xa0,0x00,0x40,0x1f,0x00,0x00, - 0x00,0x00,0x50,0x00,0x00,0x00,0x00,0x04, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01, - 0x01,0x00,0x00,0x01,0x01,0x00,0x01,0x00, + 0x00,0x00,0x50,0x00,0x00,0x00,0x00,0x28, + 0x00,0x00,0x00,0x00,0x00,0x00,0x9a,0x79, + 0x00,0x28,0x15,0x00,0x00,0x00,0x01,0x01, + 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00" } diff --git a/tools/topology/topology2/include/components/mfcc/mel80_compress.conf b/tools/topology/topology2/include/components/mfcc/mel80_compress.conf index a56c1b60e335..e104b3579e75 100644 --- a/tools/topology/topology2/include/components/mfcc/mel80_compress.conf +++ b/tools/topology/topology2/include/components/mfcc/mel80_compress.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 11-Sep-2026 +# Exported MFCC configuration 16-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " @@ -7,18 +7,18 @@ Object.Base.data."mfcc_config" { 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x74,0x00,0x00,0x00,0x00,0x02,0x00,0x04, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0xa4,0x00,0x00,0x00,0x00,0x00, + 0x33,0x03,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x80,0x3e,0x00,0x00, - 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x90,0x01,0xa0,0x00,0x40,0x1f,0x00,0x00, - 0x00,0x00,0x50,0x00,0x00,0x00,0x00,0x04, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01, - 0x01,0x00,0x00,0x01,0x01,0x00,0x01,0x01, + 0x00,0x00,0x50,0x00,0x00,0x00,0x00,0x28, + 0x00,0x00,0x00,0x00,0x00,0x00,0x9a,0x79, + 0x00,0x28,0x15,0x00,0x00,0x00,0x01,0x01, + 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x01, 0x00,0x00,0x00,0x00" } diff --git a/tools/topology/topology2/include/components/mfcc/mel80_compress_dtx.conf b/tools/topology/topology2/include/components/mfcc/mel80_compress_dtx.conf index 2264afbd1086..c3e53b28c1fb 100644 --- a/tools/topology/topology2/include/components/mfcc/mel80_compress_dtx.conf +++ b/tools/topology/topology2/include/components/mfcc/mel80_compress_dtx.conf @@ -1,4 +1,4 @@ -# Exported MFCC configuration 11-Sep-2026 +# Exported MFCC configuration 16-Sep-2026 # cd src/audio/mfcc/tune; octave setup_mfcc.m Object.Base.data."mfcc_config" { bytes " @@ -7,18 +7,18 @@ Object.Base.data."mfcc_config" { 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x74,0x00,0x00,0x00,0x00,0x02,0x00,0x04, - 0x00,0x00,0x00,0x00,0x14,0x00,0xf4,0x01, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0xa4,0x00,0x14,0x00,0xf4,0x01, + 0x33,0x03,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x80,0x3e,0x00,0x00, - 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x90,0x01,0xa0,0x00,0x40,0x1f,0x00,0x00, - 0x00,0x00,0x50,0x00,0x00,0x00,0x00,0x04, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01, - 0x01,0x00,0x00,0x01,0x01,0x01,0x01,0x01, + 0x00,0x00,0x50,0x00,0x00,0x00,0x00,0x28, + 0x00,0x00,0x00,0x00,0x00,0x00,0x9a,0x79, + 0x00,0x28,0x15,0x00,0x00,0x00,0x01,0x01, + 0x01,0x00,0x00,0x00,0x00,0x01,0x00,0x01, 0x00,0x00,0x00,0x00" } diff --git a/tools/topology/topology2/include/components/mfcc/mel80_pcan_compress.conf b/tools/topology/topology2/include/components/mfcc/mel80_pcan_compress.conf new file mode 100644 index 000000000000..4892bf431b6c --- /dev/null +++ b/tools/topology/topology2/include/components/mfcc/mel80_pcan_compress.conf @@ -0,0 +1,24 @@ +# Exported MFCC configuration 16-Sep-2026 +# cd src/audio/mfcc/tune; octave setup_mfcc.m +Object.Base.data."mfcc_config" { + bytes " + 0x53,0x4f,0x46,0x34,0x00,0x00,0x00,0x00, + 0x74,0x00,0x00,0x00,0x01,0xd0,0x01,0x03, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x74,0x00,0x00,0x00,0x00,0x02,0x00,0x04, + 0x00,0x00,0xa4,0x00,0x00,0x00,0x00,0x00, + 0x33,0x03,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x80,0x3e,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x03,0x00,0x00,0x00,0x01,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x90,0x01,0xa0,0x00,0x40,0x1f,0x00,0x00, + 0x00,0x00,0x50,0x00,0x00,0x00,0x00,0x28, + 0x00,0x00,0x00,0x00,0x00,0x00,0x9a,0x79, + 0x00,0x28,0x15,0x00,0x00,0x00,0x01,0x01, + 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x01, + 0x01,0x00,0x00,0x00" +} From 3e2934705d1eb330f68c2835e4c7c938240a7eee Mon Sep 17 00:00:00 2001 From: Seppo Ingalsuo Date: Wed, 23 Sep 2026 18:20:37 +0300 Subject: [PATCH 18/35] audio: microwakeword: implement mww component Add the microWakeWord keyword-spotting processing component. MWW runs a TensorFlow Lite Micro streaming network (based on the microWakeWord project) on MFCC features in the Data Processing domain. The component supports consuming 8-bit normalized Mel features in PCAN mode or 32-bit features with AGC. Wake word triggers notify downstream components via AMS or KPB, and Kconfig options allow selecting built-in models or runtime bytes control configuration. Signed-off-by: Seppo Ingalsuo --- src/audio/CMakeLists.txt | 3 + src/audio/Kconfig | 1 + src/audio/microwakeword/CMakeLists.txt | 375 ++++++++++ src/audio/microwakeword/Kconfig | 64 ++ src/audio/microwakeword/README.md | 151 ++++ src/audio/microwakeword/llext-wrap.c | 135 ++++ src/audio/microwakeword/llext/CMakeLists.txt | 185 +++++ src/audio/microwakeword/llext/llext.toml.h | 6 + src/audio/microwakeword/mww.c | 740 +++++++++++++++++++ src/audio/microwakeword/mww.toml | 21 + src/audio/microwakeword/mww_model.cc | 273 +++++++ src/audio/microwakeword/mww_model.h | 70 ++ uuid-registry.txt | 1 + 13 files changed, 2025 insertions(+) create mode 100644 src/audio/microwakeword/CMakeLists.txt create mode 100644 src/audio/microwakeword/Kconfig create mode 100644 src/audio/microwakeword/README.md create mode 100644 src/audio/microwakeword/llext-wrap.c create mode 100644 src/audio/microwakeword/llext/CMakeLists.txt create mode 100644 src/audio/microwakeword/llext/llext.toml.h create mode 100644 src/audio/microwakeword/mww.c create mode 100644 src/audio/microwakeword/mww.toml create mode 100644 src/audio/microwakeword/mww_model.cc create mode 100644 src/audio/microwakeword/mww_model.h diff --git a/src/audio/CMakeLists.txt b/src/audio/CMakeLists.txt index 92002c8b7c1c..1576166bf92b 100644 --- a/src/audio/CMakeLists.txt +++ b/src/audio/CMakeLists.txt @@ -56,6 +56,9 @@ if(NOT CONFIG_COMP_MODULE_SHARED_LIBRARY_BUILD) if(CONFIG_COMP_MFCC) add_subdirectory(mfcc) endif() + if(CONFIG_COMP_MWW) + add_subdirectory(microwakeword) + endif() if(CONFIG_COMP_MIXER) add_subdirectory(mixer) endif() diff --git a/src/audio/Kconfig b/src/audio/Kconfig index 8accb25738a2..a18f17523c90 100644 --- a/src/audio/Kconfig +++ b/src/audio/Kconfig @@ -142,6 +142,7 @@ rsource "eq_iir/Kconfig" rsource "google/Kconfig" rsource "igo_nr/Kconfig" rsource "mfcc/Kconfig" +rsource "microwakeword/Kconfig" rsource "mixer/Kconfig" rsource "mixin_mixout/Kconfig" rsource "module_adapter/Kconfig" diff --git a/src/audio/microwakeword/CMakeLists.txt b/src/audio/microwakeword/CMakeLists.txt new file mode 100644 index 000000000000..cd298c4ccea7 --- /dev/null +++ b/src/audio/microwakeword/CMakeLists.txt @@ -0,0 +1,375 @@ +# Copyright (c) 2026 Intel Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# Newer xt-clang LLVMs accept this to keep literals in .rodata; standard +# Clang or older xt-clang rejects the sub-option. Detect it. +set(MWW_TEXT_SECTION_LITERALS_FALSE_FLAG "") +if(CMAKE_C_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + include(CheckCXXCompilerFlag) + check_cxx_compiler_flag("-mllvm --text-section-literals=false" + MWW_HAS_MLLVM_TEXT_SECTION_LITERALS_FALSE) + if(MWW_HAS_MLLVM_TEXT_SECTION_LITERALS_FALSE) + set(MWW_TEXT_SECTION_LITERALS_FALSE_FLAG -mllvm --text-section-literals=false) + add_compile_options(${MWW_TEXT_SECTION_LITERALS_FALSE_FLAG}) + endif() +endif() + +# are we building the llext module ? +if(CONFIG_COMP_MWW STREQUAL "m" AND DEFINED CONFIG_LLEXT) + add_subdirectory(llext ${PROJECT_BINARY_DIR}/mww_llext) + add_dependencies(app mww) + return() +endif() + +# Own dedicated static-build targets (mww_tflm_lib/mww_nn_hifi_lib), distinct +# from src/audio/tensorflow/'s tflm_lib/nn_hifi_lib -- this model needs a +# materially different TFLM kernel set (resource variables, CALL_ONCE, +# Concatenation, StridedSlice, Logistic, Quantize) that tflmcly's build does +# not compile in, so the two components do not share libraries yet (see +# plan Stage 10 for a deferred shared-library refactor). +set(MWW_HAVE_NNLIB_HIFI4 FALSE) +if(CMAKE_C_COMPILER_ID STREQUAL "Clang" AND CONFIG_XTENSA_HIFI4) + set(MWW_HAVE_NNLIB_HIFI4 TRUE) +endif() + +set(NN_HIFI_PATH ${sof_top_dir}/../nnlib-hifi4/xa_nnlib) + +# paths for dependencies +set(TFLM_PATH ${sof_top_dir}/../tflite-micro) +set(FLATBUFFERS_PATH ${sof_top_dir}/../flatbuffers) +set(GEMMLOWP_PATH ${sof_top_dir}/../gemmlowp) +set(RUY_PATH ${sof_top_dir}/../ruy) + +set(MWW_TOOLCHAIN_INCLUDE_ROOT + ${ZEPHYR_SDK_INSTALL_DIR}/gnu/xtensa-${SOC_TOOLCHAIN_NAME}_zephyr-elf/xtensa-${SOC_TOOLCHAIN_NAME}_zephyr-elf/include) +set(MWW_HAL_SOC_PATH ${sof_top_dir}/../modules/hal/xtensa/zephyr/soc/${SOC_TOOLCHAIN_NAME}) + +if(MWW_HAVE_NNLIB_HIFI4) + +add_library(mww_nn_hifi_lib STATIC + ${NN_HIFI_PATH}/algo/common/src/xa_nnlib_common_api.c + ${NN_HIFI_PATH}/algo/kernels/activations/hifi4/xa_nn_activations_32_32.c + ${NN_HIFI_PATH}/algo/kernels/activations/hifi4/xa_nn_activations_f32_f32.c + ${NN_HIFI_PATH}/algo/ndsp/hifi4/src/nanf_tbl.c + ${NN_HIFI_PATH}/algo/ndsp/hifi4/src/inff_tbl.c + ${NN_HIFI_PATH}/algo/ndsp/hifi4/src/pow2f_tbl.c + ${NN_HIFI_PATH}/algo/ndsp/hifi4/src/expf_tbl.c + ${NN_HIFI_PATH}/algo/ndsp/hifi4/src/scl_sigmoidf_hifi4.c + ${NN_HIFI_PATH}/algo/ndsp/hifi4/src/vec_sigmoidf_hifi4.c + + ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_matXvec_8x8_8_circ.c + ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_conv2d_pointwise_8x16.c + ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_circ_buf.c + ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_conv2d_std_8x8.c + ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_conv2d_depthwise.c + ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_matXvec_16x16_16_circ.c + ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_conv2d_std_circ_buf.c + ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_matXvec_8x16_16_circ.c + ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_conv2d_std_8x16.c + ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_conv2d_depthwise_8x8.c + ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_conv2d_pointwise_8x8.c + ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_conv2d_std_16x16.c + ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_conv2d_pointwise_asym8xasym8.c + ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_conv2d_std_asym8xasym8.c + ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_conv2d_depthwise_asym8xasym8.c + ${NN_HIFI_PATH}/algo/kernels/fc/hifi4/xa_nn_fully_connected.c + ${NN_HIFI_PATH}/algo/kernels/pool/hifi4/xa_nn_inv_256_tbl.c + ${NN_HIFI_PATH}/algo/kernels/reorg/hifi4/xa_nn_transpose_8.c + ${NN_HIFI_PATH}/algo/kernels/reorg/hifi4/xa_nn_pad_8.c + ${NN_HIFI_PATH}/algo/kernels/reorg/hifi4/xa_nn_stride_slice_int8.c +) + +target_include_directories(mww_nn_hifi_lib PRIVATE + ${NN_HIFI_PATH}/include + ${NN_HIFI_PATH}/algo/common/include/ + ${NN_HIFI_PATH}/include/nnlib/ + ${NN_HIFI_PATH}/algo/ndsp/hifi4/include + ${MWW_TOOLCHAIN_INCLUDE_ROOT} + ${MWW_HAL_SOC_PATH} +) + +if(MWW_HAS_MLLVM_TEXT_SECTION_LITERALS_FALSE) + target_compile_options(mww_nn_hifi_lib PRIVATE + ${MWW_TEXT_SECTION_LITERALS_FALSE_FLAG} + ) +endif() + +target_compile_definitions(mww_nn_hifi_lib PRIVATE + -DHIFI4=1 + -DHAVE_VFPU=1 + -DHAVE_VFPU_SINGLE_PRECISION=1 + -DXCHAL_HAVE_HIFI4=1 + -DXCHAL_HAVE_HIFI4_VFPU=1 + + __xtensa__=1 + __XTENSA__=1 + __XCC__ + __XCC_CLANG__ + "XT_MAX(a,b)=((a)>(b)?(a):(b))" + "XT_MIN(a,b)=((a)<(b)?(a):(b))" + "AE_MOVINT16_FROMINT32(v)=((ae_int16)(v))" + "AE_CVT64F32_H(v)=((ae_int64)(int64_t)(int32_t)AE_MOVAD32_H(v))" + "AE_CVT64F32_L(v)=((ae_int64)(int64_t)(int32_t)AE_MOVAD32_L(v))" +) + +target_compile_options(mww_nn_hifi_lib PRIVATE + -fsigned-char + -fno-exceptions + -mlongcalls + -fno-zero-initialized-in-bss + -Wsign-compare + -DMODEL_INT16 + -DNNLIB_V2 + -Dhifi4 + -DTFLITE_SINGLE_ROUNDING=1 + -mcpu=${SOC_TOOLCHAIN_NAME} + "SHELL:-include xtensahifiintrin.h" + "SHELL:-include ${NN_HIFI_PATH}/algo/common/include/xa_nnlib_hifi_isa_compat.h" +) + +endif() # MWW_HAVE_NNLIB_HIFI4 + +# TFLM kernel sources beyond what tensorflow/CMakeLists.txt currently builds: +# the hey_jarvis.tflite MixConv streaming graph uses CALL_ONCE, VAR_HANDLE, +# READ_VARIABLE, ASSIGN_VARIABLE, CONCATENATION, STRIDED_SLICE, LOGISTIC and +# QUANTIZE in addition to CONV_2D/DEPTHWISE_CONV_2D/FULLY_CONNECTED/RESHAPE +# (confirmed by direct flatbuffer inspection, see plan Stage 1). +add_library(mww_tflm_lib STATIC + ${TFLM_PATH}/tensorflow/compiler/mlir/lite/core/api/error_reporter.cc + ${TFLM_PATH}/tensorflow/compiler/mlir/lite/schema/schema_utils.cc + ${TFLM_PATH}/tensorflow/lite/core/c/common.cc + ${TFLM_PATH}/tensorflow/lite/core/api/flatbuffer_conversions.cc + ${TFLM_PATH}/tensorflow/lite/core/api/tensor_utils.cc + ${TFLM_PATH}/tensorflow/lite/kernels/internal/common.cc + ${TFLM_PATH}/tensorflow/lite/kernels/internal/tensor_utils.cc + ${TFLM_PATH}/tensorflow/lite/kernels/internal/runtime_shape.cc + ${TFLM_PATH}/tensorflow/lite/kernels/internal/quantization_util.cc + ${TFLM_PATH}/tensorflow/lite/kernels/internal/tensor_ctypes.cc + ${TFLM_PATH}/tensorflow/lite/kernels/internal/portable_tensor_utils.cc + ${TFLM_PATH}/tensorflow/lite/kernels/internal/reference/comparisons.cc + ${TFLM_PATH}/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc + ${TFLM_PATH}/tensorflow/lite/kernels/kernel_util.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/kernel_util.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/call_once.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/var_handle.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/read_variable.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/assign_variable.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/concatenation.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/conv_common.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/depthwise_conv_common.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/fully_connected_common.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/logistic_common.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/quantize_common.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/reshape_common.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/strided_slice_common.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/dequantize_common.cc + # reference kernel implementations for the ops above + ${TFLM_PATH}/tensorflow/lite/micro/kernels/conv.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/depthwise_conv.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/fully_connected.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/logistic.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/quantize.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/reshape.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/strided_slice.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/dequantize.cc + ${TFLM_PATH}/tensorflow/lite/micro/mock_micro_graph.cc + ${TFLM_PATH}/tensorflow/lite/micro/flatbuffer_utils.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_interpreter_graph.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_resource_variable.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_interpreter_context.cc + ${TFLM_PATH}/tensorflow/lite/micro/fake_micro_context.cc + ${TFLM_PATH}/tensorflow/lite/micro/arena_allocator/persistent_arena_buffer_allocator.cc + ${TFLM_PATH}/tensorflow/lite/micro/arena_allocator/recording_single_arena_buffer_allocator.cc + ${TFLM_PATH}/tensorflow/lite/micro/arena_allocator/non_persistent_arena_buffer_allocator.cc + ${TFLM_PATH}/tensorflow/lite/micro/arena_allocator/single_arena_buffer_allocator.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_allocator.cc + ${TFLM_PATH}/tensorflow/lite/micro/tflite_bridge/flatbuffer_conversions_bridge.cc + ${TFLM_PATH}/tensorflow/lite/micro/tflite_bridge/micro_error_reporter.cc + ${TFLM_PATH}/tensorflow/lite/micro/system_setup.cc + ${TFLM_PATH}/tensorflow/lite/micro/test_helper_custom_ops.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_context.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_log.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_profiler.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_time.cc + # debug_log.cc is replaced by our printk-backed DebugLog() in mww.c. + #${TFLM_PATH}/tensorflow/lite/micro/debug_log.cc + ${TFLM_PATH}/tensorflow/lite/micro/test_helpers.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_op_resolver.cc + ${TFLM_PATH}/tensorflow/lite/micro/recording_micro_allocator.cc + ${TFLM_PATH}/tensorflow/lite/micro/memory_helpers.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_allocation_info.cc + ${TFLM_PATH}/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc + ${TFLM_PATH}/tensorflow/lite/micro/memory_planner/non_persistent_buffer_planner_shim.cc + ${TFLM_PATH}/tensorflow/lite/micro/memory_planner/linear_memory_planner.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_utils.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_interpreter.cc + mww_model_data.cc + mww_model.cc +) + +target_include_directories(mww_tflm_lib PRIVATE + ${TFLM_PATH} + ${FLATBUFFERS_PATH}/include + ${GEMMLOWP_PATH} + ${RUY_PATH} + ${sof_top_dir}/posix/include + ${sof_top_dir}/../modules/hal/xtensa/include + ${MWW_HAL_SOC_PATH} +) +if(MWW_HAVE_NNLIB_HIFI4) +target_include_directories(mww_tflm_lib PRIVATE + ${NN_HIFI_PATH} + ${NN_HIFI_PATH}/include +) +endif() + +if(MWW_HAVE_NNLIB_HIFI4) +target_compile_definitions(mww_tflm_lib PRIVATE + -DHIFI4=1 + -DHAVE_VFPU=1 + -DHAVE_VFPU_SINGLE_PRECISION=1 + -DXCHAL_HAVE_HIFI4=1 + -DXCHAL_HAVE_HIFI4_VFPU=1 + + __xtensa__=1 + __XTENSA__=1 + __XCC__ + __XCC_CLANG__ + "XT_MAX(a,b)=((a)>(b)?(a):(b))" + "XT_MIN(a,b)=((a)<(b)?(a):(b))" + "AE_MOVINT16_FROMINT32(v)=((ae_int16)(v))" + "AE_CVT64F32_H(v)=((ae_int64)(int64_t)(int32_t)AE_MOVAD32_H(v))" + "AE_CVT64F32_L(v)=((ae_int64)(int64_t)(int32_t)AE_MOVAD32_L(v))" +) +target_compile_options(mww_tflm_lib PRIVATE + -DHIFI4 + -DKERNELS_OPTIMIZED_FOR_SPEED + -DNNLIB_V2 +) +endif() # MWW_HAVE_NNLIB_HIFI4 + +target_compile_options(mww_tflm_lib PRIVATE + -std=c++17 + -fno-rtti + -fno-exceptions + -fno-threadsafe-statics + -Wnon-virtual-dtor + -fno-unwind-tables + -fmessage-length=0 + -DTF_LITE_STATIC_MEMORY + -DTF_LITE_DISABLE_X86_NEON + -Wsign-compare + -Wdouble-promotion + -Wunused-variable + -Wswitch + -Wvla + -Wall + -Wextra + -Wmissing-field-initializers + -Wstrict-aliasing + -Wno-unused-parameter + -DXTENSA + -DTF_LITE_MCU_DEBUG_LOG + -DTF_LITE_USE_CTIME + -mlongcalls + -Wno-shadow +) + +if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|Xtensa" OR CMAKE_C_COMPILER MATCHES "xt-clang") + target_compile_options(mww_tflm_lib PRIVATE + -stdlib=libc++ + "SHELL:-include xtensahifiintrin.h" + -fno-vectorize + -fno-slp-vectorize + ) +else() + target_compile_options(mww_tflm_lib PRIVATE + -fno-tree-vectorize + -fno-tree-slp-vectorize + ) +endif() + +if(CMAKE_C_COMPILER_ID STREQUAL "Xtensa") + target_compile_options(mww_tflm_lib PRIVATE + --xtensa-core=ace10_LX7HiFi4_2022_10 + -mcoproc + ) +elseif(CMAKE_C_COMPILER_ID STREQUAL "Clang") + target_compile_options(mww_tflm_lib PRIVATE + -mcpu=${SOC_TOOLCHAIN_NAME} + ) +endif() + +if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + target_include_directories(mww_tflm_lib SYSTEM PRIVATE + ${MWW_TOOLCHAIN_INCLUDE_ROOT}/c++/14.3.0 + ${MWW_TOOLCHAIN_INCLUDE_ROOT}/c++/14.3.0/xtensa-${SOC_TOOLCHAIN_NAME}_zephyr-elf + ${MWW_TOOLCHAIN_INCLUDE_ROOT} + ) +endif() + +add_local_sources(sof mww.c) + +if(MWW_HAVE_NNLIB_HIFI4) + zephyr_link_libraries(mww_tflm_lib mww_nn_hifi_lib) +else() + zephyr_link_libraries(mww_tflm_lib) +endif() + +# Same libc-shim rationale as src/audio/tensorflow/CMakeLists.txt: SOF's +# default CONFIG_MINIMAL_LIBC is missing abs()/libm symbols TFLM needs in a +# statically-linked (non-LLEXT) image; extract just those archive members +# from the toolchain's libc.a into a small private archive. +if(NOT CONFIG_COMP_MWW STREQUAL "m") + set(MWW_TOOLCHAIN_LIBC_ARCHIVE + ${ZEPHYR_SDK_INSTALL_DIR}/gnu/xtensa-${SOC_TOOLCHAIN_NAME}_zephyr-elf/xtensa-${SOC_TOOLCHAIN_NAME}_zephyr-elf/lib/libc.a) + set(MWW_LIBC_SHIM_MEMBERS + libc_stdlib_abs.c.o + libm_math_s_floor.c.o + libm_math_sf_exp.c.o + libm_math_sf_log.c.o + libm_math_sf_ceil.c.o + libm_math_s_frexp.c.o + libm_common_s_round.c.o + libm_common_sf_fmax.c.o + libm_common_sf_fmin.c.o + libm_common_sf_round.c.o + libm_common_sf_isnan.c.o + libm_common_sf_issignaling.c.o + libm_common_math_errf_uflowf.c.o + libm_common_math_errf_oflowf.c.o + libm_common_math_errf_divzerof.c.o + libm_common_math_errf_invalidf.c.o + ) + set(MWW_LIBC_SHIM_DIR ${CMAKE_CURRENT_BINARY_DIR}/mww_libc_shim) + set(MWW_LIBC_SHIM_ARCHIVE ${CMAKE_CURRENT_BINARY_DIR}/libmww_libc_shim.a) + file(MAKE_DIRECTORY ${MWW_LIBC_SHIM_DIR}) + add_custom_command( + OUTPUT ${MWW_LIBC_SHIM_ARCHIVE} + COMMAND ${CMAKE_AR} x ${MWW_TOOLCHAIN_LIBC_ARCHIVE} ${MWW_LIBC_SHIM_MEMBERS} + COMMAND ${CMAKE_AR} rcs ${MWW_LIBC_SHIM_ARCHIVE} ${MWW_LIBC_SHIM_MEMBERS} + WORKING_DIRECTORY ${MWW_LIBC_SHIM_DIR} + DEPENDS ${MWW_TOOLCHAIN_LIBC_ARCHIVE} + COMMENT "Extracting abs()/libm members TFLM needs from the toolchain libc.a" + ) + add_custom_target(mww_libc_shim_gen DEPENDS ${MWW_LIBC_SHIM_ARCHIVE}) + add_library(mww_libc_shim STATIC IMPORTED GLOBAL) + set_target_properties(mww_libc_shim PROPERTIES IMPORTED_LOCATION ${MWW_LIBC_SHIM_ARCHIVE}) + add_dependencies(mww_libc_shim mww_libc_shim_gen) + zephyr_link_libraries(mww_libc_shim) + + target_compile_definitions(mww_tflm_lib PRIVATE NDEBUG) +endif() +zephyr_include_directories(${TFLM_PATH}) +zephyr_include_directories(${FLATBUFFERS_PATH}/include) +zephyr_include_directories(${GEMMLOWP_PATH}) +zephyr_include_directories(${RUY_PATH}) +if(MWW_HAVE_NNLIB_HIFI4) +zephyr_include_directories(${NN_HIFI_PATH}/algo/kernels/include) +zephyr_include_directories(${NN_HIFI_PATH}/include) +endif() +target_include_directories(modules_sof SYSTEM PRIVATE + ${MWW_TOOLCHAIN_INCLUDE_ROOT}/c++/14.3.0 + ${MWW_TOOLCHAIN_INCLUDE_ROOT}/c++/14.3.0/xtensa-${SOC_TOOLCHAIN_NAME}_zephyr-elf + ${MWW_TOOLCHAIN_INCLUDE_ROOT} +) diff --git a/src/audio/microwakeword/Kconfig b/src/audio/microwakeword/Kconfig new file mode 100644 index 000000000000..4ee610b75d49 --- /dev/null +++ b/src/audio/microwakeword/Kconfig @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: BSD-3-Clause + +config COMP_MWW + tristate "microWakeWord keyword-spotting component" + depends on SOF_STAGING + depends on CPP + depends on STD_CPP17 + help + Select for microWakeWord component support. This module runs a + TensorFlow Lite Micro streaming keyword-spotting model (based on + OHF-Voice's micro-wake-word project) on MFCC features and wakes + the host via KPB/WoV when the wake word is detected. + +if COMP_MWW + +config COMP_MWW_DEBUG_TRACE + bool "Verbose per-hop / per-inference debug traces" + default n + help + Enable mww per-hop and per-inference debug prints in the hot path. + Leave disabled for production and normal capture tuning. + +config COMP_MWW_MODEL_FROM_CONTROL + bool "Load MWW model from bytes control instead of built-in header" + default n + help + When enabled, the MWW wake-word model is loaded from a runtime + or topology configuration data blob via binary kcontrol (bytes control) + instead of using the static mww_model_data embedded in the firmware image. + +choice COMP_MWW_BUILTIN_MODEL + prompt "Built-in microWakeWord model" + default COMP_MWW_MODEL_HI_INTEL + depends on !COMP_MWW_MODEL_FROM_CONTROL + help + Select which wake-word model is embedded directly into the firmware. + Selecting one keyword automatically disables the other. + +config COMP_MWW_MODEL_HI_INTEL + bool "Hi Intel" + help + Authentic speech PCAN model for "Hi Intel". + +config COMP_MWW_MODEL_HEY_JARVIS + bool "Hey Jarvis" + help + Official microWakeWord reference PCAN model for "Hey Jarvis". + +endchoice + +config COMP_MWW_PCAN + bool "Expect PCAN-normalized MFCC input" + default n + depends on COMP_MFCC_PCAN + help + Select when the upstream MFCC component runs with PCAN + (Per-Channel AGC Normalization) enabled. In that mode the Mel + filterbank energies arriving at MWW are already normalized to the + int8 dynamic range, so the in-component peak-tracking AGC and the + Q9.23->Q1.7 requantization step are compiled out and each Mel value + is clamped and cast directly to int8. Leave disabled for the + classic non-PCAN MFCC pipeline that relies on MWW's built-in AGC. + +endif # COMP_MWW diff --git a/src/audio/microwakeword/README.md b/src/audio/microwakeword/README.md new file mode 100644 index 000000000000..cf050e73ed4a --- /dev/null +++ b/src/audio/microwakeword/README.md @@ -0,0 +1,151 @@ +# microWakeWord (MWW) Architecture + +This directory provides an SOF component wrapping [OHF-Voice's +microWakeWord](https://github.com/OHF-Voice/micro-wake-word) streaming +keyword-spotting model, integrated with MFCC feature extraction and the +same Key Phrase Buffer (KPB) Wake-on-Voice (WoV) trigger infrastructure +used by `src/audio/tensorflow` (`tflmcly`). Unlike `tflmcly`'s 4-way +softmax classifier, microWakeWord outputs a single sigmoid wake-word +probability from a stateful streaming TFLM graph. + +--- + +## Overview + +`mww` consumes 40-bin mel feature hops from the upstream `mfcc` component, +requantizes them per the model's real `input_scale`/`input_zero_point`, +and runs inference via TensorFlow Lite Micro in the **Data Processing +(DP) domain**. The reference model (`hey_jarvis.tflite` v2) is a MixConv +streaming graph that keeps its own internal ring-buffer state (TFLM +resource variables + a `CALL_ONCE` init subgraph), so `mww` only needs to +supply `MWW_FEATURE_SLICE_COUNT` (3) fresh 10ms/40-bin hops per +`Invoke()` rather than maintaining a caller-side sliding window. + +When `probability >= MWW_DETECT_THRESHOLD` (0.5), `mww` acts as an **AMS +producer** of `AMS_KPD_MSG_UUID` — mirroring +`src/samples/audio/detect_test.c`'s pattern +(`ams_helper_register_producer()` / `ams_helper_prepare_payload()` / +`ams_send()`) rather than `tflmcly`'s notifier call — to signal KPB to +drain its pre-roll audio history to the host. `src/audio/kpb.c`'s +existing AMS-consumer branch requires no changes to receive this. + +--- + +## Architecture & Data Flow + +Structurally identical to `tflmcly`'s dual-path WoV topology (see +`src/audio/tensorflow/README.md`), with `mww` in place of `tflmcly` and +its own 10ms-hop MFCC profile: + +```mermaid +graph TD + subgraph Audio_Input ["HDA Analog Capture"] + HDA["HDA Analog Input (dai_type: HDA)"] + end + + subgraph KPB_Pipeline ["Capture & KPB Pipeline"] + Gain["gain.2.1 (Volume Control)"] + KPB["kpb.2.1 (Key Phrase Buffer)"] + end + + subgraph Detect_Pipeline ["Real-Time Detection Path (KPB Pin 1) - DP Domain"] + SRC["src.1.1 (Resampler: 48kHz -> 16kHz)"] + MFCC["mfcc.1.1 (Mel-40 Feature Extractor, 10ms hop)"] + MWW["mww.1.1 (microWakeWord LLEXT)"] + VSink["virtual.mww_sink (Termination)"] + end + + subgraph Host_Pipeline ["Host WoV Draining Path (KPB Pin 2)"] + Host["host-copier.0.capture (PCM Capture Stream)"] + end + + HDA --> DAI["dai-copier.HDA.Analog.capture"] + DAI --> Gain + Gain --> KPB + KPB -- Pin 1: Live Audio --> SRC + SRC --> MFCC + MFCC -- Mel-40/10ms Tensors --> MWW + MWW --> VSink + + MWW -.->|AMS_KPD_MSG_UUID producer| KPB + + KPB -- Pin 2: Pre-roll History Buffer --> Host +``` + +--- + +## Deployment Targets + +- **Aphid (PTL / ACE 3.0)**: `CONFIG_COMP_MWW=m`, built and deployed as a + real loadable `.llext` module — the target that exercises SOF's LLEXT + loading/relocation path, not just a compile check. + `app/boards/intel_adsp_ace30_ptl.conf` disables `CONFIG_COMP_TENSORFLOW` + on this board (tflmcly is not yet GNU-toolchain-clean here, and mww + builds its own independent TFLM lib copy so it doesn't need it). + +Building `mww` as a real LLEXT module (rather than statically linked) +required several fixes to SOF's and Zephyr's LLEXT support, landed +alongside this component: + +- `library_manager/llext_manager.c` — section rebase/relocation fixup + after SOF's own address rebasing, `.bss`/`DATA` region packing, and a + tracked `.exported_sym` segment. +- `zephyr/CMakeLists.txt` — `-Wl,-Bsymbolic-functions` so multi-TU C++ + internal calls (TFLM spans many `.cc` files) resolve locally. +- `src/lib/cpp_new_export.cpp` — exported `operator new`/`delete`/ + `__cxa_pure_virtual` for C++ LLEXT modules. +- A companion Zephyr fork branch (`feature/llext-stb-weak-fixes`) treats + `STB_WEAK` symbols (C++ template-instantiated vtables/typeinfo) the + same as `STB_GLOBAL` in the LLEXT export table, PLT resolution, and + `GLOB_DAT`/`JMP_SLOT` relocation — required for TFLM's vtables to + resolve correctly inside the loaded module. + +--- + +## Building and Testing + +### Building the topology target + +From `sof/tools/build_tools`: + +```bash +ninja topology2_prod_sof-ptl-hda-mww-kpb +``` + +### Deploying and testing on aphid + +```bash +# Pristine firmware + llext build +CCACHE_DISABLE=1 ./sof/scripts/xtensa-build-zephyr.py -p ptl -k sof/keys/otc_private_key_3k.pem + +# Deploy .ri / sof-ipc4-lib tree / .tplg to aphid, then reboot to load +ssh -i ~/.ssh/aphid_deploy root@aphid 'timeout 10s arecord -D hw:0,0 -f S32_LE -r 16000 -c 2 -d 10 /tmp/test.wav' +ssh -i ~/.ssh/aphid_deploy root@aphid 'timeout 10s /usr/local/bin/mtrace-reader.py' +``` + +Look for `MWW DBG feature range` and `MWW probability=` log lines in the +mtrace output. + +--- + +## Known Issues / Open Items + +--- + +## Source Files + +- **mww.c**: SOF module adapter implementation (init/prepare/process/reset, + AMS producer signaling). +- **mww_model.cc** / **mww_model.h**: TFLM C++ API bridge exposing + `MWW_SetModel()` / `MWW_InitOps()` / `MWW_ProcessClassify()`. +- **mww_model_data.cc** / **mww_model_data.h**: model flatbuffer data + (placeholder, swappable for a trained/converted checkpoint). +- **mww.toml**: rimage module manifest entry. +- **llext/**: LLEXT build scaffolding (CMakeLists.txt, llext.toml.h, + reentrant-stub shims mirroring `src/audio/template/`). +- **../../tools/topology/topology2/include/components/mww.conf**: + Topology v2 widget class definition. +- **../../tools/topology/topology2/include/pipelines/cavs/host-gateway-src-mfcc-mww-capture.conf**: + Detection pipeline template. +- **../../tools/topology/topology2/sof-hda-mww.conf**: Top-level WoV + topology configuration. diff --git a/src/audio/microwakeword/llext-wrap.c b/src/audio/microwakeword/llext-wrap.c new file mode 100644 index 000000000000..b868cea8c79d --- /dev/null +++ b/src/audio/microwakeword/llext-wrap.c @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. All rights reserved. + +#include +#include +#include +#include + +/* + * Stubs that are needed for linkage of some applications or libraries + * that come from porting userspace code. Anyone porting should + * make sure that any code does not depend on working copies of these + * reentrant functions. We will fail for any caller. + */ + +struct stat; +struct _reent; + +ssize_t _read_r(struct _reent *ptr, int fd, void *buf, size_t cnt) +{ + errno = ENOTSUP; + return -1; +} + +ssize_t _write_r(struct _reent *ptr, int fd, const void *buf, size_t cnt) +{ + errno = ENOTSUP; + return -1; +} + +void *_sbrk_r(struct _reent *ptr, ptrdiff_t incr) +{ + errno = ENOTSUP; + return (void *)-1; +} + +int _lseek_r(struct _reent *ptr, int fd, int pos, int whence) +{ + errno = ENOTSUP; + return -1; +} + +int _kill_r(struct _reent *ptr, int pid, int sig) +{ + errno = ENOTSUP; + return -1; +} + +int _getpid_r(struct _reent *ptr) +{ + errno = ENOTSUP; + return -1; +} + +int _fstat_r(struct _reent *ptr, int fd, struct stat *pstat) +{ + errno = ENOTSUP; + return -1; +} + +int _close_r(struct _reent *ptr, int fd) +{ + errno = ENOTSUP; + return -1; +} + +/* + * libc.a's fmaxf()/fminf() are not built PIC-safe: their error path + * (__math_invalidf()) calls __isnanf()/__issignalingf(), and GNU ld refuses + * those relocations ("dangerous relocation") when linking this ET_DYN/PIC + * LLEXT. TFLM's kernels use fmaxf()/fminf() for plain (non-NaN-signaling) + * min/max, so provide direct, self-contained definitions here: linked ahead + * of -lm, these satisfy the symbols outright and libm.a's versions (and the + * non-PIC-safe helpers they pull in) are never extracted. + */ +float fmaxf(float x, float y) +{ + if (x != x) + return y; + if (y != y) + return x; + return (x > y) ? x : y; +} + +float fminf(float x, float y) +{ + if (x != x) + return y; + if (y != y) + return x; + return (x < y) ? x : y; +} + +/* + * libc.a's shared float-domain-error helper __math_invalidf() (used by + * logf()/sqrtf()/powf()/and most other single-precision libm functions' + * error paths) is itself built non-PIC-safe: it calls __isnanf() via a + * direct call8, which GNU ld refuses ("dangerous relocation") once that + * non-PIC object is pulled into this ET_DYN/PIC LLEXT. Providing our own + * definition here -- linked ahead of -lm -- means libc.a's copy (and the + * relocation it can't satisfy) is never extracted, while every libm + * function that calls it keeps working via its normal linked-in code. + * Semantics mirror libc.a's own implementation: propagate x via x+x for a + * NaN input (quieting/signalling per IEEE 754), else synthesize NaN via + * 0.0f/0.0f. + */ +float __math_invalidf(float x) +{ + union { float f; uint32_t u; } v = { .f = x }; + int is_nan = (v.u & 0x7f800000u) == 0x7f800000u && (v.u & 0x007fffffu); + + if (is_nan) + return x + x; + + return 0.0f / 0.0f; +} + +/* TFLM needs exit if build as a llext module only atm */ +#if CONFIG_COMP_MWW == m +void _exit(int status) +{ + /* + * Do not call libc assert() here: it drags in __assert_no_args(), + * which calls fwrite()/abort() from libc.a. Those objects are not + * built PIC-safe, and linking them into this ET_DYN/PIC LLEXT + * triggers GNU ld "dangerous relocation" errors. The spin loop + * below already provides the required "never return" behaviour. + */ + while (1) { + /* spin forever */ + } + /* NOTREACHED */ +} +#endif diff --git a/src/audio/microwakeword/llext/CMakeLists.txt b/src/audio/microwakeword/llext/CMakeLists.txt new file mode 100644 index 000000000000..eea0e065fb96 --- /dev/null +++ b/src/audio/microwakeword/llext/CMakeLists.txt @@ -0,0 +1,185 @@ +# Copyright (c) 2026 Intel Corporation. +# SPDX-License-Identifier: Apache-2.0 +# +# LLEXT build path for the mww component (CONFIG_COMP_MWW=m). Structurally +# mirrors src/audio/tensorflow/llext/CMakeLists.txt, but targeted at aphid +# (ace30/PTL, Clang/LLVM toolchain) rather than that file's ace15_mtpm/Xtensa +# reference build -- see plan Stage 8. The toolchain paths below are derived +# from SOC_TOOLCHAIN_NAME/ZEPHYR_SDK_INSTALL_DIR rather than hardcoded, since +# this file is authored in the sof-tgl worktree before the aphid-specific +# build in sof-ptl/sof exists; re-verify against the real build-ptl-llvm +# toolchain paths when Stage 8 is actually run. + +set(TFLM_PATH ${sof_top_dir}/../tflite-micro) +set(FLATBUFFERS_PATH ${sof_top_dir}/../flatbuffers) +set(GEMMLOWP_PATH ${sof_top_dir}/../gemmlowp) +set(RUY_PATH ${sof_top_dir}/../ruy) +set(NN_HIFI_PATH ${sof_top_dir}/../nnlib-hifi4/xa_nnlib) + +set(MWW_TOOLCHAIN_ROOT "") +if(DEFINED ZEPHYR_SDK_INSTALL_DIR AND NOT "${ZEPHYR_SDK_INSTALL_DIR}" STREQUAL "") + set(MWW_TOOLCHAIN_ROOT + ${ZEPHYR_SDK_INSTALL_DIR}/gnu/xtensa-${SOC_TOOLCHAIN_NAME}_zephyr-elf/xtensa-${SOC_TOOLCHAIN_NAME}_zephyr-elf) +endif() +set(MWW_HAL_SOC_PATH ${sof_top_dir}/../modules/hal/xtensa/zephyr/soc/${SOC_TOOLCHAIN_NAME}) + +# As in tensorflow/llext/CMakeLists.txt, the HiFi4 nnlib optimized kernels +# are only pulled in for a genuine Xtensa (xcc) compiler; aphid's LLEXT +# module is built with Clang, so nn_hifi_lib is NOT built here and the +# reference (non-HiFi) TFLM kernel implementations are used instead. This +# mirrors the existing (arguably suboptimal, see plan/summary note) +# tflmcly behavior rather than diverging from it -- revisit in Stage 10. +set(MWW_LLEXT_KERNEL_SOURCES + ${TFLM_PATH}/tensorflow/lite/micro/kernels/call_once.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/var_handle.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/read_variable.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/assign_variable.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/concatenation.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/conv_common.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/depthwise_conv_common.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/fully_connected_common.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/logistic_common.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/quantize_common.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/reshape_common.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/strided_slice_common.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/dequantize_common.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/conv.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/depthwise_conv.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/fully_connected.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/logistic.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/quantize.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/reshape.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/strided_slice.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/dequantize.cc +) + +add_library(mww_tflm_lib STATIC + ${TFLM_PATH}/tensorflow/compiler/mlir/lite/core/api/error_reporter.cc + ${TFLM_PATH}/tensorflow/compiler/mlir/lite/schema/schema_utils.cc + ${TFLM_PATH}/tensorflow/lite/core/c/common.cc + ${TFLM_PATH}/tensorflow/lite/core/api/flatbuffer_conversions.cc + ${TFLM_PATH}/tensorflow/lite/core/api/tensor_utils.cc + ${TFLM_PATH}/tensorflow/lite/kernels/internal/common.cc + ${TFLM_PATH}/tensorflow/lite/kernels/internal/tensor_utils.cc + ${TFLM_PATH}/tensorflow/lite/kernels/internal/runtime_shape.cc + ${TFLM_PATH}/tensorflow/lite/kernels/internal/quantization_util.cc + ${TFLM_PATH}/tensorflow/lite/kernels/internal/tensor_ctypes.cc + ${TFLM_PATH}/tensorflow/lite/kernels/internal/portable_tensor_utils.cc + ${TFLM_PATH}/tensorflow/lite/kernels/internal/reference/comparisons.cc + ${TFLM_PATH}/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc + ${TFLM_PATH}/tensorflow/lite/kernels/kernel_util.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/kernel_util.cc + ${MWW_LLEXT_KERNEL_SOURCES} + ${TFLM_PATH}/tensorflow/lite/micro/mock_micro_graph.cc + ${TFLM_PATH}/tensorflow/lite/micro/flatbuffer_utils.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_interpreter_graph.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_resource_variable.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_interpreter_context.cc + ${TFLM_PATH}/tensorflow/lite/micro/fake_micro_context.cc + ${TFLM_PATH}/tensorflow/lite/micro/arena_allocator/persistent_arena_buffer_allocator.cc + ${TFLM_PATH}/tensorflow/lite/micro/arena_allocator/recording_single_arena_buffer_allocator.cc + ${TFLM_PATH}/tensorflow/lite/micro/arena_allocator/non_persistent_arena_buffer_allocator.cc + ${TFLM_PATH}/tensorflow/lite/micro/arena_allocator/single_arena_buffer_allocator.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_allocator.cc + ${TFLM_PATH}/tensorflow/lite/micro/tflite_bridge/flatbuffer_conversions_bridge.cc + ${TFLM_PATH}/tensorflow/lite/micro/tflite_bridge/micro_error_reporter.cc + ${TFLM_PATH}/tensorflow/lite/micro/system_setup.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_context.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_log.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_profiler.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_time.cc + # debug_log.cc is replaced by our printk-backed DebugLog() in mww.c. + #${TFLM_PATH}/tensorflow/lite/micro/debug_log.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_op_resolver.cc + ${TFLM_PATH}/tensorflow/lite/micro/memory_helpers.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_allocation_info.cc + ${TFLM_PATH}/tensorflow/lite/micro/memory_planner/greedy_memory_planner.cc + ${TFLM_PATH}/tensorflow/lite/micro/memory_planner/non_persistent_buffer_planner_shim.cc + ${TFLM_PATH}/tensorflow/lite/micro/memory_planner/linear_memory_planner.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_utils.cc + ${TFLM_PATH}/tensorflow/lite/micro/micro_interpreter.cc + ../mww_model_data.cc + ../mww_model.cc +) + +target_include_directories(mww_tflm_lib PRIVATE + ${TFLM_PATH} + ${FLATBUFFERS_PATH}/include + ${GEMMLOWP_PATH} + ${RUY_PATH} + ${sof_top_dir}/posix/include + ${sof_top_dir}/../modules/hal/xtensa/include + ${MWW_HAL_SOC_PATH} +) + +target_compile_options(mww_tflm_lib PRIVATE + -std=c++17 + -fno-rtti + -fno-exceptions + -fno-threadsafe-statics + -fno-unwind-tables + -fmessage-length=0 + -DTF_LITE_STATIC_MEMORY + -DTF_LITE_DISABLE_X86_NEON + -Wno-unused-parameter + -DXTENSA + -DTF_LITE_MCU_DEBUG_LOG + -DTF_LITE_USE_CTIME + -mlongcalls + -fPIC + # TF_LITE_STRIP_ERROR_STRINGS drops DebugLog()/DebugVsnprintf()'s calls + # into vprintf/vsnprintf; NDEBUG drops libc assert(). Both chains pull + # in libc.a objects (vfprintf/fwrite/abort) that are not built PIC-safe, + # which GNU ld refuses ("dangerous relocation") when linking this + # ET_DYN/PIC LLEXT. + -DTF_LITE_STRIP_ERROR_STRINGS + -DNDEBUG +) + +if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|Xtensa" OR CMAKE_C_COMPILER MATCHES "xt-clang") + target_compile_options(mww_tflm_lib PRIVATE + -stdlib=libc++ + -fno-vectorize + -fno-slp-vectorize + ) + if(NOT CMAKE_C_COMPILER MATCHES "xt-clang") + target_compile_options(mww_tflm_lib PRIVATE + -mcpu=${SOC_TOOLCHAIN_NAME} + ) + if(NOT "${MWW_TOOLCHAIN_ROOT}" STREQUAL "") + target_include_directories(mww_tflm_lib SYSTEM PRIVATE + ${MWW_TOOLCHAIN_ROOT}/include/c++/14.3.0 + ${MWW_TOOLCHAIN_ROOT}/include/c++/14.3.0/xtensa-${SOC_TOOLCHAIN_NAME}_zephyr-elf + ${MWW_TOOLCHAIN_ROOT}/include + ) + endif() + endif() +endif() + +set(MWW_LIBS_PATHS .) +if(NOT "${MWW_TOOLCHAIN_ROOT}" STREQUAL "") + list(APPEND MWW_LIBS_PATHS ${MWW_TOOLCHAIN_ROOT}/lib) +endif() + +sof_llext_build("mww" + SOURCES + ../mww.c + ../llext-wrap.c + INCLUDES + ${TFLM_PATH} + ${FLATBUFFERS_PATH}/include + ${GEMMLOWP_PATH} + ${RUY_PATH} + ${sof_top_dir}/posix/include + LIBS + mww_tflm_lib + LIBS_PATH + ${MWW_LIBS_PATHS} +) + +if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND NOT CMAKE_C_COMPILER MATCHES "xt-clang" AND NOT "${MWW_TOOLCHAIN_ROOT}" STREQUAL "") + llext_link_options(mww + --target=xtensa-${SOC_TOOLCHAIN_NAME}_zephyr-elf + --ld-path=${MWW_TOOLCHAIN_ROOT}-ld + ) +endif() diff --git a/src/audio/microwakeword/llext/llext.toml.h b/src/audio/microwakeword/llext/llext.toml.h new file mode 100644 index 000000000000..5929e908128b --- /dev/null +++ b/src/audio/microwakeword/llext/llext.toml.h @@ -0,0 +1,6 @@ +#include +#define LOAD_TYPE "2" +#include "../mww.toml" + +[module] +count = __COUNTER__ diff --git a/src/audio/microwakeword/mww.c b/src/audio/microwakeword/mww.c new file mode 100644 index 000000000000..1d827820aa0a --- /dev/null +++ b/src/audio/microwakeword/mww.c @@ -0,0 +1,740 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. All rights reserved. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include "mww_model.h" + +#include +#include +#include + +/* TFLM error strings land here. Route to printk/mtrace so AllocateTensors() + * and Invoke() failures print their real reason instead of vanishing. + */ +void DebugLog(const char *format, va_list args) +{ + vprintk(format, args); +} + +int DebugVsnprintf(char *buffer, size_t buf_size, const char *format, + va_list vlist) +{ + return vsnprintk(buffer, buf_size, format, vlist); +} + +#if CONFIG_AMS +#include +#include +#include +#else +#include +#endif + +/* MFCC's non-compress output prepends a struct mfcc_data_header (24 bytes) + * to each hop, followed by MWW_FEATURE_SIZE mel-log values (int8_t in PCAN + * mode, int32_t Q9.23 otherwise). + * This must match the frame size configured in the mww capture pipeline. + */ +#if CONFIG_COMP_MWW_PCAN +#define MWW_HOP_BYTES (sizeof(struct mfcc_data_header) + MWW_FEATURE_SIZE * sizeof(int8_t)) +#else +#define MWW_HOP_BYTES (sizeof(struct mfcc_data_header) + MWW_FEATURE_SIZE * sizeof(int32_t)) +#endif + +/* Pre-roll history in ms that KPB drains to host on wake-word trigger. */ +#define MWW_KPB_DRAIN_REQ_MS 1000 + +/* MFCC delivers one hop every 10 ms; MWW processes 3 hops per inference. */ +#define MWW_MFCC_HOP_MS 10 + +/* Wake-word probability threshold above which KPB draining is triggered. */ +#define MWW_DETECT_THRESHOLD 0.85f + +/* Consecutive inferences above threshold required to confirm detection (~90 ms). */ +#define MWW_CONSECUTIVE_DETECTS_REQUIRED 3 + +/* Number of startup inferences to warm up the temporal ring buffers before enabling triggers (~1 sec). */ +#define MWW_WARMUP_INFERENCES 33 + +/* Soft mel-log AGC (units: Q9.23, matches MFCC output). One decade = +10 dB. + * Target +2.5 dB (+0.25 in Q9.23), floor -20 dB. Attack: instant clamp so peak+gain + * never exceeds MEL_CLIP_MAX_Q23 (+1.0, +10 dB). Release: dual-rate additive recovery: + * - Normal release during silence (VAD == 0): ~1.0 dB/s (100 hops/s). + * - Super-slow leak during speech (VAD == 1): ~0.05 dB/s (1/20th normal release) to guarantee + * the AGC never stays permanently trapped at minimum gain even if VAD gets stuck. + */ +#define MWW_AGC_GAIN_TARGET_Q23 2097152 /* +2.5 dB (0.25 * 2^23) */ +#define MWW_AGC_GAIN_FLOOR_Q23 -16777216 /* -20 dB, int32(-20 * 0.1 * 2^23) */ +#define MWW_AGC_RELEASE_STEP_Q23 8389 /* 1.0 dB/s, int32((1.0 * 0.1 / 100) * 2^23) */ +#define MWW_AGC_RELEASE_STEP_SPEECH_Q23 419 /* 0.05 dB/s, 1/20th normal release */ + +/* The range -1.0 to +1.0 of Q9.23 Mel values is scaled to +/-1.0 Q1.7. */ +#define MEL_OFFSET_Q23 0 /* 0 */ +#define MEL_SCALE_Q30 (1 << 30) /* 1.0 in Q30 */ +#define MEL_CLIP_MAX_Q23 (1 << 23) /* +1.0 in Q23 */ +#define MEL_CLIP_MIN_Q23 (-1 << 23) /* -1.0 in Q23 */ +#define MEL_CLIP_MAX_Q7 127 +#define MEL_CLIP_MIN_Q7 -128 + +SOF_DEFINE_REG_UUID(mww); +LOG_MODULE_REGISTER(mww, CONFIG_SOF_LOG_LEVEL); +#if CONFIG_COMP_MWW_MODULE +EXPORT_SYMBOL(mww_uuid); +EXPORT_SYMBOL(log_const_mww); +#endif + +#if CONFIG_AMS +/* Key-phrase detected message, shared with src/samples/audio/detect_test.c + * and consumed by kpb.c's AMS-consumer branch -- no kpb.c changes needed. + */ +static const ams_uuid_t ams_kpd_msg_uuid = AMS_KPD_MSG_UUID; +#endif + +struct mww_comp_data { + struct comp_data_blob_handler *model_handler; +#if CONFIG_COMP_MWW_MODEL_FROM_CONTROL + /* Copy of the model blob placed in the module's own vregion so the DP + * user thread can read it during mww_prepare()/MWW_SetModel(). + * comp_get_data_blob() must not be called from the DP thread because + * the blob handler lives in kernel-only memory. + */ + void *model_data; + size_t model_size; +#endif + struct mww_classify mwc; + struct kpb_client client_data; + uint32_t drain_req_ms; +#if CONFIG_AMS + uint32_t kpd_uuid_id; +#else + struct kpb_event_data event_data; +#endif + bool initialized; + int8_t feature_buf[MWW_FEATURE_ELEM_COUNT]; + int feature_slices_filled; + + /* Bitmask of VAD flags for the MWW_FEATURE_SLICE_COUNT frames */ + uint32_t vad_history; + /* Persistent AGC gain applied to every Q9.23 mel value */ + int32_t agc_gain_q23; + + /* Hop buffer for assembling contiguous data when circular buffer wraps */ + uint8_t hop_buf[MWW_HOP_BYTES] __aligned(4); + + /* Consecutive inferences with probability >= MWW_DETECT_THRESHOLD */ + uint32_t consecutive_detects; + + /* Telemetry counters */ + uint32_t total_inferences; + uint32_t vad_gated_inferences; + uint32_t detections; + uint32_t kpb_trigger_events; +} __attribute__((aligned(8))); + +#if CONFIG_AMS +static int mww_notify_kpb(struct processing_module *mod) +{ + struct mww_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + struct ams_message_payload ams_payload; + + comp_info(dev, "MWW keyword trigger -> notifying KPB to begin draining"); + + cd->client_data.r_ptr = NULL; + cd->client_data.sink = NULL; + cd->client_data.id = 0; /**< TODO: acquire proper id from kpb */ + cd->client_data.drain_req = cd->drain_req_ms; + + dcache_writeback_region(&cd->client_data, sizeof(cd->client_data)); + + ams_helper_prepare_payload(dev, &ams_payload, cd->kpd_uuid_id, + (uint8_t *)&cd->client_data, + sizeof(struct kpb_client)); + + return ams_send(&ams_payload); +} +#else +static int mww_notify_kpb(struct processing_module *mod) +{ + struct mww_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + + comp_info(dev, "MWW keyword trigger -> notifying KPB to begin draining"); + + cd->client_data.r_ptr = NULL; + cd->client_data.sink = NULL; + cd->client_data.id = 0; + cd->client_data.drain_req = cd->drain_req_ms; + cd->event_data.event_id = KPB_EVENT_BEGIN_DRAINING; + cd->event_data.client_data = &cd->client_data; + + notifier_event(dev, NOTIFIER_ID_KPB_CLIENT_EVT, + NOTIFIER_TARGET_CORE_ALL_MASK, &cd->event_data, + sizeof(cd->event_data)); + return 0; +} +#endif /* CONFIG_AMS */ + +__cold static void mww_log_summary_at_shutdown(struct processing_module *mod) +{ + struct mww_comp_data *cd = mod ? module_get_private_data(mod) : NULL; + struct comp_dev *dev = mod ? mod->dev : NULL; + char summary_buf[256]; + + if (!cd) + return; + + snprintk(summary_buf, sizeof(summary_buf), + "[MWW STREAM SHUTDOWN SUMMARY] Total Inferences=%u | VAD Gated=%u | Detections=%u | KPB Triggers=%u | Arena Used=%zu/%zu B", + cd->total_inferences, cd->vad_gated_inferences, + cd->detections, cd->kpb_trigger_events, + MWW_ArenaUsedBytes(), MWW_ArenaCapacity()); + + if (dev) + comp_info(dev, "%s", summary_buf); + printk("%s\n", summary_buf); +} + +__cold static int mww_init(struct processing_module *mod) +{ + struct module_data *md = &mod->priv; + struct comp_dev *dev = mod->dev; + struct mww_comp_data *cd; + + assert_can_be_cold(); + + comp_info(dev, "entry"); + + cd = mod_zalloc(mod, sizeof(*cd)); + if (!cd) + return -ENOMEM; + + md->private = cd; + cd->model_handler = mod_data_blob_handler_new(mod); + if (!cd->model_handler) { + mod_free(mod, cd); + return -ENOMEM; + } + + cd->drain_req_ms = MWW_KPB_DRAIN_REQ_MS; + cd->agc_gain_q23 = MWW_AGC_GAIN_TARGET_Q23; +#if CONFIG_AMS + cd->kpd_uuid_id = AMS_INVALID_MSG_TYPE; +#endif + + return 0; +} + +#if CONFIG_COMP_MWW_DEBUG_TRACE +static uint32_t last_mww_cycle; +static uint32_t mww_call_count; +#endif + +static int mww_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct mww_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + int ret; + + comp_dbg(dev, "entry"); + + /* Set DP scheduling period to match MFCC hop cadence (10 ms); + * the sink carries feature frames, not linear PCM, so module_adapter + * cannot derive the period from nominal sink rate and frame_bytes. + */ + dev->period = MWW_MFCC_HOP_MS * 1000; + + if (cd->initialized) + return 0; + + unsigned char *model_ptr = NULL; + +#if CONFIG_COMP_MWW_MODEL_FROM_CONTROL + /* Use the cached copy that mww_set_config() placed in the module's + * vregion. Calling comp_get_data_blob() here would fault because + * mww_prepare() runs in the DP user thread when + * CONFIG_SOF_USERSPACE_APPLICATION=y, and the blob handler is + * allocated from kernel-only memory. + * + * If no model has been set yet (e.g. PipeWire probing pcm101 before + * userspace has written the byte control), skip inference setup and + * let the pipeline complete open/close without a hard error. + */ + if (!cd->model_data || !cd->model_size) { + comp_warn(dev, "MWW: no model blob set from control; passthrough"); + return 0; + } + model_ptr = cd->model_data; + comp_info(dev, "MWW: using cached model blob, size=%zu", cd->model_size); +#endif + + ret = MWW_SetModel(&cd->mwc, model_ptr); + if (ret < 0) { + comp_err(dev, "MWW_SetModel failed: %d (%s)", ret, cd->mwc.error); + return ret; + } + + ret = MWW_InitOps(&cd->mwc); + if (ret < 0) { + comp_err(dev, "MWW_InitOps failed: %d (%s)", ret, cd->mwc.error); + return ret; + } + +#if CONFIG_AMS + /* Register KD as AMS producer */ + ret = ams_helper_register_producer(dev, &cd->kpd_uuid_id, ams_kpd_msg_uuid); + if (ret) + return ret; +#endif + + cd->initialized = true; + cd->feature_slices_filled = 0; + cd->vad_history = 0; + cd->consecutive_detects = 0; + comp_info(dev, "MWW model initialized: arena_used=%zu / capacity=%zu bytes", + MWW_ArenaUsedBytes(), MWW_ArenaCapacity()); + + return 0; +} + +static int mww_process(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct mww_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + size_t bytes_to_process; + const void *data_ptr, *buf_start; + size_t buf_size; + int ret = 0; +#if CONFIG_COMP_MWW_DEBUG_TRACE + uint32_t now = k_cycle_get_32(); + uint32_t delta_cycles = now - last_mww_cycle; + int hops_processed = 0; + + last_mww_cycle = now; + mww_call_count++; +#endif + + if (!cd->initialized) { + size_t avail = source_get_data_available(sources[0]); + + if (avail > 0) { + const void *dp, *bs; + size_t bsz; + + if (source_get_data(sources[0], avail, &dp, &bs, &bsz) == 0) + source_release_data(sources[0], avail); + } + return 0; + } + + bytes_to_process = source_get_data_available(sources[0]); + if (bytes_to_process < MWW_HOP_BYTES) { +#if CONFIG_COMP_MWW_DEBUG_TRACE + comp_info(dev, "[MWW proc %u] delta=%u us (no bytes, avail=%zu)", + mww_call_count, k_cyc_to_us_near32(delta_cycles), bytes_to_process); +#endif + return 0; + } + +#if CONFIG_COMP_MWW_DEBUG_TRACE + size_t initial_bytes = bytes_to_process; +#endif + + while (bytes_to_process >= MWW_HOP_BYTES) { + const struct mfcc_data_header *hdr; + const uint8_t *hop_src; + size_t bytes_to_end; + int8_t *slice; + int i; + +#if CONFIG_COMP_MWW_DEBUG_TRACE + hops_processed++; +#endif + + ret = source_get_data(sources[0], MWW_HOP_BYTES, + &data_ptr, &buf_start, &buf_size); + if (ret != 0 || !data_ptr) + break; + + /* Assemble hop into contiguous memory if it wraps across the ring buffer boundary */ + bytes_to_end = (const uint8_t *)buf_start + buf_size - (const uint8_t *)data_ptr; + if (bytes_to_end >= MWW_HOP_BYTES) { + hop_src = (const uint8_t *)data_ptr; + } else { + memcpy(cd->hop_buf, data_ptr, bytes_to_end); + memcpy(cd->hop_buf + bytes_to_end, buf_start, MWW_HOP_BYTES - bytes_to_end); + hop_src = cd->hop_buf; + } + + hdr = (const struct mfcc_data_header *)hop_src; + slice = &cd->feature_buf[cd->feature_slices_filled * MWW_FEATURE_SIZE]; + + /* Update VAD history bitmask across MWW_FEATURE_SLICE_COUNT slices */ + cd->vad_history = ((cd->vad_history << 1) | (hdr->vad_flag ? 1U : 0U)) & + ((1U << MWW_FEATURE_SLICE_COUNT) - 1); + +#if CONFIG_COMP_MWW_PCAN + /* PCAN mode: MFCC has already normalized and quantized Mel energies + * to int8_t (Q1.7). Copy directly into the feature buffer slice. + */ + memcpy(slice, hop_src + sizeof(struct mfcc_data_header), MWW_FEATURE_SIZE); +#else + const int32_t *mel = (const int32_t *)(hop_src + sizeof(struct mfcc_data_header)); + int32_t hop_peak_q23 = mel[0]; + + for (i = 1; i < MWW_FEATURE_SIZE; i++) { + if (mel[i] > hop_peak_q23) + hop_peak_q23 = mel[i]; + } + + int32_t clip_headroom_q23 = MEL_CLIP_MAX_Q23 - hop_peak_q23; + + if (cd->agc_gain_q23 > clip_headroom_q23) + cd->agc_gain_q23 = clip_headroom_q23; + if (cd->agc_gain_q23 < MWW_AGC_GAIN_FLOOR_Q23) + cd->agc_gain_q23 = MWW_AGC_GAIN_FLOOR_Q23; + + int32_t agc_gain_q23 = cd->agc_gain_q23; + + /* Requantize to int8 (Q1.7 range matching training AGC normalization) */ + for (i = 0; i < MWW_FEATURE_SIZE; i++) { + int32_t mel_c = mel[i] + agc_gain_q23; + + /* Clamp to [-1.0, +1.0] Q9.23 range before Q1.7 conversion */ + if (mel_c > MEL_CLIP_MAX_Q23) + mel_c = MEL_CLIP_MAX_Q23; + else if (mel_c < MEL_CLIP_MIN_Q23) + mel_c = MEL_CLIP_MIN_Q23; + + /* Rescale with offset and gain */ + mel_c = Q_MULTSR_32X32((int64_t)(mel_c + MEL_OFFSET_Q23), + MEL_SCALE_Q30, 23, 30, 7); + if (mel_c > MEL_CLIP_MAX_Q7) + mel_c = MEL_CLIP_MAX_Q7; + else if (mel_c < MEL_CLIP_MIN_Q7) + mel_c = MEL_CLIP_MIN_Q7; + + slice[i] = (int8_t)mel_c; + } + + /* Release: normal recovery during silence, super-slow leak during speech */ + if (cd->agc_gain_q23 < MWW_AGC_GAIN_TARGET_Q23) { + int32_t step = hdr->vad_flag ? + MWW_AGC_RELEASE_STEP_SPEECH_Q23 : MWW_AGC_RELEASE_STEP_Q23; + + cd->agc_gain_q23 += step; + if (cd->agc_gain_q23 > MWW_AGC_GAIN_TARGET_Q23) + cd->agc_gain_q23 = MWW_AGC_GAIN_TARGET_Q23; + } +#endif /* CONFIG_COMP_MWW_PCAN */ + +#if CONFIG_COMP_MWW_DEBUG_TRACE + { + static int dbg_hop_count; + int8_t f_min = slice[0], f_max = slice[0]; + + dbg_hop_count++; + for (i = 1; i < MWW_FEATURE_SIZE; i++) { + if (slice[i] < f_min) f_min = slice[i]; + if (slice[i] > f_max) f_max = slice[i]; + } +#if CONFIG_COMP_MWW_PCAN + comp_info(dev, "[MWW DBG hop %d] vad=%d E=%d Ne=%d f_min=%d f_max=%d (pcan)", + dbg_hop_count, (int)hdr->vad_flag, + (int)hdr->energy, (int)hdr->noise_energy, + f_min, f_max); +#else + int32_t mel_min = mel[0], mel_max = mel[0]; + + for (i = 1; i < MWW_FEATURE_SIZE; i++) { + if (mel[i] < mel_min) mel_min = mel[i]; + if (mel[i] > mel_max) mel_max = mel[i]; + } + comp_info(dev, "[MWW DBG hop %d] vad=%d E=%d Ne=%d mel_min=%d mel_max=%d f_min=%d f_max=%d agc_q23=%d", + dbg_hop_count, (int)hdr->vad_flag, + (int)hdr->energy, (int)hdr->noise_energy, + mel_min, mel_max, f_min, f_max, (int)cd->agc_gain_q23); +#endif + } +#endif + + /* Copy source data to sink so downstream stages keep seeing raw MFCC hops */ + if (num_of_sinks > 0 && sinks[0]) { + void *snk_ptr, *snk_buf_start; + size_t snk_buf_size; + int sret = sink_get_buffer(sinks[0], MWW_HOP_BYTES, + &snk_ptr, &snk_buf_start, &snk_buf_size); + if (sret == 0 && snk_ptr) { + size_t snk_bytes_to_end = (uint8_t *)snk_buf_start + + snk_buf_size - (uint8_t *)snk_ptr; + if (snk_bytes_to_end >= MWW_HOP_BYTES) { + memcpy(snk_ptr, hop_src, MWW_HOP_BYTES); + } else { + memcpy(snk_ptr, hop_src, snk_bytes_to_end); + memcpy(snk_buf_start, + hop_src + snk_bytes_to_end, + MWW_HOP_BYTES - snk_bytes_to_end); + } + sink_commit_buffer(sinks[0], MWW_HOP_BYTES); + } + } + + source_release_data(sources[0], MWW_HOP_BYTES); + bytes_to_process -= MWW_HOP_BYTES; + cd->feature_slices_filled++; + + if (cd->feature_slices_filled >= MWW_FEATURE_SLICE_COUNT) { + cd->feature_slices_filled = 0; + + cd->total_inferences++; + cd->mwc.audio_features = cd->feature_buf; + cd->mwc.audio_data_size = MWW_FEATURE_ELEM_COUNT; + +#if CONFIG_COMP_MWW_DEBUG_TRACE + uint32_t c0 = k_cycle_get_32(); +#endif + ret = MWW_ProcessClassify(&cd->mwc); +#if CONFIG_COMP_MWW_DEBUG_TRACE + uint32_t c1 = k_cycle_get_32(); +#endif + if (ret < 0) { + comp_err(dev, "MWW_ProcessClassify failed: %d (%s)", + ret, cd->mwc.error); + continue; + } + +#if CONFIG_COMP_MWW_DEBUG_TRACE + comp_info(dev, "MWW probability=%d raw=%d in[0..7]=[%d,%d,%d,%d,%d,%d,%d,%d] (cycles=%u)", + (int)(cd->mwc.probability * 100.0f), (int)cd->mwc.raw_output, + (int)cd->feature_buf[0], (int)cd->feature_buf[1], + (int)cd->feature_buf[2], (int)cd->feature_buf[3], + (int)cd->feature_buf[4], (int)cd->feature_buf[5], + (int)cd->feature_buf[6], (int)cd->feature_buf[7], + c1 - c0); +#endif + + if (cd->mwc.probability >= MWW_DETECT_THRESHOLD) { + cd->consecutive_detects++; + if (cd->total_inferences > MWW_WARMUP_INFERENCES && + cd->consecutive_detects >= MWW_CONSECUTIVE_DETECTS_REQUIRED) { + cd->detections++; + comp_info(dev, "MWW keyword detected: probability=%d pct (consecutive=%u, total=%u)", + (int)(cd->mwc.probability * 100.0f), + cd->consecutive_detects, cd->detections); + cd->kpb_trigger_events++; + mww_notify_kpb(mod); + cd->consecutive_detects = 0; + } + } else { + cd->consecutive_detects = 0; + } + } + } + +#if CONFIG_COMP_MWW_DEBUG_TRACE + comp_info(dev, "[MWW proc %u] delta=%u us avail_bytes=%zu hops=%d slices=%d/%d", + mww_call_count, k_cyc_to_us_near32(delta_cycles), + initial_bytes, hops_processed, + cd->feature_slices_filled, MWW_FEATURE_SLICE_COUNT); +#endif + + return ret; +} + +static int mww_reset(struct processing_module *mod) +{ + struct mww_comp_data *cd = module_get_private_data(mod); + + comp_dbg(mod->dev, "entry"); + mww_log_summary_at_shutdown(mod); + cd->feature_slices_filled = 0; + cd->vad_history = 0; + cd->consecutive_detects = 0; + cd->total_inferences = 0; + cd->agc_gain_q23 = MWW_AGC_GAIN_TARGET_Q23; + memset(cd->feature_buf, 0, sizeof(cd->feature_buf)); + MWW_Reset(); +#if CONFIG_COMP_MWW_DEBUG_TRACE + last_mww_cycle = 0; + mww_call_count = 0; +#endif + return 0; +} + +__cold static int mww_free(struct processing_module *mod) +{ + struct mww_comp_data *cd = module_get_private_data(mod); + + assert_can_be_cold(); + + comp_dbg(mod->dev, "entry"); + mww_log_summary_at_shutdown(mod); + +#if CONFIG_AMS + if (cd->kpd_uuid_id != AMS_INVALID_MSG_TYPE) { + int ret = ams_helper_unregister_producer(mod->dev, cd->kpd_uuid_id); + + if (ret) + comp_err(mod->dev, "unregister ams error %d", ret); + } +#endif + + MWW_Free(); + +#if CONFIG_COMP_MWW_MODEL_FROM_CONTROL + if (cd->model_data) { + mod_free(mod, cd->model_data); + cd->model_data = NULL; + cd->model_size = 0; + } +#endif + + mod_data_blob_handler_free(mod, cd->model_handler); + mod_free(mod, cd); + return 0; +} + +__cold static int mww_set_config(struct processing_module *mod, uint32_t param_id, + enum module_cfg_fragment_position pos, uint32_t data_offset_size, + const uint8_t *fragment, size_t fragment_size, uint8_t *response, + size_t response_size) +{ + struct mww_comp_data *cd = module_get_private_data(mod); + int ret; + + if (mod->dev->state != COMP_STATE_INIT && mod->dev->state != COMP_STATE_READY) { + comp_warn(mod->dev, "mww_set_config(): model update ignored while not idle (state %d)", + mod->dev->state); + return 0; + } + + ret = comp_data_blob_set(cd->model_handler, pos, data_offset_size, + fragment, fragment_size); + if (ret < 0) + return ret; + +#if CONFIG_COMP_MWW_MODEL_FROM_CONTROL + /* When the blob is fully received, cache a copy in the module's own + * vregion so mww_prepare() can access it from the DP user thread. + * This function runs in the IPC (kernel) context, so it is safe to + * call comp_get_data_blob() here. + */ + if (pos == MODULE_CFG_FRAGMENT_SINGLE || pos == MODULE_CFG_FRAGMENT_LAST) { + size_t blob_size = 0; + void *blob; + + blob = comp_get_data_blob(cd->model_handler, &blob_size, NULL); + if (!blob || !blob_size) { + comp_err(mod->dev, "mww_set_config(): empty blob after set"); + return -EINVAL; + } + + if (cd->model_data) { + mod_free(mod, cd->model_data); + cd->model_data = NULL; + cd->model_size = 0; + } + + cd->model_data = mod_alloc(mod, blob_size); + if (!cd->model_data) { + comp_err(mod->dev, "mww_set_config(): model copy alloc failed (%zu B)", + blob_size); + return -ENOMEM; + } + + memcpy(cd->model_data, blob, blob_size); + cd->model_size = blob_size; + comp_info(mod->dev, "mww_set_config(): cached model blob, size=%zu", + blob_size); + } +#else + if (pos == MODULE_CFG_FRAGMENT_SINGLE || pos == MODULE_CFG_FRAGMENT_LAST) { + comp_warn(mod->dev, + "mww_set_config(): model blob received but CONFIG_COMP_MWW_MODEL_FROM_CONTROL is disabled; using built-in model"); + } +#endif + + return 0; +} + +__cold static int mww_get_config(struct processing_module *mod, + uint32_t config_id, uint32_t *data_offset_size, + uint8_t *fragment, size_t fragment_size) +{ + struct sof_ipc_ctrl_data *cdata = (struct sof_ipc_ctrl_data *)fragment; + struct mww_comp_data *cd = module_get_private_data(mod); + + return comp_data_blob_get_cmd(cd->model_handler, cdata, fragment_size); +} + +static const struct module_interface mww_interface = { + .init = mww_init, + .prepare = mww_prepare, + .process = mww_process, + .set_configuration = mww_set_config, + .get_configuration = mww_get_config, + .reset = mww_reset, + .free = mww_free +}; + +/* This controls build of the module. If COMP_MODULE is selected in kconfig + * this is build as dynamically loadable module. + */ +#if CONFIG_COMP_MWW_MODULE + +#include +#include + +static const struct sof_man_module_manifest mod_manifest __section(".module") __used = + SOF_LLEXT_MODULE_MANIFEST("MWW", &mww_interface, 1, + SOF_REG_UUID(mww), 40); + +SOF_LLEXT_BUILDINFO; + +#else + +/* Only used for the module adapter trace context, soon to be deprecated */ +DECLARE_TR_CTX(mww_tr, SOF_UUID(mww_uuid), LOG_LEVEL_INFO); +DECLARE_MODULE_ADAPTER(mww_interface, mww_uuid, mww_tr); +SOF_MODULE_INIT(mww, sys_comp_module_mww_interface_init); + +#endif diff --git a/src/audio/microwakeword/mww.toml b/src/audio/microwakeword/mww.toml new file mode 100644 index 000000000000..905880f55adf --- /dev/null +++ b/src/audio/microwakeword/mww.toml @@ -0,0 +1,21 @@ +#ifndef LOAD_TYPE +#define LOAD_TYPE "0" +#endif + + REM # microWakeWord keyword-spotting module config + [[module.entry]] + name = "MWW" + uuid = UUIDREG_STR_MWW + affinity_mask = "0x1" + instance_count = "40" + domain_types = "0" + load_type = LOAD_TYPE + module_type = "9" + auto_start = "0" + sched_caps = [1, 0x00008000] + REM # pin = [dir, type, sample rate, size, container, channel-cfg] + pin = [0, 0, 0xfeef, 0xf, 0xf, 0x45ff, 1, 0, 0xfeef, 0xf, 0xf, 0x1ff] + REM # mod_cfg [PAR_0 PAR_1 PAR_2 PAR_3 IS_BYTES CPS IBS OBS MOD_FLAGS CPC OBLS] + mod_cfg = [0, 0, 0, 0, 4096, 1000000, 128, 128, 0, 0, 0] + + index = __COUNTER__ diff --git a/src/audio/microwakeword/mww_model.cc b/src/audio/microwakeword/mww_model.cc new file mode 100644 index 000000000000..389e64d80801 --- /dev/null +++ b/src/audio/microwakeword/mww_model.cc @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. All rights reserved. + +#include +#include +#include + +#include "tensorflow/lite/core/c/common.h" +#include "tensorflow/lite/micro/micro_allocator.h" +#include "tensorflow/lite/micro/micro_interpreter.h" +#include "tensorflow/lite/micro/micro_log.h" +#include "tensorflow/lite/micro/micro_mutable_op_resolver.h" +#include "tensorflow/lite/micro/micro_resource_variable.h" +#include "mww_model.h" + +#if !CONFIG_COMP_MWW_MODEL_FROM_CONTROL +#if defined(CONFIG_COMP_MWW_MODEL_HEY_JARVIS) +#include "mww_model_data_hey_jarvis.h" +#define MWW_BUILTIN_MODEL_DATA mww_model_data_hey_jarvis +#elif defined(CONFIG_COMP_MWW_MODEL_HI_INTEL) +#include "mww_model_data_hi_intel.h" +#define MWW_BUILTIN_MODEL_DATA mww_model_data_hi_intel +#else +#include "mww_model_data.h" +#define MWW_BUILTIN_MODEL_DATA mww_model_data +#endif +#endif + +static constexpr int kFeatureSize = MWW_FEATURE_SIZE; +static constexpr int kFeatureElementCount = MWW_FEATURE_ELEM_COUNT; + +// Arena size is a generous guesstimate for the streaming MixConv graph (12 +// ops incl. Conv2D/DepthwiseConv2D/FullyConnected plus 6 persistent resource +// -variable ring buffers). Refine down using +// MicroInterpreter::arena_used_bytes() once measured on hardware (logged in +// MWW_InitOps() below). +static constexpr size_t kArenaSize = 131072; +alignas(16) static uint8_t g_arena[kArenaSize]; + +// inference +static const tflite::Model *model; +static TfLiteTensor *input; +static TfLiteTensor *output; +static tflite::MicroInterpreter *interpreter; +static tflite::MicroAllocator *allocator; +static tflite::MicroResourceVariables *resource_variables; + +// stream, stream_1..stream_5: the MixConv graph's 6 persistent ring-buffer +// state variables (VAR_HANDLE/ASSIGN_VARIABLE/READ_VARIABLE), confirmed by +// direct flatbuffer inspection (see plan Stage 1) -- CALL_ONCE invokes a +// second subgraph that ASSIGN_VARIABLEs their zero initial state. +static constexpr int kNumResourceVariables = 6; + +// Ops used by the hey_jarvis.tflite streaming graph: CALL_ONCE, VAR_HANDLE, +// READ_VARIABLE, ASSIGN_VARIABLE, RESHAPE, CONCATENATION, STRIDED_SLICE, +// CONV_2D, DEPTHWISE_CONV_2D, FULLY_CONNECTED, LOGISTIC, QUANTIZE, +// DEQUANTIZE. +using MwwOpResolver = tflite::MicroMutableOpResolver<14>; +static MwwOpResolver *op_resolver; + +int RegisterOps(MwwOpResolver *op_resolver) { + TF_LITE_ENSURE_STATUS(op_resolver->AddCallOnce()); + TF_LITE_ENSURE_STATUS(op_resolver->AddVarHandle()); + TF_LITE_ENSURE_STATUS(op_resolver->AddReadVariable()); + TF_LITE_ENSURE_STATUS(op_resolver->AddAssignVariable()); + TF_LITE_ENSURE_STATUS(op_resolver->AddReshape()); + TF_LITE_ENSURE_STATUS(op_resolver->AddConcatenation()); + TF_LITE_ENSURE_STATUS(op_resolver->AddStridedSlice()); + TF_LITE_ENSURE_STATUS(op_resolver->AddConv2D()); + TF_LITE_ENSURE_STATUS(op_resolver->AddDepthwiseConv2D()); + TF_LITE_ENSURE_STATUS(op_resolver->AddFullyConnected()); + TF_LITE_ENSURE_STATUS(op_resolver->AddLogistic()); + TF_LITE_ENSURE_STATUS(op_resolver->AddQuantize()); + TF_LITE_ENSURE_STATUS(op_resolver->AddDequantize()); + return 0; +} + +static int Init_Interpreter(struct mww_classify *mwc); + +int MWW_InitOps(struct mww_classify *mwc) +{ + op_resolver = new MwwOpResolver(); + if (!op_resolver) { + mwc->error = "op_resolver alloc failed (OOM)"; + return -ENOMEM; + } + + if (RegisterOps(op_resolver) != 0) { + mwc->error = "register ops failed"; + return -EINVAL; + } + + // VAR_HANDLE/ASSIGN_VARIABLE require an explicit MicroResourceVariables + // instance registered with the interpreter -- without one, + // VarHandlePrepare()/AssignVariable Eval() fail with kTfLiteError as soon + // as AllocateTensors() prepares the first VAR_HANDLE node (see + // tensorflow/lite/micro/kernels/var_handle.cc). Building via the + // allocator-based MicroInterpreter constructor lets us create the + // allocator once and share it with MicroResourceVariables::Create(). + allocator = tflite::MicroAllocator::Create(g_arena, kArenaSize); + if (!allocator) { + mwc->error = "allocator alloc failed (OOM)"; + delete op_resolver; + op_resolver = nullptr; + return -ENOMEM; + } + + resource_variables = tflite::MicroResourceVariables::Create(allocator, kNumResourceVariables); + if (!resource_variables) { + mwc->error = "resource_variables alloc failed (OOM)"; + delete op_resolver; + op_resolver = nullptr; + return -ENOMEM; + } + + // create the interpreter + interpreter = new tflite::MicroInterpreter(model, *op_resolver, + allocator, resource_variables); + if (!interpreter) { + mwc->error = "interpreter alloc failed (OOM)"; + delete op_resolver; + op_resolver = nullptr; + return -ENOMEM; + } + + // and allocate the tensors + if (interpreter->AllocateTensors() != kTfLiteOk) { + mwc->error = "interpreter tensor allocate failed"; + delete interpreter; + delete op_resolver; + interpreter = nullptr; + op_resolver = nullptr; + return -EINVAL; + } + + // fetch input/output tensors + quantization params once; the + // interpreter/tensors are stable for the lifetime of this instance + return Init_Interpreter(mwc); +} + +static int Init_Interpreter(struct mww_classify *mwc) +{ + input = interpreter->input(0); + if (!input) { + mwc->error = "input interpreter NULL"; + return -EINVAL; + } + + // check input tensor element count is compatible with our feature + // data size (shape is (1, MWW_FEATURE_SLICE_COUNT, MWW_FEATURE_SIZE), + // not flattened to a single trailing dim, so check the full product) + int in_elems = 1; + for (int i = 0; i < input->dims->size; i++) + in_elems *= input->dims->data[i]; + if (kFeatureElementCount != in_elems) { + mwc->error = "input interpreter shape incompatible"; + return -EINVAL; + } + + output = interpreter->output(0); + if (!output) { + mwc->error = "output interpreter NULL"; + return -EINVAL; + } + + // single sigmoid wake-word probability, quantized int8 or uint8 + if (output->type != kTfLiteInt8 && output->type != kTfLiteUInt8) { + mwc->error = "output tensor type != int8/uint8"; + return -EINVAL; + } + + // expose the model's real input quantization params so callers can + // requantize their features correctly instead of assuming a fixed + // scale/zero_point. + mwc->input_scale = input->params.scale; + mwc->input_zero_point = input->params.zero_point; + + MicroPrintf("MWW Model: in_type=%d in_scale=%f in_zp=%d out_type=%d out_scale=%f out_zp=%d", + input->type, (double)input->params.scale, input->params.zero_point, + output->type, (double)output->params.scale, output->params.zero_point); + + return 0; +} + +int MWW_SetModel(struct mww_classify *mwc, unsigned char *model_tflite) +{ +#if !CONFIG_COMP_MWW_MODEL_FROM_CONTROL + if (!model_tflite) + model_tflite = const_cast(MWW_BUILTIN_MODEL_DATA); +#endif + + if (!model_tflite) { + mwc->error = "no model provided"; + return -EINVAL; + } + + // Map the model into a usable data structure. This doesn't involve any + // copying or parsing, it's a very lightweight operation. + model = tflite::GetModel(model_tflite); + if (model->version() != TFLITE_SCHEMA_VERSION) { + mwc->error = "failed to load model"; + return -EINVAL; + } + + return 0; +} + +int MWW_ProcessClassify(struct mww_classify *mwc) +{ + float output_scale = output->params.scale; + int output_zero_point = output->params.zero_point; + + // copy features to input then invoke() + std::copy_n(mwc->audio_features, kFeatureElementCount, + tflite::GetTensorData(input)); + + // run the interpreter + if (interpreter->Invoke() != kTfLiteOk) { + mwc->error = "invoke failed"; + return -EINVAL; + } + + // Dequantize the single sigmoid probability output + float raw_val; + if (output->type == kTfLiteInt8) { + int8_t val = tflite::GetTensorData(output)[0]; + raw_val = static_cast(val); + mwc->raw_output = val; + } else { + uint8_t val = tflite::GetTensorData(output)[0]; + raw_val = static_cast(val); + mwc->raw_output = val; + } + + mwc->probability = (raw_val - output_zero_point) * output_scale; + if (mwc->probability < 0.0f) + mwc->probability = 0.0f; + else if (mwc->probability > 1.0f) + mwc->probability = 1.0f; + + return 0; +} + +int MWW_Reset(void) +{ + if (resource_variables) + resource_variables->ResetAll(); + return 0; +} + +void MWW_Free(void) +{ + delete interpreter; + delete op_resolver; + interpreter = nullptr; + op_resolver = nullptr; + allocator = nullptr; + resource_variables = nullptr; + model = nullptr; + input = nullptr; + output = nullptr; +} + +size_t MWW_ArenaUsedBytes(void) +{ + return interpreter ? interpreter->arena_used_bytes() : 0; +} + +size_t MWW_ArenaCapacity(void) +{ + return kArenaSize; +} diff --git a/src/audio/microwakeword/mww_model.h b/src/audio/microwakeword/mww_model.h new file mode 100644 index 000000000000..d8dd17673978 --- /dev/null +++ b/src/audio/microwakeword/mww_model.h @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. All rights reserved. + +#ifndef __MWW_MODEL_H__ +#define __MWW_MODEL_H__ + +#include "tensorflow/lite/core/c/common.h" + +/* microWakeWord streaming model configuration (hey_jarvis.tflite v2): + * input tensor shape (1, MWW_FEATURE_SLICE_COUNT, MWW_FEATURE_SIZE), int8, + * one new 10ms/40-bin MFCC hop per slice. The model is a stateful streaming + * graph (TFLM resource variables + CALL_ONCE init subgraph) that keeps its + * own longer-history ring buffers internally, so callers only ever need to + * supply MWW_FEATURE_SLICE_COUNT fresh hops per Invoke() rather than a + * caller-side sliding window. + */ +#define MWW_SAMPLE_RATE 16000 +#define MWW_FEATURE_SIZE 40 +#define MWW_FEATURE_SLICE_COUNT 3 +#define MWW_FEATURE_ELEM_COUNT (MWW_FEATURE_SIZE * MWW_FEATURE_SLICE_COUNT) +#define MWW_FEATURE_STRIDE_MS 10 +#define MWW_FEATURE_DURATION_MS 30 + +struct mww_classify { + int8_t *audio_features; + size_t audio_data_size; + const char *error; + float probability; /**< dequantized wake-word probability, 0..1 */ + int32_t raw_output; /**< raw int8/uint8 output tensor value */ + float input_scale; + int input_zero_point; +}; + +/* Export of C++ APIs into C namespace for linkage */ +#ifdef __cplusplus +extern "C" +{ +#endif + + /* 1st - pass in tflite flatbuffer formatted model, size is included in + * model metadata. + */ + int MWW_SetModel(struct mww_classify *mwc, unsigned char *model); + + /* 2nd - register the kernels and init TF micro for inference */ + int MWW_InitOps(struct mww_classify *mwc); + + /* 3rd - perform the inference */ + int MWW_ProcessClassify(struct mww_classify *mwc); + + /** + * \brief Reset streaming resource variables to zero. + * \return 0 on success. + */ + int MWW_Reset(void); + + /** + * \brief Free TFLite Micro interpreter and op resolver heap allocations. + */ + void MWW_Free(void); + + size_t MWW_ArenaUsedBytes(void); + size_t MWW_ArenaCapacity(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/uuid-registry.txt b/uuid-registry.txt index b235311b312c..5d581867e961 100644 --- a/uuid-registry.txt +++ b/uuid-registry.txt @@ -127,6 +127,7 @@ ee2585f2-e7d8-43dc-90ab4224e00c3e84 modules 0d9f2256-8e4f-47b3-8448239a334f1191 multiband_drc c607ff4d-9cb6-49dc-b6787da3c63ea557 mux 64ce6e35-857a-4878-ace8e2a2f42e3069 mux4 +4067fe1d-cd63-4877-966ba06ded1719ce mww f36BF24B-9AAF-83f4-8677E072E8AEADB7 notification_pool 1fb15a7a-83cd-4c2e-8b324da1b2adeeaf notifier 7ae671a7-4617-4a09-bf6d9d29c998dbc1 ns From e79fb4288334c9c8ba758ad4a9d8a37cf45f7b53 Mon Sep 17 00:00:00 2001 From: Seppo Ingalsuo Date: Thu, 27 Aug 2026 15:24:08 +0300 Subject: [PATCH 19/35] rimage: config: include microWakeWord (MWW) module in platform manifests Include audio/microwakeword/mww.toml when CONFIG_COMP_MWW is enabled across platform rimage manifest headers (tgl, tgl-h, mtl, lnl, ptl, wcl) so the MWW module UUID and entry are registered in base firmware images and loadable on target devices. Signed-off-by: Seppo Ingalsuo --- app/boards/intel_adsp_ace30_ptl.conf | 32 ++++++++++++++++++++++++++-- app/boards/intel_adsp_cavs25.conf | 11 +++++++++- tools/rimage/config/lnl.toml.h | 4 ++++ tools/rimage/config/mtl.toml.h | 4 ++++ tools/rimage/config/ptl.toml.h | 4 ++++ tools/rimage/config/tgl-h.toml.h | 4 ++++ tools/rimage/config/tgl.toml.h | 4 ++++ tools/rimage/config/wcl.toml.h | 4 ++++ 8 files changed, 64 insertions(+), 3 deletions(-) diff --git a/app/boards/intel_adsp_ace30_ptl.conf b/app/boards/intel_adsp_ace30_ptl.conf index 6697f8d5523a..ecb67d52ef77 100644 --- a/app/boards/intel_adsp_ace30_ptl.conf +++ b/app/boards/intel_adsp_ace30_ptl.conf @@ -15,6 +15,21 @@ CONFIG_FORMAT_CONVERT_HIFI3=n CONFIG_COMP_GOOGLE_RTC_AUDIO_PROCESSING=m CONFIG_GOOGLE_RTC_AUDIO_PROCESSING_MOCK=y CONFIG_COMP_STFT_PROCESS=y +CONFIG_SOF_STAGING=y +CONFIG_COMP_KPB=y +CONFIG_CPP=y +CONFIG_STD_CPP17=y +# tflmcly (stock keyword model) is not yet GNU-toolchain-clean on aphid +# (undefined sof_ut_log/__assert_no_args/abs); not required by mww, which +# builds its own independent TFLM lib copy. Disabled here to unblock the +# aphid mww LLEXT build. +CONFIG_COMP_TENSORFLOW=n +CONFIG_COMP_MFCC=y +CONFIG_COMP_MWW=m +CONFIG_AMS=y +CONFIG_STACK_SIZE_EDF=32768 +CONFIG_COMP_VOLUME=y +CONFIG_COMP_GAIN=y # SOF / infrastructure CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL=n @@ -29,7 +44,7 @@ CONFIG_COLD_STORE_EXECUTE_DRAM=y CONFIG_INTEL_MODULES=y CONFIG_LIBRARY_AUTH_SUPPORT=y CONFIG_LIBRARY_MANAGER=y -CONFIG_LIBRARY_BASE_ADDRESS=0xa0688000 +CONFIG_LIBRARY_BASE_ADDRESS=0xa0700000 CONFIG_LIBRARY_BUILD_LIB=y CONFIG_LIBRARY_DEFAULT_MODULAR=y @@ -43,11 +58,19 @@ CONFIG_SOF_LOG_LEVEL_INF=y CONFIG_DEBUG_COREDUMP_BACKEND_INTEL_ADSP_MEM_WINDOW=y CONFIG_DEBUG_COREDUMP_MEMORY_DUMP_MIN=y CONFIG_COUNTER=y -CONFIG_HEAP_MEM_POOL_SIZE=8192 +CONFIG_HEAP_MEM_POOL_SIZE=32768 +CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=32768 CONFIG_LLEXT=y CONFIG_LLEXT_STORAGE_WRITABLE=y CONFIG_LLEXT_EXPERIMENTAL=y CONFIG_LLEXT_EDK=n + +# Default LLEXT metadata heap (CONFIG_LLEXT_HEAP_SIZE, 8 KB) is sized for +# small extensions with few STB_GLOBAL-only exported symbols. mww.llext +# links in TFLM, whose template-instantiated vtables/typeinfo are emitted +# STB_WEAK -- now included in ext->sym_tab after the llext_load.c fix -- and +# blows past 8 KB (Failed to allocate extension symbol table, ret -12). +CONFIG_LLEXT_HEAP_SIZE=256 CONFIG_MODULES=y # Zephyr / device drivers @@ -83,3 +106,8 @@ CONFIG_SOF_USERSPACE_PROXY=y CONFIG_MAX_THREAD_BYTES=3 CONFIG_MAX_DOMAIN_PARTITIONS=32 + +# Debug: get faulting virtual address for openmodules-load panic +# CONFIG_XTENSA_ADSP_FATAL_BREADCRUMB_DATA_VADDR=y +CONFIG_SYS_HEAP_RUNTIME_STATS=y +CONFIG_SOF_ZEPHYR_HEAP_SIZE=0x80000 diff --git a/app/boards/intel_adsp_cavs25.conf b/app/boards/intel_adsp_cavs25.conf index 0a89f1a669a7..550804f53584 100644 --- a/app/boards/intel_adsp_cavs25.conf +++ b/app/boards/intel_adsp_cavs25.conf @@ -12,6 +12,14 @@ CONFIG_COMP_MFCC=y CONFIG_COMP_MULTIBAND_DRC=y CONFIG_COMP_VOLUME_WINDOWS_FADE=y CONFIG_FORMAT_CONVERT_HIFI3=n +CONFIG_SOF_STAGING=y +CONFIG_CPP=y +CONFIG_STD_CPP17=y +# cavs2.5 has no LLEXT/module-manager support (see CONFIG_LIBRARY_MANAGER=n +# below), so build mww in statically rather than as an LLEXT module. +CONFIG_COMP_MWW=y +CONFIG_COMP_MWW_DEBUG_TRACE=y +CONFIG_STACK_SIZE_EDF=32768 CONFIG_PCM_CONVERTER_FORMAT_S16LE=y CONFIG_PCM_CONVERTER_FORMAT_S24LE=y CONFIG_PCM_CONVERTER_FORMAT_S32LE=y @@ -37,7 +45,8 @@ CONFIG_SOF_LOG_LEVEL_INF=y # choices have to be selected per board. CONFIG_DEBUG_COREDUMP_BACKEND_INTEL_ADSP_MEM_WINDOW=y CONFIG_DEBUG_COREDUMP_MEMORY_DUMP_MIN=y -CONFIG_HEAP_MEM_POOL_SIZE=8192 +CONFIG_HEAP_MEM_POOL_SIZE=32768 +CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=32768 # Zephyr / device drivers CONFIG_DAI_INIT_PRIORITY=70 diff --git a/tools/rimage/config/lnl.toml.h b/tools/rimage/config/lnl.toml.h index faefbb8acadb..fb79be0484ad 100644 --- a/tools/rimage/config/lnl.toml.h +++ b/tools/rimage/config/lnl.toml.h @@ -166,5 +166,9 @@ #include