diff --git a/app/llext_relocatable.conf b/app/llext_relocatable.conf index 76b5339e1bb0..ce8dafe3a6aa 100644 --- a/app/llext_relocatable.conf +++ b/app/llext_relocatable.conf @@ -1 +1,2 @@ CONFIG_LLEXT_TYPE_ELF_RELOCATABLE=y +CONFIG_LLEXT_EXPORT_BUILTINS_BY_SLID=y diff --git a/scripts/llext_link_helper.py b/scripts/llext_link_helper.py index f10777c9918f..333c863d1d21 100755 --- a/scripts/llext_link_helper.py +++ b/scripts/llext_link_helper.py @@ -77,6 +77,8 @@ def main(): command = [args.command] + is_relocatable = '-r' in args.params + executable = [] writable = [] readonly = [] @@ -111,7 +113,8 @@ def main(): text_found = True text_addr = max_alignment(text_addr, 0x1000, s_alignment) text_size = s_size - command.append(f'-Wl,-Ttext=0x{text_addr:x}') + if not is_relocatable: + command.append(f'-Wl,-Ttext=0x{text_addr:x}') else: executable.append(section) @@ -164,7 +167,8 @@ def main(): dram_addr = align_up(dram_addr, s_alignment) - command.append(f'-Wl,--section-start={s_name}=0x{dram_addr:x}') + if not is_relocatable: + command.append(f'-Wl,--section-start={s_name}=0x{dram_addr:x}') dram_addr += section.header['sh_size'] @@ -177,7 +181,8 @@ def main(): dram_addr = align_up(dram_addr, s_alignment) - command.append(f'-Wl,--section-start={s_name}=0x{dram_addr:x}') + if not is_relocatable: + command.append(f'-Wl,--section-start={s_name}=0x{dram_addr:x}') dram_addr += section.header['sh_size'] @@ -189,7 +194,8 @@ def main(): start_addr = align_up(start_addr, s_alignment) - command.append(f'-Wl,--section-start={s_name}=0x{start_addr:x}') + if not is_relocatable: + command.append(f'-Wl,--section-start={s_name}=0x{start_addr:x}') start_addr += section.header['sh_size'] @@ -201,10 +207,11 @@ def main(): start_addr = align_up(start_addr, s_alignment) - if s_name == '.data': - command.append(f'-Wl,-Tdata=0x{start_addr:x}') - else: - command.append(f'-Wl,--section-start={s_name}=0x{start_addr:x}') + if not is_relocatable: + if s_name == '.data': + command.append(f'-Wl,-Tdata=0x{start_addr:x}') + else: + command.append(f'-Wl,--section-start={s_name}=0x{start_addr:x}') start_addr += section.header['sh_size'] diff --git a/scripts/llext_offset_calc.py b/scripts/llext_offset_calc.py index 0f302a8cbe12..2a07984b725f 100755 --- a/scripts/llext_offset_calc.py +++ b/scripts/llext_offset_calc.py @@ -47,6 +47,9 @@ def get_elf_size(elf_name): if section.header['sh_addr'] + section.header['sh_size'] > end: end = section.header['sh_addr'] + section.header['sh_size'] + if start == 0xffffffff: + return 0 + size = end - start return size diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h index 29c226eb61a7..87ddc082d409 100644 --- a/src/include/sof/lib_manager.h +++ b/src/include/sof/lib_manager.h @@ -116,6 +116,8 @@ struct lib_manager_module { unsigned int n_dependent; /* For auxiliary modules: number of dependents */ bool mapped; struct lib_manager_segment_desc segment[LIB_MANAGER_N_SEGMENTS]; + uintptr_t vma_base; + size_t vma_size; }; struct lib_manager_mod_ctx { diff --git a/src/library_manager/Kconfig b/src/library_manager/Kconfig index 8eac39e59970..598551a6c650 100644 --- a/src/library_manager/Kconfig +++ b/src/library_manager/Kconfig @@ -58,10 +58,4 @@ config LIBRARY_BASE_ADDRESS automatically but the beginning of that area is platform-specific and should be set by this option. -config LIBRARY_REGION_SIZE - hex "Size of memory region dedicated to loadable modules" - default 0x100000 - help - Size of the virtual memory region dedicated for loadable modules - endmenu diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 4c1e4f02d5b5..3a3aaf53cfa1 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include #include @@ -50,6 +50,131 @@ extern struct tr_ctx lib_manager_tr; #define PAGE_SZ CONFIG_MM_DRV_PAGE_SIZE +#include + +/** + * @brief Reserve virtual address space for a relocatable LLEXT image. + * @param size Total image size in bytes. + * @return Base virtual address, or zero when reservation fails. + */ +static uintptr_t llext_manager_alloc_vma(size_t size) +{ + size_t num_pages = DIV_ROUND_UP(size, PAGE_SZ); + void *vma; + + vma = vpage_reserve(num_pages); + if (!vma) { + tr_err(&lib_manager_tr, "llext_manager_alloc_vma: failed to reserve %zu pages", + num_pages); + return 0; + } + + return (uintptr_t)vma; +} + +/** + * @brief Release virtual address space reserved for a LLEXT image. + * @param vma Base virtual address returned by llext_manager_alloc_vma(). + * @param size Reserved image size in bytes. + */ +void llext_manager_free_vma(uintptr_t vma, size_t size) +{ + if (!vma || !size) + return; + + vpage_release((void *)vma); +} + +/** + * @brief Map an ELF section name and flags to a LLEXT memory region. + * @param name ELF section name. + * @param shdr ELF section header. + * @return Corresponding LLEXT memory region, or LLEXT_MEM_COUNT if unsupported. + */ +static enum llext_mem llext_manager_get_sec_mem_idx(const char *name, const elf_shdr_t *shdr) +{ + if (strcmp(name, ".exported_sym") == 0) + return LLEXT_MEM_EXPORT; + + switch (shdr->sh_type) { + case SHT_NOBITS: + return LLEXT_MEM_BSS; + case SHT_PROGBITS: + if (shdr->sh_flags & SHF_EXECINSTR) + return LLEXT_MEM_TEXT; + else if (shdr->sh_flags & SHF_WRITE) + return LLEXT_MEM_DATA; + else + return LLEXT_MEM_RODATA; + case SHT_PREINIT_ARRAY: + return LLEXT_MEM_PREINIT; + case SHT_INIT_ARRAY: + return LLEXT_MEM_INIT; + case SHT_FINI_ARRAY: + return LLEXT_MEM_FINI; + default: + return LLEXT_MEM_COUNT; + } +} + +/** + * @brief Calculate and optionally assign addresses for loadable ELF sections. + * @param elf_buf ELF image buffer. + * @param vma_base Base virtual address for relocated sections, or zero to measure only. + * @return Total virtual address space required by the loadable sections. + * + * Sections are kept grouped by their final memory permissions and each group + * starts on a page boundary. When a base address is supplied, the calculated + * addresses are written back to the ELF section headers for pre-located loading. + */ +static size_t llext_manager_layout_sections(uint8_t *elf_buf, uintptr_t vma_base) +{ + elf_ehdr_t *hdr = (elf_ehdr_t *)elf_buf; + elf_shdr_t *shdrs = (elf_shdr_t *)(elf_buf + hdr->e_shoff); + elf_shdr_t *shstr_shdr = shdrs + hdr->e_shstrndx; + const char *shstrtab = (const char *)(elf_buf + shstr_shdr->sh_offset); + + uintptr_t current_vma = vma_base; + enum llext_mem last_region = LLEXT_MEM_COUNT; + + for (int i = 0; i < hdr->e_shnum; i++) { + elf_shdr_t *shdr = shdrs + i; + + if (!(shdr->sh_flags & SHF_ALLOC) || shdr->sh_size == 0) + continue; + + const char *name = shstrtab + shdr->sh_name; + enum llext_mem s_region = llext_manager_get_sec_mem_idx(name, shdr); + + if (s_region == LLEXT_MEM_COUNT) + continue; + + /* Separate sections with different permissions into page-aligned regions. */ + if (last_region != LLEXT_MEM_COUNT && last_region != s_region) { + current_vma = ALIGN_UP(current_vma, PAGE_SZ); + } + last_region = s_region; + + /* Preserve the alignment required by the input section. */ + if (shdr->sh_addralign > 1) { + current_vma = ALIGN_UP(current_vma, shdr->sh_addralign); + } + if (vma_base) { + shdr->sh_addr = current_vma; + } + current_vma += shdr->sh_size; + } + + return current_vma - vma_base; +} + +/** + * @brief Apply memory permissions to a possibly unaligned virtual range. + * @param vma Range start address. + * @param size Range size in bytes. + * @param flags New system memory permissions. + * @return Zero on success or a negative error code. + */ static int llext_manager_update_flags(void __sparse_cache *vma, size_t size, uint32_t flags) { size_t pre_pad_size = (uintptr_t)vma & (PAGE_SZ - 1); @@ -59,6 +184,14 @@ static int llext_manager_update_flags(void __sparse_cache *vma, size_t size, uin ALIGN_UP(pre_pad_size + size, PAGE_SZ), flags); } +/** + * @brief Map a possibly unaligned virtual range after aligning its page bounds. + * @param virtual_region Shared virtual memory region used for the mapping. + * @param vma Range start address. + * @param size Range size in bytes. + * @param flags Mapping permissions. + * @return Zero on success or a negative error code. + */ static int llext_manager_align_map(const struct sys_mm_drv_region *virtual_region, void __sparse_cache *vma, size_t size, uint32_t flags) { @@ -68,6 +201,12 @@ static int llext_manager_align_map(const struct sys_mm_drv_region *virtual_regio ALIGN_UP(pre_pad_size + size, PAGE_SZ), flags); } +/** + * @brief Unmap a possibly unaligned virtual range using page-aligned bounds. + * @param vma Range start address. + * @param size Range size in bytes. + * @return Zero on success or a negative error code. + */ static int llext_manager_align_unmap(void __sparse_cache *vma, size_t size) { size_t pre_pad_size = (uintptr_t)vma & (PAGE_SZ - 1); @@ -76,6 +215,12 @@ static int llext_manager_align_unmap(void __sparse_cache *vma, size_t size) return sys_mm_drv_unmap_region(aligned_vma, ALIGN_UP(pre_pad_size + size, PAGE_SZ)); } +/** + * @brief Update permissions for sections placed outside the main VMA mapping. + * @param vma Detached section address. + * @param size Section size in bytes. + * @param flags New memory permissions. + */ static void llext_manager_detached_update_flags(void __sparse_cache *vma, size_t size, uint32_t flags) { @@ -95,6 +240,17 @@ static void llext_manager_detached_update_flags(void __sparse_cache *vma, * sections that belong to the specified 'region' and are contained in the * memory range, then remap the same area according to the 'flags' parameter. */ +/** + * @brief Map a LLEXT region, copy its sections from storage, and set permissions. + * @param virtual_region Shared virtual memory region used for mapping. + * @param ldr LLEXT buffer loader. + * @param ext Loaded LLEXT object. + * @param region LLEXT memory region to load. + * @param vma Destination virtual address. + * @param size Destination size in bytes. + * @param flags Final memory permissions. + * @return Zero on success or a negative error code. + */ static int llext_manager_load_data_from_storage(const struct sys_mm_drv_region *virtual_region, const struct llext_loader *ldr, const struct llext *ext, @@ -147,10 +303,15 @@ static int llext_manager_load_data_from_storage(const struct sys_mm_drv_region * continue; } - ret = memcpy_s((__sparse_force void *)shdr->sh_addr, size - s_offset, + /* s_offset is a file offset, so bound the copy by the mapped VMA instead */ + ret = memcpy_s((__sparse_force void *)shdr->sh_addr, + (uintptr_t)vma + size - (uintptr_t)shdr->sh_addr, (const uint8_t *)region_addr + s_offset, shdr->sh_size); - if (ret < 0) + if (ret < 0) { + tr_err(&lib_manager_tr, "cannot copy section %u to %#lx: %d", + i, (uintptr_t)shdr->sh_addr, ret); return ret; + } } /* @@ -166,6 +327,14 @@ static int llext_manager_load_data_from_storage(const struct sys_mm_drv_region * return ret; } +/** + * @brief Remove mappings for sections placed outside a main LLEXT region. + * @param ldr LLEXT buffer loader. + * @param ext Loaded LLEXT object. + * @param region LLEXT memory region being unloaded. + * @param vma Main region virtual address. + * @param size Main region size in bytes. + */ static void llext_manager_unmap_detached_sections(const struct llext_loader *ldr, const struct llext *ext, enum llext_mem region, @@ -199,6 +368,11 @@ static void llext_manager_unmap_detached_sections(const struct llext_loader *ldr #endif } +/** + * @brief Map a linked LLEXT module into virtual memory and initialize its data. + * @param mctx Library module context to load. + * @return Zero on success or a negative error code. + */ static int llext_manager_load_module(struct lib_manager_module *mctx) { /* Executable code (.text) */ @@ -222,48 +396,30 @@ static int llext_manager_load_module(struct lib_manager_module *mctx) size_t bss_size = mctx->segment[LIB_MANAGER_BSS].size; int ret; - /* Check, that .bss is within .data */ - if (bss_size && - ((uintptr_t)bss_addr + bss_size <= (uintptr_t)va_base_data || - (uintptr_t)bss_addr >= (uintptr_t)va_base_data + data_size)) { - size_t bss_align = MIN(PAGE_SZ, BIT(__builtin_ctz((uintptr_t)bss_addr))); + /* + * .bss is page-aligned into its own page by the layout, so it is usually + * not adjacent to .data. Map the range spanning both, gap included. + */ + if (bss_size) { + uintptr_t start = (uintptr_t)bss_addr; + uintptr_t end = start + bss_size; - if ((!data_size || (uintptr_t)bss_addr + bss_size == (uintptr_t)va_base_data) && - bss_align >= PAGE_SZ) { - /* - * .bss is properly aligned and either there's no writable data, - * or .bss is directly in front of writable data, prepend .bss - */ - va_base_data = bss_addr; - data_size += bss_size; - } else if ((uintptr_t)bss_addr == (uintptr_t)va_base_data + - ALIGN_UP(data_size, bss_align)) { - /* .bss directly behind writable data, append */ - data_size += bss_size; - } else { - tr_err(&lib_manager_tr, ".bss %#x @%p isn't within writable data %#x @%p!", - bss_size, (__sparse_force void *)bss_addr, - data_size, (__sparse_force void *)va_base_data); - return -EPROTO; + if (data_size) { + start = MIN(start, (uintptr_t)va_base_data); + end = MAX(end, (uintptr_t)va_base_data + data_size); } + + va_base_data = (void __sparse_cache *)start; + data_size = end - start; } const struct llext_loader *ldr = &mctx->ebl->loader; const struct llext *ext = mctx->llext; - /* find dedicated virtual memory zone */ - const struct sys_mm_drv_region *virtual_memory_regions = sys_mm_drv_query_memory_regions(); - const struct sys_mm_drv_region *virtual_region; + /* use the shared virtual page allocator's memory region */ + const struct sys_mm_drv_region *virtual_region = vpage_get_region(); - if (!virtual_memory_regions) - return -EFAULT; - - SYS_MM_DRV_MEMORY_REGION_FOREACH(virtual_memory_regions, virtual_region) { - if (virtual_region->attr == VIRTUAL_REGION_LLEXT_LIBRARIES_ATTR) - break; - } - - if (!virtual_region->size) + if (!virtual_region || !virtual_region->size) return -EFAULT; /* Copy Code */ @@ -303,6 +459,11 @@ static int llext_manager_load_module(struct lib_manager_module *mctx) return ret; } +/** + * @brief Unmap a loaded LLEXT module and its detached sections. + * @param mctx Library module context to unload. + * @return Zero on success or the first unmap error. + */ static int llext_manager_unload_module(struct lib_manager_module *mctx) { const struct llext_loader *ldr = &mctx->ebl->loader; @@ -356,11 +517,64 @@ static int llext_manager_unload_module(struct lib_manager_module *mctx) return err; } +/** + * @brief Determine whether an ELF section must remain outside the relocated VMA. + * @param shdr ELF section header. + * @return true when the section is detached from the relocated image. + */ static bool llext_manager_section_detached(const elf_shdr_t *shdr) { return shdr->sh_addr < SOF_MODULE_DRAM_LINK_END; } +/** + * @brief Measure the virtual address range occupied by one LLEXT memory region. + * @param ldr LLEXT buffer loader. + * @param ext Loaded LLEXT object. + * @param region LLEXT memory region to measure. + * @param seg Receives the region start address and size. + * + * llext sizes its region descriptors by ELF file offsets, which say nothing + * about the addresses a pre-located image is relocated to. Measuring the + * mapped sections keeps the regions from overlapping each other or reaching + * past the module's own virtual memory reservation. + */ +static void llext_manager_region_extent(const struct llext_loader *ldr, const struct llext *ext, + enum llext_mem region, + struct lib_manager_segment_desc *seg) +{ + uintptr_t start = UINTPTR_MAX, end = 0; + unsigned int i; + + for (i = 0; i < llext_section_count(ext); i++) { + const elf_shdr_t *shdr; + enum llext_mem s_region = LLEXT_MEM_COUNT; + + if (llext_get_section_info(ldr, ext, i, &shdr, &s_region, NULL) < 0 || + s_region != region || !shdr->sh_size || + llext_manager_section_detached(shdr)) + continue; + + start = MIN(start, (uintptr_t)shdr->sh_addr); + end = MAX(end, (uintptr_t)shdr->sh_addr + shdr->sh_size); + } + + seg->addr = end ? start : 0; + seg->size = end ? end - start : 0; +} + +/** + * @brief Link a LLEXT image, reusing its context when it is already linked. + * @param name Module name passed to the LLEXT loader. + * @param mctx Library module context. + * @param buildinfo Receives the module build information section. + * @param mod_manifest Receives the module manifest section. + * @return Zero on success or a negative error code. + * + * On the first link, this function lays out the ELF image in the shared VMA + * allocator and loads it at the pre-located addresses. Later instances reuse + * the retained LLEXT context and only acquire the loader reference they need. + */ static int llext_manager_link(const char *name, struct lib_manager_module *mctx, const void **buildinfo, const struct sof_man_module_manifest **mod_manifest) @@ -383,9 +597,31 @@ static int llext_manager_link(const char *name, } if (!*llext || mctx->mapped) { + if (!*llext) { + /* Measure the image before reserving and assigning its final VMA. */ + uint8_t *elf_buf = (uint8_t *)mctx->ebl->buf; + size_t total_size = llext_manager_layout_sections(elf_buf, 0); + if (total_size == 0) { + tr_err(&lib_manager_tr, "llext_manager_link: layout sections failed"); + return -EINVAL; + } + + uintptr_t vma_base = llext_manager_alloc_vma(total_size); + if (!vma_base) { + tr_err(&lib_manager_tr, "llext_manager_link: VMA allocation failed"); + return -ENOMEM; + } + + mctx->vma_base = vma_base; + mctx->vma_size = total_size; + + /* Update section addresses so llext_load() can relocate in place. */ + llext_manager_layout_sections(elf_buf, vma_base); + } + /* - * Either the very first time loading this module, or the module - * is already mapped, we just call llext_load() to refcount it + * The first load performs relocation; a mapped module only increments + * the LLEXT reference count while retaining the pre-located addresses. */ struct llext_load_param ldr_parm = { .relocate_local = !*llext, @@ -395,41 +631,44 @@ static int llext_manager_link(const char *name, }; ret = llext_load(ldr, name, llext, &ldr_parm); - if (ret) + if (ret < 0) { + tr_err(&lib_manager_tr, "llext_load failed: ret=%d", ret); + if (mctx->vma_base) { + llext_manager_free_vma(mctx->vma_base, mctx->vma_size); + mctx->vma_base = 0; + mctx->vma_size = 0; + } return ret; + } } /* All code sections */ - llext_get_region_info(ldr, *llext, LLEXT_MEM_TEXT, &hdr, NULL, NULL); - mctx->segment[LIB_MANAGER_TEXT].addr = hdr->sh_addr; - mctx->segment[LIB_MANAGER_TEXT].size = hdr->sh_size; + llext_manager_region_extent(ldr, *llext, LLEXT_MEM_TEXT, + &mctx->segment[LIB_MANAGER_TEXT]); tr_dbg(&lib_manager_tr, ".text: start: %#lx size %#x", mctx->segment[LIB_MANAGER_TEXT].addr, mctx->segment[LIB_MANAGER_TEXT].size); /* All read-only data sections */ - llext_get_region_info(ldr, *llext, LLEXT_MEM_RODATA, &hdr, NULL, NULL); - mctx->segment[LIB_MANAGER_RODATA].addr = hdr->sh_addr; - mctx->segment[LIB_MANAGER_RODATA].size = hdr->sh_size; + llext_manager_region_extent(ldr, *llext, LLEXT_MEM_RODATA, + &mctx->segment[LIB_MANAGER_RODATA]); tr_dbg(&lib_manager_tr, ".rodata: start: %#lx size %#x", mctx->segment[LIB_MANAGER_RODATA].addr, mctx->segment[LIB_MANAGER_RODATA].size); /* All writable data sections */ - llext_get_region_info(ldr, *llext, LLEXT_MEM_DATA, &hdr, NULL, NULL); - mctx->segment[LIB_MANAGER_DATA].addr = hdr->sh_addr; - mctx->segment[LIB_MANAGER_DATA].size = hdr->sh_size; + llext_manager_region_extent(ldr, *llext, LLEXT_MEM_DATA, + &mctx->segment[LIB_MANAGER_DATA]); tr_dbg(&lib_manager_tr, ".data: start: %#lx size %#x", mctx->segment[LIB_MANAGER_DATA].addr, mctx->segment[LIB_MANAGER_DATA].size); /* Writable uninitialized data section */ - llext_get_region_info(ldr, *llext, LLEXT_MEM_BSS, &hdr, NULL, NULL); - mctx->segment[LIB_MANAGER_BSS].addr = hdr->sh_addr; - mctx->segment[LIB_MANAGER_BSS].size = hdr->sh_size; + llext_manager_region_extent(ldr, *llext, LLEXT_MEM_BSS, + &mctx->segment[LIB_MANAGER_BSS]); tr_dbg(&lib_manager_tr, ".bss: start: %#lx size %#x", mctx->segment[LIB_MANAGER_BSS].addr, @@ -452,7 +691,12 @@ static int llext_manager_link(const char *name, return *buildinfo && *mod_manifest ? 0 : -EPROTO; } -/* Count "module files" in the library, allocate and initialize memory for their descriptors */ +/** + * @brief Count module files and initialize their persistent descriptors. + * @param ctx Library module context to initialize. + * @param desc Firmware library manifest. + * @return Zero on success or a negative error code. + */ static int llext_manager_mod_init(struct lib_manager_mod_ctx *ctx, const struct sof_man_fw_desc *desc) { @@ -505,7 +749,12 @@ static int llext_manager_mod_init(struct lib_manager_mod_ctx *ctx, return 0; } -/* Find a module context, containing the driver with the supplied index */ +/** + * @brief Find the module context containing a manifest entry. + * @param ctx Library module context. + * @param idx Global module manifest index. + * @return Index of the containing module context. + */ static unsigned int llext_manager_mod_find(const struct lib_manager_mod_ctx *ctx, unsigned int idx) { unsigned int i; @@ -517,6 +766,15 @@ static unsigned int llext_manager_mod_find(const struct lib_manager_mod_ctx *ctx return i - 1; } +/** + * @brief Link one module driver and return its module-context index. + * @param module_id Module identifier from the IPC configuration. + * @param desc Firmware library manifest. + * @param ctx Library module context. + * @param buildinfo Receives the build information section when first linked. + * @param mod_manifest Receives the selected module manifest. + * @return Module-context index, or a negative error code. + */ static int llext_manager_link_single(uint32_t module_id, const struct sof_man_fw_desc *desc, struct lib_manager_mod_ctx *ctx, const void **buildinfo, const struct sof_man_module_manifest **mod_manifest) @@ -622,6 +880,12 @@ static int llext_manager_link_single(uint32_t module_id, const struct sof_man_fw return mod_ctx_idx; } +/** + * @brief Find the library context for a loaded LLEXT dependency. + * @param llext LLEXT dependency to find. + * @param dep_ctx Receives the dependent module context. + * @return Library index, or a negative error code. + */ static int llext_lib_find(const struct llext *llext, struct lib_manager_module **dep_ctx) { struct ext_library *_ext_lib = ext_lib_get(); @@ -644,7 +908,11 @@ static int llext_lib_find(const struct llext *llext, struct lib_manager_module * return -ENOENT; } -/* n can be -1 */ +/** + * @brief Roll back dependency references and unload newly mapped dependencies. + * @param dep_ctx Dependency contexts to roll back. + * @param n Last dependency index to process; may be -1. + */ static void llext_manager_depend_unlink_rollback(struct lib_manager_module *dep_ctx[], int n) { for (; n >= 0; n--) @@ -654,6 +922,12 @@ static void llext_manager_depend_unlink_rollback(struct lib_manager_module *dep_ llext_manager_unload_module(dep_ctx[n]); } +/** + * @brief Link and map an IPC module, returning its entry point. + * @param ipc_config IPC module configuration. + * @param ipc_specific_config IPC-specific configuration, reserved for the loader interface. + * @return Module entry-point address, or zero on failure. + */ uintptr_t llext_manager_allocate_module(const struct comp_ipc_config *ipc_config, const void *ipc_specific_config) { @@ -748,6 +1022,14 @@ uintptr_t llext_manager_allocate_module(const struct comp_ipc_config *ipc_config } #ifdef CONFIG_USERSPACE +/** + * @brief Add a page-aligned user memory partition for a module region. + * @param domain User memory domain to update. + * @param addr Region start address. + * @param size Region size in bytes. + * @param attr Partition attributes. + * @return Zero on success or a negative error code. + */ static int llext_manager_add_partition(struct k_mem_domain *domain, uintptr_t addr, size_t size, k_mem_partition_attr_t attr) @@ -763,6 +1045,14 @@ static int llext_manager_add_partition(struct k_mem_domain *domain, return k_mem_domain_add_partition(domain, &part); } +/** + * @brief Remove a page-aligned user memory partition for a module region. + * @param domain User memory domain to update. + * @param addr Region start address. + * @param size Region size in bytes. + * @param attr Partition attributes. + * @return Zero on success or a negative error code. + */ static int llext_manager_rm_partition(struct k_mem_domain *domain, uintptr_t addr, size_t size, k_mem_partition_attr_t attr) @@ -778,6 +1068,12 @@ static int llext_manager_rm_partition(struct k_mem_domain *domain, return k_mem_domain_remove_partition(domain, &part); } +/** + * @brief Add all mapped LLEXT regions to a component's user memory domain. + * @param component_id IPC component identifier. + * @param domain User memory domain to update. + * @return Zero on success or a negative error code. + */ int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *domain) { const uint32_t module_id = IPC4_MOD_ID(component_id); @@ -914,6 +1210,12 @@ int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *d return ret; } +/** + * @brief Remove all mapped LLEXT regions from a component's user memory domain. + * @param component_id IPC component identifier. + * @param domain User memory domain to update. + * @return Zero on success or a negative error code. + */ int llext_manager_rm_domain(const uint32_t component_id, struct k_mem_domain *domain) { const uint32_t module_id = IPC4_MOD_ID(component_id); @@ -990,6 +1292,11 @@ int llext_manager_rm_domain(const uint32_t component_id, struct k_mem_domain *do } #endif +/** + * @brief Release one module instance and unload it when its last user exits. + * @param component_id IPC component identifier. + * @return Zero on success or a negative error code. + */ int llext_manager_free_module(const uint32_t component_id) { const uint32_t module_id = IPC4_MOD_ID(component_id); @@ -1057,7 +1364,11 @@ int llext_manager_free_module(const uint32_t component_id) return llext_manager_unload_module(mctx); } -/* An auxiliary library has been loaded, need to read in its exported symbols */ +/** + * @brief Link all auxiliary LLEXT modules in a library. + * @param module_id Library module identifier. + * @return Zero on success or a negative error code. + */ int llext_manager_add_library(uint32_t module_id) { struct lib_manager_mod_ctx *const ctx = lib_manager_get_mod_ctx(module_id); @@ -1094,6 +1405,11 @@ int llext_manager_add_library(uint32_t module_id) return 0; } +/** + * @brief Determine whether a component belongs to a LLEXT module. + * @param comp Component to inspect. + * @return true when the component's module manifest identifies a LLEXT module. + */ bool comp_is_llext(struct comp_dev *comp) { const uint32_t module_id = IPC4_MOD_ID(comp->ipc_config.id); @@ -1104,16 +1420,3 @@ bool comp_is_llext(struct comp_dev *comp) return mod && module_is_llext(mod); } -static int llext_memory_region_init(void) -{ - int ret; - - /* add a region for loadable libraries */ - ret = adsp_add_virtual_memory_region(CONFIG_LIBRARY_BASE_ADDRESS, - CONFIG_LIBRARY_REGION_SIZE, - VIRTUAL_REGION_LLEXT_LIBRARIES_ATTR); - - return ret; -} - -SYS_INIT(llext_memory_region_init, POST_KERNEL, 1); diff --git a/src/library_manager/llext_manager_dram.c b/src/library_manager/llext_manager_dram.c index 2f6cff2b3501..72c162178ca6 100644 --- a/src/library_manager/llext_manager_dram.c +++ b/src/library_manager/llext_manager_dram.c @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -14,6 +15,8 @@ LOG_MODULE_DECLARE(lib_manager, CONFIG_SOF_LOG_LEVEL); +void llext_manager_free_vma(uintptr_t vma, size_t size); + struct lib_manager_dram_storage { struct ext_library ext_lib; struct lib_manager_mod_ctx *ctx; @@ -232,6 +235,24 @@ int llext_manager_restore_from_dram(void) /* Not instantiated - nothing to restore */ continue; + /* + * The virtual page allocator is re-initialised empty on every + * boot, but these modules stay mapped at the addresses they + * were given before power gating, so claim them again. + */ + if (mod[k].vma_base) { + int ret = vpage_reserve_at((void *)mod[k].vma_base, + DIV_ROUND_UP(mod[k].vma_size, + CONFIG_MM_DRV_PAGE_SIZE)); + + if (ret < 0) { + tr_err(&lib_manager_tr, + "failed to re-reserve VMA %#lx size %#zx: %d", + mod[k].vma_base, mod[k].vma_size, ret); + goto nomem; + } + } + /* Loaders are supplied by the caller */ struct llext_buf_loader *bldr = rmalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, sizeof(*bldr)); @@ -332,6 +353,9 @@ int llext_manager_restore_from_dram(void) if (mod[k].llext) llext_unload(&mod[k].llext); + if (mod[k].vma_base) + llext_manager_free_vma(mod[k].vma_base, mod[k].vma_size); + if (mod[k].ebl) rfree(mod[k].ebl); } diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 4b61a9517d46..bf63b82162de 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -289,8 +289,14 @@ if (CONFIG_SOC_SERIES_INTEL_ADSP_ACE) ${SOF_PLATFORM_PATH}/novalake/lib/clk.c ) + # vpage.c is the shared virtual-page allocator: needed by the vregion + # pipeline-resource allocator (CONFIG_SOF_VREGIONS) and independently by + # the LLEXT library manager (CONFIG_LIBRARY_MANAGER) for library VMAs. + if(CONFIG_SOF_VREGIONS OR CONFIG_LIBRARY_MANAGER) + zephyr_library_sources(lib/vpage.c) + endif() + zephyr_library_sources_ifdef(CONFIG_SOF_VREGIONS - lib/vpage.c lib/vregion.c ) diff --git a/zephyr/include/sof/lib/regions_mm.h b/zephyr/include/sof/lib/regions_mm.h index abaaf158d3d4..616d70aba0bd 100644 --- a/zephyr/include/sof/lib/regions_mm.h +++ b/zephyr/include/sof/lib/regions_mm.h @@ -19,7 +19,6 @@ /* Attributes for memory regions */ #define VIRTUAL_REGION_SHARED_HEAP_ATTR 1U /*< region for shared virtual heap */ -#define VIRTUAL_REGION_LLEXT_LIBRARIES_ATTR 2U /*< region for LLEXT libraries */ #define VIRTUAL_REGION_VPAGES_ATTR 3U /*< region for virtual page allocator */ /* Dependency on ipc/topology.h created due to memory capability definitions diff --git a/zephyr/include/sof/lib/vpage.h b/zephyr/include/sof/lib/vpage.h index f3fc8b89e968..5b3d6129757b 100644 --- a/zephyr/include/sof/lib/vpage.h +++ b/zephyr/include/sof/lib/vpage.h @@ -6,6 +6,7 @@ #define __SOF_LIB_VPAGE_H__ #include +#include #include #ifdef __cplusplus @@ -31,6 +32,51 @@ void *vpage_alloc(unsigned int pages); */ void vpage_free(void *ptr); +/** + * @brief Reserve virtual pages + * Reserves a specified number of contiguous virtual memory pages from the + * shared virtual page allocator, without mapping any physical memory to + * them. Intended for callers that need to map the pages themselves, e.g. + * with per-section permissions. + * + * @param[in] pages Number of pages (usually 4kB large) to reserve. + * + * @return Pointer to the reserved virtual memory region, or NULL on failure. + */ +void *vpage_reserve(unsigned int pages); + +/** + * @brief Reserve a specific range of virtual pages + * Reserves an already occupied range of virtual memory pages, without mapping + * any physical memory to them. Intended for callers whose address was assigned + * by someone other than this allocator, e.g. modules that stay resident over a + * power gating cycle and are restored with their previous addresses. + * + * @param[in] ptr Page aligned base address of the range to reserve. + * @param[in] pages Number of pages (usually 4kB large) to reserve. + * + * @return 0 on success, negative error code otherwise. + */ +int vpage_reserve_at(void *ptr, unsigned int pages); + +/** + * @brief Release reserved virtual pages + * Releases virtual memory pages previously reserved with vpage_reserve(). + * Does not unmap any physical memory - callers that mapped the pages + * themselves must unmap them first. + * + * @param[in] ptr Pointer to the reserved memory pages to release. + */ +void vpage_release(void *ptr); + +/** + * @brief Get the shared virtual page allocator's memory region + * + * @return Pointer to the virtual memory region backing the virtual page + * allocator. + */ +const struct sys_mm_drv_region *vpage_get_region(void); + #ifdef __cplusplus } #endif diff --git a/zephyr/lib/vpage.c b/zephyr/lib/vpage.c index ce0da7b5ac97..dc5e45d27334 100644 --- a/zephyr/lib/vpage.c +++ b/zephyr/lib/vpage.c @@ -64,16 +64,16 @@ struct vpage_context { static struct vpage_context vpage_ctx; /** - * @brief Allocate and map virtual memory pages + * @brief Reserve virtual memory pages * - * Allocates memory pages from the virtual page allocator. - * Maps physical memory pages to the virtual region as needed. + * Reserves memory pages from the virtual page allocator, without mapping + * any physical memory to them. * - * @param pages Number of pages to allocate. - * @param ptr Pointer to store the address of allocated pages. + * @param pages Number of pages to reserve. + * @param ptr Pointer to store the address of the reserved pages. * @retval 0 if successful. */ -static int vpages_alloc_and_map(unsigned int pages, void **ptr) +static int vpages_reserve(unsigned int pages, void **ptr) { void *vaddr; int ret; @@ -108,16 +108,6 @@ static int vpages_alloc_and_map(unsigned int pages, void **ptr) return ret; } - /* map the virtual blocks in virtual region to free physical blocks */ - ret = sys_mm_drv_map_region_safe(vpage_ctx.virtual_region, vaddr, - 0, pages * CONFIG_MM_DRV_PAGE_SIZE, SYS_MM_MEM_PERM_RW); - if (ret < 0) { - LOG_ERR("error: failed to map virtual region %p to physical region %p, error %d", - vaddr, vpage_ctx.virtual_region->addr, ret); - sys_mem_blocks_free_contiguous(&vpage_ctx.vpage_blocks, vaddr, pages); - return ret; - } - /* success update the free pages */ vpage_ctx.free_pages -= pages; @@ -135,6 +125,157 @@ static int vpages_alloc_and_map(unsigned int pages, void **ptr) return 0; } +/** + * @brief Reserve a specific range of virtual memory pages + * + * Marks an already occupied range as allocated, for users that were given + * their address by someone other than this allocator. + * + * @param ptr Page aligned base address of the range. + * @param pages Number of pages to reserve. + * @retval 0 if successful. + */ +static int vpages_reserve_at(void *ptr, unsigned int pages) +{ + uintptr_t region_base = POINTER_TO_UINT(vpage_ctx.virtual_region->addr); + unsigned int vpage, elem_idx; + int ret; + + if (!pages) + return 0; + + CHECKIF(!IS_ALIGNED(ptr, CONFIG_MM_DRV_PAGE_SIZE)) { + LOG_ERR("error: invalid non aligned page pointer %p", ptr); + return -EINVAL; + } + + if (POINTER_TO_UINT(ptr) < region_base || + POINTER_TO_UINT(ptr) + (size_t)pages * CONFIG_MM_DRV_PAGE_SIZE > + region_base + vpage_ctx.virtual_region->size) { + LOG_ERR("error: range %p pages %u outside the virtual region", ptr, pages); + return -EINVAL; + } + + vpage = (POINTER_TO_UINT(ptr) - region_base) / CONFIG_MM_DRV_PAGE_SIZE; + + /* Several modules can share one image, so one range can be reserved repeatedly */ + for (elem_idx = 0; elem_idx < vpage_ctx.num_elems_in_use; elem_idx++) + if (vpage_ctx.velems[elem_idx].vpage == vpage) + return vpage_ctx.velems[elem_idx].pages == pages ? 0 : -EEXIST; + + if (vpage_ctx.num_elems_in_use >= VPAGE_MAX_ALLOCS) { + LOG_ERR("error: max allocation elements reached"); + return -ENOMEM; + } + + ret = sys_mem_blocks_get(&vpage_ctx.vpage_blocks, ptr, pages); + if (ret < 0) { + LOG_ERR("error: failed to reserve %u virtual pages at %p, error %d", + pages, ptr, ret); + return ret; + } + + vpage_ctx.free_pages -= pages; + + vpage_ctx.velems[vpage_ctx.num_elems_in_use].pages = pages; + vpage_ctx.velems[vpage_ctx.num_elems_in_use].vpage = vpage; + vpage_ctx.num_elems_in_use++; + + return 0; +} + +/** + * @brief Release reserved virtual memory pages + * + * @param ptr Pointer to the reserved memory pages to release. + * @retval 0 if successful. + * @retval -EINVAL if ptr is invalid. + */ +static int vpages_unreserve(void *ptr) +{ + unsigned int alloc_idx, elem_idx; + unsigned int pages = 0; + int ret; + + /* check for valid ptr which must be page aligned */ + CHECKIF(!IS_ALIGNED(ptr, CONFIG_MM_DRV_PAGE_SIZE)) { + LOG_ERR("error: invalid non aligned page pointer %p", ptr); + return -EINVAL; + } + + alloc_idx = (POINTER_TO_UINT(ptr) - POINTER_TO_UINT(vpage_ctx.virtual_region->addr)) / + CONFIG_MM_DRV_PAGE_SIZE; + + /* find the allocation element */ + for (elem_idx = 0; elem_idx < VPAGE_MAX_ALLOCS; elem_idx++) { + if (vpage_ctx.velems[elem_idx].pages > 0 && + vpage_ctx.velems[elem_idx].vpage == alloc_idx) { + pages = vpage_ctx.velems[elem_idx].pages; + break; + } + } + + /* check we found allocation element */ + CHECKIF(!pages) { + LOG_ERR("error: invalid page pointer %p not found", ptr); + return -EINVAL; + } + + /* free physical blocks */ + ret = sys_mem_blocks_free_contiguous(&vpage_ctx.vpage_blocks, ptr, pages); + if (ret < 0) { + LOG_ERR("error: failed to free %u continuous virtual page blocks at %p, error %d", + pages, ptr, ret); + return ret; + } + + /* move the last element over the released one, clear the last element */ + unsigned int last_idx = vpage_ctx.num_elems_in_use - 1; + + if (last_idx != elem_idx) + vpage_ctx.velems[elem_idx] = vpage_ctx.velems[last_idx]; + vpage_ctx.velems[last_idx].pages = 0; + vpage_ctx.velems[last_idx].vpage = 0; + vpage_ctx.num_elems_in_use--; + + /* success update the free pages */ + vpage_ctx.free_pages += pages; + + return 0; +} + +/** + * @brief Allocate and map virtual memory pages + * + * Allocates memory pages from the virtual page allocator. + * Maps physical memory pages to the virtual region as needed. + * + * @param pages Number of pages to allocate. + * @param ptr Pointer to store the address of allocated pages. + * @retval 0 if successful. + */ +static int vpages_alloc_and_map(unsigned int pages, void **ptr) +{ + int ret; + + ret = vpages_reserve(pages, ptr); + if (ret < 0 || !*ptr) + return ret; + + /* map the virtual blocks in virtual region to free physical blocks */ + ret = sys_mm_drv_map_region_safe(vpage_ctx.virtual_region, *ptr, + 0, pages * CONFIG_MM_DRV_PAGE_SIZE, SYS_MM_MEM_PERM_RW); + if (ret < 0) { + LOG_ERR("error: failed to map virtual region %p to physical region %p, error %d", + *ptr, vpage_ctx.virtual_region->addr, ret); + vpages_unreserve(*ptr); + *ptr = NULL; + return ret; + } + + return 0; +} + /** * @brief Allocate virtual memory pages * @@ -211,25 +352,7 @@ static int vpages_free_and_unmap(uintptr_t *ptr) return ret; } - /* free physical blocks */ - ret = sys_mem_blocks_free_contiguous(&vpage_ctx.vpage_blocks, ptr, pages); - if (ret < 0) { - LOG_ERR("error: failed to free %u continuous virtual page blocks at %p, error %d", - pages, ptr, ret); - return ret; - } - - /* move the last element over the released one, clear the last element */ - if (vpage_ctx.num_elems_in_use != elem_idx) - vpage_ctx.velems[elem_idx] = vpage_ctx.velems[vpage_ctx.num_elems_in_use]; - vpage_ctx.velems[vpage_ctx.num_elems_in_use].pages = 0; - vpage_ctx.velems[vpage_ctx.num_elems_in_use].vpage = 0; - vpage_ctx.num_elems_in_use--; - - /* success update the free pages */ - vpage_ctx.free_pages += pages; - - return ret; + return vpages_unreserve((void *)ptr); } /** @@ -251,6 +374,87 @@ void vpage_free(void *ptr) vpage_ctx.total_pages); } +/** + * @brief Reserve virtual memory pages + * + * Reserves virtual memory pages from the virtual page allocator, without + * mapping any physical memory to them. + * + * @param pages Number of pages (usually 4kB large) to reserve. + * @retval NULL on reservation failure. + */ +void *vpage_reserve(unsigned int pages) +{ + void *ptr = NULL; + int ret; + + k_mutex_lock(&vpage_ctx.lock, K_FOREVER); + ret = vpages_reserve(pages, &ptr); + k_mutex_unlock(&vpage_ctx.lock); + if (ret < 0) + LOG_ERR("vpage_reserve failed %d for %d pages, total %d free %d", + ret, pages, vpage_ctx.total_pages, vpage_ctx.free_pages); + else + LOG_INF("vpage_reserve ptr %p pages %u free %u/%u", ptr, pages, + vpage_ctx.free_pages, vpage_ctx.total_pages); + return ptr; +} + +/** + * @brief Reserve a specific range of virtual pages + * Reserves an already occupied range of virtual memory pages, for users that + * were given their address by someone other than this allocator. + * + * @param ptr Page aligned base address of the range to reserve. + * @param pages Number of pages (usually 4kB large) to reserve. + * @retval 0 if successful. + */ +int vpage_reserve_at(void *ptr, unsigned int pages) +{ + int ret; + + k_mutex_lock(&vpage_ctx.lock, K_FOREVER); + ret = vpages_reserve_at(ptr, pages); + k_mutex_unlock(&vpage_ctx.lock); + if (ret < 0) + LOG_ERR("vpage_reserve_at failed %d for %u pages at %p, total %d free %d", + ret, pages, ptr, vpage_ctx.total_pages, vpage_ctx.free_pages); + else + LOG_INF("vpage_reserve_at ptr %p pages %u free %u/%u", ptr, pages, + vpage_ctx.free_pages, vpage_ctx.total_pages); + return ret; +} + +/** + * @brief Release reserved virtual pages + * Releases virtual memory pages previously reserved with vpage_reserve(). + * + * @param ptr Pointer to the reserved memory pages to release. + */ +void vpage_release(void *ptr) +{ + int ret; + + k_mutex_lock(&vpage_ctx.lock, K_FOREVER); + ret = vpages_unreserve(ptr); + k_mutex_unlock(&vpage_ctx.lock); + + if (!ret) + LOG_INF("vptr %p release free/total pages %d/%d", ptr, vpage_ctx.free_pages, + vpage_ctx.total_pages); +} + +/** + * @brief Get the shared virtual page allocator's memory region + * + * @return Pointer to the virtual memory region backing the virtual page + * allocator. + */ +const struct sys_mm_drv_region *vpage_get_region(void) +{ + return vpage_ctx.virtual_region; +} + /** * @brief Initialize virtual page allocator * @@ -271,8 +475,7 @@ static int vpage_init(void) /* create the virtual memory region and add it to the system */ size_t remaining_ram = L2_SRAM_BASE + L2_SRAM_SIZE - (adsp_mm_get_unused_l2_start_aligned() + - CONFIG_SOF_ZEPHYR_VIRTUAL_HEAP_REGION_SIZE + - CONFIG_LIBRARY_REGION_SIZE); + CONFIG_SOF_ZEPHYR_VIRTUAL_HEAP_REGION_SIZE); ret = adsp_add_virtual_memory_region(adsp_mm_get_unused_l2_start_aligned() + CONFIG_SOF_ZEPHYR_VIRTUAL_HEAP_REGION_SIZE,