diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index 832d4eb12098..9b993ebf646f 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -2,6 +2,19 @@ cmake_minimum_required(VERSION 3.21.0) +# Board specific overlays (these must be before find_package(Zephyr) +# ------------------------------------------------------------------ + +# Intel PTL: use ll_userspace_overlay if building for Linux upstream +if(DEFINED EXTRA_CONF_FILE AND (BOARD MATCHES "/ptl" OR BOARD_QUALIFIERS MATCHES "/ptl")) + string(FIND "${EXTRA_CONF_FILE}" "os_linux_overlay" _linux_overlay_idx) + string(FIND "${EXTRA_CONF_FILE}" "ll_userspace_overlay" _ll_overlay_idx) + if(_linux_overlay_idx GREATER_EQUAL 0 AND _ll_overlay_idx LESS 0) + list(APPEND EXTRA_CONF_FILE + "${CMAKE_CURRENT_SOURCE_DIR}/overlays/ptl/ll_userspace_overlay.conf") + endif() +endif() + find_package(Zephyr HINTS $ENV{ZEPHYR_BASE}) project(sample_sof) diff --git a/app/overlays/ptl/ll_userspace_overlay.conf b/app/overlays/ptl/ll_userspace_overlay.conf index 28a6e4ea044b..7f7186a2dbf0 100644 --- a/app/overlays/ptl/ll_userspace_overlay.conf +++ b/app/overlays/ptl/ll_userspace_overlay.conf @@ -32,10 +32,12 @@ CONFIG_CROSS_CORE_STREAM=n CONFIG_INTEL_ADSP_MIC_PRIVACY=n CONFIG_XRUN_NOTIFICATIONS_ENABLE=n -# Extend the shared LL user-space heap to 512KiB. This -# is current maximum for PTL builds with virtual heap -# enabled and uses the vmh allocation intended for KBP. -# TODO: needs some better solution to allocate -CONFIG_SOF_ZEPHYR_SYS_USER_HEAP_SIZE=0x80000 - +# Allocate majority of heap to user-space but only +# via vregions. Keep 0x40000 for non-vregion allocs. +CONFIG_SOF_ZEPHYR_SYS_USER_HEAP_SIZE=0x40000 +CONFIG_SOF_ZEPHYR_HEAP_SIZE=0x60000 CONFIG_VIRTUAL_HEAP=n + +# sof-ptl-nocodec.tplg requires more than 64 partitions for DP +# (vregion is created for each pipeline) +CONFIG_MAX_DOMAIN_PARTITIONS=128 diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index 98234daa082c..37af16aa9e1a 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -77,11 +77,12 @@ static struct vregion *module_adapter_dp_heap_new(const struct comp_ipc_config * static struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv, const struct comp_ipc_config *config, - const struct module_ext_init_data *ext_init) + const struct module_ext_init_data *ext_init, + struct mod_alloc_ctx *ppl_alloc) { - struct k_heap *mod_heap; - struct vregion *mod_vreg; + struct k_heap *mod_heap = NULL; struct processing_module *mod; + struct mod_alloc_ctx *alloc; struct comp_dev *dev; /* * For DP shared modules the struct processing_module object must be @@ -92,49 +93,53 @@ struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv */ uint32_t flags = config->proc_domain == COMP_PROCESSING_DOMAIN_DP ? SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER; + bool use_ppl_alloc = ppl_alloc && + config->proc_domain == COMP_PROCESSING_DOMAIN_LL; - if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP && IS_ENABLED(CONFIG_SOF_VREGIONS) && - IS_ENABLED(CONFIG_USERSPACE) && !IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) { - mod_vreg = module_adapter_dp_heap_new(config, ext_init); - if (!mod_vreg) { - comp_cl_err(drv, "Failed to allocate DP module heap / vregion"); - return NULL; - } + if (use_ppl_alloc) { + /* LL modules share the pipeline's alloc context */ + alloc = ppl_alloc; + vregion_get(alloc->vreg); + } else { + struct vregion *mod_vreg = NULL; + + if (IS_ENABLED(CONFIG_SOF_VREGIONS) && IS_ENABLED(CONFIG_USERSPACE) && + !IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP) && + config->proc_domain == COMP_PROCESSING_DOMAIN_DP) { + mod_vreg = module_adapter_dp_heap_new(config, ext_init); + if (!mod_vreg) { + comp_cl_err(drv, "Failed to allocate DP module heap / vregion"); + return NULL; + } #ifdef CONFIG_SOF_USERSPACE_LL - mod_heap = sof_sys_user_heap_get(); -#else - mod_heap = NULL; + mod_heap = sof_sys_user_heap_get(); #endif - } else { + } else { #ifdef CONFIG_SOF_USERSPACE_LL - mod_heap = sof_sys_user_heap_get(); - comp_cl_dbg(drv, "using ll user heap for module"); + mod_heap = sof_sys_user_heap_get(); + comp_cl_dbg(drv, "using ll user heap for module"); #else - mod_heap = drv->user_heap; + mod_heap = drv->user_heap; #endif - mod_vreg = NULL; - } + } + alloc = sof_heap_alloc(mod_heap, flags, sizeof(*alloc), 0); + if (!alloc) { + vregion_put(mod_vreg); + return NULL; + } - if (!mod_vreg) - mod = sof_heap_alloc(mod_heap, flags, sizeof(*mod), 0); - else if (flags & SOF_MEM_FLAG_COHERENT) - mod = vregion_alloc_coherent(mod_vreg, sizeof(*mod)); - else - mod = vregion_alloc(mod_vreg, sizeof(*mod)); + memset(alloc, 0, sizeof(*alloc)); + alloc->heap = mod_heap; + alloc->vreg = mod_vreg; + } + mod = sof_ctx_alloc(alloc, flags, sizeof(*mod), 0); if (!mod) { comp_cl_err(drv, "failed to allocate memory for module"); goto emod; } - struct mod_alloc_ctx *alloc = sof_heap_alloc(mod_heap, flags, sizeof(*alloc), 0); - - if (!alloc) - goto ealloc; - memset(mod, 0, sizeof(*mod)); - alloc->heap = mod_heap; - alloc->vreg = mod_vreg; mod->priv.resources.alloc = alloc; mod_resource_init(mod); @@ -144,11 +149,7 @@ struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv * then it can be cached. Effectively it can be only cached in * single-core configurations. */ - if (mod_vreg) - dev = vregion_alloc_coherent(mod_vreg, sizeof(*dev)); - else - dev = sof_heap_alloc(mod_heap, SOF_MEM_FLAG_COHERENT, sizeof(*dev), 0); - + dev = sof_ctx_alloc(alloc, SOF_MEM_FLAG_COHERENT, sizeof(*dev), 0); if (!dev) { comp_cl_err(drv, "failed to allocate memory for comp_dev"); goto edev; @@ -163,14 +164,11 @@ struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv return mod; edev: - sof_heap_free(mod_heap, alloc); -ealloc: - if (mod_vreg) - vregion_free(mod_vreg, mod); - else - sof_heap_free(mod_heap, mod); + sof_ctx_free(alloc, mod); emod: - vregion_put(mod_vreg); + vregion_put(alloc->vreg); + if (!use_ppl_alloc) + sof_heap_free(mod_heap, alloc); return NULL; } @@ -178,26 +176,27 @@ struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv static void module_adapter_mem_free(struct processing_module *mod) { struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; - struct k_heap *mod_heap = alloc->heap; + bool ppl_alloc = mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_LL && + mod->dev->pipeline && mod->dev->pipeline->alloc == alloc; /* * In principle it shouldn't even be needed to free individual objects * on the module heap since we're freeing the heap itself too */ #if CONFIG_IPC_MAJOR_4 - sof_heap_free(mod_heap, mod->priv.cfg.input_pins); + sof_heap_free(alloc->heap, mod->priv.cfg.input_pins); #endif - if (alloc->vreg) { - struct vregion *mod_vreg = alloc->vreg; - - vregion_free(mod_vreg, mod->dev); - vregion_free(mod_vreg, mod); - if (!vregion_put(mod_vreg)) - sof_heap_free(alloc->heap, alloc); + sof_ctx_free(alloc, mod->dev); + sof_ctx_free(alloc, mod); + + if (ppl_alloc) { + /* alloc belongs to pipeline, just release vregion reference */ + vregion_put(alloc->vreg); + } else if (alloc->vreg) { + if (!vregion_put(alloc->vreg)) + rfree(alloc); } else { - sof_heap_free(mod_heap, mod->dev); - sof_heap_free(mod_heap, mod); - sof_heap_free(mod_heap, alloc); + rfree(alloc); } } @@ -248,8 +247,19 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, NULL; #endif - struct processing_module *mod = module_adapter_mem_alloc(drv, config, ext_init); + struct mod_alloc_ctx *ppl_alloc = NULL; +#if CONFIG_IPC_MAJOR_4 + struct ipc_comp_dev *ipc_pipe; + struct ipc *ipc = ipc_get(); + + /* resolve the pipeline pointer early to pass its alloc to mem_alloc */ + ipc_pipe = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, config->pipeline_id, + IPC_COMP_IGNORE_REMOTE); + if (ipc_pipe && ipc_pipe->pipeline) + ppl_alloc = ipc_pipe->pipeline->alloc; +#endif + struct processing_module *mod = module_adapter_mem_alloc(drv, config, ext_init, ppl_alloc); if (!mod) return NULL; @@ -273,6 +283,21 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, dst->ext_data = &ext_data; #endif +#if CONFIG_IPC_MAJOR_4 + /* + * Set the pipeline pointer if ipc_pipe is valid. Do this + * early so that we can use module_adapter_mem_free() in error + * handling. + */ + if (ipc_pipe) { + dev->pipeline = ipc_pipe->pipeline; + + /* LL modules have the same period as the pipeline */ + if (dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_LL) + dev->period = ipc_pipe->pipeline->period; + } +#endif + #if CONFIG_ZEPHYR_DP_SCHEDULER /* create a task for DP processing */ if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP) { @@ -306,22 +331,6 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, else goto err; -#if CONFIG_IPC_MAJOR_4 - struct ipc_comp_dev *ipc_pipe; - struct ipc *ipc = ipc_get(); - - /* set the pipeline pointer if ipc_pipe is valid */ - ipc_pipe = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, config->pipeline_id, - IPC_COMP_IGNORE_REMOTE); - if (ipc_pipe) { - dev->pipeline = ipc_pipe->pipeline; - - /* LL modules have the same period as the pipeline */ - if (dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_LL) - dev->period = ipc_pipe->pipeline->period; - } -#endif - /* Init processing module */ ret = module_init(mod); if (ret) { diff --git a/src/audio/pipeline/pipeline-graph.c b/src/audio/pipeline/pipeline-graph.c index dfefb490a09e..346ac789e24f 100644 --- a/src/audio/pipeline/pipeline-graph.c +++ b/src/audio/pipeline/pipeline-graph.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -174,6 +175,7 @@ void pipeline_posn_grant_access(struct k_thread *thread) struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_t priority, uint32_t comp_id, struct create_pipeline_params *pparams) { + struct mod_alloc_ctx *alloc; struct sof_ipc_stream_posn posn; struct pipeline *p; int ret; @@ -184,17 +186,36 @@ struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_ /* show heap status */ heap_trace_all(0); + alloc = sof_heap_alloc(heap, SOF_MEM_FLAG_USER, sizeof(*alloc), 0); + if (!alloc) { + pipe_cl_err("Failed to allocate pipeline alloc context"); + return NULL; + } + + memset(alloc, 0, sizeof(*alloc)); + alloc->heap = heap; + + /* Create vregion for pipeline and its modules if size info is available */ + if (IS_ENABLED(CONFIG_SOF_VREGIONS) && + pparams && pparams->mem_data && pparams->mem_data->heap_bytes) { + size_t buf_size = pparams->mem_data->heap_bytes; + uintptr_t vreg_start; + + alloc->vreg = vregion_create_map(&vreg_start, &buf_size); + if (!alloc->vreg) + pipe_cl_err("Failed to create pipeline vregion of %zu bytes, using heap", + pparams->mem_data->heap_bytes); + } + /* allocate new pipeline */ - p = sof_heap_alloc(heap, SOF_MEM_FLAG_USER, sizeof(*p), 0); + p = sof_ctx_zalloc(alloc, SOF_MEM_FLAG_USER, sizeof(*p), 0); if (!p) { pipe_cl_err("Out of Memory"); - return NULL; + goto free_alloc; } - memset(p, 0, sizeof(*p)); - /* init pipeline */ - p->heap = heap; + p->alloc = alloc; p->comp_id = comp_id; p->priority = priority; p->pipeline_id = pipeline_id; @@ -236,7 +257,10 @@ struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_ return p; free: - sof_heap_free(heap, p); + sof_ctx_free(alloc, p); +free_alloc: + vregion_put(alloc->vreg); + rfree(alloc); return NULL; } @@ -321,6 +345,8 @@ void pipeline_disconnect(struct comp_dev *comp, struct comp_buffer *buffer, int /* pipelines must be inactive */ int pipeline_free(struct pipeline *p) { + struct mod_alloc_ctx *alloc = p->alloc; + pipe_dbg(p, "entry"); /* @@ -336,7 +362,12 @@ int pipeline_free(struct pipeline *p) pipeline_posn_offset_put(p->posn_offset); /* now free the pipeline */ - sof_heap_free(p->heap, p); + sof_ctx_free(alloc, p); + + /* free alloc context and vregion */ + if (vregion_put(alloc->vreg)) + pipe_cl_warn("pipeline vregion still in use"); + sof_heap_free(alloc->heap, alloc); /* show heap status */ heap_trace_all(0); @@ -449,6 +480,10 @@ int pipeline_complete(struct pipeline *p, struct comp_dev *source, p->source_comp = source; p->sink_comp = sink; + + if (p->alloc && p->alloc->vreg) + vregion_set_interim(p->alloc->vreg); + p->status = COMP_STATE_READY; /* show heap status */ diff --git a/src/audio/pipeline/pipeline-schedule.c b/src/audio/pipeline/pipeline-schedule.c index 0f00dcf1701c..fc8daad6e378 100644 --- a/src/audio/pipeline/pipeline-schedule.c +++ b/src/audio/pipeline/pipeline-schedule.c @@ -349,7 +349,7 @@ static struct task *ipc4_pipeline_trigger_task_init(struct pipeline *p, uint32_t { struct task *task; - task = sof_heap_alloc(p->heap, SOF_MEM_FLAG_USER, sizeof(*task), 0); + task = sof_ctx_alloc(p->alloc, SOF_MEM_FLAG_USER, sizeof(*task), 0); if (!task) return NULL; @@ -358,7 +358,7 @@ static struct task *ipc4_pipeline_trigger_task_init(struct pipeline *p, uint32_t /* All trigger tasks use the highest priority, regardless of pipeline priority. */ if (schedule_task_init_ll(task, SOF_UUID(pipe_trigger_task_uuid), type, -1, ipc4_pipeline_trigger_task, p, p->core, 0) < 0) { - sof_heap_free(p->heap, task); + sof_ctx_free(p->alloc, task); return NULL; } @@ -370,8 +370,8 @@ static struct task *pipeline_task_init(struct pipeline *p, uint32_t type) { struct pipeline_task *task = NULL; - task = sof_heap_alloc(p->heap, SOF_MEM_FLAG_USER, - sizeof(*task), 0); + task = sof_ctx_alloc(p->alloc, SOF_MEM_FLAG_USER, + sizeof(*task), 0); if (!task) return NULL; @@ -385,7 +385,7 @@ static struct task *pipeline_task_init(struct pipeline *p, uint32_t type) ipc3_pipeline_task, #endif p, p->core, 0) < 0) { - sof_heap_free(p->heap, task); + sof_ctx_free(p->alloc, task); return NULL; } @@ -562,14 +562,14 @@ void pipeline_comp_ll_task_free(struct pipeline *p) delayed_trigger_owner[p->core] = NULL; if (p->trigger_task) - sof_heap_free(p->heap, p->trigger_task); + sof_ctx_free(p->alloc, p->trigger_task); #endif if (p->pipe_task) { #if !CONFIG_LIBRARY || UNIT_TEST schedule_task_free(p->pipe_task); #endif - sof_heap_free(p->heap, p->pipe_task); + sof_ctx_free(p->alloc, p->pipe_task); } } diff --git a/src/include/sof/audio/pipeline.h b/src/include/sof/audio/pipeline.h index 1b704ed83fe1..9e21da9bfad0 100644 --- a/src/include/sof/audio/pipeline.h +++ b/src/include/sof/audio/pipeline.h @@ -26,6 +26,7 @@ struct comp_dev; struct ipc; struct ipc_msg; struct k_heap; +struct mod_alloc_ctx; /* * Pipeline status to stop execution of current path, but to keep the @@ -53,7 +54,7 @@ struct k_heap; * Audio pipeline. */ struct pipeline { - struct k_heap *heap; /**< heap used for allocating this pipeline */ + struct mod_alloc_ctx *alloc; /**< alloc context used for allocating this pipeline */ uint32_t comp_id; /**< component id for pipeline */ uint32_t pipeline_id; /**< pipeline id */ uint32_t sched_id; /**< Scheduling component id */ diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index ec9e0dac96dd..e6a6decaccb5 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -371,7 +371,17 @@ void z_vrfy_ipc_msg_list_remove(struct ipc_msg *msg) break; } } - K_OOPS(K_SYSCALL_VERIFY(found)); + + /* + * ipc_msg_list_remove() is normally called from ipc_msg_free() to drop + * a message that may or may not still be queued. A message that has + * already been sent (or was never queued) has a self-linked, empty list + * node, so removing it via list_item_del() is a harmless no-op that only + * touches &msg->list, which was already validated above. Only reject a + * non-empty node that is not on ipc->msg_list, i.e. one whose list + * pointers would make list_item_del() corrupt unrelated memory. + */ + K_OOPS(K_SYSCALL_VERIFY(found || list_is_empty(&msg->list))); z_impl_ipc_msg_list_remove(msg); } #include diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 1d9c8954f4ac..bbb4596dc773 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -570,6 +570,7 @@ static int zephyr_ll_task_free(void *data, struct task *task) uint32_t flags; struct zephyr_ll_pdata *pdata = task->priv_data; bool must_wait, on_list = true; + int wait_ret = 0; zephyr_ll_assert_core(sch); @@ -617,10 +618,17 @@ static int zephyr_ll_task_free(void *data, struct task *task) if (must_wait) /* Wait for up to 100 periods */ - k_sem_take(pdata->sem_p, K_USEC(LL_TIMER_PERIOD_US * 100)); + wait_ret = k_sem_take(pdata->sem_p, K_USEC(LL_TIMER_PERIOD_US * 100)); /* Protect against racing with schedule_task() */ zephyr_ll_lock(sch, &flags); + + if (wait_ret && task->state != SOF_TASK_STATE_FREE) { + tr_warn(&ll_tr, "task %p still active on free (state %d, semret %d)", + task, task->state, wait_ret); + zephyr_ll_task_done(sch, task); + } + #if CONFIG_DYNAMIC_OBJECTS zephyr_ll_task_sem_free(task); #endif diff --git a/zephyr/test/userspace/test_ll_task.c b/zephyr/test/userspace/test_ll_task.c index 1e31e01538b7..f6a63b7194bd 100644 --- a/zephyr/test/userspace/test_ll_task.c +++ b/zephyr/test/userspace/test_ll_task.c @@ -113,7 +113,7 @@ static void pipeline_check(void) zassert_not_null(p, "pipeline creation failed"); /* Verify heap assignment */ - zassert_equal(p->heap, heap, "pipeline heap not equal to user heap"); + zassert_equal(p->alloc->heap, heap, "pipeline heap not equal to user heap"); /* Verify pipeline properties */ zassert_equal(p->pipeline_id, pipeline_id, "pipeline id mismatch");