diff --git a/app/boards/intel_adsp_ace30_ptl.conf b/app/boards/intel_adsp_ace30_ptl.conf index b6ac41938398..09275313ec61 100644 --- a/app/boards/intel_adsp_ace30_ptl.conf +++ b/app/boards/intel_adsp_ace30_ptl.conf @@ -21,9 +21,9 @@ CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL=n CONFIG_PROBE=y CONFIG_PROBE_DMA_MAX=2 CONFIG_SOF_TELEMETRY=y -CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=y -CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=y -CONFIG_COLD_STORE_EXECUTE_DRAM=y +CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n +CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=n +CONFIG_COLD_STORE_EXECUTE_DRAM=n # SOF / loadable modules CONFIG_INTEL_MODULES=y @@ -40,10 +40,10 @@ CONFIG_SOF_LOG_LEVEL_INF=y CONFIG_COUNTER=y CONFIG_HEAP_MEM_POOL_SIZE=8192 CONFIG_LLEXT=y -CONFIG_LLEXT_STORAGE_WRITABLE=y -CONFIG_LLEXT_EXPERIMENTAL=y +CONFIG_LLEXT_STORAGE_WRITABLE=n +CONFIG_LLEXT_EXPERIMENTAL=n CONFIG_LLEXT_EDK=n -CONFIG_MODULES=y +CONFIG_MODULES=n # Zephyr / device drivers CONFIG_DAI_INIT_PRIORITY=70 @@ -78,3 +78,24 @@ CONFIG_SOF_USERSPACE_PROXY=y CONFIG_MAX_THREAD_BYTES=3 CONFIG_MAX_DOMAIN_PARTITIONS=32 + +# Userspace LL (was app/overlays/ptl/ll_userspace_overlay.conf) +# Run Low-Latency audio pipelines in user-space threads by default. +CONFIG_SOF_USERSPACE_LL=y +CONFIG_SOF_USERSPACE_INTERFACE_DMA=y +CONFIG_DAI_USERSPACE=y + +# Settings currently required to enable user-space LL. The cold-store, +# telemetry, loadable-module and misc feature disables above/here are not +# yet user-space compatible (see the former overlay for rationale). +CONFIG_COLD_STORE_EXECUTE_DEBUG=n +CONFIG_SOF_BOOT_TEST_ALLOWED=n +CONFIG_CROSS_CORE_STREAM=n +CONFIG_INTEL_ADSP_MIC_PRIVACY=n +CONFIG_XRUN_NOTIFICATIONS_ENABLE=n +CONFIG_ZEPHYR_DP_SCHEDULER=n +# Extend the shared LL user-space heap to 0.5MB. This +# is current maximum for PTL builds with virtual heap +# enabled and uses the vmh allocation intended for KBP. +# TODO: needs some better solution to allocate +CONFIG_SOF_ZEPHYR_SYS_USER_HEAP_SIZE=0x80000 diff --git a/app/boards/intel_adsp_ace30_wcl.conf b/app/boards/intel_adsp_ace30_wcl.conf index 2196af333e65..fa845731a0c5 100644 --- a/app/boards/intel_adsp_ace30_wcl.conf +++ b/app/boards/intel_adsp_ace30_wcl.conf @@ -21,9 +21,9 @@ CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL=n CONFIG_PROBE=y CONFIG_PROBE_DMA_MAX=2 CONFIG_SOF_TELEMETRY=y -CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=y -CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=y -CONFIG_COLD_STORE_EXECUTE_DRAM=y +CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n +CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=n +CONFIG_COLD_STORE_EXECUTE_DRAM=n # SOF / loadable modules CONFIG_INTEL_MODULES=y @@ -39,10 +39,10 @@ CONFIG_SOF_LOG_LEVEL_INF=y # Zephyr / OS features CONFIG_HEAP_MEM_POOL_SIZE=8192 CONFIG_LLEXT=y -CONFIG_LLEXT_STORAGE_WRITABLE=y -CONFIG_LLEXT_EXPERIMENTAL=y +CONFIG_LLEXT_STORAGE_WRITABLE=n +CONFIG_LLEXT_EXPERIMENTAL=n CONFIG_LLEXT_EDK=n -CONFIG_MODULES=y +CONFIG_MODULES=n # Zephyr / device drivers CONFIG_DAI_INIT_PRIORITY=70 @@ -64,3 +64,37 @@ CONFIG_PM_DEVICE_RUNTIME_ASYNC=n CONFIG_LOG_BACKEND_ADSP=n CONFIG_LOG_FLUSH_SLEEP_US=5000 CONFIG_WINSTREAM_CONSOLE=n + +# Userspace base (mirrored from intel_adsp_ace30_ptl.conf) +# Required so that user-space LL (below) can actually be enabled, since +# CONFIG_SOF_USERSPACE_LL depends on CONFIG_USERSPACE. +CONFIG_USERSPACE=y +CONFIG_DYNAMIC_THREAD=y +CONFIG_DYNAMIC_THREAD_ALLOC=y +CONFIG_DYNAMIC_THREAD_PREFER_ALLOC=y +CONFIG_SOF_STACK_SIZE=8192 +CONFIG_SOF_USERSPACE_PROXY=y +CONFIG_MAX_THREAD_BYTES=3 +CONFIG_MAX_DOMAIN_PARTITIONS=32 +CONFIG_XTENSA_MMU_NUM_L2_TABLES=128 + +# Userspace LL (was app/overlays/ptl/ll_userspace_overlay.conf) +# Run Low-Latency audio pipelines in user-space threads by default. +CONFIG_SOF_USERSPACE_LL=y +CONFIG_SOF_USERSPACE_INTERFACE_DMA=y +CONFIG_DAI_USERSPACE=y + +# Settings currently required to enable user-space LL. The cold-store, +# telemetry, loadable-module and misc feature disables above/here are not +# yet user-space compatible (see the former overlay for rationale). +CONFIG_COLD_STORE_EXECUTE_DEBUG=n +CONFIG_SOF_BOOT_TEST_ALLOWED=n +CONFIG_CROSS_CORE_STREAM=n +CONFIG_INTEL_ADSP_MIC_PRIVACY=n +CONFIG_XRUN_NOTIFICATIONS_ENABLE=n +CONFIG_ZEPHYR_DP_SCHEDULER=n +# Extend the shared LL user-space heap to 0.5MB. This +# is current maximum for WCL builds with virtual heap +# enabled and uses the vmh allocation intended for KBP. +# TODO: needs some better solution to allocate +CONFIG_SOF_ZEPHYR_SYS_USER_HEAP_SIZE=0x80000 diff --git a/app/overlays/ptl/ll_userspace_overlay.conf b/app/overlays/ptl/ll_userspace_overlay.conf index 53b881320e6f..e66546b06159 100644 --- a/app/overlays/ptl/ll_userspace_overlay.conf +++ b/app/overlays/ptl/ll_userspace_overlay.conf @@ -37,3 +37,9 @@ CONFIG_CROSS_CORE_STREAM=n CONFIG_INTEL_ADSP_MIC_PRIVACY=n CONFIG_XRUN_NOTIFICATIONS_ENABLE=n CONFIG_ZEPHYR_DP_SCHEDULER=n + +# Extend the shared LL user-space heap to 0.5MB. This +# is current maximum for PTL builds with virtual heap +# enabled and uses the vmh allocation intended for KBP. +# TODO: needs some better solution to allocate +CONFIG_SOF_ZEPHYR_SYS_USER_HEAP_SIZE=0x80000 diff --git a/src/audio/buffers/comp_buffer.c b/src/audio/buffers/comp_buffer.c index 8a3d44133d4b..f64b74add169 100644 --- a/src/audio/buffers/comp_buffer.c +++ b/src/audio/buffers/comp_buffer.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -165,8 +166,7 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer) if (alloc && alloc->vreg) { vregion_free(alloc->vreg, buffer); - if (!vregion_put(alloc->vreg)) - rfree(alloc); + module_adapter_vreg_free(alloc); } else { sof_heap_free(alloc ? alloc->heap : NULL, buffer); } diff --git a/src/audio/google/google_hotword_detect.c b/src/audio/google/google_hotword_detect.c index d67a97a5823f..80b509acfb35 100644 --- a/src/audio/google/google_hotword_detect.c +++ b/src/audio/google/google_hotword_detect.c @@ -111,7 +111,7 @@ static struct comp_dev *ghd_create(const struct comp_driver *drv, cd->event.event_type = SOF_CTRL_EVENT_KD; cd->event.num_elems = 0; - cd->msg = ipc_msg_init(cd->event.rhdr.hdr.cmd, cd->event.rhdr.hdr.size); + cd->msg = ipc_msg_init(NULL, cd->event.rhdr.hdr.cmd, cd->event.rhdr.hdr.size); if (!cd->msg) { comp_err(dev, "ipc_msg_init failed"); goto cd_fail; diff --git a/src/audio/host-legacy.c b/src/audio/host-legacy.c index d17da437ffb4..591b5438f256 100644 --- a/src/audio/host-legacy.c +++ b/src/audio/host-legacy.c @@ -559,7 +559,7 @@ int host_common_new(struct host_data *hd, struct comp_dev *dev, ipc_build_stream_posn(&hd->posn, SOF_IPC_STREAM_POSITION, config_id); #if CONFIG_HOST_DMA_IPC_POSITION_UPDATES - hd->msg = ipc_msg_init(hd->posn.rhdr.hdr.cmd, hd->posn.rhdr.hdr.size); + hd->msg = ipc_msg_init(NULL, hd->posn.rhdr.hdr.cmd, hd->posn.rhdr.hdr.size); if (!hd->msg) { comp_err(dev, "ipc_msg_init failed"); dma_put(hd->dma); diff --git a/src/audio/host-zephyr.c b/src/audio/host-zephyr.c index 5f23ff405857..cf7eb1cc11f2 100644 --- a/src/audio/host-zephyr.c +++ b/src/audio/host-zephyr.c @@ -727,7 +727,7 @@ __cold int host_common_new(struct host_data *hd, struct comp_dev *dev, ipc_build_stream_posn(&hd->posn, SOF_IPC_STREAM_POSITION, config_id); #if CONFIG_HOST_DMA_IPC_POSITION_UPDATES - hd->msg = ipc_msg_init(hd->posn.rhdr.hdr.cmd, sizeof(hd->posn)); + hd->msg = ipc_msg_init(hd->heap, hd->posn.rhdr.hdr.cmd, sizeof(hd->posn)); if (!hd->msg) { comp_err(dev, "ipc_msg_init failed"); sof_dma_put(hd->dma); diff --git a/src/audio/mfcc/mfcc.c b/src/audio/mfcc/mfcc.c index 724d3d7faf06..88d5d4863d31 100644 --- a/src/audio/mfcc/mfcc.c +++ b/src/audio/mfcc/mfcc.c @@ -129,7 +129,7 @@ static int mfcc_free(struct processing_module *mod) struct mfcc_comp_data *cd = module_get_private_data(mod); comp_info(mod->dev, "entry"); - ipc_msg_free(cd->msg); + mod_ipc_msg_free(mod, cd->msg); cd->msg = NULL; mod_data_blob_handler_free(mod, cd->model_handler); mfcc_free_buffers(mod); diff --git a/src/audio/mfcc/mfcc_ipc4.c b/src/audio/mfcc/mfcc_ipc4.c index bb20d85e413b..21850be8a5d6 100644 --- a/src/audio/mfcc/mfcc_ipc4.c +++ b/src/audio/mfcc/mfcc_ipc4.c @@ -49,10 +49,10 @@ int mfcc_ipc_notification_init(struct processing_module *mod) primary->r.type = SOF_IPC4_GLB_NOTIFICATION; primary->r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REQUEST; primary->r.msg_tgt = SOF_IPC4_MESSAGE_TARGET_FW_GEN_MSG; - cd->msg = ipc_msg_w_ext_init(msg_proto.header, msg_proto.extension, - sizeof(struct sof_ipc4_notify_module_data) + - sizeof(struct sof_ipc4_control_msg_payload) + - sizeof(struct sof_ipc4_ctrl_value_chan)); + cd->msg = mod_ipc_msg_w_ext_init(mod, msg_proto.header, msg_proto.extension, + sizeof(struct sof_ipc4_notify_module_data) + + sizeof(struct sof_ipc4_control_msg_payload) + + sizeof(struct sof_ipc4_ctrl_value_chan)); if (!cd->msg) { comp_err(dev, "Failed to initialize VAD notification"); return -ENOMEM; diff --git a/src/audio/module_adapter/module/cadence_ipc4.c b/src/audio/module_adapter/module/cadence_ipc4.c index 763272964d61..bd002ea19d9f 100644 --- a/src/audio/module_adapter/module/cadence_ipc4.c +++ b/src/audio/module_adapter/module/cadence_ipc4.c @@ -233,7 +233,7 @@ static struct ipc_msg *cadence_codec_notification_init(struct processing_module primary.r.type = SOF_IPC4_GLB_NOTIFICATION; primary.r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REQUEST; primary.r.msg_tgt = SOF_IPC4_MESSAGE_TARGET_FW_GEN_MSG; - msg = ipc_msg_w_ext_init(primary.dat, 0, sizeof(*msg_module_data)); + msg = mod_ipc_msg_w_ext_init(mod, primary.dat, 0, sizeof(*msg_module_data)); if (!msg) return NULL; @@ -366,7 +366,7 @@ static int cadence_codec_init(struct processing_module *mod) if (setup_cfg) mod_free(mod, setup_cfg->data); free_notification: - ipc_msg_free(cd->msg); + mod_ipc_msg_free(mod, cd->msg); free_cd: mod_free(mod, cd); @@ -568,7 +568,7 @@ static int ipc4_cadence_codec_free(struct processing_module *mod) { struct cadence_codec_data *cd = module_get_private_data(mod); - ipc_msg_free(cd->msg); + mod_ipc_msg_free(mod, cd->msg); return cadence_codec_free(mod); } diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index 995501bfbe0a..db1148113f16 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -51,14 +51,83 @@ struct comp_dev *module_adapter_new(const struct comp_driver *drv, return module_adapter_new_ext(drv, config, spec, NULL, NULL, NULL); } -static struct vregion *module_adapter_dp_heap_new(const struct comp_ipc_config *config, - size_t *heap_size) +struct vregion *z_impl_module_adapter_vreg_new(uintptr_t *vreg_start, size_t *vreg_size) { /* src-lite with 8 channels has been seen allocating 14k in one go */ /* FIXME: the size will be derived from configuration */ const size_t buf_size = 28 * 1024; + struct vregion *vr = vregion_create(buf_size); - return vregion_create(buf_size); + if (!vr) + return NULL; + +#ifdef CONFIG_SOF_USERSPACE_LL + vregion_mem_info(vr, vreg_size, vreg_start); + + /* + * In the userspace LL case allocations are also performed by the + * userspace IPC thread, which is also the one, executing this syscall + */ + struct k_mem_partition cached_part = { + .start = *vreg_start, + .size = *vreg_size, + .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, + }; + int ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cached_part); + + if (ret < 0) { + vregion_put(vr); + return NULL; + } + + struct k_mem_partition uncached_part = { + .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)cached_part.start), + .size = cached_part.size, + .attr = K_MEM_PARTITION_P_RW_U_RW, + }; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &uncached_part); + if (ret < 0) { + k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &cached_part); + vregion_put(vr); + return NULL; + } +#else + (void)vreg_start; + (void)vreg_size; +#endif + + return vr; +} + +void z_impl_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc) +{ +#ifdef CONFIG_SOF_USERSPACE_LL + struct k_mem_partition part = { + .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, + }; + + vregion_mem_info(alloc->vreg, &part.size, &part.start); + + k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &part); + + part.start = (uintptr_t)sys_cache_uncached_ptr_get((void *)part.start); + part.attr = K_MEM_PARTITION_P_RW_U_RW; + + k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &part); +#else + (void)alloc; +#endif +} + +void module_adapter_vreg_free(struct mod_alloc_ctx *alloc) +{ + if (vregion_put(alloc->vreg)) + return; + + module_adapter_vreg_unmap(alloc); + + sof_heap_free(alloc->heap, alloc); } static struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv, @@ -77,11 +146,12 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv */ uint32_t flags = config->proc_domain == COMP_PROCESSING_DOMAIN_DP ? SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER; - size_t heap_size; + size_t vreg_size; + uintptr_t vreg_start; if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP && IS_ENABLED(CONFIG_SOF_VREGIONS) && IS_ENABLED(CONFIG_USERSPACE) && !IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) { - mod_vreg = module_adapter_dp_heap_new(config, &heap_size); + mod_vreg = module_adapter_vreg_new(&vreg_start, &vreg_size); if (!mod_vreg) { comp_cl_err(drv, "Failed to allocate DP module heap / vregion"); return NULL; @@ -98,7 +168,8 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv #else mod_heap = drv->user_heap; #endif - heap_size = 0; + vreg_size = 0; + vreg_start = 0; mod_vreg = NULL; } @@ -162,6 +233,34 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv return NULL; } +#ifdef CONFIG_USERSPACE +#include +struct vregion *z_vrfy_module_adapter_vreg_new(uintptr_t *vreg_start, size_t *vreg_size) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(vreg_start, sizeof(*vreg_start))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(vreg_size, sizeof(*vreg_size))); + return z_impl_module_adapter_vreg_new(vreg_start, vreg_size); +} +#include +void z_vrfy_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc) +{ + /* + * Protect against rogue module attempting to unmap another module's + * vregion. It could pass an "alloc" pointer, belonging to another + * module (assuming it somehow could get it), but then it wouldn't have + * access rights to it. And with the "alloc" pointer to which it has + * access rights, it can only unmap a vregion, of which the pointer is + * the owner. + */ + K_OOPS(K_SYSCALL_MEMORY_READ(alloc, sizeof(*alloc))); + if (!vregion_verify(alloc->vreg)) + return; + K_OOPS(vregion_owner_get(alloc->vreg) != alloc); + z_impl_module_adapter_vreg_unmap(alloc); +} +#include +#endif + static void module_adapter_mem_free(struct processing_module *mod) { struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; @@ -179,8 +278,7 @@ static void module_adapter_mem_free(struct processing_module *mod) vregion_free(mod_vreg, mod->dev); vregion_free(mod_vreg, mod); - if (!vregion_put(mod_vreg)) - sof_heap_free(alloc->heap, alloc); + module_adapter_vreg_free(alloc); } else { sof_heap_free(mod_heap, mod->dev); sof_heap_free(mod_heap, mod); @@ -199,6 +297,7 @@ static void module_adapter_mem_free(struct processing_module *mod) * * Note: Use the ext version if you need to set the module's private data before calling * the create method. + * Note 2: ATM runs in privileged / kernel mode for DP modules */ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, const struct comp_ipc_config *config, @@ -244,12 +343,15 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, #if CONFIG_ZEPHYR_DP_SCHEDULER /* create a task for DP processing */ if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP) { + struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; + /* All data allocated, create a thread */ ret = pipeline_comp_dp_task_init(dev); if (ret) { comp_cl_err(drv, "DP task creation failed with error %d.", ret); goto err; } + vregion_owner_set(alloc->vreg, alloc); } #endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ diff --git a/src/audio/pipeline/pipeline-graph.c b/src/audio/pipeline/pipeline-graph.c index 670df08e9383..e6f56eaf7096 100644 --- a/src/audio/pipeline/pipeline-graph.c +++ b/src/audio/pipeline/pipeline-graph.c @@ -227,7 +227,7 @@ struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_ ipc_build_stream_posn(&posn, SOF_IPC_STREAM_TRIG_XRUN, p->comp_id); if (posn.rhdr.hdr.size) { - p->msg = ipc_msg_init(posn.rhdr.hdr.cmd, posn.rhdr.hdr.size); + p->msg = ipc_msg_init(heap, posn.rhdr.hdr.cmd, posn.rhdr.hdr.size); if (!p->msg) { pipe_err(p, "ipc_msg_init failed"); goto free; diff --git a/src/audio/sound_dose/sound_dose-ipc4.c b/src/audio/sound_dose/sound_dose-ipc4.c index fd3a8151e984..3c3f537dc338 100644 --- a/src/audio/sound_dose/sound_dose-ipc4.c +++ b/src/audio/sound_dose/sound_dose-ipc4.c @@ -32,9 +32,9 @@ static struct ipc_msg *sound_dose_notification_init(struct processing_module *mo primary->r.type = SOF_IPC4_GLB_NOTIFICATION; primary->r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REQUEST; primary->r.msg_tgt = SOF_IPC4_MESSAGE_TARGET_FW_GEN_MSG; - msg = ipc_msg_w_ext_init(msg_proto.header, msg_proto.extension, - sizeof(struct sof_ipc4_notify_module_data) + - sizeof(struct sof_ipc4_control_msg_payload)); + msg = mod_ipc_msg_w_ext_init(mod, msg_proto.header, msg_proto.extension, + sizeof(struct sof_ipc4_notify_module_data) + + sizeof(struct sof_ipc4_control_msg_payload)); if (!msg) return NULL; diff --git a/src/audio/sound_dose/sound_dose.c b/src/audio/sound_dose/sound_dose.c index c28b55baee4d..e0767981b943 100644 --- a/src/audio/sound_dose/sound_dose.c +++ b/src/audio/sound_dose/sound_dose.c @@ -327,7 +327,7 @@ __cold static int sound_dose_free(struct processing_module *mod) comp_dbg(mod->dev, "entry"); sound_dose_filters_free(cd); - ipc_msg_free(cd->msg); + mod_ipc_msg_free(mod, cd->msg); rfree(cd->abi); rfree(cd); return 0; diff --git a/src/audio/tdfb/tdfb.c b/src/audio/tdfb/tdfb.c index bf3d1dd8ff5e..eed22f395632 100644 --- a/src/audio/tdfb/tdfb.c +++ b/src/audio/tdfb/tdfb.c @@ -676,7 +676,7 @@ static int tdfb_init(struct processing_module *mod) err: /* These are null if not used for IPC version */ mod_free(mod, cd->ctrl_data); - ipc_msg_free(cd->msg); + mod_ipc_msg_free(mod, cd->msg); mod_data_blob_handler_free(mod, cd->model_handler); err_free_cd: @@ -691,7 +691,7 @@ static int tdfb_free(struct processing_module *mod) comp_dbg(mod->dev, "entry"); - ipc_msg_free(cd->msg); + mod_ipc_msg_free(mod, cd->msg); tdfb_free_delaylines(mod); mod_data_blob_handler_free(mod, cd->model_handler); tdfb_direction_free(mod); diff --git a/src/audio/tdfb/tdfb_direction.c b/src/audio/tdfb/tdfb_direction.c index bd435fa2ebd3..b81bb2e77477 100644 --- a/src/audio/tdfb/tdfb_direction.c +++ b/src/audio/tdfb/tdfb_direction.c @@ -53,7 +53,7 @@ * from output files tdfb_iir_emphasis_48k.h and tdfb_iir_emphasis_16k.h. */ -uint32_t iir_emphasis_48k[20] = { +static const uint32_t iir_emphasis_48k[20] = { 0x00000002, 0x00000002, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xc8cf47b5, 0x7689916a, 0x1dc95968, 0xc46d4d30, 0x1dc95968, 0x00000000, @@ -61,7 +61,7 @@ uint32_t iir_emphasis_48k[20] = { 0x032cc4ce, 0x01966267, 0xfffffffe, 0x00004222 }; -uint32_t iir_emphasis_16k[20] = { +static const uint32_t iir_emphasis_16k[20] = { 0x00000002, 0x00000002, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xd6f418ae, 0x63e7b85c, 0x19ae069b, 0xcca3f2ca, 0x19ae069b, 0x00000000, diff --git a/src/audio/tdfb/tdfb_ipc3.c b/src/audio/tdfb/tdfb_ipc3.c index 1ebdb9641ccf..f7b655065674 100644 --- a/src/audio/tdfb/tdfb_ipc3.c +++ b/src/audio/tdfb/tdfb_ipc3.c @@ -36,7 +36,7 @@ static int init_get_ctl_ipc(struct processing_module *mod) cd->ctrl_data->rhdr.hdr.cmd = SOF_IPC_GLB_COMP_MSG | SOF_IPC_COMP_GET_VALUE | comp_id; cd->ctrl_data->rhdr.hdr.size = TDFB_GET_CTRL_DATA_SIZE; - cd->msg = ipc_msg_init(cd->ctrl_data->rhdr.hdr.cmd, cd->ctrl_data->rhdr.hdr.size); + cd->msg = mod_ipc_msg_init(mod, cd->ctrl_data->rhdr.hdr.cmd, cd->ctrl_data->rhdr.hdr.size); cd->ctrl_data->comp_id = comp_id; cd->ctrl_data->type = SOF_CTRL_TYPE_VALUE_CHAN_GET; diff --git a/src/audio/tdfb/tdfb_ipc4.c b/src/audio/tdfb/tdfb_ipc4.c index 2f7e7e865214..a7b4e34977c7 100644 --- a/src/audio/tdfb/tdfb_ipc4.c +++ b/src/audio/tdfb/tdfb_ipc4.c @@ -48,8 +48,8 @@ static struct ipc_msg *tdfb_notification_init(struct processing_module *mod, primary->r.type = SOF_IPC4_GLB_NOTIFICATION; primary->r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REQUEST; primary->r.msg_tgt = SOF_IPC4_MESSAGE_TARGET_FW_GEN_MSG; - msg = ipc_msg_w_ext_init(msg_proto.header, msg_proto.extension, - sizeof(struct sof_ipc4_notify_module_data) + + msg = mod_ipc_msg_w_ext_init(mod, msg_proto.header, msg_proto.extension, + sizeof(struct sof_ipc4_notify_module_data) + sizeof(struct sof_ipc4_control_msg_payload) + sizeof(struct sof_ipc4_ctrl_value_chan)); if (!msg) diff --git a/src/include/ipc4/notification.h b/src/include/ipc4/notification.h index 614a926d16ad..bad08a39082c 100644 --- a/src/include/ipc4/notification.h +++ b/src/include/ipc4/notification.h @@ -297,4 +297,21 @@ void send_mixer_underrun_notif_msg(uint32_t resource_id, uint32_t eos_flag, uint uint32_t expected_data_mixed); void ipc4_update_notification_mask(uint32_t ntfy_mask, uint32_t enabled_mask); +#ifdef CONFIG_SOF_USERSPACE_LL + +__syscall bool send_resource_notif(uint32_t resource_id, uint32_t event_type, + uint32_t resource_type, void *data, uint32_t data_size); + +bool z_impl_send_resource_notif(uint32_t resource_id, uint32_t event_type, + uint32_t resource_type, void *data, uint32_t data_size); + +#include + +#else + +bool send_resource_notif(uint32_t resource_id, uint32_t event_type, + uint32_t resource_type, void *data, uint32_t data_size); + +#endif /* CONFIG_SOF_USERSPACE_LL */ + #endif /* __IPC4_NOTIFICATION_H__ */ diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index 625b3a180a60..c496f9369fcb 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -19,6 +19,7 @@ #include #include #include +#include #include "module_interface.h" #include @@ -192,16 +193,23 @@ void *z_impl_mod_balloc_align(struct processing_module *mod, size_t size, size_t #endif void mod_resource_init(struct processing_module *mod); void mod_heap_info(struct processing_module *mod, size_t *size, uintptr_t *start); +void module_adapter_vreg_free(struct mod_alloc_ctx *alloc); #if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) +__syscall struct vregion *module_adapter_vreg_new(uintptr_t *vreg_start, size_t *vreg_size); +__syscall void module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc); __syscall void *mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size, size_t alignment); __syscall int mod_free(struct processing_module *mod, const void *ptr); __syscall void mod_free_all(struct processing_module *mod); #else +struct vregion *z_impl_module_adapter_vreg_new(uintptr_t *vreg_start, size_t *vreg_size); +void z_impl_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc); void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size, size_t alignment); int z_impl_mod_free(struct processing_module *mod, const void *ptr); void z_impl_mod_free_all(struct processing_module *mod); +#define module_adapter_vreg_new z_impl_module_adapter_vreg_new +#define module_adapter_vreg_unmap z_impl_module_adapter_vreg_unmap #define mod_alloc_ext z_impl_mod_alloc_ext #define mod_free z_impl_mod_free #define mod_free_all z_impl_mod_free_all @@ -241,6 +249,65 @@ static inline void *mod_zalloc(struct processing_module *mod, size_t size) return ret; } +/** + * \brief Initialize a new IPC message using the module allocator. + * @param mod Module to allocate from + * @param header Message header metadata + * @param extension Message header extension metadata + * @param size Message data size in bytes. + * @return New IPC message. + */ +static inline struct ipc_msg *mod_ipc_msg_w_ext_init(struct processing_module *mod, + uint32_t header, + uint32_t extension, + uint32_t size) +{ + struct ipc_msg *msg; + + msg = mod_zalloc(mod, sizeof(*msg)); + if (!msg) + return NULL; + + if (size) { + msg->tx_data = mod_zalloc(mod, size); + if (!msg->tx_data) { + mod_free(mod, msg); + return NULL; + } + } + + msg->header = header; + msg->extension = extension; + msg->tx_size = size; + list_init(&msg->list); + + return msg; +} + +static inline struct ipc_msg *mod_ipc_msg_init(struct processing_module *mod, + uint32_t header, uint32_t size) +{ + return mod_ipc_msg_w_ext_init(mod, header, 0, size); +} + +/** + * \brief Free an IPC message allocated with mod_ipc_msg_init() or + * mod_ipc_msg_w_ext_init(). + * @param mod Module that owns the message. + * @param msg Message to free (may be NULL). + */ +static inline void mod_ipc_msg_free(struct processing_module *mod, + struct ipc_msg *msg) +{ + if (!msg) + return; + + ipc_msg_list_remove(msg); + + mod_free(mod, msg->tx_data); + mod_free(mod, msg); +} + #if CONFIG_COMP_BLOB #if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) __syscall struct comp_data_blob_handler *mod_data_blob_handler_new(struct processing_module *mod); diff --git a/src/include/sof/ipc/ipc_msg_list_remove.h b/src/include/sof/ipc/ipc_msg_list_remove.h new file mode 100644 index 000000000000..0d22c6f0704a --- /dev/null +++ b/src/include/sof/ipc/ipc_msg_list_remove.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. All rights reserved. + */ + +#ifndef __SOF_IPC_IPC_MSG_LIST_REMOVE_H__ +#define __SOF_IPC_IPC_MSG_LIST_REMOVE_H__ + +struct ipc_msg; + +/** + * \brief Remove an IPC message from the send queue. + * + * Acquires the IPC lock and removes the message from its list. + * Safe to call from userspace. + * + * @param msg The IPC message to remove from the queue. + */ +#if defined(__ZEPHYR__) && defined(CONFIG_SOF_USERSPACE_LL) +__syscall void ipc_msg_list_remove(struct ipc_msg *msg); +#else +void z_impl_ipc_msg_list_remove(struct ipc_msg *msg); +#define ipc_msg_list_remove z_impl_ipc_msg_list_remove +#endif + +#if defined(__ZEPHYR__) && defined(CONFIG_SOF_USERSPACE_LL) +#include +#endif + +#endif /* __SOF_IPC_IPC_MSG_LIST_REMOVE_H__ */ diff --git a/src/include/sof/ipc/ipc_msg_send.h b/src/include/sof/ipc/ipc_msg_send.h new file mode 100644 index 000000000000..c5b9b4a4448a --- /dev/null +++ b/src/include/sof/ipc/ipc_msg_send.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. All rights reserved. + */ + +#ifndef __SOF_IPC_IPC_MSG_SEND_H__ +#define __SOF_IPC_IPC_MSG_SEND_H__ + +#include + +struct ipc_msg; + +/** + * \brief Queues an IPC message for transmission. + * @param msg The IPC message. + * @param data The message data. + * @param high_priority True if a high priority message. + */ +#if defined(__ZEPHYR__) && defined(CONFIG_SOF_USERSPACE_LL) +__syscall void ipc_msg_send(struct ipc_msg *msg, void *data, + bool high_priority); +#else +void z_impl_ipc_msg_send(struct ipc_msg *msg, void *data, + bool high_priority); +#define ipc_msg_send z_impl_ipc_msg_send +#endif + +#if defined(__ZEPHYR__) && defined(CONFIG_SOF_USERSPACE_LL) +#include +#endif + +#endif /* __SOF_IPC_IPC_MSG_SEND_H__ */ diff --git a/src/include/sof/ipc/msg.h b/src/include/sof/ipc/msg.h index 160a9be9ec7c..25f13592a23c 100644 --- a/src/include/sof/ipc/msg.h +++ b/src/include/sof/ipc/msg.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,7 @@ struct dai_config; struct dma; struct dma_sg_elem_array; +struct k_heap; struct ipc_msg { uint32_t header; /* specific to platform */ @@ -40,46 +42,27 @@ struct ipc_msg { }; /** - * \brief Initialize a new IPC message. + * \brief Initialize a new IPC message using a specific heap. + * @param heap Heap to allocate from (NULL = default kernel heap) * @param header Message header metadata * @param extension Message header extension metadata * @param size Message data size in bytes. * @return New IPC message. */ -static inline struct ipc_msg *ipc_msg_w_ext_init(uint32_t header, uint32_t extension, - uint32_t size) -{ - struct ipc_msg *msg; - - msg = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*msg)); - if (!msg) - return NULL; - - if (size) { - msg->tx_data = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, size); - if (!msg->tx_data) { - rfree(msg); - return NULL; - } - } - - msg->header = header; - msg->extension = extension; - msg->tx_size = size; - list_init(&msg->list); - - return msg; -} +struct ipc_msg *ipc_msg_w_ext_init(struct k_heap *heap, uint32_t header, + uint32_t extension, uint32_t size); /** - * \brief Initialise a new IPC message. + * \brief Initialize a new IPC message using a specific heap. + * @param heap Heap to allocate from (NULL = default kernel heap) * @param header Message header metadata * @param size Message data size in bytes. * @return New IPC message. */ -static inline struct ipc_msg *ipc_msg_init(uint32_t header, uint32_t size) +static inline struct ipc_msg *ipc_msg_init(struct k_heap *heap, + uint32_t header, uint32_t size) { - return ipc_msg_w_ext_init(header, 0, size); + return ipc_msg_w_ext_init(heap, header, 0, size); } /** @@ -91,16 +74,9 @@ static inline void ipc_msg_free(struct ipc_msg *msg) if (!msg) return; - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - - key = k_spin_lock(&ipc->lock); - - list_item_del(&msg->list); + ipc_msg_list_remove(msg); rfree(msg->tx_data); rfree(msg); - - k_spin_unlock(&ipc->lock, key); } /** @@ -108,13 +84,7 @@ static inline void ipc_msg_free(struct ipc_msg *msg) */ void ipc_send_queued_msg(void); -/** - * \brief Queues an IPC message for transmission. - * @param msg The IPC message to be freed. - * @param data The message data. - * @param high_priority True if a high priortity message. - */ -void ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority); +#include /** * \brief Send an IPC message directly for emergency. diff --git a/src/include/sof/lib/vregion.h b/src/include/sof/lib/vregion.h index 5c066c90dbc8..52055d373716 100644 --- a/src/include/sof/lib/vregion.h +++ b/src/include/sof/lib/vregion.h @@ -6,6 +6,8 @@ #define __SOF_LIB_VREGION_H__ #include +#include +#include #ifdef __cplusplus extern "C" { @@ -49,7 +51,7 @@ struct vregion *vregion_create(size_t memsize); * * @param[in] vr Pointer to the virtual region instance. */ -void vregion_set_interim(struct vregion *vr); +__syscall void vregion_set_interim(struct vregion *vr); /** * @brief Increment virtual region's user count. @@ -60,7 +62,7 @@ void vregion_set_interim(struct vregion *vr); * @param[in] vr Pointer to the virtual region instance to release. * @return struct vregion* Pointer to the virtual region instance. */ -struct vregion *vregion_get(struct vregion *vr); +__syscall struct vregion *vregion_get(struct vregion *vr); /** * @brief Decrement virtual region's user count or destroy it. @@ -71,7 +73,7 @@ struct vregion *vregion_get(struct vregion *vr); * @param[in] vr Pointer to the virtual region instance to release. * @return struct vregion* Pointer to the virtual region instance or NULL if it has been destroyed. */ -struct vregion *vregion_put(struct vregion *vr); +__syscall struct vregion *vregion_put(struct vregion *vr); /** * @brief Allocate memory from the specified virtual region. @@ -80,12 +82,16 @@ struct vregion *vregion_put(struct vregion *vr); * @param[in] size Size of memory to allocate in bytes. * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc(struct vregion *vr, size_t size); +__syscall void *vregion_alloc(struct vregion *vr, size_t size); + +void *z_impl_vregion_alloc(struct vregion *vr, size_t size); /** * @brief like vregion_alloc() but allocates coherent memory */ -void *vregion_alloc_coherent(struct vregion *vr, size_t size); +__syscall void *vregion_alloc_coherent(struct vregion *vr, size_t size); + +void *z_impl_vregion_alloc_coherent(struct vregion *vr, size_t size); /** * @brief Allocate aligned memory from the specified virtual region. @@ -98,12 +104,16 @@ void *vregion_alloc_coherent(struct vregion *vr, size_t size); * @param[in] alignment Alignment of memory to allocate in bytes. * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment); +__syscall void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment); + +void *z_impl_vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment); /** * @brief like vregion_alloc_align() but allocates coherent memory */ -void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment); +__syscall void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment); + +void *z_impl_vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment); /** * @brief Free memory allocated from the specified virtual region. @@ -113,7 +123,9 @@ void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t align * @param[in] vr Pointer to the virtual region instance. * @param[in] ptr Pointer to the memory to free. */ -void vregion_free(struct vregion *vr, void *ptr); +__syscall void vregion_free(struct vregion *vr, void *ptr); + +void z_impl_vregion_free(struct vregion *vr, void *ptr); /** * @brief Log virtual region memory usage. @@ -131,6 +143,12 @@ void vregion_info(struct vregion *vr); */ void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start); +void vregion_owner_set(struct vregion *vr, void *owner); +void *vregion_owner_get(struct vregion *vr); +bool vregion_verify(struct vregion *vr); + +#include + #else /* CONFIG_SOF_VREGIONS */ struct vregion { @@ -174,6 +192,9 @@ static inline void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t if (size) *size = 0; } +static inline void vregion_owner_set(struct vregion *vr, void *owner) {} +static inline void *vregion_owner_get(struct vregion *vr) {return NULL;} +static inline bool vregion_verify(struct vregion *vr) {return false;} #endif /* CONFIG_SOF_VREGIONS */ diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index c5e2727ab3cb..3eda6ae36ef5 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -39,6 +40,7 @@ #ifdef __ZEPHYR__ #include +#include #endif #ifdef CONFIG_SOF_USERSPACE_LL @@ -64,6 +66,50 @@ struct ipc *ipc_get(void) } #endif +struct ipc_msg *ipc_msg_w_ext_init(struct k_heap *heap, uint32_t header, + uint32_t extension, uint32_t size) +{ + struct ipc_msg *msg; + + if (heap) { + msg = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*msg), 0); + if (msg) + memset(msg, 0, sizeof(*msg)); + } else { + msg = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*msg)); + } + if (!msg) + return NULL; + + if (size) { + if (heap) { + msg->tx_data = sof_heap_alloc(heap, + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + size, 0); + if (msg->tx_data) + memset(msg->tx_data, 0, size); + } else { + msg->tx_data = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + size); + } + if (!msg->tx_data) { + if (heap) + sof_heap_free(heap, msg); + else + rfree(msg); + return NULL; + } + } + + msg->header = header; + msg->extension = extension; + msg->tx_size = size; + list_init(&msg->list); + + return msg; +} + int ipc_process_on_core(uint32_t core, bool blocking) { struct ipc *ipc = ipc_get(); @@ -237,7 +283,11 @@ __cold void ipc_msg_send_direct(struct ipc_msg *msg, void *data) k_spin_unlock(&ipc->lock, key); } +#ifdef CONFIG_SOF_USERSPACE_LL +void z_impl_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) +#else void ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) +#endif { struct ipc *ipc = ipc_get(); k_spinlock_key_t key; @@ -283,6 +333,53 @@ void ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) } EXPORT_SYMBOL(ipc_msg_send); +#ifdef CONFIG_SOF_USERSPACE_LL +static inline bool z_vrfy_ipc_msg_send_check_data(struct ipc_msg *msg, void *data) +{ + K_OOPS(K_SYSCALL_VERIFY(msg->tx_size <= SOF_IPC_MSG_MAX_SIZE)); + + if (msg->tx_size > 0) { + K_OOPS(K_SYSCALL_VERIFY(msg->tx_data != NULL)); + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg->tx_data, msg->tx_size)); + } + + /* If data != NULL and tx_size > 0, verify the data buffer */ + if (data && msg->tx_size > 0) + K_OOPS(K_SYSCALL_MEMORY_READ(data, msg->tx_size)); + + return true; +} + +void z_vrfy_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); + + z_vrfy_ipc_msg_send_check_data(msg, data); + + z_impl_ipc_msg_send(msg, data, high_priority); +} +#include +#endif + +void z_impl_ipc_msg_list_remove(struct ipc_msg *msg) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + + key = k_spin_lock(&ipc->lock); + list_item_del(&msg->list); + k_spin_unlock(&ipc->lock, key); +} + +#ifdef CONFIG_SOF_USERSPACE_LL +void z_vrfy_ipc_msg_list_remove(struct ipc_msg *msg) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); + z_impl_ipc_msg_list_remove(msg); +} +#include +#endif + #ifdef __ZEPHYR__ static void ipc_work_handler(struct k_work *work) { diff --git a/src/ipc/ipc4/CMakeLists.txt b/src/ipc/ipc4/CMakeLists.txt index 360a4e988650..c9f9fcb5710d 100644 --- a/src/ipc/ipc4/CMakeLists.txt +++ b/src/ipc/ipc4/CMakeLists.txt @@ -7,6 +7,7 @@ add_local_sources(sof helper.c logging.c notification.c + notification-user.c ) # The DAI interface is not implemented in library builds and diff --git a/src/ipc/ipc4/notification-user.c b/src/ipc/ipc4/notification-user.c new file mode 100644 index 000000000000..37a881e32829 --- /dev/null +++ b/src/ipc/ipc4/notification-user.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2023 Intel Corporation. All rights reserved. + * + * Author: Piotr Makaruk + * Adrian Warecki + */ + +#include +#include +#include + +#include + +static enum sof_ipc4_resource_event_type dir_to_xrun_event(enum sof_ipc_stream_direction dir) +{ + return (dir == SOF_IPC_STREAM_PLAYBACK) ? SOF_IPC4_GATEWAY_UNDERRUN_DETECTED : + SOF_IPC4_GATEWAY_OVERRUN_DETECTED; +} + +bool send_copier_gateway_xrun_notif_msg(uint32_t pipeline_id, enum sof_ipc_stream_direction dir) +{ + return send_resource_notif(pipeline_id, dir_to_xrun_event(dir), SOF_IPC4_PIPELINE, NULL, + 0); +} + +bool send_gateway_xrun_notif_msg(uint32_t resource_id, enum sof_ipc_stream_direction dir) +{ + return send_resource_notif(resource_id, dir_to_xrun_event(dir), SOF_IPC4_GATEWAY, NULL, 0); +} + +void send_mixer_underrun_notif_msg(uint32_t resource_id, uint32_t eos_flag, uint32_t data_mixed, + uint32_t expected_data_mixed) +{ + struct ipc4_mixer_underrun_event_data mixer_underrun_data; + + mixer_underrun_data.eos_flag = eos_flag; + mixer_underrun_data.data_mixed = data_mixed; + mixer_underrun_data.expected_data_mixed = expected_data_mixed; + + send_resource_notif(resource_id, SOF_IPC4_MIXER_UNDERRUN_DETECTED, SOF_IPC4_PIPELINE, + &mixer_underrun_data, sizeof(mixer_underrun_data)); +} +EXPORT_SYMBOL(send_mixer_underrun_notif_msg); + +void send_process_data_error_notif_msg(uint32_t resource_id, uint32_t error_code) +{ + struct ipc4_process_data_error_event_data error_data; + + error_data.error_code = error_code; + + send_resource_notif(resource_id, SOF_IPC4_PROCESS_DATA_ERROR, SOF_IPC4_MODULE_INSTANCE, + &error_data, sizeof(error_data)); +} diff --git a/src/ipc/ipc4/notification.c b/src/ipc/ipc4/notification.c index 9fd566ee1f93..485b875f8468 100644 --- a/src/ipc/ipc4/notification.c +++ b/src/ipc/ipc4/notification.c @@ -12,7 +12,9 @@ #include #include -#include +#ifdef CONFIG_SOF_USERSPACE_LL +#include +#endif static uint32_t notification_mask = 0xFFFFFFFF; @@ -37,8 +39,13 @@ static bool is_notif_filtered_out(uint32_t event_type) return (notification_mask & BIT(notif_idx)) == 0; } -static bool send_resource_notif(uint32_t resource_id, uint32_t event_type, uint32_t resource_type, - void *data, uint32_t data_size) +#ifdef CONFIG_SOF_USERSPACE_LL +bool z_impl_send_resource_notif(uint32_t resource_id, uint32_t event_type, + uint32_t resource_type, void *data, uint32_t data_size) +#else +bool send_resource_notif(uint32_t resource_id, uint32_t event_type, + uint32_t resource_type, void *data, uint32_t data_size) +#endif { struct ipc_msg *msg; @@ -80,49 +87,34 @@ static bool send_resource_notif(uint32_t resource_id, uint32_t event_type, uint3 return true; } -static enum sof_ipc4_resource_event_type dir_to_xrun_event(enum sof_ipc_stream_direction dir) -{ - return (dir == SOF_IPC_STREAM_PLAYBACK) ? SOF_IPC4_GATEWAY_UNDERRUN_DETECTED : - SOF_IPC4_GATEWAY_OVERRUN_DETECTED; -} - void ipc4_update_notification_mask(uint32_t ntfy_mask, uint32_t enabled_mask) { notification_mask &= enabled_mask | (~ntfy_mask); notification_mask |= enabled_mask & ntfy_mask; } -bool send_copier_gateway_xrun_notif_msg(uint32_t pipeline_id, enum sof_ipc_stream_direction dir) +#ifdef CONFIG_SOF_USERSPACE_LL +static inline bool z_vrfy_send_resource_notif(uint32_t resource_id, uint32_t event_type, + uint32_t resource_type, void *data, + uint32_t data_size) { - return send_resource_notif(pipeline_id, dir_to_xrun_event(dir), SOF_IPC4_PIPELINE, NULL, - 0); -} + /* Validate event_type is a known resource event */ + K_OOPS(K_SYSCALL_VERIFY(event_type < SOF_IPC4_INVALID_RESOURCE_EVENT_TYPE)); -bool send_gateway_xrun_notif_msg(uint32_t resource_id, enum sof_ipc_stream_direction dir) -{ - return send_resource_notif(resource_id, dir_to_xrun_event(dir), SOF_IPC4_GATEWAY, NULL, 0); -} + /* Validate resource_type is a known resource type */ + K_OOPS(K_SYSCALL_VERIFY(resource_type < SOF_IPC4_INVALID_RESOURCE_TYPE)); -void send_mixer_underrun_notif_msg(uint32_t resource_id, uint32_t eos_flag, uint32_t data_mixed, - uint32_t expected_data_mixed) -{ - struct ipc4_mixer_underrun_event_data mixer_underrun_data; - - mixer_underrun_data.eos_flag = eos_flag; - mixer_underrun_data.data_mixed = data_mixed; - mixer_underrun_data.expected_data_mixed = expected_data_mixed; + /* data and data_size must be consistent */ + K_OOPS(K_SYSCALL_VERIFY((!data && !data_size) || (data && data_size))); - send_resource_notif(resource_id, SOF_IPC4_MIXER_UNDERRUN_DETECTED, SOF_IPC4_PIPELINE, - &mixer_underrun_data, sizeof(mixer_underrun_data)); -} -EXPORT_SYMBOL(send_mixer_underrun_notif_msg); - -void send_process_data_error_notif_msg(uint32_t resource_id, uint32_t error_code) -{ - struct ipc4_process_data_error_event_data error_data; + /* Payload must fit in the event_data union and the IPC message */ + K_OOPS(K_SYSCALL_VERIFY(data_size <= sizeof(union ipc4_resource_event_data))); + K_OOPS(K_SYSCALL_VERIFY(data_size <= SOF_IPC_MSG_MAX_SIZE)); - error_data.error_code = error_code; + if (data && data_size) + K_OOPS(K_SYSCALL_MEMORY_READ(data, data_size)); - send_resource_notif(resource_id, SOF_IPC4_PROCESS_DATA_ERROR, SOF_IPC4_MODULE_INSTANCE, - &error_data, sizeof(error_data)); + return z_impl_send_resource_notif(resource_id, event_type, resource_type, data, data_size); } +#include +#endif diff --git a/src/library_manager/lib_notification.c b/src/library_manager/lib_notification.c index 36aa85835e70..0feab9660c3e 100644 --- a/src/library_manager/lib_notification.c +++ b/src/library_manager/lib_notification.c @@ -53,7 +53,7 @@ struct ipc_msg *lib_notif_msg_init(uint32_t header, uint32_t size) sizeof(*msg_pool_elem)); if (!msg_pool_elem) return NULL; - msg = ipc_msg_init(header, SRAM_OUTBOX_SIZE); + msg = ipc_msg_init(NULL, header, SRAM_OUTBOX_SIZE); if (!msg) { rfree(msg_pool_elem); return NULL; diff --git a/src/samples/audio/detect_test.c b/src/samples/audio/detect_test.c index 2ce5f01f3999..4783e2b183c2 100644 --- a/src/samples/audio/detect_test.c +++ b/src/samples/audio/detect_test.c @@ -472,7 +472,7 @@ static struct ipc_msg *ipc4_kd_notification_init(uint32_t word_id, notif.extension.r.sv_score = score; - msg = ipc_msg_w_ext_init(notif.primary.dat, + msg = ipc_msg_w_ext_init(NULL, notif.primary.dat, notif.extension.dat, 0); if (!msg) @@ -731,7 +731,7 @@ static struct comp_dev *test_keyword_new(const struct comp_driver *drv, cd->msg = ipc4_kd_notification_init(NOTIFICATION_DEFAULT_WORD_ID, NOTIFICATION_DEFAULT_SCORE); #else - cd->msg = ipc_msg_init(cd->event.rhdr.hdr.cmd, sizeof(cd->event)); + cd->msg = ipc_msg_init(NULL, cd->event.rhdr.hdr.cmd, sizeof(cd->event)); #endif /* CONFIG_IPC_MAJOR_4 */ if (!cd->msg) { diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index e7a72e6db028..58eca5cd8477 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -570,6 +570,7 @@ static int zephyr_ll_task_free(void *data, struct task *task) uint32_t flags; struct zephyr_ll_pdata *pdata = task->priv_data; bool must_wait, on_list = true; + int wait_ret = 0; zephyr_ll_assert_core(sch); @@ -617,10 +618,17 @@ static int zephyr_ll_task_free(void *data, struct task *task) if (must_wait) /* Wait for up to 100 periods */ - k_sem_take(pdata->sem_p, K_USEC(LL_TIMER_PERIOD_US * 100)); + wait_ret = k_sem_take(pdata->sem_p, K_USEC(LL_TIMER_PERIOD_US * 100)); /* Protect against racing with schedule_task() */ zephyr_ll_lock(sch, &flags); + + if (wait_ret && task->state != SOF_TASK_STATE_FREE) { + tr_warn(&ll_tr, "task %p still active on free (state %d, semret %d)", + task, task->state, wait_ret); + zephyr_ll_task_done(sch, task); + } + #if CONFIG_DYNAMIC_OBJECTS zephyr_ll_task_sem_free(task); #endif diff --git a/src/trace/dma-trace.c b/src/trace/dma-trace.c index bfa359b4ba74..57db2da2c36d 100644 --- a/src/trace/dma-trace.c +++ b/src/trace/dma-trace.c @@ -160,7 +160,7 @@ int dma_trace_init_early(struct sof *sof) k_spinlock_init(&sof->dmat->lock); ipc_build_trace_posn(&sof->dmat->posn); - sof->dmat->msg = ipc_msg_init(sof->dmat->posn.rhdr.hdr.cmd, + sof->dmat->msg = ipc_msg_init(NULL, sof->dmat->posn.rhdr.hdr.cmd, sof->dmat->posn.rhdr.hdr.size); if (!sof->dmat->msg) { ret = -ENOMEM; diff --git a/test/cmocka/src/audio/volume/CMakeLists.txt b/test/cmocka/src/audio/volume/CMakeLists.txt index 0385441e5878..199790c330d8 100644 --- a/test/cmocka/src/audio/volume/CMakeLists.txt +++ b/test/cmocka/src/audio/volume/CMakeLists.txt @@ -42,6 +42,7 @@ add_library(audio_for_volume STATIC ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-stream.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-xrun.c ${PROJECT_SOURCE_DIR}/src/audio/component.c + ${PROJECT_SOURCE_DIR}/src/audio/data_blob.c ${PROJECT_SOURCE_DIR}/src/math/numbers.c ) sof_append_relative_path_definitions(audio_for_volume) diff --git a/test/cmocka/src/common_mocks.c b/test/cmocka/src/common_mocks.c index 633108a0730e..983364a2c1ef 100644 --- a/test/cmocka/src/common_mocks.c +++ b/test/cmocka/src/common_mocks.c @@ -141,6 +141,11 @@ void WEAK sof_heap_free(struct k_heap *heap, void *addr) free(addr); } +void WEAK module_adapter_vreg_free(struct mod_alloc_ctx *alloc) +{ + (void)alloc; +} + int WEAK memcpy_s(void *dest, size_t dest_size, const void *src, size_t count) { diff --git a/west.yml b/west.yml index a684deb8871a..2572630b44b5 100644 --- a/west.yml +++ b/west.yml @@ -11,6 +11,8 @@ manifest: url-base: https://github.com/thesofproject - name: zephyrproject url-base: https://github.com/zephyrproject-rtos + - name: kvzephyrproject + url-base: https://github.com/kv2019i # When upgrading projects here please run git log --oneline in the # project and if not too long then include the output in your commit @@ -43,8 +45,8 @@ manifest: - name: zephyr repo-path: zephyr - revision: fbd71c30f7b9083de9826577edb57d360efc01ed - remote: zephyrproject + revision: e7650c8c4a7077dd649b42a5b99f8e12b8cca3b3 + remote: kvzephyrproject # Import some projects listed in zephyr/west.yml@revision # diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index e0e7e8bfb302..b2ffcec80d3d 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -627,13 +627,19 @@ zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/fast-get.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/ipc/ipc_reply.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/schedule/ll_schedule_domain.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/ipc4/handler.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/ipc/ipc_msg_send.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/ipc/ipc_msg_list_remove.h) zephyr_syscall_header(include/rtos/alloc.h) zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_ALLOC syscall/alloc.c) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/vregion.h) +zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_VREGION syscall/vregion.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/dai-zephyr.h) zephyr_library_sources_ifdef(CONFIG_USERSPACE syscall/dai.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/user/debug_stream_slot.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/ipc4/notification.h) + zephyr_library_link_libraries(SOF) target_link_libraries(SOF INTERFACE zephyr_interface) diff --git a/zephyr/Kconfig b/zephyr/Kconfig index ab88efcb9a6a..2d03a6d6f7ae 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -36,11 +36,20 @@ config SOF_USERSPACE_INTERFACE_ALLOC Allow user-space threads to use sof_heap_alloc/sof_heap_free as Zephyr system calls. +config SOF_USERSPACE_INTERFACE_VREGION + bool "Enable SOF vregion interface to userspace threads" + depends on USERSPACE + depends on SOF_VREGIONS + help + Allow user-space threads to use vregion_alloc/vregion_free + and their variants as Zephyr system calls. + config SOF_USERSPACE_LL bool "Run Low-Latency pipelines in userspace threads" depends on USERSPACE select SOF_USERSPACE_INTERFACE_ALLOC select SOF_USERSPACE_INTERFACE_DMA + select SOF_USERSPACE_INTERFACE_VREGION if SOF_VREGIONS help Run Low-Latency (LL) pipelines in userspace threads. This adds memory protection between operating system resources and diff --git a/zephyr/lib/userspace_helper.c b/zephyr/lib/userspace_helper.c index ea9db4181b9d..e205fc256308 100644 --- a/zephyr/lib/userspace_helper.c +++ b/zephyr/lib/userspace_helper.c @@ -110,7 +110,7 @@ int user_access_to_mailbox(struct k_mem_domain *domain, k_tid_t thread_id) if (ret < 0) return ret; -#if defined(CONFIG_SOF_USERSPACE_LL) && defined(CONFIG_IPC_MAJOR_4) +#ifdef CONFIG_IPC_MAJOR_4 /* HOSTBOX partitions for IPC4 module init parameter block reads. * comp_new_ipc4() accesses MAILBOX_HOSTBOX_BASE directly to get * the module configuration data sent by the host. @@ -141,9 +141,7 @@ int user_access_to_mailbox(struct k_mem_domain *domain, k_tid_t thread_id) ret = k_mem_domain_add_partition(domain, &hostbox_partition); if (ret < 0) return ret; -#endif /* CONFIG_IPC_MAJOR_4 */ - -#ifndef CONFIG_IPC_MAJOR_4 +#else /* CONFIG_IPC_MAJOR_4 */ /* * Next mailbox_stream (not available in IPC4). Stream access is cached, * so different mapping this time. diff --git a/zephyr/lib/vregion.c b/zephyr/lib/vregion.c index 9c8c94c23f97..bcb477c88ba8 100644 --- a/zephyr/lib/vregion.c +++ b/zephyr/lib/vregion.c @@ -87,6 +87,7 @@ struct vregion { unsigned int pages; /* size of whole region in pages */ struct k_mutex lock; /* protect vregion heaps and use-count */ unsigned int use_count; + void *owner; /* current allocation mode */ enum vregion_mem_type type; /* LIFETIME at creation, switch to INTERIM */ @@ -144,6 +145,7 @@ struct vregion *vregion_create(size_t memsize) vr->base = vregion_base; vr->size = total_size; vr->pages = pages; + vr->owner = NULL; /* lifetime linear allocator starts at the beginning of the vregion memory */ vr->lifetime.base = vregion_base; @@ -161,12 +163,12 @@ struct vregion *vregion_create(size_t memsize) /* log the new vregion */ LOG_INF("new at base %p size %#zx pages %u metadata at %p", - (void *)vr->base, total_size, pages, (void *)vr); + (void *)vregion_base, total_size, pages, (void *)vr); return vr; } -struct vregion *vregion_get(struct vregion *vr) +struct vregion *z_impl_vregion_get(struct vregion *vr) { if (!vr) return NULL; @@ -184,7 +186,7 @@ struct vregion *vregion_get(struct vregion *vr) * @param[in] vr Pointer to the virtual region instance to release. * @return struct vregion* Pointer to the virtual region instance or NULL if it has been destroyed. */ -struct vregion *vregion_put(struct vregion *vr) +struct vregion *z_impl_vregion_put(struct vregion *vr) { unsigned int use_count; @@ -259,7 +261,7 @@ static void interim_heap_init(struct vregion *vr) vr->lifetime.used = (uint8_t *)vr->lifetime.ptr - (uint8_t *)vr->lifetime.base; } -void vregion_set_interim(struct vregion *vr) +void z_impl_vregion_set_interim(struct vregion *vr) { if (!vr) return; @@ -365,7 +367,7 @@ static void lifetime_free(struct vlinear_heap *heap, void *ptr) * @param vr Pointer to the virtual region instance. * @param ptr Pointer to the memory to free. */ -void vregion_free(struct vregion *vr, void *ptr) +void z_impl_vregion_free(struct vregion *vr, void *ptr) { if (!vr || !ptr) return; @@ -390,7 +392,7 @@ void vregion_free(struct vregion *vr, void *ptr) k_mutex_unlock(&vr->lock); } -EXPORT_SYMBOL(vregion_free); +EXPORT_SYMBOL(z_impl_vregion_free); /** * @brief Allocate memory from the virtual region. @@ -401,7 +403,8 @@ EXPORT_SYMBOL(vregion_free); * * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment) +void *z_impl_vregion_alloc_align(struct vregion *vr, + size_t size, size_t alignment) { void *p; @@ -429,7 +432,7 @@ void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment) return p; } -EXPORT_SYMBOL(vregion_alloc_align); +EXPORT_SYMBOL(z_impl_vregion_alloc_align); /** * @brief Allocate memory from the virtual region. @@ -437,17 +440,17 @@ EXPORT_SYMBOL(vregion_alloc_align); * @param[in] size Size of the allocation. * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc(struct vregion *vr, size_t size) +void *z_impl_vregion_alloc(struct vregion *vr, size_t size) { - return vregion_alloc_align(vr, size, 0); + return z_impl_vregion_alloc_align(vr, size, 0); } -EXPORT_SYMBOL(vregion_alloc); +EXPORT_SYMBOL(z_impl_vregion_alloc); -void *vregion_alloc_coherent(struct vregion *vr, size_t size) +void *z_impl_vregion_alloc_coherent(struct vregion *vr, size_t size) { size = ALIGN_UP(size, CONFIG_DCACHE_LINE_SIZE); - void *p = vregion_alloc_align(vr, size, CONFIG_DCACHE_LINE_SIZE); + void *p = z_impl_vregion_alloc_align(vr, size, CONFIG_DCACHE_LINE_SIZE); if (!p) return NULL; @@ -456,14 +459,15 @@ void *vregion_alloc_coherent(struct vregion *vr, size_t size) return sys_cache_uncached_ptr_get(p); } +EXPORT_SYMBOL(z_impl_vregion_alloc_coherent); -void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment) +void *z_impl_vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment) { if (alignment < CONFIG_DCACHE_LINE_SIZE) alignment = CONFIG_DCACHE_LINE_SIZE; size = ALIGN_UP(size, CONFIG_DCACHE_LINE_SIZE); - void *p = vregion_alloc_align(vr, size, alignment); + void *p = z_impl_vregion_alloc_align(vr, size, alignment); if (!p) return NULL; @@ -472,6 +476,7 @@ void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t align return sys_cache_uncached_ptr_get(p); } +EXPORT_SYMBOL(z_impl_vregion_alloc_coherent_align); /** * @brief Log virtual region memory usage. @@ -488,7 +493,6 @@ void vregion_info(struct vregion *vr) LOG_INF("lifetime used %#zx free count %d", vr->lifetime.used, vr->lifetime.free_count); } -EXPORT_SYMBOL(vregion_info); void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start) { @@ -498,3 +502,38 @@ void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start) if (start) *start = (uintptr_t)vr->base; } + +void vregion_owner_set(struct vregion *vr, void *owner) +{ + if (vr->owner) + LOG_ERR("vregion %p already owned by %p", vr, vr->owner); + else + vr->owner = owner; +} + +void *vregion_owner_get(struct vregion *vr) +{ + return vr->owner; +} + +#if CONFIG_SOF_VREGIONS && CONFIG_USERSPACE +#include + +bool vregion_verify(struct vregion *vr) +{ + if (!vr) + return false; + + /* vregion instances must not be accessible to the userspace. */ + K_OOPS(!K_SYSCALL_MEMORY_READ(vr, sizeof(*vr))); + + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(vr, &vr_size, &vr_start); + if (vr_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); + + return true; +} +#endif diff --git a/zephyr/syscall/vregion.c b/zephyr/syscall/vregion.c new file mode 100644 index 000000000000..abc912e88c29 --- /dev/null +++ b/zephyr/syscall/vregion.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. + +#include +#include +#include + +static inline void *z_vrfy_vregion_alloc(struct vregion *vr, size_t size) +{ + if (vregion_verify(vr)) + return z_impl_vregion_alloc(vr, size); + + return NULL; +} +#include + +static inline void *z_vrfy_vregion_alloc_coherent(struct vregion *vr, size_t size) +{ + if (vregion_verify(vr)) + return z_impl_vregion_alloc_coherent(vr, size); + + return NULL; +} +#include + +static inline void *z_vrfy_vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment) +{ + if (vregion_verify(vr)) + return z_impl_vregion_alloc_align(vr, size, alignment); + + return NULL; +} +#include + +static inline void *z_vrfy_vregion_alloc_coherent_align(struct vregion *vr, + size_t size, size_t alignment) +{ + if (vregion_verify(vr)) + return z_impl_vregion_alloc_coherent_align(vr, size, alignment); + + return NULL; +} +#include + +static inline void z_vrfy_vregion_free(struct vregion *vr, void *ptr) +{ + if (vregion_verify(vr)) + z_impl_vregion_free(vr, ptr); +} +#include + +struct vregion *z_vrfy_vregion_get(struct vregion *vr) +{ + if (vregion_verify(vr)) + return z_impl_vregion_get(vr); + return NULL; +} +#include + +struct vregion *z_vrfy_vregion_put(struct vregion *vr) +{ + if (vregion_verify(vr)) + return z_impl_vregion_put(vr); + return NULL; +} +#include + +void z_vrfy_vregion_set_interim(struct vregion *vr) +{ + if (vregion_verify(vr)) + z_impl_vregion_set_interim(vr); +} +#include