WIP: schedule: add multi-core support for user-space LL scheduler

- add arch_schedulers_get_for_core() for task-carried core routing
- route 8 task-aware scheduler callers via task->core under CONFIG_SOF_USERSPACE_LL
- fix zephyr_domain_thread_tid() to accept core parameter instead of hardcoding core 0
- create domain threads for secondary cores in secondary_core_init()
- remove FIXME workaround from arch_schedulers_get()

Signed-off-by: Kai Vehmanen <kai.vehmanen@linux.intel.com>

Author: kv2019i

src/audio/pipeline/pipeline-schedule.c

@@ -291,7 +291,9 @@ void pipeline_schedule_triggered(struct pipeline_walk_context *ctx,
 	 * while pipeline state is being updated. The k_mutex is re-entrant
 	 * so schedule_task() calls inside the critical section are safe.
 	 */
-	zephyr_ll_lock_sched();
+	int sched_core = ppl_data->start->ipc_config.core;
+
+	zephyr_ll_lock_sched(sched_core);
 #else
 	uint32_t flags;
 
@@ -358,7 +360,7 @@ void pipeline_schedule_triggered(struct pipeline_walk_context *ctx,
 		}
 	}
 #ifdef CONFIG_SOF_USERSPACE_LL
-	zephyr_ll_unlock_sched();
+	zephyr_ll_unlock_sched(sched_core);
 #else
 	irq_local_enable(flags);
 #endif

src/include/sof/schedule/ll_schedule_domain.h

@@ -117,8 +117,8 @@ void zephyr_ll_task_free(struct task *task);
 struct k_heap *zephyr_ll_user_heap(void);
 void zephyr_ll_user_resources_init(void);
 void zephyr_ll_grant_access(struct k_thread *thread);
-void zephyr_ll_lock_sched(void);
-void zephyr_ll_unlock_sched(void);
+void zephyr_ll_lock_sched(int core);
+void zephyr_ll_unlock_sched(int core);
 #endif /* CONFIG_SOF_USERSPACE_LL */
 
 static inline struct ll_schedule_domain *domain_init
@@ -317,7 +317,7 @@ struct ll_schedule_domain *zephyr_ll_domain(void);
 struct ll_schedule_domain *zephyr_domain_init(int clk);
 #define timer_domain_init(timer, clk) zephyr_domain_init(clk)
 #ifdef CONFIG_SOF_USERSPACE_LL
-struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain);
+struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain, int core);
 struct k_mem_domain *zephyr_ll_mem_domain(void);
 #endif /* CONFIG_SOF_USERSPACE_LL */
 #endif /* __ZEPHYR__ */

src/include/sof/schedule/schedule.h

@@ -198,6 +198,10 @@ struct schedulers {
  */
 struct schedulers **arch_schedulers_get(void);
 
+#if CONFIG_SOF_USERSPACE_LL
+struct schedulers **arch_schedulers_get_for_core(int core);
+#endif
+
 /**
  * Retrieves scheduler's data.
  * @param type SOF_SCHEDULE_ type.
@@ -218,10 +222,39 @@ static inline void *scheduler_get_data(uint16_t type)
 	return NULL;
 }
 
+#if CONFIG_SOF_USERSPACE_LL
+/**
+ * Retrieves scheduler's data for a specific core.
+ * @param type SOF_SCHEDULE_ type.
+ * @param core Core ID to get scheduler data for.
+ * @return Pointer to scheduler's data.
+ *
+ * Safe to call from user-space context — does not use cpu_get_id().
+ */
+static inline void *scheduler_get_data_for_core(uint16_t type, int core)
+{
+	struct schedulers *schedulers = *arch_schedulers_get_for_core(core);
+	struct schedule_data *sch;
+	struct list_item *slist;
+
+	list_for_item(slist, &schedulers->list) {
+		sch = container_of(slist, struct schedule_data, list);
+		if (type == sch->type)
+			return sch->data;
+	}
+
+	return NULL;
+}
+#endif
+
 /** See scheduler_ops::schedule_task_running */
 static inline int schedule_task_running(struct task *task)
 {
+#if CONFIG_SOF_USERSPACE_LL
+	struct schedulers *schedulers = *arch_schedulers_get_for_core(task->core);
+#else
 	struct schedulers *schedulers = *arch_schedulers_get();
+#endif
 	struct schedule_data *sch;
 	struct list_item *slist;
 
@@ -243,7 +276,11 @@ static inline int schedule_task_running(struct task *task)
 static inline int schedule_task(struct task *task, uint64_t start,
 				uint64_t period)
 {
+#if CONFIG_SOF_USERSPACE_LL
+	struct schedulers *schedulers = *arch_schedulers_get_for_core(task->core);
+#else
 	struct schedulers *schedulers = *arch_schedulers_get();
+#endif
 	struct schedule_data *sch;
 	struct list_item *slist;
 
@@ -264,7 +301,11 @@ static inline int schedule_task(struct task *task, uint64_t start,
 static inline int schedule_task_before(struct task *task, uint64_t start,
 				       uint64_t period, struct task *before)
 {
+#if CONFIG_SOF_USERSPACE_LL
+	struct schedulers *schedulers = *arch_schedulers_get_for_core(task->core);
+#else
 	struct schedulers *schedulers = *arch_schedulers_get();
+#endif
 	struct schedule_data *sch;
 	struct list_item *slist;
 
@@ -290,7 +331,11 @@ static inline int schedule_task_before(struct task *task, uint64_t start,
 static inline int schedule_task_after(struct task *task, uint64_t start,
 				      uint64_t period, struct task *after)
 {
+#if CONFIG_SOF_USERSPACE_LL
+	struct schedulers *schedulers = *arch_schedulers_get_for_core(task->core);
+#else
 	struct schedulers *schedulers = *arch_schedulers_get();
+#endif
 	struct schedule_data *sch;
 	struct list_item *slist;
 
@@ -315,7 +360,11 @@ static inline int schedule_task_after(struct task *task, uint64_t start,
 /** See scheduler_ops::reschedule_task */
 static inline int reschedule_task(struct task *task, uint64_t start)
 {
+#if CONFIG_SOF_USERSPACE_LL
+	struct schedulers *schedulers = *arch_schedulers_get_for_core(task->core);
+#else
 	struct schedulers *schedulers = *arch_schedulers_get();
+#endif
 	struct schedule_data *sch;
 	struct list_item *slist;
 
@@ -337,7 +386,11 @@ static inline int reschedule_task(struct task *task, uint64_t start)
 /** See scheduler_ops::schedule_task_cancel */
 static inline int schedule_task_cancel(struct task *task)
 {
+#if CONFIG_SOF_USERSPACE_LL
+	struct schedulers *schedulers = *arch_schedulers_get_for_core(task->core);
+#else
 	struct schedulers *schedulers = *arch_schedulers_get();
+#endif
 	struct schedule_data *sch;
 	struct list_item *slist;
 
@@ -353,7 +406,11 @@ static inline int schedule_task_cancel(struct task *task)
 /** See scheduler_ops::schedule_task_free */
 static inline int schedule_task_free(struct task *task)
 {
+#if CONFIG_SOF_USERSPACE_LL
+	struct schedulers *schedulers = *arch_schedulers_get_for_core(task->core);
+#else
 	struct schedulers *schedulers = *arch_schedulers_get();
+#endif
 	struct schedule_data *sch;
 	struct list_item *slist;
 
@@ -402,7 +459,11 @@ static inline int schedulers_restore(void)
 /** See scheduler_ops::scheduler_init_context */
 static inline struct k_thread *scheduler_init_context(struct task *task)
 {
+#if CONFIG_SOF_USERSPACE_LL
+	struct schedulers *schedulers = *arch_schedulers_get_for_core(task->core);
+#else
 	struct schedulers *schedulers = *arch_schedulers_get();
+#endif
 	struct schedule_data *sch;
 	struct list_item *slist;
 

src/init/init.c

@@ -48,6 +48,10 @@
 
 LOG_MODULE_REGISTER(init, CONFIG_SOF_LOG_LEVEL);
 
+#if CONFIG_SOF_USERSPACE_LL
+SOF_DEFINE_REG_UUID(sec_core_init);
+#endif
+
 /* main firmware context */
 static struct sof sof;
 
@@ -136,6 +140,18 @@ __cold int secondary_core_init(struct sof *sof)
 		return err;
 #endif /* CONFIG_ZEPHYR_DP_SCHEDULER */
 
+#if CONFIG_SOF_USERSPACE_LL
+	/* Create domain thread for this secondary core's LL scheduler */
+	{
+		struct task *task = zephyr_ll_task_alloc();
+
+		schedule_task_init_ll(task, SOF_UUID(sec_core_init_uuid),
+				      SOF_SCHEDULE_LL_TIMER,
+				      0, NULL, NULL, cpu_get_id(), 0);
+		scheduler_init_context(task);
+	}
+#endif
+
 	/* initialize IDC mechanism */
 	trace_point(TRACE_BOOT_PLATFORM_IDC);
 	err = idc_init();

src/ipc/ipc4/helper.c

@@ -613,15 +613,15 @@ __cold static struct comp_buffer *ipc4_create_buffer(struct comp_dev *src, bool
 		if (cross_core_bind) \
 			domain_block(sof_get()->platform_timer_domain); \
 		else \
-			zephyr_ll_lock_sched(); \
+			zephyr_ll_lock_sched(cpu_get_id()); \
 	} while (0)
 
 #define ll_unblock(cross_core_bind, flags) \
 	do { \
 		if (cross_core_bind) \
 			domain_unblock(sof_get()->platform_timer_domain); \
 		else \
-			zephyr_ll_unlock_sched(); \
+			zephyr_ll_unlock_sched(cpu_get_id()); \
 	} while (0)
 #else
 #define ll_block(cross_core_bind, flags) \
@@ -672,8 +672,8 @@ static int ll_wait_finished_on_core(struct comp_dev *dev)
 #else
 
 #if CONFIG_SOF_USERSPACE_LL
-#define ll_block(cross_core_bind, flags)	zephyr_ll_lock_sched()
-#define ll_unblock(cross_core_bind, flags)	zephyr_ll_unlock_sched()
+#define ll_block(cross_core_bind, flags)	zephyr_ll_lock_sched(cpu_get_id())
+#define ll_unblock(cross_core_bind, flags)	zephyr_ll_unlock_sched(cpu_get_id())
 #else
 #define ll_block(cross_core_bind, flags)	irq_local_disable(flags)
 #define ll_unblock(cross_core_bind, flags)	irq_local_enable(flags)

src/schedule/zephyr_domain.c

@@ -477,13 +477,12 @@ static void zephyr_domain_thread_free(struct ll_schedule_domain *domain,
 	tr_info(&ll_tr, "thread_free done, core %d", core);
 }
 
-struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain)
+struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain, int core)
 {
 	struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain);
-	int core = 0; /* cpu_get_id(); */
 	struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core;
 
-	tr_dbg(&ll_tr, "entry");
+	tr_dbg(&ll_tr, "entry core %d", core);
 
 	return dt->ll_thread;
 }

src/schedule/zephyr_ll.c

@@ -66,8 +66,15 @@ static void zephyr_ll_unlock(struct zephyr_ll *sch, uint32_t *flags)
 
 static void zephyr_ll_assert_core(const struct zephyr_ll *sch)
 {
-	assert(CONFIG_CORE_COUNT == 1 || IS_ENABLED(CONFIG_SOF_USERSPACE_LL) ||
-	       sch->core == cpu_get_id());
+#if CONFIG_SOF_USERSPACE_LL
+	/* In user-space mode, cpu_get_id() is not available.
+	 * Core correctness is ensured by task->core routing in
+	 * schedule.h and verified at task schedule time.
+	 */
+	(void)sch;
+#else
+	assert(CONFIG_CORE_COUNT == 1 || sch->core == cpu_get_id());
+#endif
 }
 
 /* Locking: caller should hold the domain lock */
@@ -533,15 +540,16 @@ struct k_thread *zephyr_ll_init_context(void *data, struct task *task)
 	}
 
 	if (!k_is_user_context()) {
-		k_thread_access_grant(zephyr_domain_thread_tid(sch->ll_domain), sch->lock);
+		k_thread_access_grant(zephyr_domain_thread_tid(sch->ll_domain, task->core),
+				      sch->lock);
 
-		tr_dbg(&ll_tr, "granting access to lock %p for thread %p", sch->lock,
-			zephyr_domain_thread_tid(sch->ll_domain));
+		tr_dbg(&ll_tr, "granting access to lock %p for thread %p on core %d", sch->lock,
+			zephyr_domain_thread_tid(sch->ll_domain, task->core), task->core);
 		tr_dbg(&ll_tr, "granting access to domain lock %p for thread %p", &sch->ll_domain->lock,
-			zephyr_domain_thread_tid(sch->ll_domain));
+			zephyr_domain_thread_tid(sch->ll_domain, task->core));
 	}
 
-	return zephyr_domain_thread_tid(sch->ll_domain);
+	return zephyr_domain_thread_tid(sch->ll_domain, task->core);
 }
 
 void zephyr_ll_free_context(void *data)
@@ -592,19 +600,21 @@ void zephyr_ll_grant_access(struct k_thread *thread)
  * schedule_task() calls within the locked section will not deadlock.
  * Must be paired with zephyr_ll_unlock_sched().
  */
-void zephyr_ll_lock_sched(void)
+void zephyr_ll_lock_sched(int core)
 {
-	struct zephyr_ll *sch = (struct zephyr_ll *)scheduler_get_data(SOF_SCHEDULE_LL_TIMER);
+	struct zephyr_ll *sch = (struct zephyr_ll *)scheduler_get_data_for_core(SOF_SCHEDULE_LL_TIMER,
+										core);
 
 	k_mutex_lock(sch->lock, K_FOREVER);
 }
 
 /**
  * Unlock the LL scheduler after a previous zephyr_ll_lock_sched() call.
  */
-void zephyr_ll_unlock_sched(void)
+void zephyr_ll_unlock_sched(int core)
 {
-	struct zephyr_ll *sch = (struct zephyr_ll *)scheduler_get_data(SOF_SCHEDULE_LL_TIMER);
+	struct zephyr_ll *sch = (struct zephyr_ll *)scheduler_get_data_for_core(SOF_SCHEDULE_LL_TIMER,
+										core);
 
 	k_mutex_unlock(sch->lock);
 }

uuid-registry.txt

@@ -146,6 +146,7 @@ d7f6712d-131c-45a7-82ed6aa9dc2291ea pm_runtime
 9302adf5-88be-4234-a0a7dca538ef81f4 sai
 3dee06de-f25a-4e10-ae1fabc9573873ea schedule
 70d223ef-2b91-4aac-b444d89a0db2793a sdma
+bdcb1461-34f5-4047-b9cc70fdf8dfb234 sec_core_init
 55a88ed5-3d18-46ca-88f10ee6eae9930f selector
 32fe92c1-1e17-4fc2-9758c7f3542e980a selector4
 cf90d851-68a2-4987-a2de85aed0c8531c sgen_mt8186

zephyr/schedule.c

@@ -22,11 +22,21 @@ static APP_TASK_BSS struct schedulers *_schedulers[CONFIG_CORE_COUNT];
  */
 struct schedulers **arch_schedulers_get(void)
 {
-	if (k_is_user_context()) {
-		printk("FIXME: using core0 scheduler\n");
-		return _schedulers;
-	}
-
 	return _schedulers + cpu_get_id();
 }
 EXPORT_SYMBOL(arch_schedulers_get);
+
+#if CONFIG_SOF_USERSPACE_LL
+/**
+ * Retrieves registered schedulers for a specific core.
+ * @param core Core ID to get schedulers for.
+ * @return List of registered schedulers for the specified core.
+ *
+ * Safe to call from user-space context — does not use cpu_get_id().
+ */
+struct schedulers **arch_schedulers_get_for_core(int core)
+{
+	return _schedulers + core;
+}
+EXPORT_SYMBOL(arch_schedulers_get_for_core);
+#endif