3 * \brief Threads implementation.
7 * Copyright (c) 2007, 2008, 2009, 2010, 2011, 2012, ETH Zurich.
10 * This file is distributed under the terms in the attached LICENSE file.
11 * If you do not find this file, copies can be found by writing to:
12 * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
18 #include <barrelfish/barrelfish.h>
19 #include <barrelfish/dispatch.h>
20 #include <barrelfish/dispatcher_arch.h>
21 #include <barrelfish/debug.h>
22 #include <barrelfish/slab.h>
23 #include <barrelfish/caddr.h>
24 #include <barrelfish/curdispatcher_arch.h>
25 #include <barrelfish/vspace_mmu_aware.h>
26 #include <barrelfish_kpi/cpu_arch.h>
27 #include <barrelfish_kpi/domain_params.h>
28 #include <arch/registers.h>
29 #include <trace/trace.h>
31 #include <trace_definitions/trace_defs.h>
33 #include "arch/threads.h"
34 #include "threads_priv.h"
37 #if defined(__x86_64__)
38 # include "arch/ldt.h"
41 #ifdef FPU_LAZY_CONTEXT_SWITCH
42 # include <arch/fpu.h>
45 /// Maximum number of threads in a domain, used to size VM region for thread structures
46 // there is no point having MAX_THREADS > LDT_NENTRIES on x86 (see ldt.c)
47 #define MAX_THREADS 256
49 /// 16-byte alignment required for x86-64
50 // FIXME: this should be in an arch header
51 #define STACK_ALIGNMENT (sizeof(uint64_t) * 2)
53 /// Static stack and storage for a bootstrap/cleanup thread
54 // XXX: 16-byte aligned for x86-64
55 static uintptr_t staticstack[THREADS_DEFAULT_STACK_BYTES / sizeof(uintptr_t)]
56 __attribute__((aligned(STACK_ALIGNMENT)));
57 static struct thread staticthread = {
59 .stack_top = (char *)staticstack + sizeof(staticstack)
61 static struct thread_mutex staticthread_lock = THREAD_MUTEX_INITIALIZER;
63 /// Storage metadata for thread structures (and TLS data)
64 static struct slab_allocator thread_slabs;
65 static struct vspace_mmu_aware thread_slabs_vm;
67 // XXX: mutex and spinlock protecting thread slabs in spanned domains
68 /* This ought to be just a mutex. However, thread_create() is called on the
69 * inter-disp message handler thread, and if it blocks in a mutex, there is no
70 * way to wake it up and we will deadlock. This is a quick-fix workaround:
71 * The spinlock protects the data structure
72 * The mutex avoids unneccessary spinning (it is acquired first when safe)
74 static spinlock_t thread_slabs_spinlock;
75 static struct thread_mutex thread_slabs_mutex = THREAD_MUTEX_INITIALIZER;
77 /// Base and size of the original ("pristine") thread-local storage init data
78 static void *tls_block_init_base;
79 static size_t tls_block_init_len;
80 static size_t tls_block_total_len;
82 /// Warning already issued about RSP usage. (Prevent repeated warnings
83 /// from the same domain -- e.g., when using THC whose stacks appear
85 __attribute__((unused)) static bool stack_warned=0;
87 /// Wrapper function for most threads, runs given function then deletes itself
88 static void thread_entry(thread_func_t start_func, void *start_data)
90 assert((lvaddr_t)start_func >= BASE_PAGE_SIZE);
91 start_func(start_data);
93 assert(!"thread_exit returned");
97 /// Debugging assertions on thread queues
98 static void check_queue(struct thread *queue)
103 struct thread *q = queue;
107 assert_disabled(q != NULL);
109 // check for NULL next and prev pointers
110 assert_disabled((lvaddr_t)q->next > BASE_PAGE_SIZE);
111 assert_disabled((lvaddr_t)q->prev > BASE_PAGE_SIZE);
113 // check that next and prev pointers are sane
114 assert_disabled(q->next->prev == q);
115 assert_disabled(q->prev->next == q);
117 // advance to next elem
120 assert_disabled(i < MAX_THREADS);
121 } while (q != queue);
123 #else /* NDEBUG version */
124 static inline void check_queue(struct thread *queue) {}
128 * \brief Enqueue a thread in the given queue
130 * For safety, should only happen while disabled.
132 void thread_enqueue(struct thread *thread, struct thread **queue)
134 assert_disabled(thread != NULL);
135 assert_disabled(queue != NULL);
137 if (*queue == NULL) {
138 *queue = thread->prev = thread->next = thread;
140 assert_disabled((*queue)->prev != NULL);
141 thread->prev = (*queue)->prev;
142 thread->next = *queue;
143 (*queue)->prev = thread;
144 assert_disabled(thread->prev != NULL);
145 thread->prev->next = thread;
152 * \brief Dequeue the first thread on the given queue
154 * For safety, should only happen while disabled.
156 * \returns Pointer to thread that was dequeued
158 struct thread *thread_dequeue(struct thread **queue)
160 assert_disabled(queue != NULL);
161 struct thread *thread = *queue;
162 assert_disabled(thread != NULL);
164 if (thread->prev == thread) {
165 assert_disabled(thread->next == thread);
168 thread->prev->next = thread->next;
169 thread->next->prev = thread->prev;
170 *queue = thread->next;
174 thread->prev = thread->next = NULL;
180 * \brief Remove a specific thread from a queue
182 * Does not check that the thread is in the given queue, which it must be.
183 * For safety, should only happen while disabled.
185 void thread_remove_from_queue(struct thread **queue, struct thread *thread)
187 assert_disabled(queue != NULL);
188 assert_disabled(thread != NULL);
190 if (thread->prev == thread) {
191 assert_disabled(thread->next == thread);
192 assert_disabled(*queue == thread);
195 thread->prev->next = thread->next;
196 thread->next->prev = thread->prev;
197 if (*queue == thread) {
198 *queue = thread->next;
203 thread->prev = thread->next = NULL;
207 /// Refill backing storage for thread region
208 static errval_t refill_thread_slabs(struct slab_allocator *slabs)
210 assert(slabs == &thread_slabs);
217 err = slot_alloc(&frame);
218 if (err_is_fail(err)) {
219 return err_push(err, LIB_ERR_SLOT_ALLOC);
222 size_t blocksize = sizeof(struct thread) + tls_block_total_len;
223 err = vspace_mmu_aware_map(&thread_slabs_vm, frame, blocksize, &buf, &size);
224 if (err_is_fail(err)) {
225 return err_push(err, LIB_ERR_VSPACE_MMU_AWARE_MAP);
228 slab_grow(slabs, buf, size);
233 /// Initialise the state of a new thread structure
234 static void thread_init(dispatcher_handle_t disp, struct thread *newthread)
236 newthread->self = newthread;
238 newthread->next = newthread->prev = NULL;
240 newthread->tls_dtv = NULL;
241 newthread->disp = disp;
242 newthread->coreid = get_dispatcher_generic(disp)->core_id;
243 newthread->userptr = NULL;
244 memset(newthread->userptrs, 0, sizeof(newthread->userptrs));
245 newthread->yield_epoch = 0;
246 newthread->wakeup_reason = NULL;
247 newthread->return_value = 0;
248 thread_cond_init(&newthread->exit_condition);
249 thread_mutex_init(&newthread->exit_lock);
250 newthread->state = THREAD_STATE_RUNNABLE;
251 newthread->detached = false;
252 newthread->joining = false;
253 newthread->in_exception = false;
254 newthread->used_fpu = false;
255 newthread->paused = false;
256 newthread->slab = NULL;
260 * \brief Handle FPU context switching
262 static void fpu_context_switch(struct dispatcher_generic *disp_gen,
263 struct thread * next)
265 #ifdef FPU_LAZY_CONTEXT_SWITCH
266 if(disp_gen->fpu_thread == NULL) {
267 // FPU wasn't used -- no switching necessary
271 // Switch FPU trap back on if we switch away from FPU thread
272 if(disp_gen->fpu_thread == disp_gen->current &&
273 disp_gen->fpu_thread != next) {
280 * \brief Returns false if the stack pointer is out of bounds.
282 static bool thread_check_stack_bounds(struct thread *thread,
283 arch_registers_state_t *archregs) {
284 lvaddr_t sp = (lvaddr_t) registers_get_sp(archregs);
285 return sp > (lvaddr_t)thread->stack ||
286 sp <= (lvaddr_t)thread->stack_top;
290 * \brief Schedule and run the next active thread, or yield the dispatcher.
292 * This may only be called from the dispatcher (on its stack and while
295 * \param disp Dispatcher pointer
297 void thread_run_disabled(dispatcher_handle_t handle)
299 struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle);
300 struct dispatcher_shared_generic *disp =
301 get_dispatcher_shared_generic(handle);
302 arch_registers_state_t *enabled_area =
303 dispatcher_get_enabled_save_area(handle);
305 if (disp_gen->current != NULL) {
306 assert_disabled(disp_gen->runq != NULL);
308 // check stack bounds
309 warn_disabled(&stack_warned,
310 thread_check_stack_bounds(disp_gen->current, enabled_area));
312 struct thread *next = disp_gen->current->next;
313 assert_disabled(next != NULL);
314 if (next != disp_gen->current) {
315 fpu_context_switch(disp_gen, next);
317 // save previous thread's state
318 arch_registers_state_t *cur_regs = &disp_gen->current->regs;
319 memcpy(cur_regs, enabled_area, sizeof(arch_registers_state_t));
320 disp_gen->current = next;
321 disp_resume(handle, &next->regs);
323 // same thread as before
324 disp_resume(handle, enabled_area);
326 } else if (disp_gen->runq != NULL) {
327 fpu_context_switch(disp_gen, disp_gen->runq);
328 disp_gen->current = disp_gen->runq;
329 disp->haswork = true;
330 disp_resume(handle, &disp_gen->runq->regs);
332 // kernel gave us the CPU when we have nothing to do. block!
333 disp->haswork = havework_disabled(handle);
334 disp_gen->current = NULL;
335 disp_yield_disabled(handle);
339 /** Free all heap/slab-allocated state associated with a thread */
340 static void free_thread(struct thread *thread)
342 #if defined(__x86_64__) // XXX: gungy segment selector stuff
343 assert(thread->thread_seg_selector != 0);
345 __asm("mov %%fs, %0" : "=r" (fs));
346 if (thread->thread_seg_selector == fs) {
347 assert(thread->disp == curdispatcher());
348 struct dispatcher_x86_64 *disp_priv = get_dispatcher_x86_64(thread->disp);
349 // we're freeing the current thread; make sure we reload a valid segment
350 // selector so that curdispatcher() keeps working!
351 __asm volatile("mov %%ax, %%fs"
353 : "a" (disp_priv->disp_seg_selector));
355 ldt_free_segment(thread->thread_seg_selector);
359 if (thread->tls_dtv != NULL) {
360 free(thread->tls_dtv);
363 thread_mutex_lock(&thread_slabs_mutex);
364 acquire_spinlock(&thread_slabs_spinlock);
365 slab_free(&thread_slabs, thread->slab); // frees thread itself
366 release_spinlock(&thread_slabs_spinlock);
367 thread_mutex_unlock(&thread_slabs_mutex);
371 * \brief Creates a new thread that will not be runnable
373 * \param start_func Function to run on the new thread
374 * \param arg Argument to pass to function
375 * \param stacksize Size of stack, in bytes
377 * \returns Thread pointer on success, NULL on failure
379 struct thread *thread_create_unrunnable(thread_func_t start_func, void *arg,
383 assert((stacksize % sizeof(uintptr_t)) == 0);
384 void *stack = malloc(stacksize);
389 // allocate space for TCB + initial TLS data
390 // no mutex as it may deadlock: see comment for thread_slabs_spinlock
391 // thread_mutex_lock(&thread_slabs_mutex);
392 acquire_spinlock(&thread_slabs_spinlock);
393 void *space = slab_alloc(&thread_slabs);
394 release_spinlock(&thread_slabs_spinlock);
395 // thread_mutex_unlock(&thread_slabs_mutex);
401 // split space into TLS data followed by TCB
402 // XXX: this layout is specific to the x86 ABIs! once other (saner)
403 // architectures support TLS, we'll need to break out the logic.
404 void *tls_data = space;
405 struct thread *newthread = (void *)((uintptr_t)space + tls_block_total_len);
408 thread_init(curdispatcher(), newthread);
409 newthread->slab = space;
411 if (tls_block_total_len > 0) {
412 // populate initial TLS data from pristine copy
413 assert(tls_block_init_len <= tls_block_total_len);
414 memcpy(tls_data, tls_block_init_base, tls_block_init_len);
416 // zero-fill remainder
417 memset((char *)tls_data + tls_block_init_len, 0,
418 tls_block_total_len - tls_block_init_len);
420 // create a TLS thread vector
421 struct tls_dtv *dtv = malloc(sizeof(struct tls_dtv) + 1 * sizeof(void *));
425 dtv->dtv[0] = tls_data;
426 newthread->tls_dtv = dtv;
429 // FIXME: make arch-specific
430 #if defined(__x86_64__) || defined(__k1om__)
431 // create segment for TCB
432 errval_t err = ldt_alloc_segment(newthread, &newthread->thread_seg_selector);
433 if (err_is_fail(err)) {
434 DEBUG_ERR(err, "error allocating LDT segment for new thread");
435 free_thread(newthread);
441 newthread->stack = stack;
442 newthread->stack_top = (char *)stack + stacksize;
444 // waste space for alignment, if malloc gave us an unaligned stack
445 newthread->stack_top = (char *)newthread->stack_top
446 - (lvaddr_t)newthread->stack_top % STACK_ALIGNMENT;
449 registers_set_initial(&newthread->regs, newthread, (lvaddr_t)thread_entry,
450 (lvaddr_t)newthread->stack_top,
451 (lvaddr_t)start_func, (lvaddr_t)arg, 0, 0);
457 * \brief Creates a new thread, and makes it runnable
459 * \param start_func Function to run on the new thread
460 * \param arg Argument to pass to function
461 * \param stacksize Size of stack, in bytes
463 * \returns Thread pointer on success, NULL on failure
465 struct thread *thread_create_varstack(thread_func_t start_func, void *arg,
468 struct thread *newthread = thread_create_unrunnable(start_func, arg, stacksize);
471 dispatcher_handle_t handle = disp_disable();
472 struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle);
473 newthread->disp = handle;
474 thread_enqueue(newthread, &disp_gen->runq);
481 * \brief Creates a new thread, and makes it runnable
483 * \param start_func Function to run on the new thread
484 * \param arg Argument to pass to function
486 * \returns Thread pointer on success, NULL on failure
488 struct thread *thread_create(thread_func_t start_func, void *arg)
490 return thread_create_varstack(start_func, arg, THREADS_DEFAULT_STACK_BYTES);
494 * \brief Wait for termination of another thread
496 * \param thread Pointer to thread to wait for
497 * \param retval Pointer to variable to hold return value of thread, or NULL
499 * \returns SYS_ERR_OK on success, error code on error.
501 errval_t thread_join(struct thread *thread, int *retval)
503 assert(thread != NULL);
505 thread_mutex_lock(&thread->exit_lock);
506 if(thread->detached) {
507 // Thread is detached and thus not joinable
508 thread_mutex_unlock(&thread->exit_lock);
509 return LIB_ERR_THREAD_JOIN_DETACHED;
512 if(thread->joining) {
513 // Someone else already joins, that's an error
514 thread_mutex_unlock(&thread->exit_lock);
515 return LIB_ERR_THREAD_JOIN;
518 thread->joining = true;
519 if(thread->state != THREAD_STATE_EXITED) { // Possibly wait for thread exit
520 thread_cond_wait(&thread->exit_condition, &thread->exit_lock);
524 *retval = thread->return_value;
527 thread_mutex_unlock(&thread->exit_lock); // Not really needed
534 * \brief Detach a thread. Free its state when it terminates.
536 * \param thread Pointer to thread to detach
538 * \return SYS_ERR_OK on success.
540 errval_t thread_detach(struct thread *thread)
542 assert(thread != NULL);
543 thread_mutex_lock(&thread->exit_lock);
545 if(thread->joining) {
546 // Someone else already joins, that's an error
547 thread_mutex_unlock(&thread->exit_lock);
548 return LIB_ERR_THREAD_JOIN;
551 if(!thread->detached) {
552 thread->detached = true;
554 // Detaching more than once is an error
555 thread_mutex_unlock(&thread->exit_lock);
556 return LIB_ERR_THREAD_DETACHED;
559 if(thread->state == THREAD_STATE_EXITED) {
560 // Thread already exited before we detached, clean it up
565 thread_mutex_unlock(&thread->exit_lock);
570 * \brief Returns the thread pointer to the currently-running thread
572 struct thread *thread_self(void)
575 #if defined(__x86_64__) // XXX: AB's silly little arch-specific optimisation
576 __asm("movq %%fs:0, %0" : "=r" (me));
578 // it's not necessary to disable, but might be once we do migration
579 dispatcher_handle_t handle = disp_disable();
580 struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle);
581 me = disp_gen->current;
587 uintptr_t thread_id(void)
589 return thread_self()->id;
592 void thread_set_id(uintptr_t id)
594 struct thread *me = thread_self();
599 * \brief Yield the calling thread
601 * Switches to the next runnable thread in this dispatcher, or if none is
602 * available, yields the dispatcher.
604 void thread_yield(void)
606 dispatcher_handle_t handle = disp_disable();
607 struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle);
608 struct dispatcher_shared_generic *disp =
609 get_dispatcher_shared_generic(handle);
610 arch_registers_state_t *enabled_area =
611 dispatcher_get_enabled_save_area(handle);
613 struct thread *me = disp_gen->current;
614 struct thread *next = me;
615 me->yield_epoch = disp_gen->timeslice;
618 assert_disabled(next != NULL);
621 break; // Everybody yielded this timeslice
623 } while(next->yield_epoch == disp_gen->timeslice);
626 fpu_context_switch(disp_gen, next);
627 disp_gen->current = next;
628 disp_switch(handle, &me->regs, &next->regs);
630 assert_disabled(disp_gen->runq != NULL);
631 assert_disabled(disp->haswork);
632 trace_event(TRACE_SUBSYS_THREADS, TRACE_EVENT_THREADS_C_DISP_SAVE, 3);
633 disp_save(handle, enabled_area, true, CPTR_NULL);
638 * \brief Yield both the calling thread, and the dispatcher to another domain
640 * \param endpoint Endpoint cap to which we wish to yield, or #CAP_NULL
641 * for an undirected yield
643 * Yields the dispatcher, optionally to another specified dispatcher.
645 void thread_yield_dispatcher(struct capref endpoint)
647 dispatcher_handle_t handle = disp_disable();
648 struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle);
649 struct dispatcher_shared_generic *disp =
650 get_dispatcher_shared_generic(handle);
651 arch_registers_state_t *enabled_area =
652 dispatcher_get_enabled_save_area(handle);
654 assert_disabled(disp_gen->runq != NULL);
655 assert_disabled(disp->haswork);
657 trace_event(TRACE_SUBSYS_THREADS, TRACE_EVENT_THREADS_C_DISP_SAVE, 1);
658 disp_save(handle, enabled_area, true, get_cap_addr(endpoint));
661 /// Function that runs on the static thread/stack to clean up a "real" (alloced) thread
662 static int cleanup_thread(void *arg)
664 struct thread *thread = arg;
666 // free old thread and its stack
667 if (thread != NULL) {
671 // disable and release static thread
672 dispatcher_handle_t handle = disp_disable();
673 struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle);
674 struct dispatcher_shared_generic *disp =
675 get_dispatcher_shared_generic(handle);
676 struct thread *me = disp_gen->current;
678 thread_mutex_unlock_disabled(handle, &disp_gen->cleanupthread_lock);
681 // run the next thread, if any
682 struct thread *next = me->next;
683 thread_remove_from_queue(&disp_gen->runq, me);
685 disp_gen->current = next;
686 disp_resume(handle, &next->regs);
688 disp_gen->current = NULL;
689 disp->haswork = havework_disabled(handle);
690 disp_yield_disabled(handle);
697 * \brief Terminate the calling thread
699 void thread_exit(void)
701 struct thread *me = thread_self();
703 thread_mutex_lock(&me->exit_lock);
705 // if this is the static thread, we don't need to do anything but cleanup
706 if (me == &staticthread) {
707 assert(me->detached);
708 // disable and release static thread
709 dispatcher_handle_t handle = disp_disable();
710 struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle);
711 struct dispatcher_shared_generic *disp =
712 get_dispatcher_shared_generic(handle);
713 assert_disabled(me == &staticthread);
714 assert_disabled(me->stack == staticstack);
716 thread_mutex_unlock_disabled(handle, &staticthread_lock);
719 #ifdef FPU_LAZY_CONTEXT_SWITCH
720 // No more FPU usage from here on
721 if(disp_gen->fpu_thread == me) {
722 disp_gen->fpu_thread = NULL;
727 // run the next thread, if any
728 struct thread *next = me->next;
729 thread_remove_from_queue(&disp_gen->runq, me);
731 fpu_context_switch(disp_gen, next);
732 disp_gen->current = next;
733 disp_resume(handle, &next->regs);
735 disp_gen->current = NULL;
736 disp->haswork = havework_disabled(handle);
737 disp_yield_disabled(handle);
742 // otherwise, we use a dispatcher-local thread to perform cleanup
743 struct dispatcher_generic *dg = get_dispatcher_generic(curdispatcher());
744 thread_mutex_lock(&dg->cleanupthread_lock);
745 if(dg->cleanupthread == NULL) {
747 thread_create_unrunnable(cleanup_thread, me,
748 THREADS_DEFAULT_STACK_BYTES);
750 thread_init(curdispatcher(), dg->cleanupthread);
752 registers_set_initial(&dg->cleanupthread->regs, dg->cleanupthread,
753 (lvaddr_t)cleanup_thread,
754 (lvaddr_t)dg->cleanupthread->stack_top, (lvaddr_t)me,
757 // Switch to it (on this dispatcher)
758 dispatcher_handle_t handle = disp_disable();
759 struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle);
761 #ifdef FPU_LAZY_CONTEXT_SWITCH
762 // No more FPU usage from here on
763 if(disp_gen->fpu_thread == me) {
764 disp_gen->fpu_thread = NULL;
769 thread_remove_from_queue(&disp_gen->runq, me);
770 thread_enqueue(dg->cleanupthread, &disp_gen->runq);
771 disp_gen->cleanupthread->disp = handle;
772 fpu_context_switch(disp_gen, dg->cleanupthread);
773 disp_gen->current = dg->cleanupthread;
774 disp_resume(handle, &dg->cleanupthread->regs);
776 // We're not detached -- wakeup joiner
777 me->return_value = 0; // XXX: Should be an argument to thread_exit()
778 me->state = THREAD_STATE_EXITED;
779 thread_cond_signal(&me->exit_condition);
781 // Disable and unlock exit lock
782 dispatcher_handle_t handle = disp_disable();
783 struct thread *wakeup =
784 thread_mutex_unlock_disabled(handle, &me->exit_lock);
785 struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle);
786 struct dispatcher_shared_generic *disp =
787 get_dispatcher_shared_generic(handle);
789 assert_disabled(wakeup == NULL);
791 #ifdef FPU_LAZY_CONTEXT_SWITCH
792 // No more FPU usage from here on
793 if(disp_gen->fpu_thread == me) {
794 disp_gen->fpu_thread = NULL;
799 // run the next thread, if any
800 struct thread *next = me->next;
801 thread_remove_from_queue(&disp_gen->runq, me);
803 fpu_context_switch(disp_gen, next);
804 disp_gen->current = next;
805 disp_resume(handle, &next->regs);
807 disp_gen->current = NULL;
808 disp->haswork = havework_disabled(handle);
809 disp_yield_disabled(handle);
813 USER_PANIC("should never be reached");
817 * \brief Block the caller, and optionally release a spinlock, while disabled
819 * The caller is unconditionally blocked, and placed into the given queue
820 * pending a call that will unblock it. After manipulating the queues, and
821 * before switching threds, the given spinlock, if specified, is unlocked.
822 * This function must only be called while disabled.
824 * This function is intended for use by multi-processor thread synchronisation
827 * \param disp Dispatcher pointer
828 * \param queue (Optional) Queue of threads in which to place caller
829 * \param spinlock (Optional) pointer to spinlock
831 * \returns Argument passed to thread_unblock, when unblocked
833 void *thread_block_and_release_spinlock_disabled(dispatcher_handle_t handle,
834 struct thread **queue,
835 spinlock_t *spinlock)
837 struct dispatcher_shared_generic *disp =
838 get_dispatcher_shared_generic(handle);
839 struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle);
840 struct thread *me = disp_gen->current;
841 struct thread *next = me->next;
842 assert_disabled(next != NULL);
844 assert_disabled(me->state == THREAD_STATE_RUNNABLE);
845 me->state = THREAD_STATE_BLOCKED;
847 thread_remove_from_queue(&disp_gen->runq, me);
849 thread_enqueue(me, queue);
852 if (spinlock != NULL) {
853 release_spinlock(spinlock);
857 assert_disabled(disp_gen->runq != NULL);
858 fpu_context_switch(disp_gen, next);
859 disp_gen->current = next;
860 disp_switch(handle, &me->regs, &next->regs);
862 assert_disabled(disp_gen->runq == NULL);
863 disp_gen->current = NULL;
864 disp->haswork = havework_disabled(handle);
865 trace_event(TRACE_SUBSYS_THREADS, TRACE_EVENT_THREADS_C_DISP_SAVE, 2);
866 disp_save(handle, &me->regs, true, CPTR_NULL);
869 assert(me->disp == handle); // didn't migrate while asleep
870 return me->wakeup_reason;
874 * \brief Block the calling thread, while disabled
876 * The caller is unconditionally blocked, and placed into the given queue
877 * pending a call that will unblock it.
878 * This function must only be called while disabled.
880 * \param disp Dispatcher pointer
881 * \param queue Queue of threads in which to place caller
883 * \returns Argument passed to thread_unblock, when unblocked
885 void *thread_block_disabled(dispatcher_handle_t disp, struct thread **queue)
887 return thread_block_and_release_spinlock_disabled(disp, queue, NULL);
891 * \brief Block the calling thread, while enabled
893 * The caller is unconditionally blocked, and placed into the given queue
894 * pending a call that will unblock it.
895 * This function must only be called while enabled.
897 * \param queue Queue of threads in which to place caller
899 * \returns Argument passed to thread_unblock, when unblocked
901 void *thread_block(struct thread **queue)
903 return thread_block_disabled(disp_disable(), queue);
907 * \brief Unblock a single thread from a given queue, while disabled
909 * A single thread is removed from the queue of blocked threads, and awoken.
910 * This function must only be called while disabled.
912 * \param disp Dispatcher pointer
913 * \param queue Queue of threads from which to unblock one
914 * \param reason Value to be returned from thread_block()
916 * \returns Pointer to thread to be woken on a foreign dispatcher
918 struct thread *thread_unblock_one_disabled(dispatcher_handle_t handle,
919 struct thread **queue,
922 assert_disabled(queue != NULL);
924 // Any threads in queue?
925 if (*queue == NULL) {
929 // Wakeup one waiting thread
930 struct thread *wakeup = thread_dequeue(queue);
931 wakeup->wakeup_reason = reason;
932 assert_disabled(wakeup->state == THREAD_STATE_BLOCKED);
933 wakeup->state = THREAD_STATE_RUNNABLE;
935 /* enqueue on run queue if it's "our" thread, and not paused */
936 if (wakeup->disp == handle) {
937 if (!wakeup->paused) {
938 struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle);
939 thread_enqueue(wakeup, &disp_gen->runq);
948 * \brief Unblock a single thread from a given queue, while enabled
950 * A single thread is removed from the queue of blocked threads, and awoken.
951 * This function must only be called while enabled.
953 * \param queue Queue of threads from which to unblock one
954 * \param reason Value to be returned from thread_block()
956 * \returns Pointer to thread to be woken on a foreign dispatcher
958 struct thread *thread_unblock_one(struct thread **queue, void *reason)
960 return thread_unblock_one_disabled(disp_disable(), queue, reason);
964 * \brief Unblock all threads on a given queue, while disabled
966 * All threads on the queue of blocked threads are awoken.
967 * This function must only be called while disabled.
969 * \param disp Dispatcher pointer
970 * \param queue Queue of threads to unblock
971 * \param reason Value to be returned from thread_block()
973 * \returns Pointer to list of threads to be woken on a foreign dispatcher
975 struct thread *thread_unblock_all_disabled(dispatcher_handle_t handle,
976 struct thread **queue, void *reason)
978 assert_disabled(queue != NULL);
979 struct thread *wakeupq = NULL;
981 // Wakeup all waiting threads
982 while (*queue != NULL) {
983 struct thread *wakeup = thread_unblock_one_disabled(handle, queue, reason);
984 if (wakeup != NULL) {
985 wakeup->next = wakeupq;
993 extern int _main(int argc, const char *argv[]);
995 /// Thread created in new domain that runs main()
996 static int main_thread(void *params)
998 struct spawn_domain_params *p = params;
999 exit(_main(p->argc, p->argv));
1000 return EXIT_FAILURE;
1003 static bool init_domain_global; // XXX
1005 /// Thread created on static stack in new domain that runs init code
1006 static int bootstrap_thread(struct spawn_domain_params *params)
1007 //int bootstrap_thread(struct spawn_domain_params *params);
1008 //int bootstrap_thread(struct spawn_domain_params *params)
1012 // Set libc function pointers
1013 barrelfish_libc_glue_init();
1015 if (params == NULL) {
1016 printf("%s: error in creating a thread, NULL parameters given\n",
1019 assert(params != NULL);
1021 // Do we have TLS data?
1022 tls_block_init_base = params->tls_init_base;
1023 tls_block_init_len = params->tls_init_len;
1024 tls_block_total_len = params->tls_total_len;
1026 // Initialize subsystems
1027 err = barrelfish_init_onthread(params);
1028 if (err_is_fail(err)) {
1029 DEBUG_ERR(err, "error during libbarrelfish init");
1031 assert(!"exit returned!");
1034 // Allocate storage region for real threads
1035 size_t blocksize = sizeof(struct thread) + tls_block_total_len;
1036 err = vspace_mmu_aware_init(&thread_slabs_vm, MAX_THREADS * blocksize);
1037 if (err_is_fail(err)) {
1038 USER_PANIC_ERR(err, "vspace_mmu_aware_init for thread region failed\n");
1040 slab_init(&thread_slabs, blocksize, refill_thread_slabs);
1042 if (init_domain_global) {
1043 // run main() on this thread, since we can't allocate
1044 if (tls_block_total_len > 0) {
1045 USER_PANIC("unsupported: use of TLS data in bootstrap domain\n");
1047 main_thread(params);
1049 // Start real thread to run main()
1050 struct thread *thread = thread_create(main_thread, params);
1051 assert(thread != NULL);
1054 return 0; // ignored
1058 * \brief Initialise thread system while still disabled
1060 * This function initialises the thread system while the dispatcher is still
1061 * disabled, before enabling the dispatcher, running the general initialisation
1062 * code, and calling main().
1064 * \param disp Dispatcher pointer
1065 * \param init_domain True if we are a bootstrap domain
1067 void thread_init_disabled(dispatcher_handle_t handle, bool init_domain)
1069 struct dispatcher_shared_generic *disp =
1070 get_dispatcher_shared_generic(handle);
1071 struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle);
1072 arch_registers_state_t *enabled_area =
1073 dispatcher_get_enabled_save_area(handle);
1075 init_domain_global = init_domain;
1077 // Create the first thread manually
1078 struct thread *thread = &staticthread;
1079 staticthread_lock.locked = true; // XXX: safe while disabled
1081 // waste space for alignment, if unaligned
1082 thread->stack_top = (char *)thread->stack_top
1083 - (lvaddr_t)thread->stack_top % STACK_ALIGNMENT;
1085 // Initialise the first (static) thread
1086 thread_init(handle, thread);
1087 thread->detached = true;
1089 #if defined(__x86_64__)
1090 // create segment for TCB
1091 errval_t err = ldt_alloc_segment_disabled(handle, thread,
1092 &thread->thread_seg_selector);
1093 if (err_is_fail(err)) {
1094 USER_PANIC_ERR(err, "error allocating LDT segment for first thread");
1099 registers_get_param(enabled_area, ¶m);
1101 registers_set_initial(&thread->regs, thread, (lvaddr_t)thread_entry,
1102 /* TODO: pass stack base and limit, choose in arch
1103 * code (possibly setting up other hints on stack) */
1104 (lvaddr_t)thread->stack_top,
1105 (lvaddr_t)bootstrap_thread, param, 0, 0);
1107 // Switch to it (always on this dispatcher)
1108 thread->disp = handle;
1109 thread_enqueue(thread, &disp_gen->runq);
1110 disp_gen->current = thread;
1111 disp->haswork = true;
1112 disp_resume(handle, &thread->regs);
1116 * \brief Called on the remote core when spanning a domain across cores
1118 * Runs the provided thread after enqueuing it and enabling the dispatcher
1120 void thread_init_remote(dispatcher_handle_t handle, struct thread *thread)
1122 struct dispatcher_shared_generic *disp =
1123 get_dispatcher_shared_generic(handle);
1124 struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle);
1125 thread_enqueue(thread, &disp_gen->runq);
1126 disp_gen->current = thread;
1127 disp->haswork = true;
1128 disp_resume(handle, &thread->regs);
1132 * \brief Prepare to span the current domain
1134 * This is a kludge. It is called from domain.c when creating a new dispatcher,
1135 * and is responsible for pre-allocating all the storage that might be needed
1136 * for thread metadata in the slab allocator. It can go away once we sanely
1137 * manage the vspace across multiple dispatchers in a domain.
1139 void threads_prepare_to_span(dispatcher_handle_t newdh)
1146 thread_mutex_lock(&thread_slabs_mutex);
1147 acquire_spinlock(&thread_slabs_spinlock);
1149 while (slab_freecount(&thread_slabs) < MAX_THREADS - 1) {
1150 struct capref frame;
1155 err = slot_alloc(&frame);
1156 if (err_is_fail(err)) {
1157 USER_PANIC_ERR(err, "in slot_alloc while prefilling thread slabs\n");
1160 size_t blocksize = sizeof(struct thread) + tls_block_total_len;
1161 err = vspace_mmu_aware_map(&thread_slabs_vm, frame, blocksize,
1163 if (err_is_fail(err)) {
1165 if (err_no(err) == LIB_ERR_VSPACE_MMU_AWARE_NO_SPACE) {
1166 // we've wasted space with fragmentation
1167 // cross our fingers and hope for the best...
1170 USER_PANIC_ERR(err, "in vspace_mmu_aware_map while prefilling "
1174 slab_grow(&thread_slabs, buf, size);
1177 release_spinlock(&thread_slabs_spinlock);
1178 thread_mutex_unlock(&thread_slabs_mutex);
1183 * \brief Pause (suspend execution of) the given thread, and optionally capture its register state
1185 * The thread will not be run, until a subsequent call to thread_resume()
1187 void thread_pause_and_capture_state(struct thread *thread,
1188 arch_registers_state_t **ret_regs,
1189 arch_registers_fpu_state_t **ret_fpuregs)
1191 assert(thread != NULL);
1192 dispatcher_handle_t dh = disp_disable();
1193 struct dispatcher_generic *disp = get_dispatcher_generic(dh);
1194 if (thread->disp == dh) {
1195 if (!thread->paused) {
1196 thread->paused = true;
1197 if (thread == disp->current) { // doesn't make much sense...
1198 sys_print("Warning: pausing current thread!\n",100);
1199 assert_disabled(thread->state == THREAD_STATE_RUNNABLE);
1200 thread_block_disabled(dh, NULL);
1201 } else if (thread->state == THREAD_STATE_RUNNABLE) {
1202 thread_remove_from_queue(&disp->runq, thread);
1205 if (ret_regs != NULL) {
1206 *ret_regs = &thread->regs;
1208 if (ret_fpuregs != NULL) {
1209 if (thread->used_fpu) {
1210 // FIXME: this may not be the right FPU state?
1211 *ret_fpuregs = &thread->fpu_state;
1213 *ret_fpuregs = NULL;
1217 USER_PANIC("NYI: remote dispatcher thread_pause()");
1223 * \brief Pause (suspend execution of) the given thread
1225 * The thread will not be run, until a subsequent call to thread_resume()
1227 void thread_pause(struct thread *thread)
1229 thread_pause_and_capture_state(thread, NULL, NULL);
1233 * \brief Resume execution of a thread previously suspended by thread_pause()
1235 void thread_resume(struct thread *thread)
1237 assert(thread != NULL);
1238 dispatcher_handle_t dh = disp_disable();
1239 struct dispatcher_generic *disp = get_dispatcher_generic(dh);
1240 if (thread->disp == dh) {
1241 if (thread->paused) {
1242 thread->paused = false;
1243 if (thread->state == THREAD_STATE_RUNNABLE) {
1244 thread_enqueue(thread, &disp->runq);
1248 USER_PANIC("NYI: remote dispatcher thread_resume()");
1254 * \brief Set old-style thread-local storage pointer.
1255 * \param p User's pointer
1257 void thread_set_tls(void *p)
1259 struct thread *me = thread_self();
1263 void thread_set_tls_key(int key, void *p)
1265 struct thread *me = thread_self();
1266 me->userptrs[key] = p;
1270 * \brief Return old-style thread-local storage pointer.
1271 * \return User's pointer, previously passed to thread_set_tls()
1273 void *thread_get_tls(void)
1275 struct thread *me = thread_self();
1279 void *thread_get_tls_key(int key)
1281 struct thread *me = thread_self();
1282 return me->userptrs[key];
1286 * \brief Set the exception handler function for the current thread.
1287 * Optionally also change its stack, and return the old values.
1289 * \param newhandler New exception handler. Pass NULL to disable an existing handler.
1290 * \param oldhandler If non-NULL, returns previous exception handler
1291 * \param new_stack_base If non-NULL, sets a new exception handler stack (base)
1292 * \param new_stack_top If non-NULL, sets a new exception handler stack (top)
1293 * \param old_stack_base If non-NULL, returns previous stack base
1294 * \param old_stack_top If non-NULL, returns previous stack top
1296 errval_t thread_set_exception_handler(exception_handler_fn newhandler,
1297 exception_handler_fn *oldhandler,
1298 void *new_stack_base, void *new_stack_top,
1299 void **old_stack_base, void **old_stack_top)
1301 struct thread *me = thread_self();
1303 if (oldhandler != NULL) {
1304 *oldhandler = me->exception_handler;
1307 if (old_stack_base != NULL) {
1308 *old_stack_base = me->exception_stack;
1311 if (old_stack_top != NULL) {
1312 *old_stack_top = me->exception_stack_top;
1315 me->exception_handler = newhandler;
1317 if (new_stack_base != NULL && new_stack_top != NULL) {
1318 me->exception_stack = new_stack_base;
1319 me->exception_stack_top = new_stack_top;
1325 static void exception_handler_wrapper(arch_registers_state_t *cpuframe,
1326 arch_registers_fpu_state_t *fpuframe,
1327 uintptr_t hack_arg, void *addr)
1329 struct thread *me = thread_self();
1331 assert(me->in_exception);
1332 assert(me->exception_handler != NULL);
1334 // XXX: unpack hack arg
1335 enum exception_type type = hack_arg >> 16;
1336 int subtype = hack_arg & 0xffff;
1339 me->exception_handler(type, subtype, addr, cpuframe, fpuframe);
1342 dispatcher_handle_t dh = disp_disable();
1343 struct dispatcher_generic *disp_gen = get_dispatcher_generic(dh);
1344 //memcpy(&me->regs, cpuframe, sizeof(arch_registers_state_t));
1346 #ifdef FPU_LAZY_CONTEXT_SWITCH
1347 if (fpuframe != NULL) {
1348 assert_disabled(me->used_fpu);
1349 arch_registers_fpu_state_t *dest;
1350 if (disp_gen->fpu_thread == me) {
1351 dest = dispatcher_get_enabled_fpu_save_area(dh);
1353 dest = &me->fpu_state;
1355 fpu_copy(dest, fpuframe);
1360 assert_disabled(me->in_exception);
1361 me->in_exception = false;
1363 assert_disabled(disp_gen->current == me);
1364 disp_resume(dh, cpuframe);
1368 void thread_debug_regs(struct thread *t);
1369 void thread_debug_regs(struct thread *t)
1371 printf("%d: RIP = %lx, RSP = %lx\n", disp_get_domain_id(),
1372 t->regs.rip, t->regs.rsp);
1373 uint64_t *stack = (uint64_t *)t->regs.rsp;
1374 printf("%d: ", disp_get_domain_id());
1375 for(int i = 0; i < 30; i++) {
1376 printf("%lx ", stack[i]);
1383 * \brief Deliver an exception to the current thread, and resume.
1385 * This may only be called from the dispatcher (on its stack and while
1388 * \param handle Dispatcher handle
1389 * \param type Exception type
1390 * \param subtype Exception subtype
1391 * \param addr Exception address
1392 * \param regs CPU register state at time of exception
1394 void thread_deliver_exception_disabled(dispatcher_handle_t handle,
1395 enum exception_type type, int subtype,
1396 void *addr, arch_registers_state_t *regs)
1398 struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle);
1399 struct thread *thread = disp_gen->current;
1400 assert_disabled(thread != NULL);
1401 assert_disabled(disp_gen->runq != NULL);
1403 // can we deliver the exception?
1404 if (thread->exception_handler == NULL || thread->exception_stack_top == NULL
1405 || thread->in_exception) {
1406 if (thread->in_exception) {
1407 sys_print("Can't deliver exception to thread: already in handler\n",
1410 sys_print("Can't deliver exception to thread: handler not set\n",
1414 // warn on stack overflow.
1415 lvaddr_t sp = (lvaddr_t) registers_get_sp(regs);
1416 if (sp < (lvaddr_t)thread->stack ||
1417 sp > (lvaddr_t)thread->stack_top) {
1419 snprintf(str, sizeof(str), "Error: stack bounds exceeded: sp = 0x%"
1420 PRIxPTR " but [bottom, top] = [0x%" PRIxPTR ", 0x%"
1421 PRIxPTR "]\n", (lvaddr_t) sp, (lvaddr_t) thread->stack,
1422 (lvaddr_t) thread->stack_top);
1423 sys_print(str, sizeof(str));
1426 // TODO: actually delete the thread!
1427 disp_gen->current = NULL;
1428 thread_remove_from_queue(&disp_gen->runq, thread);
1432 thread->in_exception = true;
1434 lvaddr_t stack_top = (lvaddr_t)thread->exception_stack_top;
1436 // save thread's state at time of fault on top of exception stack
1437 stack_top -= sizeof(arch_registers_state_t);
1438 arch_registers_state_t *cpuframe = (void *)stack_top;
1439 memcpy(cpuframe, regs, sizeof(arch_registers_state_t));
1441 arch_registers_fpu_state_t *fpuframe = NULL;
1442 #ifdef FPU_LAZY_CONTEXT_SWITCH
1443 if (thread->used_fpu) {
1444 stack_top -= sizeof(arch_registers_fpu_state_t);
1445 fpuframe = (void *)stack_top;
1446 fpu_copy(fpuframe, &thread->fpu_state);
1451 stack_top -= stack_top % STACK_ALIGNMENT;
1453 // XXX: sanity-check to ensure we have a sensible amount of exception stack left
1454 assert_disabled(stack_top > (lvaddr_t)thread->exception_stack + 8192);
1456 // XXX: pack two small ints together to fit into a single register
1457 uintptr_t hack_arg = (uintptr_t)type << 16 | (subtype & 0xffff);
1459 registers_set_initial(&thread->regs, thread,
1460 (lvaddr_t)exception_handler_wrapper,
1461 stack_top, (lvaddr_t)cpuframe, (lvaddr_t)fpuframe,
1462 hack_arg, (lvaddr_t)addr);
1464 disp_resume(handle, &thread->regs);