Add per-spawnd message queues to the process manager.
authorRazvan Damachi <razvan.damachi@gmail.com>
Sun, 2 Jul 2017 14:25:53 +0000 (16:25 +0200)
committerSimon Gerber <simon.gerber@inf.ethz.ch>
Thu, 31 Aug 2017 14:35:08 +0000 (16:35 +0200)
The process manager will now enqueue requests to be sent to spawnds. There is
one queue per spawnd server that the process manager is connected to. The
change is meant so that if multiple clients send requests meant to be served
by the same spawnd (e.g. spawn multiple domains on the same core), the process
manager will not return an error if the target spawnd's Flounder queue is full,
but instead queue up the requests to be sent later.

In order to accommodate for this change, the spawn API exposed to the process
manager now features a specific reply for each request type. The proces
manager handles every reply type separately, meaning that every client can now
send up to 1 request of each type exposed by the proc_mgmt API, up from a max
of 1 request of any type at any given time.

Signed-off-by: Razvan Damachi <razvan.damachi@gmail.com>

if/proc_mgmt.if
if/spawn.if
lib/barrelfish/proc_mgmt_client.c
usr/proc_mgmt/pending_clients.c
usr/proc_mgmt/pending_clients.h
usr/proc_mgmt/service.c
usr/proc_mgmt/spawnd_state.h
usr/spawnd/service.c

index 6ff2980..4b02a0c 100644 (file)
@@ -39,7 +39,8 @@ interface proc_mgmt "Process management service" {
   rpc kill(in cap domain_cap, out errval err);
 
   // Let the process manager know the caller has finished execution.
-  message exit(cap domain_cap, uint8 status);
+  //message exit(cap domain_cap, uint8 status);
+  rpc exit(in cap domain_cap, in uint8 status);
 
   rpc wait(in cap domain_wap, out errval err, out uint8 status);
 
index eedfa28..a501255 100644 (file)
@@ -27,6 +27,8 @@ interface spawn "Interface to spawn domains" {
                           char argvbuf[argvbytes, 2048],
                           char envbuf[envbytes, 2048],
                           uint8 flags);
+    message spawn_reply(cap domain_cap, errval err);
+
     message spawn_with_caps_request(cap procmng_cap,
                                     cap domain_cap,
                                     String path[2048],
@@ -35,11 +37,20 @@ interface spawn "Interface to spawn domains" {
                                     cap inheritcn_cap,
                                     cap argcn_cap,
                                     uint8 flags);
+    message spawn_with_caps_reply(cap domain_cap, errval err);
+
     message span_request(cap procmng_cap, cap domain_cap, cap vroot,
                          cap dispframe);
+    message span_reply(cap domain_cap, errval err);
+
     message kill_request(cap procmng_cap, cap domain_cap);
+    message kill_reply(cap domain_cap, errval err);
+
+    message exit_request(cap procmng_cap, cap domain_cap);
+    message exit_reply(cap domain_cap, errval err);
+
     message cleanup_request(cap procmng_cap, cap domain_cap);
-    message spawn_reply(cap domain_cap, errval err);
+    message cleanup_reply(cap domain_cap, errval err);
 
     rpc spawn_proc_mgmt_domain(in cap domain_cap,
                                in String path[2048],
index 3dfc77d..8ab4406 100644 (file)
@@ -491,7 +491,7 @@ errval_t proc_mgmt_kill(struct capref domain_cap)
 /**
  * \brief Inform the process manager about exiting execution.
  */
-errval_t proc_mgmt_exit(uint8_t status )
+errval_t proc_mgmt_exit(uint8_t status)
 {
     errval_t err = proc_mgmt_bind_client();
     if (err_is_fail(err)) {
@@ -501,7 +501,7 @@ errval_t proc_mgmt_exit(uint8_t status )
     struct proc_mgmt_binding *b = get_proc_mgmt_binding();
     assert(b != NULL);
 
-    err = b->tx_vtbl.exit(b, NOP_CONT, cap_domainid, status);
+    err = b->rpc_tx_vtbl.exit(b, cap_domainid, status);
     if (err_is_fail(err)) {
         return err;
     }
index 999bdcf..c9ddc7a 100644 (file)
 #include "domain.h"
 #include "pending_clients.h"
 
-static collections_hash_table* pending_clients_table = NULL;
+static collections_hash_table *spawn_table = NULL;
+static collections_hash_table *spawn_with_caps_table = NULL;
+static collections_hash_table *span_table = NULL;
+static collections_hash_table *kill_table = NULL;
+static collections_hash_table *exit_table = NULL;
+static collections_hash_table *cleanup_table = NULL;
 
 errval_t pending_clients_add(struct capref domain_cap,
                              struct proc_mgmt_binding *b, enum ClientType type,
                              coreid_t core_id)
 {
-    if (pending_clients_table == NULL) {
-        collections_hash_create_with_buckets(&pending_clients_table,
-                                             HASH_INDEX_BUCKETS, NULL);
-        if (pending_clients_table == NULL) {
+    collections_hash_table **table;
+    switch (type) {
+        case ClientType_Spawn:
+            table = &spawn_table;
+            break;
+        case ClientType_SpawnWithCaps:
+            table = &spawn_with_caps_table;
+            break;
+        case ClientType_Span:
+            table = &span_table;
+            break;
+        case ClientType_Kill:
+            table = &kill_table;
+            break;
+        case ClientType_Exit:
+            table = &exit_table;
+            break;
+        case ClientType_Cleanup:
+            table = &cleanup_table;
+            break;
+        default:
+            USER_PANIC("Unhandled client type %d\n", type);
+    }
+
+    if (*table == NULL) {
+        collections_hash_create_with_buckets(table, HASH_INDEX_BUCKETS, NULL);
+        if (*table == NULL) {
             return PROC_MGMT_ERR_CREATE_CLIENTS_TABLE;
         }
     }
@@ -34,18 +62,32 @@ errval_t pending_clients_add(struct capref domain_cap,
         return err;
     }
 
-    struct pending_client *client = (struct pending_client*) malloc(
+    struct pending_client *cl = (struct pending_client*) malloc(
             sizeof(struct pending_client));
-    client->b = b;
-    client->core_id = core_id;
-    client->type = type;
-    collections_hash_insert(pending_clients_table, key, client);
+    cl->b = b;
+    cl->domain_cap = domain_cap;
+    cl->core_id = core_id;
+    cl->type = type;
+    cl->next = NULL;
+
+    if (type == ClientType_Kill) {
+        // Special case: multiple clients might have issued a kill for some
+        // domain. Need to chain them together.
+        void *entry = collections_hash_find(*table, key);
+        if (entry != NULL) {
+            struct pending_client* old = (struct pending_client*) entry;
+            collections_hash_delete(*table, key);
+            cl->next = old;
+        }
+    }
+    
+    collections_hash_insert(*table, key, cl);
 
     return SYS_ERR_OK;
 }
 
-errval_t pending_clients_release(struct capref domain_cap,
-                                        struct pending_client **ret_cl)
+errval_t pending_clients_release(struct capref domain_cap, enum ClientType type,
+                                 struct pending_client **ret_cl)
 {
     uint64_t key;
     errval_t err = domain_cap_hash(domain_cap, &key);
@@ -53,15 +95,114 @@ errval_t pending_clients_release(struct capref domain_cap,
         return err;
     }
 
-    void *table_entry = collections_hash_find(pending_clients_table, key);
-    if (table_entry == NULL) {
+    collections_hash_table **table;
+    switch (type) {
+        case ClientType_Spawn:
+            table = &spawn_table;
+            break;
+        case ClientType_SpawnWithCaps:
+            table = &spawn_with_caps_table;
+            break;
+        case ClientType_Span:
+            table = &span_table;
+            break;
+        case ClientType_Kill:
+            table = &kill_table;
+            break;
+        case ClientType_Exit:
+            table = &exit_table;
+            break;
+        case ClientType_Cleanup:
+            table = &cleanup_table;
+            break;
+        default:
+            USER_PANIC("Unhandled client type %d\n", type);
+    }
+
+    void *entry = collections_hash_find(*table, key);
+    if (entry == NULL) {
         return PROC_MGMT_ERR_CLIENTS_TABLE_FIND;
     }
+    struct pending_client *cl = (struct pending_client*) entry;
     if (ret_cl != NULL) {
-        *ret_cl = (struct pending_client*) table_entry;
+        *ret_cl = cl;
+    } else {
+        free(cl);
     }
 
-    collections_hash_delete(pending_clients_table, key);
+    collections_hash_delete(*table, key);
+
+    return SYS_ERR_OK;
+}
+
+errval_t pending_clients_release_one(struct capref domain_cap,
+                                     enum ClientType type,
+                                     struct proc_mgmt_binding *b,
+                                     struct pending_client **ret_cl)
+{
+    uint64_t key;
+    errval_t err = domain_cap_hash(domain_cap, &key);
+    if (err_is_fail(err)) {
+        return err;
+    }
+
+    collections_hash_table **table;
+    switch (type) {
+        case ClientType_Spawn:
+            table = &spawn_table;
+            break;
+        case ClientType_SpawnWithCaps:
+            table = &spawn_with_caps_table;
+            break;
+        case ClientType_Span:
+            table = &span_table;
+            break;
+        case ClientType_Kill:
+            table = &kill_table;
+            break;
+        case ClientType_Exit:
+            table = &exit_table;
+            break;
+        case ClientType_Cleanup:
+            table = &cleanup_table;
+            break;
+        default:
+            USER_PANIC("Unhandled client type %d\n", type);
+    }
+
+    void *entry = collections_hash_find(*table, key);
+    if (entry == NULL) {
+        return PROC_MGMT_ERR_CLIENTS_TABLE_FIND;
+    }
+    struct pending_client *cl = (struct pending_client*) entry;
+    if (cl->b == b) {
+        struct pending_client *tmp = cl;
+        cl = cl->next;
+        if (ret_cl != NULL) {
+            *ret_cl = tmp;
+        } else {
+            free(tmp);
+        }
+    } else {
+        while (cl->next != NULL) {
+            if (cl->next->b == b) {
+                struct pending_client *tmp = cl->next;
+                cl->next = cl->next->next;
+                if (ret_cl != NULL) {
+                    *ret_cl = tmp;
+                } else {
+                    free(tmp);
+                }
+                break;
+            }
+            cl = cl->next;
+        }
+    }
+
+    collections_hash_delete(*table, key);
+    if (cl != NULL) {
+        collections_hash_insert(*table, key, cl);
+    }
 
     return SYS_ERR_OK;
 }
index 94d239e..234b80f 100644 (file)
@@ -13,7 +13,9 @@
 #define PENDING_CLIENTS_H
 
 #include <barrelfish/barrelfish.h>
+#include <barrelfish/event_queue.h>
 #include <if/proc_mgmt_defs.h>
+#include <if/spawn_defs.h>
 
 #define HASH_INDEX_BUCKETS 6151
 
@@ -23,20 +25,70 @@ enum ClientType {
        ClientType_Span,
        ClientType_Kill,
        ClientType_Exit,
-       ClientType_Wait
+       ClientType_Cleanup
        // TODO(razvan): Others?
 };
 
+struct pending_spawn {
+    struct capref domain_cap;
+
+       struct spawn_binding *b;
+       coreid_t core_id;
+       
+       const char *path;
+       
+       const char *argvbuf;
+       size_t argvbytes;
+       const char *envbuf;
+       size_t envbytes;
+       
+       struct capref inheritcn_cap;
+       struct capref argcn_cap;
+       
+       uint8_t flags;
+
+       struct event_queue_node qn;
+};
+
+struct pending_span {
+    struct capref domain_cap;
+
+       struct spawn_binding *b;
+
+       coreid_t core_id;
+       struct capref vroot;
+       struct capref dispframe;
+
+       struct event_queue_node qn;
+};
+
+struct pending_kill_exit_cleanup {
+    struct capref domain_cap;
+       struct spawn_binding *sb;
+       struct proc_mgmt_binding *pmb;
+       
+       struct event_queue_node qn;
+};
+
 struct pending_client {
     struct proc_mgmt_binding *b;
+
+    struct capref domain_cap;
+
     coreid_t core_id;
     enum ClientType type;
+
+    struct pending_client *next;
 };
 
 errval_t pending_clients_add(struct capref domain_cap,
                              struct proc_mgmt_binding *b, enum ClientType type,
                              coreid_t core_id);
-errval_t pending_clients_release(struct capref domain_cap,
+errval_t pending_clients_release(struct capref domain_cap, enum ClientType type,
                                  struct pending_client **ret_cl);
+errval_t pending_clients_release_one(struct capref domain_cap,
+                                        enum ClientType type,
+                                        struct proc_mgmt_binding *b,
+                                     struct pending_client **ret_cl);
 
 #endif  // PENDING_CLIENTS_H
\ No newline at end of file
index 66fc779..41df760 100644 (file)
@@ -58,262 +58,559 @@ static void add_spawnd_handler_non_monitor(struct proc_mgmt_binding *b,
 }
 
 static void spawn_reply_handler(struct spawn_binding *b,
-                                struct capref domain_cap, errval_t spawn_err)
+                                struct capref domain_cap, errval_t spawn_err);
+static void spawn_with_caps_reply_handler(struct spawn_binding *b,
+                                          struct capref domain_cap,
+                                          errval_t spawn_err);
+static void span_reply_handler(struct spawn_binding *b,
+                               struct capref domain_cap, errval_t span_err);
+static void kill_reply_handler(struct spawn_binding *b,
+                               struct capref domain_cap, errval_t kill_err);
+static void exit_reply_handler(struct spawn_binding *b,
+                               struct capref domain_cap, errval_t exit_err);
+static void cleanup_reply_handler(struct spawn_binding *b,
+                                  struct capref domain_cap,
+                                  errval_t cleanup_err);
+
+static void spawn_request_sender(void *arg)
 {
-    struct pending_client *cl;
-    errval_t err = pending_clients_release(domain_cap, &cl);
-    if (err_is_fail(err)) {
-        // This might be a kill request issued after a successful spawn/span
-        // followed by a local error in the process manager (see below). If that
-        // is the case, then we won't have a client, as it has already been
-        // released.
-        DEBUG_ERR(err, "failed to retrieve pending client based on domain cap "
-                  "returned by spawnd");
-        return;
+    struct pending_spawn *spawn = (struct pending_spawn*) arg;
+
+    errval_t err;
+    bool with_caps = !(capref_is_null(spawn->inheritcn_cap) &&
+                       capref_is_null(spawn->argcn_cap));
+    if (with_caps) {
+        spawn->b->rx_vtbl.spawn_with_caps_reply = spawn_with_caps_reply_handler;
+        err = spawn->b->tx_vtbl.spawn_with_caps_request(spawn->b, NOP_CONT,
+                                                        cap_procmng,
+                                                        spawn->domain_cap,
+                                                        spawn->path,
+                                                        spawn->argvbuf,
+                                                        spawn->argvbytes,
+                                                        spawn->envbuf,
+                                                        spawn->envbytes,
+                                                        spawn->inheritcn_cap,
+                                                        spawn->argcn_cap,
+                                                        spawn->flags);
+    } else {
+        spawn->b->rx_vtbl.spawn_reply = spawn_reply_handler;
+        err = spawn->b->tx_vtbl.spawn_request(spawn->b, NOP_CONT, cap_procmng,
+                                              spawn->domain_cap, spawn->path,
+                                              spawn->argvbuf, spawn->argvbytes,
+                                              spawn->envbuf, spawn->envbytes,
+                                              spawn->flags);
     }
-
-    errval_t resp_err = SYS_ERR_OK;
-    struct domain_entry *entry;
-    switch (cl->type) {
-        case ClientType_Spawn:
-            err = spawn_err;
-            if (err_is_ok(spawn_err)) {
-                err = domain_spawn(domain_cap, cl->core_id);
-            }
-            resp_err = cl->b->tx_vtbl.spawn_response(cl->b, NOP_CONT, err,
-                                                     domain_cap);
-            break;
-
-        case ClientType_SpawnWithCaps:
-            err = spawn_err;
-            if (err_is_ok(spawn_err)) {
-                err = domain_spawn(domain_cap, cl->core_id);
-            }
-            resp_err = cl->b->tx_vtbl.spawn_with_caps_response(cl->b, NOP_CONT,
-                                                               err, domain_cap);
-            break;
-
-        case ClientType_Span:
-            err = spawn_err;
-            if (err_is_ok(spawn_err)) {
-                err = domain_span(domain_cap, cl->core_id);
-            }
-            resp_err = cl->b->tx_vtbl.span_response(cl->b, NOP_CONT, err);
-            break;
-
-        case ClientType_Kill:
-            if (err_is_fail(spawn_err)) {
-                // Looks like some spawnd was unable to successfully kill
-                // its dispatcher for this domain. Not much the process
-                // manager can do about it; return the error to the client.
-                resp_err = cl->b->tx_vtbl.kill_response(cl->b, NOP_CONT,
-                                                        err);
-                break;
+    if (err_is_ok(err)) {
+        free(spawn);
+    } else {
+        if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
+            err = spawn->b->register_send(spawn->b, spawn->b->waitset,
+                                          MKCONT(spawn_request_sender, arg));
+            if (err_is_fail(err)) {
+                DEBUG_ERR(err, "registering for spawn request");
+                pending_clients_release(spawn->domain_cap,
+                                        with_caps ? ClientType_SpawnWithCaps
+                                                  : ClientType_Spawn,
+                                        NULL);
+                event_mutex_unlock(&spawn->b->mutex);
+                free(spawn);
             }
+        } else {
+            DEBUG_ERR(err, "sending spawn request");
+            pending_clients_release(spawn->domain_cap,
+                                    with_caps ? ClientType_SpawnWithCaps
+                                              : ClientType_Spawn,
+                                    NULL);
+            event_mutex_unlock(&spawn->b->mutex);
+            free(spawn);
+        }
+    }
+}
 
-            err = domain_get_by_cap(domain_cap, &entry);
+static void span_request_sender(void *arg)
+{
+    struct pending_span *span = (struct pending_span*) arg;
+
+    errval_t err;
+    span->b->rx_vtbl.span_reply = span_reply_handler;
+    err = span->b->tx_vtbl.span_request(span->b, NOP_CONT, cap_procmng,
+                                        span->domain_cap, span->vroot,
+                                        span->dispframe);
+    if (err_is_ok(err)) {
+        err = domain_span(span->domain_cap, span->core_id);
+        if (err_is_fail(err)) {
+            DEBUG_ERR(err, "failed domain_span to core %u\n", span->core_id);
+        }
+        free(span);
+    } else {
+        if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
+            err = span->b->register_send(span->b, span->b->waitset,
+                                         MKCONT(span_request_sender, arg));
             if (err_is_fail(err)) {
-                DEBUG_ERR(err, "failed to retrieve domain by domain_cap "
-                          "returned by spawnd after kill");
-                break;
+                DEBUG_ERR(err, "registering for span request");
+                pending_clients_release(span->domain_cap, ClientType_Span,
+                                        NULL);
+                event_mutex_unlock(&span->b->mutex);
+                free(span);
             }
+        } else {
+            DEBUG_ERR(err, "sending span request");
+            pending_clients_release(span->domain_cap, ClientType_Span, NULL);
+            event_mutex_unlock(&span->b->mutex);
+            free(span);
+        }
+    }
+}
 
-            assert(entry->num_spawnds_resources > 0 ||
-                   entry->num_spawnds_running > 0);
-            assert(entry->status != DOMAIN_STATUS_CLEANED);
-
-            if (entry->num_spawnds_running > 0) {
-                --entry->num_spawnds_running;
-                    
-                err = pending_clients_add(domain_cap, cl->b,
-                                          ClientType_Kill, MAX_COREID);
-                if (err_is_fail(err)) {
-                    DEBUG_ERR(err, "pending_clients_add in reply handler");
-                }
-
-                if (entry->num_spawnds_running == 0) {
-                    entry->status = DOMAIN_STATUS_STOPPED;
-                    entry->exit_status = EXIT_STATUS_KILLED;
-
-                    // TODO(razvan): Might it be more sane if we respond back
-                    // to the client after the domain has been cleaned up (i.e.
-                    // the cspace root has been revoked for all dispatchers)?
-                    resp_err = cl->b->tx_vtbl.kill_response(cl->b, NOP_CONT,
-                                                           err);
-                    
-                    // TODO(razvan): Same problem applies to the waiters: would
-                    // it be better if we sent them wait_responses after the
-                    // cspace root has been revoked, too? (here and in the exit
-                    // case).
-                    struct domain_waiter *waiter = entry->waiters;
-                    while (waiter != NULL) {
-                        waiter->b->tx_vtbl.wait_response(waiter->b, NOP_CONT,
-                                                         SYS_ERR_OK,
-                                                         entry->exit_status);
-                        struct domain_waiter *aux = waiter;
-                        waiter = waiter->next;
-                        free(aux);
-                    }
+static void kill_request_sender(void *arg)
+{
+    struct pending_kill_exit_cleanup *kill = (struct pending_kill_exit_cleanup*) arg;
 
-                    for (coreid_t i = 0; i < MAX_COREID; ++i) {
-                        if (entry->spawnds[i] == NULL) {
-                            continue;
-                        }
-
-                        struct spawn_binding *spb = entry->spawnds[i]->b;
-                        spb->rx_vtbl.spawn_reply = spawn_reply_handler;
-                        errval_t req_err = spb->tx_vtbl.cleanup_request(spb,
-                                NOP_CONT, cap_procmng, domain_cap);
-                        if (err_is_fail(req_err)) {
-                            DEBUG_ERR(req_err, "failed to send cleanup_request "
-                                      "to spawnd %u\n", i);
-                        }
+    errval_t err;
+    kill->sb->rx_vtbl.kill_reply = kill_reply_handler;
+    err = kill->sb->tx_vtbl.kill_request(kill->sb, NOP_CONT, cap_procmng,
+                                        kill->domain_cap);
+    if (err_is_ok(err)) {
+        free(kill);
+    } else {
+        if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
+            err = kill->sb->register_send(kill->sb, kill->sb->waitset,
+                                         MKCONT(kill_request_sender, arg));
+            if (err_is_fail(err)) {
+                DEBUG_ERR(err, "registering for kill request");
+
+                struct pending_client *cl;
+                err = pending_clients_release_one(kill->domain_cap,
+                                                  ClientType_Kill,
+                                                  kill->pmb, &cl);
+                if (err_is_ok(err)) {
+                    while (cl != NULL) {
+                        struct pending_client *tmp = cl;
+                        cl = cl->next;
+                        free(tmp);
                     }
                 }
-            } else {
-                --entry->num_spawnds_resources;
-
-                if (entry->num_spawnds_resources == 0) {
-                    entry->status = DOMAIN_STATUS_CLEANED;
-
-                    // At this point, the domain exists in state CLEANED for
-                    // history reasons. For instance, if some other domain
-                    // issues a wait call for this one, the process manager can
-                    // return the exit status directly.
-                    // At some point, however, we might want to just clean up
-                    // the domain entry and recycle the domain cap.
-                } else {
-                    // Expecting to receive further cleanup replies from other
-                    // spawnds for the same domain cap, hence re-add the
-                    // pending client.
-                    err = pending_clients_add(domain_cap, cl->b,
-                                              ClientType_Exit, MAX_COREID);
-                    if (err_is_fail(err)) {
-                        DEBUG_ERR(err, "pending_clients_add in reply handler");
-                    }
+
+                event_mutex_unlock(&kill->sb->mutex);
+                free(kill);
+            }
+        } else {
+            DEBUG_ERR(err, "sending kill request");
+            
+            struct pending_client *cl;
+            err = pending_clients_release_one(kill->domain_cap,
+                                              ClientType_Kill,
+                                              kill->pmb, &cl);
+            if (err_is_ok(err)) {
+                while (cl != NULL) {
+                    struct pending_client *tmp = cl;
+                    cl = cl->next;
+                    free(tmp);
                 }
             }
-            break;
-
-        case ClientType_Exit:
-            if (err_is_fail(spawn_err)) {
-                // Looks like some spawnd was unable to successfully kill
-                // its dispatcher for this domain. Not much the process
-                // manager can do about it. Furthermore, this was an exit call,
-                // so there's no client to reply back to.
-                break;
+
+            event_mutex_unlock(&kill->sb->mutex);
+            free(kill);
+        }
+    }
+}
+
+static void exit_request_sender(void *arg)
+{
+    struct pending_kill_exit_cleanup *exit = (struct pending_kill_exit_cleanup*) arg;
+
+    errval_t err;
+    exit->sb->rx_vtbl.exit_reply = exit_reply_handler;
+    err = exit->sb->tx_vtbl.exit_request(exit->sb, NOP_CONT, cap_procmng,
+                                        exit->domain_cap);
+    if (err_is_ok(err)) {
+        free(exit);
+    } else {
+        if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
+            err = exit->sb->register_send(exit->sb, exit->sb->waitset,
+                                         MKCONT(exit_request_sender, arg));
+            if (err_is_fail(err)) {
+                DEBUG_ERR(err, "registering for exit request");
+                err = pending_clients_release(exit->domain_cap, ClientType_Exit,
+                                              NULL);
+                event_mutex_unlock(&exit->sb->mutex);
+                free(exit);
             }
+        } else {
+            DEBUG_ERR(err, "sending exit request");
+            err = pending_clients_release(exit->domain_cap, ClientType_Exit,
+                                          NULL);
+            event_mutex_unlock(&exit->sb->mutex);
+            free(exit);
+        }
+    }
+}
 
-            err = domain_get_by_cap(domain_cap, &entry);
+static void cleanup_request_sender(void *arg)
+{
+    struct pending_kill_exit_cleanup *cleanup = (struct pending_kill_exit_cleanup*) arg;
+
+    errval_t err;
+    cleanup->sb->rx_vtbl.cleanup_reply = cleanup_reply_handler;
+    err = cleanup->sb->tx_vtbl.cleanup_request(cleanup->sb, NOP_CONT, cap_procmng,
+                                              cleanup->domain_cap);
+    if (err_is_ok(err)) {
+        free(cleanup);
+    } else {
+        if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
+            err = cleanup->sb->register_send(cleanup->sb, cleanup->sb->waitset,
+                                            MKCONT(cleanup_request_sender, arg));
             if (err_is_fail(err)) {
-                DEBUG_ERR(err, "failed to retrieve domain by domain_cap "
-                          "returned by spawnd after kill");
-                break;
+                DEBUG_ERR(err, "registering for cleanup request");
+                pending_clients_release(cleanup->domain_cap, ClientType_Cleanup,
+                                        NULL);
+                event_mutex_unlock(&cleanup->sb->mutex);
+                free(cleanup);
             }
+        } else {
+            DEBUG_ERR(err, "sending cleanup request");
+            pending_clients_release(cleanup->domain_cap, ClientType_Cleanup,
+                                    NULL);
+            event_mutex_unlock(&cleanup->sb->mutex);
+            free(cleanup);
+        }
+    }
+}
 
-            assert(entry->num_spawnds_resources > 0 ||
-                   entry->num_spawnds_running > 0);
-            assert(entry->status != DOMAIN_STATUS_CLEANED);
+static void spawn_reply_handler(struct spawn_binding *b,
+                                struct capref domain_cap, errval_t spawn_err)
+{
+    event_mutex_unlock(&b->mutex);
 
-            if (entry->num_spawnds_running > 0) {
-                --entry->num_spawnds_running;
+    struct pending_client *cl;
+    errval_t err = pending_clients_release(domain_cap, ClientType_Spawn, &cl);
+    if (err_is_fail(err)) {
+        DEBUG_ERR(err, "failed to retrieve pending spawn client based on domain"
+                  " cap");
+        return;
+    }
 
-                err = pending_clients_add(domain_cap, cl->b,
-                                          ClientType_Exit, MAX_COREID);
-                if (err_is_fail(err)) {
-                    DEBUG_ERR(err, "pending_clients_add in reply handler");
-                }
+    err = spawn_err;
+    if (err_is_ok(spawn_err)) {
+        err = domain_spawn(domain_cap, cl->core_id);
+    }
 
-                if (entry->num_spawnds_running == 0) {
-                    entry->status = DOMAIN_STATUS_STOPPED;
-
-                    struct domain_waiter *waiter = entry->waiters;
-                    while (waiter != NULL) {
-                        waiter->b->tx_vtbl.wait_response(waiter->b, NOP_CONT,
-                                                         SYS_ERR_OK,
-                                                         entry->exit_status);
-                        struct domain_waiter *aux = waiter;
-                        waiter = waiter->next;
-                        free(aux);
-                    }
+    errval_t resp_err = cl->b->tx_vtbl.spawn_response(cl->b, NOP_CONT, err,
+                                                      domain_cap);
+    if (err_is_fail(resp_err)) {
+        DEBUG_ERR(resp_err, "failed to send spawn_response to client");
+    }
+    
+    free(cl);
+}
 
-                    for (coreid_t i = 0; i < MAX_COREID; ++i) {
-                        if (entry->spawnds[i] == NULL) {
-                            continue;
-                        }
-
-                        struct spawn_binding *spb = entry->spawnds[i]->b;
-                        spb->rx_vtbl.spawn_reply = spawn_reply_handler;
-                        errval_t req_err = spb->tx_vtbl.cleanup_request(spb,
-                                NOP_CONT, cap_procmng, domain_cap);
-                        if (err_is_fail(req_err)) {
-                            DEBUG_ERR(req_err, "failed to send cleanup_request "
-                                      "to spawnd %u\n", i);
-                        }
-                    }
-                }
-            } else {
-                --entry->num_spawnds_resources;
-
-                if (entry->num_spawnds_resources == 0) {
-                    entry->status = DOMAIN_STATUS_CLEANED;
-
-                    // At this point, the domain exists in state CLEANED for
-                    // history reasons. For instance, if some other domain
-                    // issues a wait call for this one, the process manager can
-                    // return the exit status directly.
-                    // At some point, however, we might want to just clean up
-                    // the domain entry and recycle the domain cap.
-                } else {
-                    // Expecting to receive further cleanup replies from other
-                    // spawnds for the same domain cap, hence re-add the
-                    // pending client.
-                    err = pending_clients_add(domain_cap, cl->b,
-                                              ClientType_Exit, MAX_COREID);
-                    if (err_is_fail(err)) {
-                        DEBUG_ERR(err, "pending_clients_add in reply handler");
-                    }
-                }
+static void spawn_with_caps_reply_handler(struct spawn_binding *b,
+                                          struct capref domain_cap,
+                                          errval_t spawn_err)
+{
+    event_mutex_unlock(&b->mutex);
+    
+    struct pending_client *cl;
+    errval_t err = pending_clients_release(domain_cap, ClientType_SpawnWithCaps,
+                                           &cl);
+    if (err_is_fail(err)) {
+        DEBUG_ERR(err, "failed to retrieve pending spawn_with_caps client based"
+                  " on domain cap");
+        return;
+    }
+
+    err = spawn_err;
+    if (err_is_ok(spawn_err)) {
+        err = domain_spawn(domain_cap, cl->core_id);
+    }
+
+    errval_t resp_err = cl->b->tx_vtbl.spawn_with_caps_response(cl->b, NOP_CONT,
+                                                                err,
+                                                                domain_cap);
+    if (err_is_fail(resp_err)) {
+        DEBUG_ERR(resp_err, "failed to send spawn_with_caps_response to "
+                  "client");
+    }
+    
+    free(cl);
+}
+
+static void span_reply_handler(struct spawn_binding *b,
+                               struct capref domain_cap, errval_t span_err)
+{
+    event_mutex_unlock(&b->mutex);
+
+    struct pending_client *cl;
+    errval_t err = pending_clients_release(domain_cap, ClientType_Span, &cl);
+    if (err_is_fail(err)) {
+        DEBUG_ERR(err, "failed to retrieve pending span client based on domain"
+                  " cap");
+        return;
+    }
+
+    struct domain_entry *entry;
+    err = domain_get_by_cap(cl->domain_cap, &entry);
+    if (err_is_fail(err)) {
+        DEBUG_ERR(err, "failed to retrieve span client by domain cap");
+        return;
+    }
+
+    if (entry->status != DOMAIN_STATUS_RUNNING) {
+        // Domain has been stopped while we were serving the request; there's
+        // no one to respond to.
+        free(cl);
+        return;
+    }
+
+    err = cl->b->tx_vtbl.span_response(cl->b, NOP_CONT, span_err);
+    if (err_is_fail(err)) {
+        DEBUG_ERR(err, "failed to send span_response to client");
+    }
+    
+    free(cl);
+}
+
+static void cleanup_reply_handler(struct spawn_binding *b,
+                                  struct capref domain_cap,
+                                  errval_t cleanup_err)
+{
+    event_mutex_unlock(&b->mutex);
+
+    struct pending_client *cl;
+    errval_t err = pending_clients_release(domain_cap, ClientType_Cleanup, &cl);
+    if (err_is_fail(err)) {
+        DEBUG_ERR(err, "failed to retrieve pending cleanup client based on "
+                  "domain cap");
+        return;
+    }
+
+    if (err_is_fail(cleanup_err)) {
+        // TODO(razvan): Here, spawnd has failed deleting its local cspace.
+        // Should we send another cleanup message, until it might succeed?
+        free(cl);
+        return;
+    }
+
+    struct domain_entry *entry;
+    err = domain_get_by_cap(domain_cap, &entry);
+    if (err_is_fail(err)) {
+        DEBUG_ERR(err, "failed to retrieve domain by cap returned by spawnd "
+                  "after cleanup");
+        return;
+    }
+
+    assert(entry->num_spawnds_resources > 0);
+    assert(entry->status != DOMAIN_STATUS_CLEANED);
+
+    --entry->num_spawnds_resources;
+
+    if (entry->num_spawnds_resources == 0) {
+        entry->status = DOMAIN_STATUS_CLEANED;
+
+        // At this point, the domain exists in state CLEANED for
+        // history reasons. For instance, if some other domain
+        // issues a wait call for this one, the process manager can
+        // return the exit status directly.
+        // At some point, however, we might want to just clean up
+        // the domain entry and recycle the domain cap.
+    } else {
+        // Expecting to receive further cleanup replies from other
+        // spawnds for the same domain cap, hence re-add the
+        // pending client.
+        err = pending_clients_add(domain_cap, cl->b,
+                                  ClientType_Cleanup, MAX_COREID);
+        if (err_is_fail(err)) {
+            DEBUG_ERR(err, "pending_clients_add in cleanup_reply_handler");
+        }
+    }
+}
+
+static void kill_reply_handler(struct spawn_binding *b,
+                               struct capref domain_cap, errval_t kill_err)
+{
+    event_mutex_unlock(&b->mutex);
+
+    struct pending_client *cl;
+    errval_t err = pending_clients_release(domain_cap, ClientType_Kill, &cl);
+    if (err_is_fail(err)) {
+        DEBUG_ERR(err, "failed to retrieve pending kill client based on domain "
+                  "cap");
+        return;
+    }
+
+    errval_t resp_err;
+    if (err_is_fail(kill_err)) {
+        // TODO(razvan): Here, spawnd has failed deleting its local dispatcher.
+        // Should we send another kill message, until it might succeed?
+        while (cl != NULL) {
+            resp_err = cl->b->tx_vtbl.kill_response(cl->b, NOP_CONT,
+                                                     kill_err);
+            if (err_is_fail(resp_err)) {
+                DEBUG_ERR(resp_err, "failed to send kill_response to client");
             }
-            break;
-
-        default:
-            // TODO(razvan): Handle the other cases, e.g. wait.
-            debug_printf("Unknown client type %u\n", cl->type);
-            return;
-    }
-
-    if (err_is_ok(spawn_err) && err_is_fail(err)) {
-        // Spawnd has successfully completed its end of the operation, but
-        // there's been an error in the process manager's book-keeping
-        // of domains. Therefore, if the request was a spawn or span one, spawnd
-        // needs to be asked to stop the dispatcher which it has just enqueued.
-        if (cl->type == ClientType_Spawn ||
-            cl->type == ClientType_SpawnWithCaps ||
-            cl->type == ClientType_Span) {
-            struct spawnd_state *state = spawnd_state_get(cl->core_id);
-            assert(state != NULL);
-            struct spawn_binding *spb = state->b;
-            assert(spb != NULL);
-
-            err = spb->tx_vtbl.kill_request(spb, NOP_CONT, cap_procmng, 
-                                            domain_cap);
-            if (err_is_fail(err)) {
-                // XXX: How severe is this? Maybe we want something more
-                // assertive than logging an error message.
-                DEBUG_ERR(err, "failed to send kill request for dangling "
-                          "dispatcher");
-            } else {
-                pending_clients_add(domain_cap, cl->b, ClientType_Kill,
-                                    MAX_COREID);
+            struct pending_client *tmp = cl;
+            cl = cl->next;
+            free(tmp);
+        }
+        return;
+    }
+
+    struct domain_entry *entry;
+    err = domain_get_by_cap(domain_cap, &entry);
+    if (err_is_fail(err)) {
+        DEBUG_ERR(err, "failed to retrieve domain by cap returned by spawnd "
+                  "after kill");
+        return;
+    }
+
+    assert(entry->num_spawnds_running > 0);
+    assert(entry->status != DOMAIN_STATUS_STOPPED);
+
+    --entry->num_spawnds_running;
+
+    if (entry->num_spawnds_running == 0) {
+        entry->status = DOMAIN_STATUS_STOPPED;
+        entry->exit_status = EXIT_STATUS_KILLED;
+
+        err = pending_clients_add(domain_cap, NULL, ClientType_Cleanup,
+                                  MAX_COREID);
+        if (err_is_fail(err)) {
+            DEBUG_ERR(err, "pending_clients_add in kill_reply_handler");
+        }
+
+        // TODO(razvan): Might it be more sane if we respond back
+        // to the client after the domain has been cleaned up (i.e.
+        // the cspace root has been revoked for all dispatchers)?
+        while (cl != NULL) {
+            resp_err = cl->b->tx_vtbl.kill_response(cl->b, NOP_CONT,
+                                                     kill_err);
+            if (err_is_fail(resp_err)) {
+                DEBUG_ERR(resp_err, "failed to send kill_response to client");
             }
+            struct pending_client *tmp = cl;
+            cl = cl->next;
+            free(tmp);
+        }
+        
+        // TODO(razvan): Same problem applies to the waiters: would
+        // it be better if we sent them wait_responses after the
+        // cspace root has been revoked, too? (here and in the exit
+        // case).
+        struct domain_waiter *waiter = entry->waiters;
+        while (waiter != NULL) {
+            waiter->b->tx_vtbl.wait_response(waiter->b, NOP_CONT,
+                                             SYS_ERR_OK,
+                                             entry->exit_status);
+            struct domain_waiter *tmp = waiter;
+            waiter = waiter->next;
+            free(tmp);
+        }
+
+        for (coreid_t i = 0; i < MAX_COREID; ++i) {
+            if (entry->spawnds[i] == NULL) {
+                continue;
+            }
+
+            struct spawn_binding *spb = entry->spawnds[i]->b;
+
+            struct pending_kill_exit_cleanup *cleanup = (struct pending_kill_exit_cleanup*) malloc(
+                    sizeof(struct pending_kill_exit_cleanup));
+            cleanup->sb = spb;
+            cleanup->domain_cap = domain_cap;
+
+            spb->rx_vtbl.cleanup_reply = cleanup_reply_handler;
+            event_mutex_enqueue_lock(&spb->mutex,
+                                     &cleanup->qn,
+                                     (struct event_closure) {
+                                         .handler = cleanup_request_sender,
+                                         .arg = cleanup });
+        }
+    } else {
+        err = pending_clients_add(domain_cap, cl->b, ClientType_Kill,
+                                  MAX_COREID);
+        if (err_is_fail(err)) {
+            DEBUG_ERR(err, "pending_clients_add in kill_reply_handler");
         }
     }
+}
 
-    free(cl);
+static void exit_reply_handler(struct spawn_binding *b,
+                               struct capref domain_cap, errval_t exit_err)
+{
+    event_mutex_unlock(&b->mutex);
 
-    if (err_is_fail(resp_err)) {
-        DEBUG_ERR(resp_err, "failed to send response to client");
+    struct pending_client *cl;
+    errval_t err = pending_clients_release(domain_cap, ClientType_Exit, &cl);
+    if (err_is_fail(err)) {
+        DEBUG_ERR(err, "failed to retrieve pending exit client based on domain "
+                  "cap");
+        return;
+    }
+
+    if (err_is_fail(exit_err)) {
+        // TODO(razvan): Here, spawnd has failed deleting its local dispatcher.
+        // Should we send another kill message, until it might succeed?
+        free(cl);
+        return;
+    }
+
+    struct domain_entry *entry;
+    err = domain_get_by_cap(domain_cap, &entry);
+    if (err_is_fail(err)) {
+        DEBUG_ERR(err, "failed to retrieve domain by cap returned by spawnd "
+                  "after exit");
+        return;
+    }
+
+    assert(entry->num_spawnds_running > 0);
+    assert(entry->status != DOMAIN_STATUS_STOPPED);
+
+    --entry->num_spawnds_running;
+
+    if (entry->num_spawnds_running == 0) {
+        entry->status = DOMAIN_STATUS_STOPPED;
+        
+        free(cl);
+
+        // TODO(razvan): Same problem applies to the waiters: would
+        // it be better if we sent them wait_responses after the
+        // cspace root has been revoked, too? (here and in the exit
+        // case).
+        struct domain_waiter *waiter = entry->waiters;
+        while (waiter != NULL) {
+            waiter->b->tx_vtbl.wait_response(waiter->b, NOP_CONT,
+                                             SYS_ERR_OK,
+                                             entry->exit_status);
+            struct domain_waiter *tmp = waiter;
+            waiter = waiter->next;
+            free(tmp);
+        }
+
+        for (coreid_t i = 0; i < MAX_COREID; ++i) {
+            if (entry->spawnds[i] == NULL) {
+                continue;
+            }
+
+            struct spawn_binding *spb = entry->spawnds[i]->b;
+
+            struct pending_kill_exit_cleanup *cleanup = (struct pending_kill_exit_cleanup*) malloc(
+                    sizeof(struct pending_kill_exit_cleanup));
+            cleanup->sb = spb;
+            cleanup->domain_cap = domain_cap;
+
+            spb->rx_vtbl.cleanup_reply = cleanup_reply_handler;
+            event_mutex_enqueue_lock(&spb->mutex,
+                                     &cleanup->qn,
+                                     (struct event_closure) {
+                                         .handler = cleanup_request_sender,
+                                         .arg = cleanup });
+        }
+    } else {
+        err = pending_clients_add(domain_cap, cl->b, ClientType_Exit,
+                                  MAX_COREID);
+        if (err_is_fail(err)) {
+            DEBUG_ERR(err, "pending_clients_add in kill_reply_handler");
+        }
     }
 }
 
@@ -323,11 +620,8 @@ static errval_t spawn_handler_common(struct proc_mgmt_binding *b,
                                      const char *argvbuf, size_t argvbytes,
                                      const char *envbuf, size_t envbytes,
                                      struct capref inheritcn_cap,
-                                     struct capref argcn_cap, uint8_t flags,
-                                     struct capref *ret_domain_cap)
+                                     struct capref argcn_cap, uint8_t flags)
 {
-    assert(ret_domain_cap != NULL);
-
     if (!spawnd_state_exists(core_id)) {
         return PROC_MGMT_ERR_INVALID_SPAWND;
     }
@@ -355,23 +649,24 @@ static errval_t spawn_handler_common(struct proc_mgmt_binding *b,
         return err;
     }
 
-    cl->rx_vtbl.spawn_reply = spawn_reply_handler;
-    if (capref_is_null(inheritcn_cap) && capref_is_null(argcn_cap)) {
-        err = cl->tx_vtbl.spawn_request(cl, NOP_CONT, cap_procmng, domain_cap,
-                                        path, argvbuf, argvbytes, envbuf,
-                                        envbytes, flags);
-    } else {
-        err = cl->tx_vtbl.spawn_with_caps_request(cl, NOP_CONT, cap_procmng,
-                                                  domain_cap, path, argvbuf,
-                                                  argvbytes, envbuf, envbytes,
-                                                  inheritcn_cap, argcn_cap,
-                                                  flags);
-    }
-    if (err_is_fail(err)) {
-        DEBUG_ERR(err, "sending spawn request");
-        pending_clients_release(domain_cap, NULL);
-        return err_push(err, PROC_MGMT_ERR_SPAWND_REQUEST);
-    }
+    struct pending_spawn *spawn = (struct pending_spawn*) malloc(
+            sizeof(struct pending_spawn));
+    spawn->domain_cap = domain_cap;
+    spawn->b = cl;
+    spawn->core_id = core_id;
+    spawn->path = path;
+    spawn->argvbuf = argvbuf;
+    spawn->argvbytes = argvbytes;
+    spawn->envbuf = envbuf;
+    spawn->envbytes = envbytes;
+    spawn->inheritcn_cap = inheritcn_cap;
+    spawn->argcn_cap = argcn_cap;
+    spawn->flags = flags;
+
+    event_mutex_enqueue_lock(&cl->mutex, &spawn->qn,
+                             (struct event_closure) {
+                                 .handler = spawn_request_sender,
+                                 .arg = spawn });
 
     return SYS_ERR_OK;
 }
@@ -382,10 +677,9 @@ static void spawn_handler(struct proc_mgmt_binding *b, coreid_t core_id,
                           uint8_t flags)
 {
     errval_t err, resp_err;
-    struct capref domain_cap;
     err = spawn_handler_common(b, ClientType_Spawn, core_id, path, argvbuf,
                                argvbytes, envbuf, envbytes, NULL_CAP, NULL_CAP,
-                               flags, &domain_cap);
+                               flags);
     if (err_is_ok(err)) {
         // Will respond to client when we get the reply from spawnd.
         return;
@@ -405,10 +699,9 @@ static void spawn_with_caps_handler(struct proc_mgmt_binding *b,
                                     struct capref argcn_cap, uint8_t flags)
 {
     errval_t err, resp_err;
-    struct capref domain_cap;
     err = spawn_handler_common(b, ClientType_SpawnWithCaps, core_id, path,
                                argvbuf, argvbytes, envbuf, envbytes,
-                               inheritcn_cap, argcn_cap, flags, &domain_cap);
+                               inheritcn_cap, argcn_cap, flags);
     if (err_is_ok(err)) {
         // Will respond to client when we get the reply from spawnd.
         return;
@@ -446,17 +739,18 @@ static void span_handler(struct proc_mgmt_binding *b, struct capref domain_cap,
         goto respond_with_err;
     }
 
-    cl->rx_vtbl.spawn_reply = spawn_reply_handler;
-    err = cl->tx_vtbl.span_request(cl, NOP_CONT, cap_procmng, domain_cap, vroot,
-                                   dispframe);
-    if (err_is_ok(err)) {
-        // Will respond to client when we get the reply from spawnd.
-        return;
-    } else {
-        DEBUG_ERR(err, "sending span request");
-        pending_clients_release(domain_cap, NULL);
-        err = err_push(err, PROC_MGMT_ERR_SPAWND_REQUEST);
-    }
+    struct pending_span *span = (struct pending_span*) malloc(
+            sizeof(struct pending_span));
+    span->domain_cap = domain_cap;
+    span->b = cl;
+    span->core_id = core_id;
+    span->vroot = vroot;
+    span->dispframe = dispframe;
+
+    event_mutex_enqueue_lock(&cl->mutex, &span->qn,
+                             (struct event_closure) {
+                                 .handler = span_request_sender,
+                                 .arg = span });
 
 respond_with_err:
     resp_err = b->tx_vtbl.span_response(b, NOP_CONT, err);
@@ -490,20 +784,42 @@ static errval_t kill_handler_common(struct proc_mgmt_binding *b,
         }
 
         struct spawn_binding *spb = entry->spawnds[i]->b;
-        spb->rx_vtbl.spawn_reply = spawn_reply_handler;
-        errval_t req_err = spb->tx_vtbl.kill_request(spb, NOP_CONT, cap_procmng,
-                                                     domain_cap);
-        if (err_is_fail(req_err)) {
-            DEBUG_ERR(req_err, "failed to send kill_request to spawnd %u\n", i);
+
+        struct pending_kill_exit_cleanup *cmd = (struct pending_kill_exit_cleanup*) malloc(
+                sizeof(struct pending_kill_exit_cleanup));
+        cmd->domain_cap = domain_cap;
+        cmd->sb = spb;
+
+        switch (type) {
+            case ClientType_Kill:
+                cmd->pmb = b;
+
+                event_mutex_enqueue_lock(&spb->mutex,
+                                         &cmd->qn,
+                                         (struct event_closure) {
+                                            .handler = kill_request_sender,
+                                            .arg = cmd });
+                break;
+
+            case ClientType_Exit:
+                event_mutex_enqueue_lock(&spb->mutex,
+                                         &cmd->qn,
+                                         (struct event_closure) {
+                                            .handler = exit_request_sender,
+                                            .arg = cmd });
+                break;
+            default:
+                USER_PANIC("invalid client type for kill: %u\n", type);
         }
     }
 
     return SYS_ERR_OK;
 }
 
-static void kill_handler(struct proc_mgmt_binding *b, struct capref domain_cap)
+static void kill_handler(struct proc_mgmt_binding *b,
+                         struct capref victim_domain_cap)
 {
-    errval_t err = kill_handler_common(b, domain_cap, ClientType_Kill,
+    errval_t err = kill_handler_common(b, victim_domain_cap, ClientType_Kill,
                                        EXIT_STATUS_KILLED);
     if (err_is_fail(err)) {
         errval_t resp_err = b->tx_vtbl.kill_response(b, NOP_CONT, err);
@@ -522,7 +838,7 @@ static void exit_handler(struct proc_mgmt_binding *b, struct capref domain_cap,
         DEBUG_ERR(err, "processing exit_handler for requesting domain, exit "
                   "code %u", exit_status);
     }
-    // Error or not, there's no client to reply to anymore.
+    // Error or not, there's no client to respond to anymore.
 }
 
 static void wait_handler(struct proc_mgmt_binding *b, struct capref domain_cap)
@@ -560,7 +876,7 @@ static struct proc_mgmt_rx_vtbl monitor_vtbl = {
     .spawn_with_caps_call = spawn_with_caps_handler,
     .span_call            = span_handler,
     .kill_call            = kill_handler,
-    .exit                 = exit_handler,
+    .exit_call            = exit_handler,
     .wait_call            = wait_handler
 };
 
@@ -570,7 +886,7 @@ static struct proc_mgmt_rx_vtbl non_monitor_vtbl = {
     .spawn_with_caps_call = spawn_with_caps_handler,
     .span_call            = span_handler,
     .kill_call            = kill_handler,
-    .exit                 = exit_handler,
+    .exit_call            = exit_handler,
     .wait_call            = wait_handler
 };
 
index ce5ae27..ad2e845 100644 (file)
 #include <barrelfish/barrelfish.h>
 
 struct spawnd_state {
-       coreid_t core_id;
+    coreid_t core_id;
     struct spawn_binding *b;
-
-    // TODO(razvan): will need more state here.
 };
 
 errval_t spawnd_state_alloc(coreid_t core_id, struct spawn_binding *b);
index a1c9b7e..5fea71c 100644 (file)
@@ -386,9 +386,9 @@ static void spawn_with_caps_request_handler(struct spawn_binding *b,
                                  &dummy_domain_id);
 
 reply:
-    reply_err = b->tx_vtbl.spawn_reply(b, NOP_CONT, domain_cap, err);
+    reply_err = b->tx_vtbl.spawn_with_caps_reply(b, NOP_CONT, domain_cap, err);
     if (err_is_fail(reply_err)) {
-        DEBUG_ERR(err, "failed to send spawn_reply");
+        DEBUG_ERR(err, "failed to send spawn_with_caps_reply");
     }
 }
 
@@ -564,9 +564,9 @@ static void span_request_handler(struct spawn_binding *b,
     }
 
 reply:
-    reply_err = b->tx_vtbl.spawn_reply(b, NOP_CONT, domain_cap, err);
+    reply_err = b->tx_vtbl.span_reply(b, NOP_CONT, domain_cap, err);
     if (err_is_fail(reply_err)) {
-        DEBUG_ERR(err, "failed to send spawn_reply");
+        DEBUG_ERR(err, "failed to send span_reply");
     }
 }
 
@@ -584,36 +584,50 @@ static void cleanup_cap(struct capref cap)
     }
 }
 
-static void kill_request_handler(struct spawn_binding *b,
-                                 struct capref procmng_cap,
-                                 struct capref domain_cap)
+static errval_t kill_handler_common(struct capref procmng_cap,
+                                    struct capref domain_cap)
 {
-    errval_t err, reply_err;
     struct capability ret;
-    err = monitor_cap_identify_remote(procmng_cap, &ret);
+    errval_t err = monitor_cap_identify_remote(procmng_cap, &ret);
     if (err_is_fail(err)) {
-        err = err_push(err, SPAWN_ERR_IDENTIFY_PROC_MNGR_CAP);
-        goto reply;
+        return err_push(err, SPAWN_ERR_IDENTIFY_PROC_MNGR_CAP);
     }
 
     if (ret.type != ObjType_ProcessManager) {
-        err = SPAWN_ERR_NOT_PROC_MNGR;
-        goto reply;
+        return SPAWN_ERR_NOT_PROC_MNGR;
     }
 
     struct ps_entry *pe;
     err = ps_get_domain(domain_cap, &pe, NULL);
     if (err_is_fail(err)) {
-        err = err_push(err, SPAWN_ERR_DOMAIN_NOTFOUND);
-        goto reply;
+        return err_push(err, SPAWN_ERR_DOMAIN_NOTFOUND);
     }
 
     cleanup_cap(pe->dcb);
 
-reply:
-    reply_err = b->tx_vtbl.spawn_reply(b, NOP_CONT, domain_cap, err);
+    return SYS_ERR_OK;
+}
+
+static void kill_request_handler(struct spawn_binding *b,
+                                 struct capref procmng_cap,
+                                 struct capref victim_domain_cap)
+{
+    errval_t err = kill_handler_common(procmng_cap, victim_domain_cap);
+    errval_t reply_err = b->tx_vtbl.kill_reply(b, NOP_CONT, victim_domain_cap,
+                                               err);
     if (err_is_fail(reply_err)) {
-        DEBUG_ERR(err, "failed to send spawn_reply");
+        DEBUG_ERR(err, "failed to send kill_reply");
+    }
+}
+
+static void exit_request_handler(struct spawn_binding *b,
+                                 struct capref procmng_cap,
+                                 struct capref domain_cap)
+{
+    errval_t err = kill_handler_common(procmng_cap, domain_cap);
+    errval_t reply_err = b->tx_vtbl.exit_reply(b, NOP_CONT, domain_cap, err);
+    if (err_is_fail(reply_err)) {
+        DEBUG_ERR(err, "failed to send exit_reply");
     }
 }
 
@@ -650,9 +664,9 @@ static void cleanup_request_handler(struct spawn_binding *b,
     free(pe);
 
 reply:
-    reply_err = b->tx_vtbl.spawn_reply(b, NOP_CONT, domain_cap, err);
+    reply_err = b->tx_vtbl.cleanup_reply(b, NOP_CONT, domain_cap, err);
     if (err_is_fail(reply_err)) {
-        DEBUG_ERR(err, "failed to send spawn_reply");
+        DEBUG_ERR(err, "failed to send cleanup_reply");
     }
 }
 
@@ -851,6 +865,7 @@ static struct spawn_rx_vtbl rx_vtbl = {
     .spawn_with_caps_request = spawn_with_caps_request_handler,
     .span_request            = span_request_handler,
     .kill_request            = kill_request_handler,
+    .exit_request            = exit_request_handler,
     .cleanup_request         = cleanup_request_handler,
 
     .use_local_memserv_call = use_local_memserv_handler,