41df760aac19744824cc7aedcee2c97535f85726
[barrelfish] / usr / proc_mgmt / service.c
1 /**
2  * \file
3  * \brief Process management service.
4  */
5
6 /*
7  * Copyright (c) 2017, ETH Zurich.
8  * All rights reserved.
9  *
10  * This file is distributed under the terms in the attached LICENSE file.
11  * If you do not find this file, copies can be found by writing to:
12  * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
13  */
14
15 #include <barrelfish/barrelfish.h>
16 #include <barrelfish/nameservice_client.h>
17 #include <barrelfish/proc_mgmt_client.h>
18 #include <barrelfish/spawn_client.h>
19 #include <if/monitor_defs.h>
20 #include <if/proc_mgmt_defs.h>
21 #include <if/spawn_defs.h>
22
23 #include "domain.h"
24 #include "internal.h"
25 #include "pending_clients.h"
26 #include "spawnd_state.h"
27
28 static void add_spawnd_handler(struct proc_mgmt_binding *b, coreid_t core_id,
29                                iref_t iref)
30 {
31     if (spawnd_state_exists(core_id)) {
32         DEBUG_ERR(PROC_MGMT_ERR_SPAWND_EXISTS, "spawnd_state_exists");
33         return;
34     }
35
36     // Bind with the spawnd.
37     struct spawn_binding *spawnb;
38     errval_t err = spawn_bind_iref(iref, &spawnb);
39     if (err_is_fail(err)) {
40         DEBUG_ERR(err, "spawn_bind_iref");
41         return;
42     }
43
44     err = spawnd_state_alloc(core_id, spawnb);
45     if (err_is_fail(err)) {
46         DEBUG_ERR(err, "spawnd_state_alloc");
47     }
48
49     debug_printf("Process manager bound with spawnd.%u on iref %u\n", core_id,
50             iref);
51 }
52
53 static void add_spawnd_handler_non_monitor(struct proc_mgmt_binding *b,
54                                            coreid_t core_id, iref_t iref)
55 {
56     // debug_printf("Ignoring add_spawnd call: %s\n",
57     //              err_getstring(PROC_MGMT_ERR_NOT_MONITOR));
58 }
59
60 static void spawn_reply_handler(struct spawn_binding *b,
61                                 struct capref domain_cap, errval_t spawn_err);
62 static void spawn_with_caps_reply_handler(struct spawn_binding *b,
63                                           struct capref domain_cap,
64                                           errval_t spawn_err);
65 static void span_reply_handler(struct spawn_binding *b,
66                                struct capref domain_cap, errval_t span_err);
67 static void kill_reply_handler(struct spawn_binding *b,
68                                struct capref domain_cap, errval_t kill_err);
69 static void exit_reply_handler(struct spawn_binding *b,
70                                struct capref domain_cap, errval_t exit_err);
71 static void cleanup_reply_handler(struct spawn_binding *b,
72                                   struct capref domain_cap,
73                                   errval_t cleanup_err);
74
75 static void spawn_request_sender(void *arg)
76 {
77     struct pending_spawn *spawn = (struct pending_spawn*) arg;
78
79     errval_t err;
80     bool with_caps = !(capref_is_null(spawn->inheritcn_cap) &&
81                        capref_is_null(spawn->argcn_cap));
82     if (with_caps) {
83         spawn->b->rx_vtbl.spawn_with_caps_reply = spawn_with_caps_reply_handler;
84         err = spawn->b->tx_vtbl.spawn_with_caps_request(spawn->b, NOP_CONT,
85                                                         cap_procmng,
86                                                         spawn->domain_cap,
87                                                         spawn->path,
88                                                         spawn->argvbuf,
89                                                         spawn->argvbytes,
90                                                         spawn->envbuf,
91                                                         spawn->envbytes,
92                                                         spawn->inheritcn_cap,
93                                                         spawn->argcn_cap,
94                                                         spawn->flags);
95     } else {
96         spawn->b->rx_vtbl.spawn_reply = spawn_reply_handler;
97         err = spawn->b->tx_vtbl.spawn_request(spawn->b, NOP_CONT, cap_procmng,
98                                               spawn->domain_cap, spawn->path,
99                                               spawn->argvbuf, spawn->argvbytes,
100                                               spawn->envbuf, spawn->envbytes,
101                                               spawn->flags);
102     }
103     if (err_is_ok(err)) {
104         free(spawn);
105     } else {
106         if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
107             err = spawn->b->register_send(spawn->b, spawn->b->waitset,
108                                           MKCONT(spawn_request_sender, arg));
109             if (err_is_fail(err)) {
110                 DEBUG_ERR(err, "registering for spawn request");
111                 pending_clients_release(spawn->domain_cap,
112                                         with_caps ? ClientType_SpawnWithCaps
113                                                   : ClientType_Spawn,
114                                         NULL);
115                 event_mutex_unlock(&spawn->b->mutex);
116                 free(spawn);
117             }
118         } else {
119             DEBUG_ERR(err, "sending spawn request");
120             pending_clients_release(spawn->domain_cap,
121                                     with_caps ? ClientType_SpawnWithCaps
122                                               : ClientType_Spawn,
123                                     NULL);
124             event_mutex_unlock(&spawn->b->mutex);
125             free(spawn);
126         }
127     }
128 }
129
130 static void span_request_sender(void *arg)
131 {
132     struct pending_span *span = (struct pending_span*) arg;
133
134     errval_t err;
135     span->b->rx_vtbl.span_reply = span_reply_handler;
136     err = span->b->tx_vtbl.span_request(span->b, NOP_CONT, cap_procmng,
137                                         span->domain_cap, span->vroot,
138                                         span->dispframe);
139     if (err_is_ok(err)) {
140         err = domain_span(span->domain_cap, span->core_id);
141         if (err_is_fail(err)) {
142             DEBUG_ERR(err, "failed domain_span to core %u\n", span->core_id);
143         }
144         free(span);
145     } else {
146         if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
147             err = span->b->register_send(span->b, span->b->waitset,
148                                          MKCONT(span_request_sender, arg));
149             if (err_is_fail(err)) {
150                 DEBUG_ERR(err, "registering for span request");
151                 pending_clients_release(span->domain_cap, ClientType_Span,
152                                         NULL);
153                 event_mutex_unlock(&span->b->mutex);
154                 free(span);
155             }
156         } else {
157             DEBUG_ERR(err, "sending span request");
158             pending_clients_release(span->domain_cap, ClientType_Span, NULL);
159             event_mutex_unlock(&span->b->mutex);
160             free(span);
161         }
162     }
163 }
164
165 static void kill_request_sender(void *arg)
166 {
167     struct pending_kill_exit_cleanup *kill = (struct pending_kill_exit_cleanup*) arg;
168
169     errval_t err;
170     kill->sb->rx_vtbl.kill_reply = kill_reply_handler;
171     err = kill->sb->tx_vtbl.kill_request(kill->sb, NOP_CONT, cap_procmng,
172                                         kill->domain_cap);
173     if (err_is_ok(err)) {
174         free(kill);
175     } else {
176         if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
177             err = kill->sb->register_send(kill->sb, kill->sb->waitset,
178                                          MKCONT(kill_request_sender, arg));
179             if (err_is_fail(err)) {
180                 DEBUG_ERR(err, "registering for kill request");
181
182                 struct pending_client *cl;
183                 err = pending_clients_release_one(kill->domain_cap,
184                                                   ClientType_Kill,
185                                                   kill->pmb, &cl);
186                 if (err_is_ok(err)) {
187                     while (cl != NULL) {
188                         struct pending_client *tmp = cl;
189                         cl = cl->next;
190                         free(tmp);
191                     }
192                 }
193
194                 event_mutex_unlock(&kill->sb->mutex);
195                 free(kill);
196             }
197         } else {
198             DEBUG_ERR(err, "sending kill request");
199             
200             struct pending_client *cl;
201             err = pending_clients_release_one(kill->domain_cap,
202                                               ClientType_Kill,
203                                               kill->pmb, &cl);
204             if (err_is_ok(err)) {
205                 while (cl != NULL) {
206                     struct pending_client *tmp = cl;
207                     cl = cl->next;
208                     free(tmp);
209                 }
210             }
211
212             event_mutex_unlock(&kill->sb->mutex);
213             free(kill);
214         }
215     }
216 }
217
218 static void exit_request_sender(void *arg)
219 {
220     struct pending_kill_exit_cleanup *exit = (struct pending_kill_exit_cleanup*) arg;
221
222     errval_t err;
223     exit->sb->rx_vtbl.exit_reply = exit_reply_handler;
224     err = exit->sb->tx_vtbl.exit_request(exit->sb, NOP_CONT, cap_procmng,
225                                         exit->domain_cap);
226     if (err_is_ok(err)) {
227         free(exit);
228     } else {
229         if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
230             err = exit->sb->register_send(exit->sb, exit->sb->waitset,
231                                          MKCONT(exit_request_sender, arg));
232             if (err_is_fail(err)) {
233                 DEBUG_ERR(err, "registering for exit request");
234                 err = pending_clients_release(exit->domain_cap, ClientType_Exit,
235                                               NULL);
236                 event_mutex_unlock(&exit->sb->mutex);
237                 free(exit);
238             }
239         } else {
240             DEBUG_ERR(err, "sending exit request");
241             err = pending_clients_release(exit->domain_cap, ClientType_Exit,
242                                           NULL);
243             event_mutex_unlock(&exit->sb->mutex);
244             free(exit);
245         }
246     }
247 }
248
249 static void cleanup_request_sender(void *arg)
250 {
251     struct pending_kill_exit_cleanup *cleanup = (struct pending_kill_exit_cleanup*) arg;
252
253     errval_t err;
254     cleanup->sb->rx_vtbl.cleanup_reply = cleanup_reply_handler;
255     err = cleanup->sb->tx_vtbl.cleanup_request(cleanup->sb, NOP_CONT, cap_procmng,
256                                               cleanup->domain_cap);
257     if (err_is_ok(err)) {
258         free(cleanup);
259     } else {
260         if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
261             err = cleanup->sb->register_send(cleanup->sb, cleanup->sb->waitset,
262                                             MKCONT(cleanup_request_sender, arg));
263             if (err_is_fail(err)) {
264                 DEBUG_ERR(err, "registering for cleanup request");
265                 pending_clients_release(cleanup->domain_cap, ClientType_Cleanup,
266                                         NULL);
267                 event_mutex_unlock(&cleanup->sb->mutex);
268                 free(cleanup);
269             }
270         } else {
271             DEBUG_ERR(err, "sending cleanup request");
272             pending_clients_release(cleanup->domain_cap, ClientType_Cleanup,
273                                     NULL);
274             event_mutex_unlock(&cleanup->sb->mutex);
275             free(cleanup);
276         }
277     }
278 }
279
280 static void spawn_reply_handler(struct spawn_binding *b,
281                                 struct capref domain_cap, errval_t spawn_err)
282 {
283     event_mutex_unlock(&b->mutex);
284
285     struct pending_client *cl;
286     errval_t err = pending_clients_release(domain_cap, ClientType_Spawn, &cl);
287     if (err_is_fail(err)) {
288         DEBUG_ERR(err, "failed to retrieve pending spawn client based on domain"
289                   " cap");
290         return;
291     }
292
293     err = spawn_err;
294     if (err_is_ok(spawn_err)) {
295         err = domain_spawn(domain_cap, cl->core_id);
296     }
297
298     errval_t resp_err = cl->b->tx_vtbl.spawn_response(cl->b, NOP_CONT, err,
299                                                       domain_cap);
300     if (err_is_fail(resp_err)) {
301         DEBUG_ERR(resp_err, "failed to send spawn_response to client");
302     }
303     
304     free(cl);
305 }
306
307 static void spawn_with_caps_reply_handler(struct spawn_binding *b,
308                                           struct capref domain_cap,
309                                           errval_t spawn_err)
310 {
311     event_mutex_unlock(&b->mutex);
312     
313     struct pending_client *cl;
314     errval_t err = pending_clients_release(domain_cap, ClientType_SpawnWithCaps,
315                                            &cl);
316     if (err_is_fail(err)) {
317         DEBUG_ERR(err, "failed to retrieve pending spawn_with_caps client based"
318                   " on domain cap");
319         return;
320     }
321
322     err = spawn_err;
323     if (err_is_ok(spawn_err)) {
324         err = domain_spawn(domain_cap, cl->core_id);
325     }
326
327     errval_t resp_err = cl->b->tx_vtbl.spawn_with_caps_response(cl->b, NOP_CONT,
328                                                                 err,
329                                                                 domain_cap);
330     if (err_is_fail(resp_err)) {
331         DEBUG_ERR(resp_err, "failed to send spawn_with_caps_response to "
332                   "client");
333     }
334     
335     free(cl);
336 }
337
338 static void span_reply_handler(struct spawn_binding *b,
339                                struct capref domain_cap, errval_t span_err)
340 {
341     event_mutex_unlock(&b->mutex);
342
343     struct pending_client *cl;
344     errval_t err = pending_clients_release(domain_cap, ClientType_Span, &cl);
345     if (err_is_fail(err)) {
346         DEBUG_ERR(err, "failed to retrieve pending span client based on domain"
347                   " cap");
348         return;
349     }
350
351     struct domain_entry *entry;
352     err = domain_get_by_cap(cl->domain_cap, &entry);
353     if (err_is_fail(err)) {
354         DEBUG_ERR(err, "failed to retrieve span client by domain cap");
355         return;
356     }
357
358     if (entry->status != DOMAIN_STATUS_RUNNING) {
359         // Domain has been stopped while we were serving the request; there's
360         // no one to respond to.
361         free(cl);
362         return;
363     }
364
365     err = cl->b->tx_vtbl.span_response(cl->b, NOP_CONT, span_err);
366     if (err_is_fail(err)) {
367         DEBUG_ERR(err, "failed to send span_response to client");
368     }
369     
370     free(cl);
371 }
372
373 static void cleanup_reply_handler(struct spawn_binding *b,
374                                   struct capref domain_cap,
375                                   errval_t cleanup_err)
376 {
377     event_mutex_unlock(&b->mutex);
378
379     struct pending_client *cl;
380     errval_t err = pending_clients_release(domain_cap, ClientType_Cleanup, &cl);
381     if (err_is_fail(err)) {
382         DEBUG_ERR(err, "failed to retrieve pending cleanup client based on "
383                   "domain cap");
384         return;
385     }
386
387     if (err_is_fail(cleanup_err)) {
388         // TODO(razvan): Here, spawnd has failed deleting its local cspace.
389         // Should we send another cleanup message, until it might succeed?
390         free(cl);
391         return;
392     }
393
394     struct domain_entry *entry;
395     err = domain_get_by_cap(domain_cap, &entry);
396     if (err_is_fail(err)) {
397         DEBUG_ERR(err, "failed to retrieve domain by cap returned by spawnd "
398                   "after cleanup");
399         return;
400     }
401
402     assert(entry->num_spawnds_resources > 0);
403     assert(entry->status != DOMAIN_STATUS_CLEANED);
404
405     --entry->num_spawnds_resources;
406
407     if (entry->num_spawnds_resources == 0) {
408         entry->status = DOMAIN_STATUS_CLEANED;
409
410         // At this point, the domain exists in state CLEANED for
411         // history reasons. For instance, if some other domain
412         // issues a wait call for this one, the process manager can
413         // return the exit status directly.
414         // At some point, however, we might want to just clean up
415         // the domain entry and recycle the domain cap.
416     } else {
417         // Expecting to receive further cleanup replies from other
418         // spawnds for the same domain cap, hence re-add the
419         // pending client.
420         err = pending_clients_add(domain_cap, cl->b,
421                                   ClientType_Cleanup, MAX_COREID);
422         if (err_is_fail(err)) {
423             DEBUG_ERR(err, "pending_clients_add in cleanup_reply_handler");
424         }
425     }
426 }
427
428 static void kill_reply_handler(struct spawn_binding *b,
429                                struct capref domain_cap, errval_t kill_err)
430 {
431     event_mutex_unlock(&b->mutex);
432
433     struct pending_client *cl;
434     errval_t err = pending_clients_release(domain_cap, ClientType_Kill, &cl);
435     if (err_is_fail(err)) {
436         DEBUG_ERR(err, "failed to retrieve pending kill client based on domain "
437                   "cap");
438         return;
439     }
440
441     errval_t resp_err;
442     if (err_is_fail(kill_err)) {
443         // TODO(razvan): Here, spawnd has failed deleting its local dispatcher.
444         // Should we send another kill message, until it might succeed?
445         while (cl != NULL) {
446             resp_err = cl->b->tx_vtbl.kill_response(cl->b, NOP_CONT,
447                                                      kill_err);
448             if (err_is_fail(resp_err)) {
449                 DEBUG_ERR(resp_err, "failed to send kill_response to client");
450             }
451             struct pending_client *tmp = cl;
452             cl = cl->next;
453             free(tmp);
454         }
455         return;
456     }
457
458     struct domain_entry *entry;
459     err = domain_get_by_cap(domain_cap, &entry);
460     if (err_is_fail(err)) {
461         DEBUG_ERR(err, "failed to retrieve domain by cap returned by spawnd "
462                   "after kill");
463         return;
464     }
465
466     assert(entry->num_spawnds_running > 0);
467     assert(entry->status != DOMAIN_STATUS_STOPPED);
468
469     --entry->num_spawnds_running;
470
471     if (entry->num_spawnds_running == 0) {
472         entry->status = DOMAIN_STATUS_STOPPED;
473         entry->exit_status = EXIT_STATUS_KILLED;
474
475         err = pending_clients_add(domain_cap, NULL, ClientType_Cleanup,
476                                   MAX_COREID);
477         if (err_is_fail(err)) {
478             DEBUG_ERR(err, "pending_clients_add in kill_reply_handler");
479         }
480
481         // TODO(razvan): Might it be more sane if we respond back
482         // to the client after the domain has been cleaned up (i.e.
483         // the cspace root has been revoked for all dispatchers)?
484         while (cl != NULL) {
485             resp_err = cl->b->tx_vtbl.kill_response(cl->b, NOP_CONT,
486                                                      kill_err);
487             if (err_is_fail(resp_err)) {
488                 DEBUG_ERR(resp_err, "failed to send kill_response to client");
489             }
490             struct pending_client *tmp = cl;
491             cl = cl->next;
492             free(tmp);
493         }
494         
495         // TODO(razvan): Same problem applies to the waiters: would
496         // it be better if we sent them wait_responses after the
497         // cspace root has been revoked, too? (here and in the exit
498         // case).
499         struct domain_waiter *waiter = entry->waiters;
500         while (waiter != NULL) {
501             waiter->b->tx_vtbl.wait_response(waiter->b, NOP_CONT,
502                                              SYS_ERR_OK,
503                                              entry->exit_status);
504             struct domain_waiter *tmp = waiter;
505             waiter = waiter->next;
506             free(tmp);
507         }
508
509         for (coreid_t i = 0; i < MAX_COREID; ++i) {
510             if (entry->spawnds[i] == NULL) {
511                 continue;
512             }
513
514             struct spawn_binding *spb = entry->spawnds[i]->b;
515
516             struct pending_kill_exit_cleanup *cleanup = (struct pending_kill_exit_cleanup*) malloc(
517                     sizeof(struct pending_kill_exit_cleanup));
518             cleanup->sb = spb;
519             cleanup->domain_cap = domain_cap;
520
521             spb->rx_vtbl.cleanup_reply = cleanup_reply_handler;
522             event_mutex_enqueue_lock(&spb->mutex,
523                                      &cleanup->qn,
524                                      (struct event_closure) {
525                                          .handler = cleanup_request_sender,
526                                          .arg = cleanup });
527         }
528     } else {
529         err = pending_clients_add(domain_cap, cl->b, ClientType_Kill,
530                                   MAX_COREID);
531         if (err_is_fail(err)) {
532             DEBUG_ERR(err, "pending_clients_add in kill_reply_handler");
533         }
534     }
535 }
536
537 static void exit_reply_handler(struct spawn_binding *b,
538                                struct capref domain_cap, errval_t exit_err)
539 {
540     event_mutex_unlock(&b->mutex);
541
542     struct pending_client *cl;
543     errval_t err = pending_clients_release(domain_cap, ClientType_Exit, &cl);
544     if (err_is_fail(err)) {
545         DEBUG_ERR(err, "failed to retrieve pending exit client based on domain "
546                   "cap");
547         return;
548     }
549
550     if (err_is_fail(exit_err)) {
551         // TODO(razvan): Here, spawnd has failed deleting its local dispatcher.
552         // Should we send another kill message, until it might succeed?
553         free(cl);
554         return;
555     }
556
557     struct domain_entry *entry;
558     err = domain_get_by_cap(domain_cap, &entry);
559     if (err_is_fail(err)) {
560         DEBUG_ERR(err, "failed to retrieve domain by cap returned by spawnd "
561                   "after exit");
562         return;
563     }
564
565     assert(entry->num_spawnds_running > 0);
566     assert(entry->status != DOMAIN_STATUS_STOPPED);
567
568     --entry->num_spawnds_running;
569
570     if (entry->num_spawnds_running == 0) {
571         entry->status = DOMAIN_STATUS_STOPPED;
572         
573         free(cl);
574
575         // TODO(razvan): Same problem applies to the waiters: would
576         // it be better if we sent them wait_responses after the
577         // cspace root has been revoked, too? (here and in the exit
578         // case).
579         struct domain_waiter *waiter = entry->waiters;
580         while (waiter != NULL) {
581             waiter->b->tx_vtbl.wait_response(waiter->b, NOP_CONT,
582                                              SYS_ERR_OK,
583                                              entry->exit_status);
584             struct domain_waiter *tmp = waiter;
585             waiter = waiter->next;
586             free(tmp);
587         }
588
589         for (coreid_t i = 0; i < MAX_COREID; ++i) {
590             if (entry->spawnds[i] == NULL) {
591                 continue;
592             }
593
594             struct spawn_binding *spb = entry->spawnds[i]->b;
595
596             struct pending_kill_exit_cleanup *cleanup = (struct pending_kill_exit_cleanup*) malloc(
597                     sizeof(struct pending_kill_exit_cleanup));
598             cleanup->sb = spb;
599             cleanup->domain_cap = domain_cap;
600
601             spb->rx_vtbl.cleanup_reply = cleanup_reply_handler;
602             event_mutex_enqueue_lock(&spb->mutex,
603                                      &cleanup->qn,
604                                      (struct event_closure) {
605                                          .handler = cleanup_request_sender,
606                                          .arg = cleanup });
607         }
608     } else {
609         err = pending_clients_add(domain_cap, cl->b, ClientType_Exit,
610                                   MAX_COREID);
611         if (err_is_fail(err)) {
612             DEBUG_ERR(err, "pending_clients_add in kill_reply_handler");
613         }
614     }
615 }
616
617 static errval_t spawn_handler_common(struct proc_mgmt_binding *b,
618                                      enum ClientType type,
619                                      coreid_t core_id, const char *path,
620                                      const char *argvbuf, size_t argvbytes,
621                                      const char *envbuf, size_t envbytes,
622                                      struct capref inheritcn_cap,
623                                      struct capref argcn_cap, uint8_t flags)
624 {
625     if (!spawnd_state_exists(core_id)) {
626         return PROC_MGMT_ERR_INVALID_SPAWND;
627     }
628
629     struct spawnd_state *state = spawnd_state_get(core_id);
630     assert(state != NULL);
631     struct spawn_binding *cl = state->b;
632     assert(cl != NULL);
633
634     struct capref domain_cap;
635     errval_t err = slot_alloc(&domain_cap);
636     if (err_is_fail(err)) {
637         DEBUG_ERR(err, "slot_alloc domain_cap");
638         return err_push(err, PROC_MGMT_ERR_CREATE_DOMAIN_CAP);
639     }
640     err = cap_retype(domain_cap, cap_procmng, 0, ObjType_Domain, 0, 1);
641     if (err_is_fail(err)) {
642         DEBUG_ERR(err, "cap_retype domain_cap");
643         return err_push(err, PROC_MGMT_ERR_CREATE_DOMAIN_CAP);
644     }
645
646     err = pending_clients_add(domain_cap, b, type, core_id);
647     if (err_is_fail(err)) {
648         DEBUG_ERR(err, "pending_clients_add");
649         return err;
650     }
651
652     struct pending_spawn *spawn = (struct pending_spawn*) malloc(
653             sizeof(struct pending_spawn));
654     spawn->domain_cap = domain_cap;
655     spawn->b = cl;
656     spawn->core_id = core_id;
657     spawn->path = path;
658     spawn->argvbuf = argvbuf;
659     spawn->argvbytes = argvbytes;
660     spawn->envbuf = envbuf;
661     spawn->envbytes = envbytes;
662     spawn->inheritcn_cap = inheritcn_cap;
663     spawn->argcn_cap = argcn_cap;
664     spawn->flags = flags;
665
666     event_mutex_enqueue_lock(&cl->mutex, &spawn->qn,
667                              (struct event_closure) {
668                                  .handler = spawn_request_sender,
669                                  .arg = spawn });
670
671     return SYS_ERR_OK;
672 }
673
674 static void spawn_handler(struct proc_mgmt_binding *b, coreid_t core_id,
675                           const char *path, const char *argvbuf,
676                           size_t argvbytes, const char *envbuf, size_t envbytes,
677                           uint8_t flags)
678 {
679     errval_t err, resp_err;
680     err = spawn_handler_common(b, ClientType_Spawn, core_id, path, argvbuf,
681                                argvbytes, envbuf, envbytes, NULL_CAP, NULL_CAP,
682                                flags);
683     if (err_is_ok(err)) {
684         // Will respond to client when we get the reply from spawnd.
685         return;
686     }
687
688     resp_err = b->tx_vtbl.spawn_response(b, NOP_CONT, err, NULL_CAP);
689     if (err_is_fail(resp_err)) {
690         DEBUG_ERR(resp_err, "failed to send spawn_response");
691     }
692 }
693
694 static void spawn_with_caps_handler(struct proc_mgmt_binding *b,
695                                     coreid_t core_id, const char *path,
696                                     const char *argvbuf, size_t argvbytes,
697                                     const char *envbuf, size_t envbytes,
698                                     struct capref inheritcn_cap,
699                                     struct capref argcn_cap, uint8_t flags)
700 {
701     errval_t err, resp_err;
702     err = spawn_handler_common(b, ClientType_SpawnWithCaps, core_id, path,
703                                argvbuf, argvbytes, envbuf, envbytes,
704                                inheritcn_cap, argcn_cap, flags);
705     if (err_is_ok(err)) {
706         // Will respond to client when we get the reply from spawnd.
707         return;
708     }
709
710     resp_err = b->tx_vtbl.spawn_with_caps_response(b, NOP_CONT, err,
711                                                             NULL_CAP);
712     if (err_is_fail(resp_err)) {
713         DEBUG_ERR(resp_err, "failed to send spawn_with_caps_response");
714     }
715 }
716
717 static void span_handler(struct proc_mgmt_binding *b, struct capref domain_cap,
718                          coreid_t core_id, struct capref vroot,
719                          struct capref dispframe)
720 {
721     errval_t err, resp_err;
722     err = domain_can_span(domain_cap, core_id);
723     if (err_is_fail(err)) {
724         goto respond_with_err;
725     }
726
727     if (!spawnd_state_exists(core_id)) {
728         err = PROC_MGMT_ERR_INVALID_SPAWND;
729         goto respond_with_err;
730     }
731
732     struct spawnd_state *state = spawnd_state_get(core_id);
733     assert(state != NULL);
734     struct spawn_binding *cl = state->b;
735     assert(cl != NULL);
736
737     err = pending_clients_add(domain_cap, b, ClientType_Span, core_id);
738     if (err_is_fail(err)) {
739         goto respond_with_err;
740     }
741
742     struct pending_span *span = (struct pending_span*) malloc(
743             sizeof(struct pending_span));
744     span->domain_cap = domain_cap;
745     span->b = cl;
746     span->core_id = core_id;
747     span->vroot = vroot;
748     span->dispframe = dispframe;
749
750     event_mutex_enqueue_lock(&cl->mutex, &span->qn,
751                              (struct event_closure) {
752                                  .handler = span_request_sender,
753                                  .arg = span });
754
755 respond_with_err:
756     resp_err = b->tx_vtbl.span_response(b, NOP_CONT, err);
757     if (err_is_fail(resp_err)) {
758         DEBUG_ERR(resp_err, "failed to send span_response");
759     }
760 }
761
762 static errval_t kill_handler_common(struct proc_mgmt_binding *b,
763                                     struct capref domain_cap,
764                                     enum ClientType type,
765                                     uint8_t exit_status)
766 {
767     errval_t err = pending_clients_add(domain_cap, b, type, MAX_COREID);
768     if (err_is_fail(err)) {
769         return err;
770     }
771
772     struct domain_entry *entry;
773     err = domain_get_by_cap(domain_cap, &entry);
774     if (err_is_fail(err)) {
775         return err;
776     }
777
778     entry->exit_status = exit_status;
779     domain_stop_pending(entry);
780
781     for (coreid_t i = 0; i < MAX_COREID; ++i) {
782         if (entry->spawnds[i] == NULL) {
783             continue;
784         }
785
786         struct spawn_binding *spb = entry->spawnds[i]->b;
787
788         struct pending_kill_exit_cleanup *cmd = (struct pending_kill_exit_cleanup*) malloc(
789                 sizeof(struct pending_kill_exit_cleanup));
790         cmd->domain_cap = domain_cap;
791         cmd->sb = spb;
792
793         switch (type) {
794             case ClientType_Kill:
795                 cmd->pmb = b;
796
797                 event_mutex_enqueue_lock(&spb->mutex,
798                                          &cmd->qn,
799                                          (struct event_closure) {
800                                             .handler = kill_request_sender,
801                                             .arg = cmd });
802                 break;
803
804             case ClientType_Exit:
805                 event_mutex_enqueue_lock(&spb->mutex,
806                                          &cmd->qn,
807                                          (struct event_closure) {
808                                             .handler = exit_request_sender,
809                                             .arg = cmd });
810                 break;
811             default:
812                 USER_PANIC("invalid client type for kill: %u\n", type);
813         }
814     }
815
816     return SYS_ERR_OK;
817 }
818
819 static void kill_handler(struct proc_mgmt_binding *b,
820                          struct capref victim_domain_cap)
821 {
822     errval_t err = kill_handler_common(b, victim_domain_cap, ClientType_Kill,
823                                        EXIT_STATUS_KILLED);
824     if (err_is_fail(err)) {
825         errval_t resp_err = b->tx_vtbl.kill_response(b, NOP_CONT, err);
826         if (err_is_fail(resp_err)) {
827             DEBUG_ERR(resp_err, "failed to send kill_response");
828         }
829     }
830 }
831
832 static void exit_handler(struct proc_mgmt_binding *b, struct capref domain_cap,
833                          uint8_t exit_status)
834 {
835     errval_t err = kill_handler_common(b, domain_cap, ClientType_Exit,
836                                        exit_status);
837     if (err_is_fail(err)) {
838         DEBUG_ERR(err, "processing exit_handler for requesting domain, exit "
839                   "code %u", exit_status);
840     }
841     // Error or not, there's no client to respond to anymore.
842 }
843
844 static void wait_handler(struct proc_mgmt_binding *b, struct capref domain_cap)
845 {
846     errval_t err, resp_err;
847     struct domain_entry *entry;
848     err = domain_get_by_cap(domain_cap, &entry);
849     if (err_is_fail(err)) {
850         goto respond;
851     }
852
853     if (entry->status == DOMAIN_STATUS_STOPPED) {
854         // Domain has already been stopped, so just reply with exit status.
855         goto respond;
856     }
857
858     struct domain_waiter *waiter = (struct domain_waiter*) malloc(
859             sizeof(struct domain_waiter));
860     waiter->b = b;
861     waiter->next = entry->waiters;
862     entry->waiters = waiter;
863     // Will respond when domain is stopped.
864     return;
865
866 respond:
867     resp_err = b->tx_vtbl.wait_response(b, NOP_CONT, err, entry->exit_status);
868     if (err_is_fail(resp_err)) {
869         DEBUG_ERR(resp_err, "failed to send wait_response");
870     }
871 }
872
873 static struct proc_mgmt_rx_vtbl monitor_vtbl = {
874     .add_spawnd           = add_spawnd_handler,
875     .spawn_call           = spawn_handler,
876     .spawn_with_caps_call = spawn_with_caps_handler,
877     .span_call            = span_handler,
878     .kill_call            = kill_handler,
879     .exit_call            = exit_handler,
880     .wait_call            = wait_handler
881 };
882
883 static struct proc_mgmt_rx_vtbl non_monitor_vtbl = {
884     .add_spawnd           = add_spawnd_handler_non_monitor,
885     .spawn_call           = spawn_handler,
886     .spawn_with_caps_call = spawn_with_caps_handler,
887     .span_call            = span_handler,
888     .kill_call            = kill_handler,
889     .exit_call            = exit_handler,
890     .wait_call            = wait_handler
891 };
892
893 static errval_t alloc_ep_for_monitor(struct capref *ep)
894 {
895     struct proc_mgmt_lmp_binding *lmpb =
896         malloc(sizeof(struct proc_mgmt_lmp_binding));
897     assert(lmpb != NULL);
898
899     // setup our end of the binding
900     errval_t err = proc_mgmt_client_lmp_accept(lmpb, get_default_waitset(),
901                                                DEFAULT_LMP_BUF_WORDS);
902     if (err_is_fail(err)) {
903         free(lmpb);
904         return err_push(err, LIB_ERR_PROC_MGMT_CLIENT_ACCEPT);
905     }
906
907     *ep = lmpb->chan.local_cap;
908     lmpb->b.rx_vtbl = monitor_vtbl;
909
910     return SYS_ERR_OK;
911 }
912
913 static void export_cb(void *st, errval_t err, iref_t iref)
914 {
915     if (err_is_fail(err)) {
916         USER_PANIC_ERR(err, "export failed");
917     }
918
919     // Allocate an endpoint for the local monitor, who will use it to inform
920     // us about new spawnd irefs on behalf of other monitors.
921     struct capref ep;
922     err = alloc_ep_for_monitor(&ep);
923     if (err_is_fail(err)) {
924         USER_PANIC_ERR(err, "failed to allocate LMP EP for local monitor");
925     }
926
927     // Send the endpoint to the monitor, so it can finish the handshake.
928     struct monitor_binding *mb = get_monitor_binding();
929     err = mb->tx_vtbl.set_proc_mgmt_ep_request(mb, NOP_CONT, ep);
930     if (err_is_fail(err)) {
931         USER_PANIC_ERR(err, "failed to send set_proc_mgmt_ep_request to "
932                        "monitor");
933     }
934
935     // Also register this iref with the name service, for arbitrary client
936     // domains to use for spawn-related ops.
937     err = nameservice_register(SERVICE_BASENAME, iref);
938     if (err_is_fail(err)) {
939         USER_PANIC_ERR(err, "nameservice_register failed");
940     }
941 }
942
943 static errval_t connect_cb(void *st, struct proc_mgmt_binding *b)
944 {
945     b->rx_vtbl = non_monitor_vtbl;
946     return SYS_ERR_OK;
947 }
948
949 errval_t start_service(void)
950 {
951     return proc_mgmt_export(NULL, export_cb, connect_cb, get_default_waitset(),
952             IDC_EXPORT_FLAGS_DEFAULT);
953 }