Break the spawnd kill API into kill + cleanup.
[barrelfish] / usr / proc_mgmt / service.c
1 /**
2  * \file
3  * \brief Process management service.
4  */
5
6 /*
7  * Copyright (c) 2017, ETH Zurich.
8  * All rights reserved.
9  *
10  * This file is distributed under the terms in the attached LICENSE file.
11  * If you do not find this file, copies can be found by writing to:
12  * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
13  */
14
15 #include <barrelfish/barrelfish.h>
16 #include <barrelfish/nameservice_client.h>
17 #include <barrelfish/proc_mgmt_client.h>
18 #include <barrelfish/spawn_client.h>
19 #include <if/monitor_defs.h>
20 #include <if/proc_mgmt_defs.h>
21 #include <if/spawn_defs.h>
22
23 #include "domain.h"
24 #include "internal.h"
25 #include "pending_clients.h"
26 #include "spawnd_state.h"
27
28 static void add_spawnd_handler(struct proc_mgmt_binding *b, coreid_t core_id,
29                                iref_t iref)
30 {
31     if (spawnd_state_exists(core_id)) {
32         DEBUG_ERR(PROC_MGMT_ERR_SPAWND_EXISTS, "spawnd_state_exists");
33         return;
34     }
35
36     // Bind with the spawnd.
37     struct spawn_binding *spawnb;
38     errval_t err = spawn_bind_iref(iref, &spawnb);
39     if (err_is_fail(err)) {
40         DEBUG_ERR(err, "spawn_bind_iref");
41         return;
42     }
43
44     err = spawnd_state_alloc(core_id, spawnb);
45     if (err_is_fail(err)) {
46         DEBUG_ERR(err, "spawnd_state_alloc");
47     }
48
49     debug_printf("Process manager bound with spawnd.%u on iref %u\n", core_id,
50             iref);
51 }
52
53 static void add_spawnd_handler_non_monitor(struct proc_mgmt_binding *b,
54                                            coreid_t core_id, iref_t iref)
55 {
56     // debug_printf("Ignoring add_spawnd call: %s\n",
57     //              err_getstring(PROC_MGMT_ERR_NOT_MONITOR));
58 }
59
60 static void spawn_reply_handler(struct spawn_binding *b,
61                                 struct capref domain_cap, errval_t spawn_err)
62 {
63     struct pending_client *cl;
64     errval_t err = pending_clients_release(domain_cap, &cl);
65     if (err_is_fail(err)) {
66         // This might be a kill request issued after a successful spawn/span
67         // followed by a local error in the process manager (see below). If that
68         // is the case, then we won't have a client, as it has already been
69         // released.
70         DEBUG_ERR(err, "failed to retrieve pending client based on domain cap "
71                   "returned by spawnd");
72         return;
73     }
74
75     errval_t resp_err = SYS_ERR_OK;
76     struct domain_entry *entry;
77     switch (cl->type) {
78         case ClientType_Spawn:
79             err = spawn_err;
80             if (err_is_ok(spawn_err)) {
81                 err = domain_spawn(domain_cap, cl->core_id);
82             }
83             resp_err = cl->b->tx_vtbl.spawn_response(cl->b, NOP_CONT, err,
84                                                      domain_cap);
85             break;
86
87         case ClientType_SpawnWithCaps:
88             err = spawn_err;
89             if (err_is_ok(spawn_err)) {
90                 err = domain_spawn(domain_cap, cl->core_id);
91             }
92             resp_err = cl->b->tx_vtbl.spawn_with_caps_response(cl->b, NOP_CONT,
93                                                                err, domain_cap);
94             break;
95
96         case ClientType_Span:
97             err = spawn_err;
98             if (err_is_ok(spawn_err)) {
99                 err = domain_span(domain_cap, cl->core_id);
100             }
101             resp_err = cl->b->tx_vtbl.span_response(cl->b, NOP_CONT, err);
102             break;
103
104         case ClientType_Kill:
105             if (err_is_fail(spawn_err)) {
106                 // Looks like some spawnd was unable to successfully kill
107                 // its dispatcher for this domain. Not much the process
108                 // manager can do about it; return the error to the client.
109                 resp_err = cl->b->tx_vtbl.kill_response(cl->b, NOP_CONT,
110                                                         err);
111                 break;
112             }
113
114             err = domain_get_by_cap(domain_cap, &entry);
115             if (err_is_fail(err)) {
116                 DEBUG_ERR(err, "failed to retrieve domain by domain_cap "
117                           "returned by spawnd after kill");
118                 break;
119             }
120
121             assert(entry->num_spawnds_resources > 0 ||
122                    entry->num_spawnds_running > 0);
123             assert(entry->status != DOMAIN_STATUS_CLEANED);
124
125             if (entry->num_spawnds_running > 0) {
126                 --entry->num_spawnds_running;
127                     
128                 err = pending_clients_add(domain_cap, cl->b,
129                                           ClientType_Kill, MAX_COREID);
130                 if (err_is_fail(err)) {
131                     DEBUG_ERR(err, "pending_clients_add in reply handler");
132                 }
133
134                 if (entry->num_spawnds_running == 0) {
135                     entry->status = DOMAIN_STATUS_STOPPED;
136                     entry->exit_status = EXIT_STATUS_KILLED;
137
138                     // TODO(razvan): Might it be more sane if we respond back
139                     // to the client after the domain has been cleaned up (i.e.
140                     // the cspace root has been revoked for all dispatchers)?
141                     resp_err = cl->b->tx_vtbl.kill_response(cl->b, NOP_CONT,
142                                                            err);
143                     
144                     // TODO(razvan): Same problem applies to the waiters: would
145                     // it be better if we sent them wait_responses after the
146                     // cspace root has been revoked, too? (here and in the exit
147                     // case).
148                     struct domain_waiter *waiter = entry->waiters;
149                     while (waiter != NULL) {
150                         waiter->b->tx_vtbl.wait_response(waiter->b, NOP_CONT,
151                                                          SYS_ERR_OK,
152                                                          entry->exit_status);
153                         struct domain_waiter *aux = waiter;
154                         waiter = waiter->next;
155                         free(aux);
156                     }
157
158                     for (coreid_t i = 0; i < MAX_COREID; ++i) {
159                         if (entry->spawnds[i] == NULL) {
160                             continue;
161                         }
162
163                         struct spawn_binding *spb = entry->spawnds[i]->b;
164                         spb->rx_vtbl.spawn_reply = spawn_reply_handler;
165                         errval_t req_err = spb->tx_vtbl.cleanup_request(spb,
166                                 NOP_CONT, cap_procmng, domain_cap);
167                         if (err_is_fail(req_err)) {
168                             DEBUG_ERR(req_err, "failed to send cleanup_request "
169                                       "to spawnd %u\n", i);
170                         }
171                     }
172                 }
173             } else {
174                 --entry->num_spawnds_resources;
175
176                 if (entry->num_spawnds_resources == 0) {
177                     entry->status = DOMAIN_STATUS_CLEANED;
178
179                     // At this point, the domain exists in state CLEANED for
180                     // history reasons. For instance, if some other domain
181                     // issues a wait call for this one, the process manager can
182                     // return the exit status directly.
183                     // At some point, however, we might want to just clean up
184                     // the domain entry and recycle the domain cap.
185                 } else {
186                     // Expecting to receive further cleanup replies from other
187                     // spawnds for the same domain cap, hence re-add the
188                     // pending client.
189                     err = pending_clients_add(domain_cap, cl->b,
190                                               ClientType_Exit, MAX_COREID);
191                     if (err_is_fail(err)) {
192                         DEBUG_ERR(err, "pending_clients_add in reply handler");
193                     }
194                 }
195             }
196             break;
197
198         case ClientType_Exit:
199             if (err_is_fail(spawn_err)) {
200                 // Looks like some spawnd was unable to successfully kill
201                 // its dispatcher for this domain. Not much the process
202                 // manager can do about it. Furthermore, this was an exit call,
203                 // so there's no client to reply back to.
204                 break;
205             }
206
207             err = domain_get_by_cap(domain_cap, &entry);
208             if (err_is_fail(err)) {
209                 DEBUG_ERR(err, "failed to retrieve domain by domain_cap "
210                           "returned by spawnd after kill");
211                 break;
212             }
213
214             assert(entry->num_spawnds_resources > 0 ||
215                    entry->num_spawnds_running > 0);
216             assert(entry->status != DOMAIN_STATUS_CLEANED);
217
218             if (entry->num_spawnds_running > 0) {
219                 --entry->num_spawnds_running;
220
221                 err = pending_clients_add(domain_cap, cl->b,
222                                           ClientType_Exit, MAX_COREID);
223                 if (err_is_fail(err)) {
224                     DEBUG_ERR(err, "pending_clients_add in reply handler");
225                 }
226
227                 if (entry->num_spawnds_running == 0) {
228                     entry->status = DOMAIN_STATUS_STOPPED;
229
230                     struct domain_waiter *waiter = entry->waiters;
231                     while (waiter != NULL) {
232                         waiter->b->tx_vtbl.wait_response(waiter->b, NOP_CONT,
233                                                          SYS_ERR_OK,
234                                                          entry->exit_status);
235                         struct domain_waiter *aux = waiter;
236                         waiter = waiter->next;
237                         free(aux);
238                     }
239
240                     for (coreid_t i = 0; i < MAX_COREID; ++i) {
241                         if (entry->spawnds[i] == NULL) {
242                             continue;
243                         }
244
245                         struct spawn_binding *spb = entry->spawnds[i]->b;
246                         spb->rx_vtbl.spawn_reply = spawn_reply_handler;
247                         errval_t req_err = spb->tx_vtbl.cleanup_request(spb,
248                                 NOP_CONT, cap_procmng, domain_cap);
249                         if (err_is_fail(req_err)) {
250                             DEBUG_ERR(req_err, "failed to send cleanup_request "
251                                       "to spawnd %u\n", i);
252                         }
253                     }
254                 }
255             } else {
256                 --entry->num_spawnds_resources;
257
258                 if (entry->num_spawnds_resources == 0) {
259                     entry->status = DOMAIN_STATUS_CLEANED;
260
261                     // At this point, the domain exists in state CLEANED for
262                     // history reasons. For instance, if some other domain
263                     // issues a wait call for this one, the process manager can
264                     // return the exit status directly.
265                     // At some point, however, we might want to just clean up
266                     // the domain entry and recycle the domain cap.
267                 } else {
268                     // Expecting to receive further cleanup replies from other
269                     // spawnds for the same domain cap, hence re-add the
270                     // pending client.
271                     err = pending_clients_add(domain_cap, cl->b,
272                                               ClientType_Exit, MAX_COREID);
273                     if (err_is_fail(err)) {
274                         DEBUG_ERR(err, "pending_clients_add in reply handler");
275                     }
276                 }
277             }
278             break;
279
280         default:
281             // TODO(razvan): Handle the other cases, e.g. wait.
282             debug_printf("Unknown client type %u\n", cl->type);
283             return;
284     }
285
286     if (err_is_ok(spawn_err) && err_is_fail(err)) {
287         // Spawnd has successfully completed its end of the operation, but
288         // there's been an error in the process manager's book-keeping
289         // of domains. Therefore, if the request was a spawn or span one, spawnd
290         // needs to be asked to stop the dispatcher which it has just enqueued.
291         if (cl->type == ClientType_Spawn ||
292             cl->type == ClientType_SpawnWithCaps ||
293             cl->type == ClientType_Span) {
294             struct spawnd_state *state = spawnd_state_get(cl->core_id);
295             assert(state != NULL);
296             struct spawn_binding *spb = state->b;
297             assert(spb != NULL);
298
299             err = spb->tx_vtbl.kill_request(spb, NOP_CONT, cap_procmng, 
300                                             domain_cap);
301             if (err_is_fail(err)) {
302                 // XXX: How severe is this? Maybe we want something more
303                 // assertive than logging an error message.
304                 DEBUG_ERR(err, "failed to send kill request for dangling "
305                           "dispatcher");
306             } else {
307                 pending_clients_add(domain_cap, cl->b, ClientType_Kill,
308                                     MAX_COREID);
309             }
310         }
311     }
312
313     free(cl);
314
315     if (err_is_fail(resp_err)) {
316         DEBUG_ERR(resp_err, "failed to send response to client");
317     }
318 }
319
320 static errval_t spawn_handler_common(struct proc_mgmt_binding *b,
321                                      enum ClientType type,
322                                      coreid_t core_id, const char *path,
323                                      const char *argvbuf, size_t argvbytes,
324                                      const char *envbuf, size_t envbytes,
325                                      struct capref inheritcn_cap,
326                                      struct capref argcn_cap, uint8_t flags,
327                                      struct capref *ret_domain_cap)
328 {
329     assert(ret_domain_cap != NULL);
330
331     if (!spawnd_state_exists(core_id)) {
332         return PROC_MGMT_ERR_INVALID_SPAWND;
333     }
334
335     struct spawnd_state *state = spawnd_state_get(core_id);
336     assert(state != NULL);
337     struct spawn_binding *cl = state->b;
338     assert(cl != NULL);
339
340     struct capref domain_cap;
341     errval_t err = slot_alloc(&domain_cap);
342     if (err_is_fail(err)) {
343         DEBUG_ERR(err, "slot_alloc domain_cap");
344         return err_push(err, PROC_MGMT_ERR_CREATE_DOMAIN_CAP);
345     }
346     err = cap_retype(domain_cap, cap_procmng, 0, ObjType_Domain, 0, 1);
347     if (err_is_fail(err)) {
348         DEBUG_ERR(err, "cap_retype domain_cap");
349         return err_push(err, PROC_MGMT_ERR_CREATE_DOMAIN_CAP);
350     }
351
352     err = pending_clients_add(domain_cap, b, type, core_id);
353     if (err_is_fail(err)) {
354         DEBUG_ERR(err, "pending_clients_add");
355         return err;
356     }
357
358     cl->rx_vtbl.spawn_reply = spawn_reply_handler;
359     if (capref_is_null(inheritcn_cap) && capref_is_null(argcn_cap)) {
360         err = cl->tx_vtbl.spawn_request(cl, NOP_CONT, cap_procmng, domain_cap,
361                                         path, argvbuf, argvbytes, envbuf,
362                                         envbytes, flags);
363     } else {
364         err = cl->tx_vtbl.spawn_with_caps_request(cl, NOP_CONT, cap_procmng,
365                                                   domain_cap, path, argvbuf,
366                                                   argvbytes, envbuf, envbytes,
367                                                   inheritcn_cap, argcn_cap,
368                                                   flags);
369     }
370     if (err_is_fail(err)) {
371         DEBUG_ERR(err, "sending spawn request");
372         pending_clients_release(domain_cap, NULL);
373         return err_push(err, PROC_MGMT_ERR_SPAWND_REQUEST);
374     }
375
376     return SYS_ERR_OK;
377 }
378
379 static void spawn_handler(struct proc_mgmt_binding *b, coreid_t core_id,
380                           const char *path, const char *argvbuf,
381                           size_t argvbytes, const char *envbuf, size_t envbytes,
382                           uint8_t flags)
383 {
384     errval_t err, resp_err;
385     struct capref domain_cap;
386     err = spawn_handler_common(b, ClientType_Spawn, core_id, path, argvbuf,
387                                argvbytes, envbuf, envbytes, NULL_CAP, NULL_CAP,
388                                flags, &domain_cap);
389     if (err_is_ok(err)) {
390         // Will respond to client when we get the reply from spawnd.
391         return;
392     }
393
394     resp_err = b->tx_vtbl.spawn_response(b, NOP_CONT, err, NULL_CAP);
395     if (err_is_fail(resp_err)) {
396         DEBUG_ERR(resp_err, "failed to send spawn_response");
397     }
398 }
399
400 static void spawn_with_caps_handler(struct proc_mgmt_binding *b,
401                                     coreid_t core_id, const char *path,
402                                     const char *argvbuf, size_t argvbytes,
403                                     const char *envbuf, size_t envbytes,
404                                     struct capref inheritcn_cap,
405                                     struct capref argcn_cap, uint8_t flags)
406 {
407     errval_t err, resp_err;
408     struct capref domain_cap;
409     err = spawn_handler_common(b, ClientType_SpawnWithCaps, core_id, path,
410                                argvbuf, argvbytes, envbuf, envbytes,
411                                inheritcn_cap, argcn_cap, flags, &domain_cap);
412     if (err_is_ok(err)) {
413         // Will respond to client when we get the reply from spawnd.
414         return;
415     }
416
417     resp_err = b->tx_vtbl.spawn_with_caps_response(b, NOP_CONT, err,
418                                                             NULL_CAP);
419     if (err_is_fail(resp_err)) {
420         DEBUG_ERR(resp_err, "failed to send spawn_with_caps_response");
421     }
422 }
423
424 static void span_handler(struct proc_mgmt_binding *b, struct capref domain_cap,
425                          coreid_t core_id, struct capref vroot,
426                          struct capref dispframe)
427 {
428     errval_t err, resp_err;
429     err = domain_can_span(domain_cap, core_id);
430     if (err_is_fail(err)) {
431         goto respond_with_err;
432     }
433
434     if (!spawnd_state_exists(core_id)) {
435         err = PROC_MGMT_ERR_INVALID_SPAWND;
436         goto respond_with_err;
437     }
438
439     struct spawnd_state *state = spawnd_state_get(core_id);
440     assert(state != NULL);
441     struct spawn_binding *cl = state->b;
442     assert(cl != NULL);
443
444     err = pending_clients_add(domain_cap, b, ClientType_Span, core_id);
445     if (err_is_fail(err)) {
446         goto respond_with_err;
447     }
448
449     cl->rx_vtbl.spawn_reply = spawn_reply_handler;
450     err = cl->tx_vtbl.span_request(cl, NOP_CONT, cap_procmng, domain_cap, vroot,
451                                    dispframe);
452     if (err_is_ok(err)) {
453         // Will respond to client when we get the reply from spawnd.
454         return;
455     } else {
456         DEBUG_ERR(err, "sending span request");
457         pending_clients_release(domain_cap, NULL);
458         err = err_push(err, PROC_MGMT_ERR_SPAWND_REQUEST);
459     }
460
461 respond_with_err:
462     resp_err = b->tx_vtbl.span_response(b, NOP_CONT, err);
463     if (err_is_fail(resp_err)) {
464         DEBUG_ERR(resp_err, "failed to send span_response");
465     }
466 }
467
468 static errval_t kill_handler_common(struct proc_mgmt_binding *b,
469                                     struct capref domain_cap,
470                                     enum ClientType type,
471                                     uint8_t exit_status)
472 {
473     errval_t err = pending_clients_add(domain_cap, b, type, MAX_COREID);
474     if (err_is_fail(err)) {
475         return err;
476     }
477
478     struct domain_entry *entry;
479     err = domain_get_by_cap(domain_cap, &entry);
480     if (err_is_fail(err)) {
481         return err;
482     }
483
484     entry->exit_status = exit_status;
485     domain_stop_pending(entry);
486
487     for (coreid_t i = 0; i < MAX_COREID; ++i) {
488         if (entry->spawnds[i] == NULL) {
489             continue;
490         }
491
492         struct spawn_binding *spb = entry->spawnds[i]->b;
493         spb->rx_vtbl.spawn_reply = spawn_reply_handler;
494         errval_t req_err = spb->tx_vtbl.kill_request(spb, NOP_CONT, cap_procmng,
495                                                      domain_cap);
496         if (err_is_fail(req_err)) {
497             DEBUG_ERR(req_err, "failed to send kill_request to spawnd %u\n", i);
498         }
499     }
500
501     return SYS_ERR_OK;
502 }
503
504 static void kill_handler(struct proc_mgmt_binding *b, struct capref domain_cap)
505 {
506     errval_t err = kill_handler_common(b, domain_cap, ClientType_Kill,
507                                        EXIT_STATUS_KILLED);
508     if (err_is_fail(err)) {
509         errval_t resp_err = b->tx_vtbl.kill_response(b, NOP_CONT, err);
510         if (err_is_fail(resp_err)) {
511             DEBUG_ERR(resp_err, "failed to send kill_response");
512         }
513     }
514 }
515
516 static void exit_handler(struct proc_mgmt_binding *b, struct capref domain_cap,
517                          uint8_t exit_status)
518 {
519     errval_t err = kill_handler_common(b, domain_cap, ClientType_Exit,
520                                        exit_status);
521     if (err_is_fail(err)) {
522         DEBUG_ERR(err, "processing exit_handler for requesting domain, exit "
523                   "code %u", exit_status);
524     }
525     // Error or not, there's no client to reply to anymore.
526 }
527
528 static void wait_handler(struct proc_mgmt_binding *b, struct capref domain_cap)
529 {
530     errval_t err, resp_err;
531     struct domain_entry *entry;
532     err = domain_get_by_cap(domain_cap, &entry);
533     if (err_is_fail(err)) {
534         goto respond;
535     }
536
537     if (entry->status == DOMAIN_STATUS_STOPPED) {
538         // Domain has already been stopped, so just reply with exit status.
539         goto respond;
540     }
541
542     struct domain_waiter *waiter = (struct domain_waiter*) malloc(
543             sizeof(struct domain_waiter));
544     waiter->b = b;
545     waiter->next = entry->waiters;
546     entry->waiters = waiter;
547     // Will respond when domain is stopped.
548     return;
549
550 respond:
551     resp_err = b->tx_vtbl.wait_response(b, NOP_CONT, err, entry->exit_status);
552     if (err_is_fail(resp_err)) {
553         DEBUG_ERR(resp_err, "failed to send wait_response");
554     }
555 }
556
557 static struct proc_mgmt_rx_vtbl monitor_vtbl = {
558     .add_spawnd           = add_spawnd_handler,
559     .spawn_call           = spawn_handler,
560     .spawn_with_caps_call = spawn_with_caps_handler,
561     .span_call            = span_handler,
562     .kill_call            = kill_handler,
563     .exit                 = exit_handler,
564     .wait_call            = wait_handler
565 };
566
567 static struct proc_mgmt_rx_vtbl non_monitor_vtbl = {
568     .add_spawnd           = add_spawnd_handler_non_monitor,
569     .spawn_call           = spawn_handler,
570     .spawn_with_caps_call = spawn_with_caps_handler,
571     .span_call            = span_handler,
572     .kill_call            = kill_handler,
573     .exit                 = exit_handler,
574     .wait_call            = wait_handler
575 };
576
577 static errval_t alloc_ep_for_monitor(struct capref *ep)
578 {
579     struct proc_mgmt_lmp_binding *lmpb =
580         malloc(sizeof(struct proc_mgmt_lmp_binding));
581     assert(lmpb != NULL);
582
583     // setup our end of the binding
584     errval_t err = proc_mgmt_client_lmp_accept(lmpb, get_default_waitset(),
585                                                DEFAULT_LMP_BUF_WORDS);
586     if (err_is_fail(err)) {
587         free(lmpb);
588         return err_push(err, LIB_ERR_PROC_MGMT_CLIENT_ACCEPT);
589     }
590
591     *ep = lmpb->chan.local_cap;
592     lmpb->b.rx_vtbl = monitor_vtbl;
593
594     return SYS_ERR_OK;
595 }
596
597 static void export_cb(void *st, errval_t err, iref_t iref)
598 {
599     if (err_is_fail(err)) {
600         USER_PANIC_ERR(err, "export failed");
601     }
602
603     // Allocate an endpoint for the local monitor, who will use it to inform
604     // us about new spawnd irefs on behalf of other monitors.
605     struct capref ep;
606     err = alloc_ep_for_monitor(&ep);
607     if (err_is_fail(err)) {
608         USER_PANIC_ERR(err, "failed to allocate LMP EP for local monitor");
609     }
610
611     // Send the endpoint to the monitor, so it can finish the handshake.
612     struct monitor_binding *mb = get_monitor_binding();
613     err = mb->tx_vtbl.set_proc_mgmt_ep_request(mb, NOP_CONT, ep);
614     if (err_is_fail(err)) {
615         USER_PANIC_ERR(err, "failed to send set_proc_mgmt_ep_request to "
616                        "monitor");
617     }
618
619     // Also register this iref with the name service, for arbitrary client
620     // domains to use for spawn-related ops.
621     err = nameservice_register(SERVICE_BASENAME, iref);
622     if (err_is_fail(err)) {
623         USER_PANIC_ERR(err, "nameservice_register failed");
624     }
625 }
626
627 static errval_t connect_cb(void *st, struct proc_mgmt_binding *b)
628 {
629     b->rx_vtbl = non_monitor_vtbl;
630     return SYS_ERR_OK;
631 }
632
633 errval_t start_service(void)
634 {
635     return proc_mgmt_export(NULL, export_cb, connect_cb, get_default_waitset(),
636             IDC_EXPORT_FLAGS_DEFAULT);
637 }