git.barrelfish.org Git - barrelfish/blob - usr/proc_mgmt/service.c

   1 /**
   2  * \file
   3  * \brief Process management service.
   4  */
   5
   6 /*
   7  * Copyright (c) 2017, ETH Zurich.
   8  * All rights reserved.
   9  *
  10  * This file is distributed under the terms in the attached LICENSE file.
  11  * If you do not find this file, copies can be found by writing to:
  12  * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
  13  */
  14
  15 #include <barrelfish/barrelfish.h>
  16 #include <barrelfish/nameservice_client.h>
  17 #include <barrelfish/proc_mgmt_client.h>
  18 #include <barrelfish/spawn_client.h>
  19 #include <if/monitor_defs.h>
  20 #include <if/proc_mgmt_defs.h>
  21 #include <if/spawn_defs.h>
  22
  23 #include "domain.h"
  24 #include "internal.h"
  25 #include "pending_clients.h"
  26 #include "spawnd_state.h"
  27
  28 static void add_spawnd_handler(struct proc_mgmt_binding *b, coreid_t core_id,
  29                                iref_t iref)
  30 {
  31     if (spawnd_state_exists(core_id)) {
  32         DEBUG_ERR(PROC_MGMT_ERR_SPAWND_EXISTS, "spawnd_state_exists");
  33         return;
  34     }
  35
  36     // Bind with the spawnd.
  37     struct spawn_binding *spawnb;
  38     errval_t err = spawn_bind_iref(iref, &spawnb);
  39     if (err_is_fail(err)) {
  40         DEBUG_ERR(err, "spawn_bind_iref");
  41         return;
  42     }
  43
  44     err = spawnd_state_alloc(core_id, spawnb);
  45     if (err_is_fail(err)) {
  46         DEBUG_ERR(err, "spawnd_state_alloc");
  47     }
  48
  49     debug_printf("Process manager bound with spawnd.%u on iref %u\n", core_id,
  50             iref);
  51 }
  52
  53 static void add_spawnd_handler_non_monitor(struct proc_mgmt_binding *b,
  54                                            coreid_t core_id, iref_t iref)
  55 {
  56     // debug_printf("Ignoring add_spawnd call: %s\n",
  57     //              err_getstring(PROC_MGMT_ERR_NOT_MONITOR));
  58 }
  59
  60 static void spawn_reply_handler(struct spawn_binding *b,
  61                                 struct capref domain_cap, errval_t spawn_err);
  62 static void spawn_with_caps_reply_handler(struct spawn_binding *b,
  63                                           struct capref domain_cap,
  64                                           errval_t spawn_err);
  65 static void span_reply_handler(struct spawn_binding *b,
  66                                struct capref domain_cap, errval_t span_err);
  67 static void kill_reply_handler(struct spawn_binding *b,
  68                                struct capref domain_cap, errval_t kill_err);
  69 static void exit_reply_handler(struct spawn_binding *b,
  70                                struct capref domain_cap, errval_t exit_err);
  71 static void cleanup_reply_handler(struct spawn_binding *b,
  72                                   struct capref domain_cap,
  73                                   errval_t cleanup_err);
  74
  75 static void spawn_request_sender(void *arg)
  76 {
  77     struct pending_spawn *spawn = (struct pending_spawn*) arg;
  78
  79     errval_t err;
  80     bool with_caps = !(capref_is_null(spawn->inheritcn_cap) &&
  81                        capref_is_null(spawn->argcn_cap));
  82     if (with_caps) {
  83         spawn->b->rx_vtbl.spawn_with_caps_reply = spawn_with_caps_reply_handler;
  84         err = spawn->b->tx_vtbl.spawn_with_caps_request(spawn->b, NOP_CONT,
  85                                                         cap_procmng,
  86                                                         spawn->domain_cap,
  87                                                         spawn->path,
  88                                                         spawn->argvbuf,
  89                                                         spawn->argvbytes,
  90                                                         spawn->envbuf,
  91                                                         spawn->envbytes,
  92                                                         spawn->inheritcn_cap,
  93                                                         spawn->argcn_cap,
  94                                                         spawn->flags);
  95     } else {
  96         spawn->b->rx_vtbl.spawn_reply = spawn_reply_handler;
  97         err = spawn->b->tx_vtbl.spawn_request(spawn->b, NOP_CONT, cap_procmng,
  98                                               spawn->domain_cap, spawn->path,
  99                                               spawn->argvbuf, spawn->argvbytes,
 100                                               spawn->envbuf, spawn->envbytes,
 101                                               spawn->flags);
 102     }
 103     if (err_is_ok(err)) {
 104         free(spawn);
 105     } else {
 106         if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
 107             err = spawn->b->register_send(spawn->b, spawn->b->waitset,
 108                                           MKCONT(spawn_request_sender, arg));
 109             if (err_is_fail(err)) {
 110                 DEBUG_ERR(err, "registering for spawn request");
 111                 pending_clients_release(spawn->domain_cap,
 112                                         with_caps ? ClientType_SpawnWithCaps
 113                                                   : ClientType_Spawn,
 114                                         NULL);
 115                 free(spawn);
 116             }
 117         } else {
 118             DEBUG_ERR(err, "sending spawn request");
 119             pending_clients_release(spawn->domain_cap,
 120                                     with_caps ? ClientType_SpawnWithCaps
 121                                               : ClientType_Spawn,
 122                                     NULL);
 123             free(spawn);
 124         }
 125     }
 126 }
 127
 128 static void span_request_sender(void *arg)
 129 {
 130     struct pending_span *span = (struct pending_span*) arg;
 131
 132     errval_t err;
 133     span->b->rx_vtbl.span_reply = span_reply_handler;
 134     err = span->b->tx_vtbl.span_request(span->b, NOP_CONT, cap_procmng,
 135                                         span->domain_cap, span->vroot,
 136                                         span->dispframe);
 137     if (err_is_ok(err)) {
 138         err = domain_span(span->domain_cap, span->core_id);
 139         if (err_is_fail(err)) {
 140             DEBUG_ERR(err, "failed domain_span to core %u\n", span->core_id);
 141         }
 142         free(span);
 143     } else {
 144         if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
 145             err = span->b->register_send(span->b, span->b->waitset,
 146                                          MKCONT(span_request_sender, arg));
 147             if (err_is_fail(err)) {
 148                 DEBUG_ERR(err, "registering for span request");
 149                 pending_clients_release(span->domain_cap, ClientType_Span,
 150                                         NULL);
 151                 free(span);
 152             }
 153         } else {
 154             DEBUG_ERR(err, "sending span request");
 155             pending_clients_release(span->domain_cap, ClientType_Span, NULL);
 156             free(span);
 157         }
 158     }
 159 }
 160
 161 static void kill_request_sender(void *arg)
 162 {
 163     struct pending_kill_exit_cleanup *kill = (struct pending_kill_exit_cleanup*) arg;
 164
 165     errval_t err;
 166     kill->sb->rx_vtbl.kill_reply = kill_reply_handler;
 167     err = kill->sb->tx_vtbl.kill_request(kill->sb, NOP_CONT, cap_procmng,
 168                                         kill->domain_cap);
 169     if (err_is_ok(err)) {
 170         free(kill);
 171     } else {
 172         if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
 173             err = kill->sb->register_send(kill->sb, kill->sb->waitset,
 174                                          MKCONT(kill_request_sender, arg));
 175             if (err_is_fail(err)) {
 176                 DEBUG_ERR(err, "registering for kill request");
 177
 178                 struct pending_client *cl;
 179                 err = pending_clients_release_one(kill->domain_cap,
 180                                                   ClientType_Kill,
 181                                                   kill->pmb, &cl);
 182                 if (err_is_ok(err)) {
 183                     while (cl != NULL) {
 184                         struct pending_client *tmp = cl;
 185                         cl = cl->next;
 186                         free(tmp);
 187                     }
 188                 }
 189
 190                 free(kill);
 191             }
 192         } else {
 193             DEBUG_ERR(err, "sending kill request");
 194
 195             struct pending_client *cl;
 196             err = pending_clients_release_one(kill->domain_cap,
 197                                               ClientType_Kill,
 198                                               kill->pmb, &cl);
 199             if (err_is_ok(err)) {
 200                 while (cl != NULL) {
 201                     struct pending_client *tmp = cl;
 202                     cl = cl->next;
 203                     free(tmp);
 204                 }
 205             }
 206
 207             free(kill);
 208         }
 209     }
 210 }
 211
 212 static void exit_request_sender(void *arg)
 213 {
 214     struct pending_kill_exit_cleanup *exit = (struct pending_kill_exit_cleanup*) arg;
 215
 216     errval_t err;
 217     exit->sb->rx_vtbl.exit_reply = exit_reply_handler;
 218     err = exit->sb->tx_vtbl.exit_request(exit->sb, NOP_CONT, cap_procmng,
 219                                         exit->domain_cap);
 220     if (err_is_ok(err)) {
 221         free(exit);
 222     } else {
 223         if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
 224             err = exit->sb->register_send(exit->sb, exit->sb->waitset,
 225                                          MKCONT(exit_request_sender, arg));
 226             if (err_is_fail(err)) {
 227                 DEBUG_ERR(err, "registering for exit request");
 228                 err = pending_clients_release(exit->domain_cap, ClientType_Exit,
 229                                               NULL);
 230                 free(exit);
 231             }
 232         } else {
 233             DEBUG_ERR(err, "sending exit request");
 234             err = pending_clients_release(exit->domain_cap, ClientType_Exit,
 235                                           NULL);
 236             free(exit);
 237         }
 238     }
 239 }
 240
 241 static void cleanup_request_sender(void *arg)
 242 {
 243     struct pending_kill_exit_cleanup *cleanup = (struct pending_kill_exit_cleanup*) arg;
 244
 245     errval_t err;
 246     cleanup->sb->rx_vtbl.cleanup_reply = cleanup_reply_handler;
 247     err = cleanup->sb->tx_vtbl.cleanup_request(cleanup->sb, NOP_CONT, cap_procmng,
 248                                               cleanup->domain_cap);
 249     if (err_is_ok(err)) {
 250         free(cleanup);
 251     } else {
 252         if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
 253             err = cleanup->sb->register_send(cleanup->sb, cleanup->sb->waitset,
 254                                             MKCONT(cleanup_request_sender, arg));
 255             if (err_is_fail(err)) {
 256                 DEBUG_ERR(err, "registering for cleanup request");
 257                 pending_clients_release(cleanup->domain_cap, ClientType_Cleanup,
 258                                         NULL);
 259                 free(cleanup);
 260             }
 261         } else {
 262             DEBUG_ERR(err, "sending cleanup request");
 263             pending_clients_release(cleanup->domain_cap, ClientType_Cleanup,
 264                                     NULL);
 265             free(cleanup);
 266         }
 267     }
 268 }
 269
 270 static void spawn_reply_handler(struct spawn_binding *b,
 271                                 struct capref domain_cap, errval_t spawn_err)
 272 {
 273     struct pending_client *cl;
 274     errval_t err = pending_clients_release(domain_cap, ClientType_Spawn, &cl);
 275     if (err_is_fail(err)) {
 276         DEBUG_ERR(err, "failed to retrieve pending spawn client based on domain"
 277                   " cap");
 278         return;
 279     }
 280
 281     err = spawn_err;
 282     if (err_is_ok(spawn_err)) {
 283         err = domain_spawn(domain_cap, cl->core_id);
 284     }
 285
 286     errval_t resp_err = cl->b->tx_vtbl.spawn_response(cl->b, NOP_CONT, err,
 287                                                       domain_cap);
 288     if (err_is_fail(resp_err)) {
 289         DEBUG_ERR(resp_err, "failed to send spawn_response to client");
 290     }
 291
 292     free(cl);
 293 }
 294
 295 static void spawn_with_caps_reply_handler(struct spawn_binding *b,
 296                                           struct capref domain_cap,
 297                                           errval_t spawn_err)
 298 {
 299     struct pending_client *cl;
 300     errval_t err = pending_clients_release(domain_cap, ClientType_SpawnWithCaps,
 301                                            &cl);
 302     if (err_is_fail(err)) {
 303         DEBUG_ERR(err, "failed to retrieve pending spawn_with_caps client based"
 304                   " on domain cap");
 305         return;
 306     }
 307
 308     err = spawn_err;
 309     if (err_is_ok(spawn_err)) {
 310         err = domain_spawn(domain_cap, cl->core_id);
 311     }
 312
 313     errval_t resp_err = cl->b->tx_vtbl.spawn_with_caps_response(cl->b, NOP_CONT,
 314                                                                 err,
 315                                                                 domain_cap);
 316     if (err_is_fail(resp_err)) {
 317         DEBUG_ERR(resp_err, "failed to send spawn_with_caps_response to "
 318                   "client");
 319     }
 320
 321     free(cl);
 322 }
 323
 324 static void span_reply_handler(struct spawn_binding *b,
 325                                struct capref domain_cap, errval_t span_err)
 326 {
 327     struct pending_client *cl;
 328     errval_t err = pending_clients_release(domain_cap, ClientType_Span, &cl);
 329     if (err_is_fail(err)) {
 330         DEBUG_ERR(err, "failed to retrieve pending span client based on domain"
 331                   " cap");
 332         return;
 333     }
 334
 335     struct domain_entry *entry;
 336     err = domain_get_by_cap(cl->domain_cap, &entry);
 337     if (err_is_fail(err)) {
 338         DEBUG_ERR(err, "failed to retrieve span client by domain cap");
 339         return;
 340     }
 341
 342     if (entry->status != DOMAIN_STATUS_RUNNING) {
 343         // Domain has been stopped while we were serving the request; there's
 344         // no one to respond to.
 345         free(cl);
 346         return;
 347     }
 348
 349     err = cl->b->tx_vtbl.span_response(cl->b, NOP_CONT, span_err);
 350     if (err_is_fail(err)) {
 351         DEBUG_ERR(err, "failed to send span_response to client");
 352     }
 353
 354     free(cl);
 355 }
 356
 357 static void cleanup_reply_handler(struct spawn_binding *b,
 358                                   struct capref domain_cap,
 359                                   errval_t cleanup_err)
 360 {
 361     struct pending_client *cl;
 362     errval_t err = pending_clients_release(domain_cap, ClientType_Cleanup, &cl);
 363     if (err_is_fail(err)) {
 364         DEBUG_ERR(err, "failed to retrieve pending cleanup client based on "
 365                   "domain cap");
 366         return;
 367     }
 368
 369     if (err_is_fail(cleanup_err)) {
 370         // TODO(razvan): Here, spawnd has failed deleting its local cspace.
 371         // Should we send another cleanup message, until it might succeed?
 372         free(cl);
 373         return;
 374     }
 375
 376     struct domain_entry *entry;
 377     err = domain_get_by_cap(domain_cap, &entry);
 378     if (err_is_fail(err)) {
 379         DEBUG_ERR(err, "failed to retrieve domain by cap returned by spawnd "
 380                   "after cleanup");
 381         return;
 382     }
 383
 384     assert(entry->num_spawnds_resources > 0);
 385     assert(entry->status != DOMAIN_STATUS_CLEANED);
 386
 387     --entry->num_spawnds_resources;
 388
 389     if (entry->num_spawnds_resources == 0) {
 390         entry->status = DOMAIN_STATUS_CLEANED;
 391
 392         // At this point, the domain exists in state CLEANED for
 393         // history reasons. For instance, if some other domain
 394         // issues a wait call for this one, the process manager can
 395         // return the exit status directly.
 396         // At some point, however, we might want to just clean up
 397         // the domain entry and recycle the domain cap.
 398     } else {
 399         // Expecting to receive further cleanup replies from other
 400         // spawnds for the same domain cap, hence re-add the
 401         // pending client.
 402         err = pending_clients_add(domain_cap, cl->b,
 403                                   ClientType_Cleanup, MAX_COREID);
 404         if (err_is_fail(err)) {
 405             DEBUG_ERR(err, "pending_clients_add in cleanup_reply_handler");
 406         }
 407     }
 408 }
 409
 410 static void kill_reply_handler(struct spawn_binding *b,
 411                                struct capref domain_cap, errval_t kill_err)
 412 {
 413     struct pending_client *cl;
 414     errval_t err = pending_clients_release(domain_cap, ClientType_Kill, &cl);
 415     if (err_is_fail(err)) {
 416         DEBUG_ERR(err, "failed to retrieve pending kill client based on domain "
 417                   "cap");
 418         return;
 419     }
 420
 421     errval_t resp_err;
 422     if (err_is_fail(kill_err)) {
 423         // TODO(razvan): Here, spawnd has failed deleting its local dispatcher.
 424         // Should we send another kill message, until it might succeed?
 425         while (cl != NULL) {
 426             resp_err = cl->b->tx_vtbl.kill_response(cl->b, NOP_CONT,
 427                                                      kill_err);
 428             if (err_is_fail(resp_err)) {
 429                 DEBUG_ERR(resp_err, "failed to send kill_response to client");
 430             }
 431             struct pending_client *tmp = cl;
 432             cl = cl->next;
 433             free(tmp);
 434         }
 435         return;
 436     }
 437
 438     struct domain_entry *entry;
 439     err = domain_get_by_cap(domain_cap, &entry);
 440     if (err_is_fail(err)) {
 441         DEBUG_ERR(err, "failed to retrieve domain by cap returned by spawnd "
 442                   "after kill");
 443         return;
 444     }
 445
 446     assert(entry->num_spawnds_running > 0);
 447     assert(entry->status != DOMAIN_STATUS_STOPPED);
 448
 449     --entry->num_spawnds_running;
 450
 451     if (entry->num_spawnds_running == 0) {
 452         entry->status = DOMAIN_STATUS_STOPPED;
 453         entry->exit_status = EXIT_STATUS_KILLED;
 454
 455         err = pending_clients_add(domain_cap, NULL, ClientType_Cleanup,
 456                                   MAX_COREID);
 457         if (err_is_fail(err)) {
 458             DEBUG_ERR(err, "pending_clients_add in kill_reply_handler");
 459         }
 460
 461         // TODO(razvan): Might it be more sane if we respond back
 462         // to the client after the domain has been cleaned up (i.e.
 463         // the cspace root has been revoked for all dispatchers)?
 464         while (cl != NULL) {
 465             resp_err = cl->b->tx_vtbl.kill_response(cl->b, NOP_CONT,
 466                                                      kill_err);
 467             if (err_is_fail(resp_err)) {
 468                 DEBUG_ERR(resp_err, "failed to send kill_response to client");
 469             }
 470             struct pending_client *tmp = cl;
 471             cl = cl->next;
 472             free(tmp);
 473         }
 474
 475         // TODO(razvan): Same problem applies to the waiters: would
 476         // it be better if we sent them wait_responses after the
 477         // cspace root has been revoked, too? (here and in the exit
 478         // case).
 479         struct domain_waiter *waiter = entry->waiters;
 480         while (waiter != NULL) {
 481             waiter->b->tx_vtbl.wait_response(waiter->b, NOP_CONT,
 482                                              SYS_ERR_OK,
 483                                              entry->exit_status);
 484             struct domain_waiter *tmp = waiter;
 485             waiter = waiter->next;
 486             free(tmp);
 487         }
 488
 489         for (coreid_t i = 0; i < MAX_COREID; ++i) {
 490             if (entry->spawnds[i] == NULL) {
 491                 continue;
 492             }
 493
 494             struct spawn_binding *spb = entry->spawnds[i]->b;
 495
 496             struct pending_kill_exit_cleanup *cleanup = (struct pending_kill_exit_cleanup*) malloc(
 497                     sizeof(struct pending_kill_exit_cleanup));
 498             cleanup->sb = spb;
 499             cleanup->domain_cap = domain_cap;
 500
 501             spb->rx_vtbl.cleanup_reply = cleanup_reply_handler;
 502             err = spb->register_send(spb, spb->waitset,
 503                                      MKCONT(cleanup_request_sender, cleanup));
 504             if (err_is_fail(err)) {
 505                 DEBUG_ERR(err, "registering for cleanup request");
 506                 free(cleanup);
 507             }
 508         }
 509     } else {
 510         err = pending_clients_add(domain_cap, cl->b, ClientType_Kill,
 511                                   MAX_COREID);
 512         if (err_is_fail(err)) {
 513             DEBUG_ERR(err, "pending_clients_add in kill_reply_handler");
 514         }
 515     }
 516 }
 517
 518 static void exit_reply_handler(struct spawn_binding *b,
 519                                struct capref domain_cap, errval_t exit_err)
 520 {
 521     struct pending_client *cl;
 522     errval_t err = pending_clients_release(domain_cap, ClientType_Exit, &cl);
 523     if (err_is_fail(err)) {
 524         DEBUG_ERR(err, "failed to retrieve pending exit client based on domain "
 525                   "cap");
 526         return;
 527     }
 528
 529     if (err_is_fail(exit_err)) {
 530         // TODO(razvan): Here, spawnd has failed deleting its local dispatcher.
 531         // Should we send another kill message, until it might succeed?
 532         free(cl);
 533         return;
 534     }
 535
 536     struct domain_entry *entry;
 537     err = domain_get_by_cap(domain_cap, &entry);
 538     if (err_is_fail(err)) {
 539         DEBUG_ERR(err, "failed to retrieve domain by cap returned by spawnd "
 540                   "after exit");
 541         return;
 542     }
 543
 544     assert(entry->num_spawnds_running > 0);
 545     assert(entry->status != DOMAIN_STATUS_STOPPED);
 546
 547     --entry->num_spawnds_running;
 548
 549     if (entry->num_spawnds_running == 0) {
 550         entry->status = DOMAIN_STATUS_STOPPED;
 551
 552         err = pending_clients_add(domain_cap, NULL, ClientType_Cleanup,
 553                                   MAX_COREID);
 554         if (err_is_fail(err)) {
 555             DEBUG_ERR(err, "pending_clients_add in exit_reply_handler");
 556         }
 557
 558         free(cl);
 559
 560         // TODO(razvan): Same problem applies to the waiters: would
 561         // it be better if we sent them wait_responses after the
 562         // cspace root has been revoked, too? (here and in the exit
 563         // case).
 564         struct domain_waiter *waiter = entry->waiters;
 565         while (waiter != NULL) {
 566             waiter->b->tx_vtbl.wait_response(waiter->b, NOP_CONT,
 567                                              SYS_ERR_OK,
 568                                              entry->exit_status);
 569             struct domain_waiter *tmp = waiter;
 570             waiter = waiter->next;
 571             free(tmp);
 572         }
 573
 574         for (coreid_t i = 0; i < MAX_COREID; ++i) {
 575             if (entry->spawnds[i] == NULL) {
 576                 continue;
 577             }
 578
 579             struct spawn_binding *spb = entry->spawnds[i]->b;
 580
 581             struct pending_kill_exit_cleanup *cleanup = (struct pending_kill_exit_cleanup*) malloc(
 582                     sizeof(struct pending_kill_exit_cleanup));
 583             cleanup->sb = spb;
 584             cleanup->domain_cap = domain_cap;
 585
 586             spb->rx_vtbl.cleanup_reply = cleanup_reply_handler;
 587             err = spb->register_send(spb, spb->waitset,
 588                                      MKCONT(cleanup_request_sender, cleanup));
 589             if (err_is_fail(err)) {
 590                 DEBUG_ERR(err, "registering for cleanup request");
 591                 free(cleanup);
 592             }
 593         }
 594     } else {
 595         err = pending_clients_add(domain_cap, cl->b, ClientType_Exit,
 596                                   MAX_COREID);
 597         if (err_is_fail(err)) {
 598             DEBUG_ERR(err, "pending_clients_add in kill_reply_handler");
 599         }
 600     }
 601 }
 602
 603 static errval_t spawn_handler_common(struct proc_mgmt_binding *b,
 604                                      enum ClientType type,
 605                                      coreid_t core_id, const char *path,
 606                                      const char *argvbuf, size_t argvbytes,
 607                                      const char *envbuf, size_t envbytes,
 608                                      struct capref inheritcn_cap,
 609                                      struct capref argcn_cap, uint8_t flags)
 610 {
 611     if (!spawnd_state_exists(core_id)) {
 612         return PROC_MGMT_ERR_INVALID_SPAWND;
 613     }
 614
 615     struct spawnd_state *state = spawnd_state_get(core_id);
 616     assert(state != NULL);
 617     struct spawn_binding *cl = state->b;
 618     assert(cl != NULL);
 619
 620     struct capref domain_cap;
 621     errval_t err = slot_alloc(&domain_cap);
 622     if (err_is_fail(err)) {
 623         DEBUG_ERR(err, "slot_alloc domain_cap");
 624         return err_push(err, PROC_MGMT_ERR_CREATE_DOMAIN_CAP);
 625     }
 626     err = cap_retype(domain_cap, cap_procmng, 0, ObjType_Domain, 0, 1);
 627     if (err_is_fail(err)) {
 628         DEBUG_ERR(err, "cap_retype domain_cap");
 629         return err_push(err, PROC_MGMT_ERR_CREATE_DOMAIN_CAP);
 630     }
 631
 632     err = pending_clients_add(domain_cap, b, type, core_id);
 633     if (err_is_fail(err)) {
 634         DEBUG_ERR(err, "pending_clients_add");
 635         return err;
 636     }
 637
 638     struct pending_spawn *spawn = (struct pending_spawn*) malloc(
 639             sizeof(struct pending_spawn));
 640     spawn->domain_cap = domain_cap;
 641     spawn->b = cl;
 642     spawn->core_id = core_id;
 643     spawn->path = path;
 644     spawn->argvbuf = argvbuf;
 645     spawn->argvbytes = argvbytes;
 646     spawn->envbuf = envbuf;
 647     spawn->envbytes = envbytes;
 648     spawn->inheritcn_cap = inheritcn_cap;
 649     spawn->argcn_cap = argcn_cap;
 650     spawn->flags = flags;
 651
 652     err = cl->register_send(cl, cl->waitset,
 653                             MKCONT(spawn_request_sender, spawn));
 654     if (err_is_fail(err)) {
 655         DEBUG_ERR(err, "registering for spawn request");
 656         free(spawn);
 657     }
 658
 659     return SYS_ERR_OK;
 660 }
 661
 662 static void spawn_handler(struct proc_mgmt_binding *b, coreid_t core_id,
 663                           const char *path, const char *argvbuf,
 664                           size_t argvbytes, const char *envbuf, size_t envbytes,
 665                           uint8_t flags)
 666 {
 667     errval_t err, resp_err;
 668     err = spawn_handler_common(b, ClientType_Spawn, core_id, path, argvbuf,
 669                                argvbytes, envbuf, envbytes, NULL_CAP, NULL_CAP,
 670                                flags);
 671     if (err_is_ok(err)) {
 672         // Will respond to client when we get the reply from spawnd.
 673         return;
 674     }
 675
 676     resp_err = b->tx_vtbl.spawn_response(b, NOP_CONT, err, NULL_CAP);
 677     if (err_is_fail(resp_err)) {
 678         DEBUG_ERR(resp_err, "failed to send spawn_response");
 679     }
 680 }
 681
 682 static void spawn_with_caps_handler(struct proc_mgmt_binding *b,
 683                                     coreid_t core_id, const char *path,
 684                                     const char *argvbuf, size_t argvbytes,
 685                                     const char *envbuf, size_t envbytes,
 686                                     struct capref inheritcn_cap,
 687                                     struct capref argcn_cap, uint8_t flags)
 688 {
 689     errval_t err, resp_err;
 690     err = spawn_handler_common(b, ClientType_SpawnWithCaps, core_id, path,
 691                                argvbuf, argvbytes, envbuf, envbytes,
 692                                inheritcn_cap, argcn_cap, flags);
 693     if (err_is_ok(err)) {
 694         // Will respond to client when we get the reply from spawnd.
 695         return;
 696     }
 697
 698     resp_err = b->tx_vtbl.spawn_with_caps_response(b, NOP_CONT, err,
 699                                                             NULL_CAP);
 700     if (err_is_fail(resp_err)) {
 701         DEBUG_ERR(resp_err, "failed to send spawn_with_caps_response");
 702     }
 703 }
 704
 705 static void span_handler(struct proc_mgmt_binding *b, struct capref domain_cap,
 706                          coreid_t core_id, struct capref vroot,
 707                          struct capref dispframe)
 708 {
 709     errval_t err, resp_err;
 710     err = domain_can_span(domain_cap, core_id);
 711     if (err_is_fail(err)) {
 712         goto respond_with_err;
 713     }
 714
 715     if (!spawnd_state_exists(core_id)) {
 716         err = PROC_MGMT_ERR_INVALID_SPAWND;
 717         goto respond_with_err;
 718     }
 719
 720     struct spawnd_state *state = spawnd_state_get(core_id);
 721     assert(state != NULL);
 722     struct spawn_binding *cl = state->b;
 723     assert(cl != NULL);
 724
 725     err = pending_clients_add(domain_cap, b, ClientType_Span, core_id);
 726     if (err_is_fail(err)) {
 727         goto respond_with_err;
 728     }
 729
 730     struct pending_span *span = (struct pending_span*) malloc(
 731             sizeof(struct pending_span));
 732     span->domain_cap = domain_cap;
 733     span->b = cl;
 734     span->core_id = core_id;
 735     span->vroot = vroot;
 736     span->dispframe = dispframe;
 737
 738     err = cl->register_send(cl, cl->waitset,
 739                             MKCONT(span_request_sender, span));
 740     if (err_is_fail(err)) {
 741         DEBUG_ERR(err, "registering for span request");
 742         free(span);
 743     }
 744
 745 respond_with_err:
 746     resp_err = b->tx_vtbl.span_response(b, NOP_CONT, err);
 747     if (err_is_fail(resp_err)) {
 748         DEBUG_ERR(resp_err, "failed to send span_response");
 749     }
 750 }
 751
 752 static errval_t kill_handler_common(struct proc_mgmt_binding *b,
 753                                     struct capref domain_cap,
 754                                     enum ClientType type,
 755                                     uint8_t exit_status)
 756 {
 757     errval_t err = pending_clients_add(domain_cap, b, type, MAX_COREID);
 758     if (err_is_fail(err)) {
 759         return err;
 760     }
 761
 762     struct domain_entry *entry;
 763     err = domain_get_by_cap(domain_cap, &entry);
 764     if (err_is_fail(err)) {
 765         return err;
 766     }
 767
 768     entry->exit_status = exit_status;
 769     domain_stop_pending(entry);
 770
 771     for (coreid_t i = 0; i < MAX_COREID; ++i) {
 772         if (entry->spawnds[i] == NULL) {
 773             continue;
 774         }
 775
 776         struct spawn_binding *spb = entry->spawnds[i]->b;
 777
 778         struct pending_kill_exit_cleanup *cmd = (struct pending_kill_exit_cleanup*) malloc(
 779                 sizeof(struct pending_kill_exit_cleanup));
 780         cmd->domain_cap = domain_cap;
 781         cmd->sb = spb;
 782
 783         switch (type) {
 784             case ClientType_Kill:
 785                 cmd->pmb = b;
 786                 err = spb->register_send(spb, spb->waitset,
 787                                          MKCONT(kill_request_sender, cmd));
 788                 if (err_is_fail(err)) {
 789                     DEBUG_ERR(err, "registering for kill request");
 790                     free(cmd);
 791                 }
 792                 break;
 793
 794             case ClientType_Exit:
 795                 err = spb->register_send(spb, spb->waitset,
 796                                          MKCONT(exit_request_sender, cmd));
 797                 if (err_is_fail(err)) {
 798                     DEBUG_ERR(err, "registering for exit request");
 799                     free(cmd);
 800                 }
 801                 break;
 802             default:
 803                 USER_PANIC("invalid client type for kill: %u\n", type);
 804         }
 805     }
 806
 807     return SYS_ERR_OK;
 808 }
 809
 810 static void kill_handler(struct proc_mgmt_binding *b,
 811                          struct capref victim_domain_cap)
 812 {
 813     errval_t err = kill_handler_common(b, victim_domain_cap, ClientType_Kill,
 814                                        EXIT_STATUS_KILLED);
 815     if (err_is_fail(err)) {
 816         errval_t resp_err = b->tx_vtbl.kill_response(b, NOP_CONT, err);
 817         if (err_is_fail(resp_err)) {
 818             DEBUG_ERR(resp_err, "failed to send kill_response");
 819         }
 820     }
 821 }
 822
 823 static void exit_handler(struct proc_mgmt_binding *b, struct capref domain_cap,
 824                          uint8_t exit_status)
 825 {
 826     errval_t err = kill_handler_common(b, domain_cap, ClientType_Exit,
 827                                        exit_status);
 828     if (err_is_fail(err)) {
 829         DEBUG_ERR(err, "processing exit_handler for requesting domain, exit "
 830                   "code %u", exit_status);
 831     }
 832     // Error or not, there's no client to respond to anymore.
 833 }
 834
 835 static void wait_handler(struct proc_mgmt_binding *b, struct capref domain_cap)
 836 {
 837     errval_t err, resp_err;
 838     struct domain_entry *entry;
 839     err = domain_get_by_cap(domain_cap, &entry);
 840     if (err_is_fail(err)) {
 841         goto respond;
 842     }
 843
 844     if (entry->status == DOMAIN_STATUS_STOPPED) {
 845         // Domain has already been stopped, so just reply with exit status.
 846         goto respond;
 847     }
 848
 849     struct domain_waiter *waiter = (struct domain_waiter*) malloc(
 850             sizeof(struct domain_waiter));
 851     waiter->b = b;
 852     waiter->next = entry->waiters;
 853     entry->waiters = waiter;
 854     // Will respond when domain is stopped.
 855     return;
 856
 857 respond:
 858     resp_err = b->tx_vtbl.wait_response(b, NOP_CONT, err, entry->exit_status);
 859     if (err_is_fail(resp_err)) {
 860         DEBUG_ERR(resp_err, "failed to send wait_response");
 861     }
 862 }
 863
 864 static struct proc_mgmt_rx_vtbl monitor_vtbl = {
 865     .add_spawnd           = add_spawnd_handler,
 866     .spawn_call           = spawn_handler,
 867     .spawn_with_caps_call = spawn_with_caps_handler,
 868     .span_call            = span_handler,
 869     .kill_call            = kill_handler,
 870     .exit_call            = exit_handler,
 871     .wait_call            = wait_handler
 872 };
 873
 874 static struct proc_mgmt_rx_vtbl non_monitor_vtbl = {
 875     .add_spawnd           = add_spawnd_handler_non_monitor,
 876     .spawn_call           = spawn_handler,
 877     .spawn_with_caps_call = spawn_with_caps_handler,
 878     .span_call            = span_handler,
 879     .kill_call            = kill_handler,
 880     .exit_call            = exit_handler,
 881     .wait_call            = wait_handler
 882 };
 883
 884 static errval_t alloc_ep_for_monitor(struct capref *ep)
 885 {
 886     struct proc_mgmt_lmp_binding *lmpb =
 887         malloc(sizeof(struct proc_mgmt_lmp_binding));
 888     assert(lmpb != NULL);
 889
 890     // setup our end of the binding
 891     errval_t err = proc_mgmt_client_lmp_accept(lmpb, get_default_waitset(),
 892                                                DEFAULT_LMP_BUF_WORDS);
 893     if (err_is_fail(err)) {
 894         free(lmpb);
 895         return err_push(err, LIB_ERR_PROC_MGMT_CLIENT_ACCEPT);
 896     }
 897
 898     *ep = lmpb->chan.local_cap;
 899     lmpb->b.rx_vtbl = monitor_vtbl;
 900
 901     return SYS_ERR_OK;
 902 }
 903
 904 static void export_cb(void *st, errval_t err, iref_t iref)
 905 {
 906     if (err_is_fail(err)) {
 907         USER_PANIC_ERR(err, "export failed");
 908     }
 909
 910     // Allocate an endpoint for the local monitor, who will use it to inform
 911     // us about new spawnd irefs on behalf of other monitors.
 912     struct capref ep;
 913     err = alloc_ep_for_monitor(&ep);
 914     if (err_is_fail(err)) {
 915         USER_PANIC_ERR(err, "failed to allocate LMP EP for local monitor");
 916     }
 917
 918     // Send the endpoint to the monitor, so it can finish the handshake.
 919     struct monitor_binding *mb = get_monitor_binding();
 920     err = mb->tx_vtbl.set_proc_mgmt_ep_request(mb, NOP_CONT, ep);
 921     if (err_is_fail(err)) {
 922         USER_PANIC_ERR(err, "failed to send set_proc_mgmt_ep_request to "
 923                        "monitor");
 924     }
 925
 926     // Also register this iref with the name service, for arbitrary client
 927     // domains to use for spawn-related ops.
 928     err = nameservice_register(SERVICE_BASENAME, iref);
 929     if (err_is_fail(err)) {
 930         USER_PANIC_ERR(err, "nameservice_register failed");
 931     }
 932 }
 933
 934 static errval_t connect_cb(void *st, struct proc_mgmt_binding *b)
 935 {
 936     b->rx_vtbl = non_monitor_vtbl;
 937     return SYS_ERR_OK;
 938 }
 939
 940 errval_t start_service(void)
 941 {
 942     return proc_mgmt_export(NULL, export_cb, connect_cb, get_default_waitset(),
 943             IDC_EXPORT_FLAGS_DEFAULT);
 944 }