git.barrelfish.org Git - barrelfish/blob - usr/proc_mgmt/service.c

   1 /**
   2  * \file
   3  * \brief Process management service.
   4  */
   5
   6 /*
   7  * Copyright (c) 2017, ETH Zurich.
   8  * All rights reserved.
   9  *
  10  * This file is distributed under the terms in the attached LICENSE file.
  11  * If you do not find this file, copies can be found by writing to:
  12  * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
  13  */
  14
  15 #include <barrelfish/barrelfish.h>
  16 #include <barrelfish/nameservice_client.h>
  17 #include <barrelfish/proc_mgmt_client.h>
  18 #include <barrelfish/spawn_client.h>
  19 #include <if/monitor_defs.h>
  20 #include <if/proc_mgmt_defs.h>
  21 #include <if/spawn_defs.h>
  22
  23 #include "domain.h"
  24 #include "internal.h"
  25 #include "pending_clients.h"
  26 #include "spawnd_state.h"
  27
  28 static void add_spawnd_handler(struct proc_mgmt_binding *b, coreid_t core_id,
  29                                iref_t iref)
  30 {
  31     if (spawnd_state_exists(core_id)) {
  32         DEBUG_ERR(PROC_MGMT_ERR_SPAWND_EXISTS, "spawnd_state_exists");
  33         return;
  34     }
  35
  36     // Bind with the spawnd.
  37     struct spawn_binding *spawnb;
  38     errval_t err = spawn_bind_iref(iref, &spawnb);
  39     if (err_is_fail(err)) {
  40         DEBUG_ERR(err, "spawn_bind_iref");
  41         return;
  42     }
  43
  44     err = spawnd_state_alloc(core_id, spawnb);
  45     if (err_is_fail(err)) {
  46         DEBUG_ERR(err, "spawnd_state_alloc");
  47     }
  48
  49     debug_printf("Process manager bound with spawnd.%u on iref %u\n", core_id,
  50             iref);
  51 }
  52
  53 static void add_spawnd_handler_non_monitor(struct proc_mgmt_binding *b,
  54                                            coreid_t core_id, iref_t iref)
  55 {
  56     // debug_printf("Ignoring add_spawnd call: %s\n",
  57     //              err_getstring(PROC_MGMT_ERR_NOT_MONITOR));
  58 }
  59
  60 static bool cleanup_request_sender(struct msg_queue_elem *m);
  61
  62
  63 #define PROC_MGMT_BENCH 1
  64 #define PROC_MGMT_BENCH_MIN_RUNS 150
  65
  66 #ifdef PROC_MGMT_BENCH
  67 #include <bench/bench.h>
  68
  69 static inline cycles_t calculate_time(cycles_t tsc_start, cycles_t tsc_end)
  70 {
  71     cycles_t result;
  72     if (tsc_end < tsc_start) {
  73         result = (LONG_MAX - tsc_start) + tsc_end - bench_tscoverhead();
  74     } else {
  75         result = (tsc_end - tsc_start - bench_tscoverhead());
  76     }
  77     return result;
  78 }
  79
  80 static bench_ctl_t *req_ctl_1;
  81 static bench_ctl_t *req_ctl_2;
  82 static bench_ctl_t *req_ctl_3;
  83 static bench_ctl_t *req_ctl_4;
  84 static bench_ctl_t *req_ctl_5;
  85 static bench_ctl_t *resp_ctl;
  86 static uint64_t tscperus;
  87 #endif
  88
  89 static void spawn_reply_handler(struct spawn_binding *b, errval_t spawn_err)
  90 {
  91 #ifdef PROC_MGMT_BENCH
  92     cycles_t tsc_start, tsc_end;
  93     cycles_t result;
  94
  95     tsc_start = bench_tsc();
  96 #endif
  97
  98     struct pending_client *cl =
  99             (struct pending_client*) spawnd_state_dequeue_recv(b->st);
 100
 101     struct pending_spawn *spawn = NULL;
 102     struct pending_span *span = NULL;
 103     struct pending_kill_cleanup *kc = NULL;
 104
 105     struct domain_entry *entry;
 106
 107     errval_t err, resp_err;
 108
 109     switch (cl->type) {
 110         case ClientType_Spawn:
 111         case ClientType_SpawnWithCaps:
 112             spawn = (struct pending_spawn*) cl->st;
 113             err = spawn_err;
 114             if (err_is_ok(spawn_err)) {
 115                 err = domain_spawn(spawn->cap_node, spawn->core_id);
 116                 if (cl->type == ClientType_Spawn) {
 117                     resp_err = cl->b->tx_vtbl.spawn_response(cl->b, NOP_CONT,
 118                                                              err,
 119                                                              spawn->cap_node->domain_cap);
 120                 } else {
 121                     resp_err = cl->b->tx_vtbl.spawn_with_caps_response(cl->b,
 122                                                                        NOP_CONT,
 123                                                                        err,
 124                                                                        spawn->cap_node->domain_cap);
 125                 }
 126             }
 127
 128             free(spawn);
 129
 130 #ifdef PROC_MGMT_BENCH
 131             tsc_end = bench_tsc();
 132             result = calculate_time(tsc_start, tsc_end);
 133
 134             if (resp_ctl != NULL && bench_ctl_add_run(resp_ctl, &result)) {
 135                 bench_ctl_dump_analysis(resp_ctl, 0, "proc_mgmt_resp",
 136                                         tscperus);
 137                 bench_ctl_destroy(resp_ctl);
 138                 resp_ctl = NULL;
 139             }
 140 #endif
 141             break;
 142
 143         case ClientType_Span:
 144             span = (struct pending_span*) cl->st;
 145             entry = span->entry;
 146             if (entry->status == DOMAIN_STATUS_RUNNING) {
 147                 resp_err = cl->b->tx_vtbl.span_response(cl->b, NOP_CONT,
 148                                                         spawn_err);
 149             }
 150
 151             free(span);
 152             break;
 153
 154         case ClientType_Cleanup:
 155             kc = (struct pending_kill_cleanup*) cl->st;
 156             entry = kc->entry;
 157
 158             assert(entry->num_spawnds_resources > 0);
 159             assert(entry->status != DOMAIN_STATUS_CLEANED);
 160
 161             --entry->num_spawnds_resources;
 162             if (entry->num_spawnds_resources == 0) {
 163                 entry->status = DOMAIN_STATUS_CLEANED;
 164
 165                 // At this point, the domain exists in state CLEANED for history
 166                 // reasons. For instance, if some other domain issues a wait
 167                 // call for this one, the process manager can return the exit
 168                 // status directly. At some point, however, we might want to
 169                 // just clean up the domain entry and recycle the domain cap.
 170             }
 171
 172             free(kc);
 173             break;
 174
 175         case ClientType_Kill:
 176         case ClientType_Exit:
 177             kc = (struct pending_kill_cleanup*) cl->st;
 178             entry = kc->entry;
 179
 180             assert(entry->num_spawnds_running > 0);
 181             assert(entry->status != DOMAIN_STATUS_STOPPED);
 182
 183             --entry->num_spawnds_running;
 184
 185             if (entry->num_spawnds_running == 0) {
 186                 entry->status = DOMAIN_STATUS_STOPPED;
 187
 188                 if (cl->type == ClientType_Kill) {
 189                     entry->exit_status = EXIT_STATUS_KILLED;
 190                     resp_err = cl->b->tx_vtbl.kill_response(cl->b, NOP_CONT,
 191                                                             spawn_err);
 192                 }
 193
 194                 struct domain_waiter *waiter = entry->waiters;
 195                 while (waiter != NULL) {
 196                     waiter->b->tx_vtbl.wait_response(waiter->b, NOP_CONT,
 197                                                      SYS_ERR_OK,
 198                                                      entry->exit_status);
 199                     struct domain_waiter *tmp = waiter;
 200                     waiter = waiter->next;
 201                     free(tmp);
 202                 }
 203
 204                 for (coreid_t i = 0; i < MAX_COREID; ++i) {
 205                     if (entry->spawnds[i] == NULL) {
 206                         continue;
 207                     }
 208
 209                     struct spawn_binding *spb = entry->spawnds[i]->b;
 210
 211                     struct pending_kill_cleanup *cleanup =
 212                             (struct pending_kill_cleanup*) malloc(
 213                                     sizeof(struct pending_kill_cleanup));
 214                     cleanup->b = spb;
 215                     cleanup->domain_cap = kc->domain_cap;
 216                     cleanup->entry = entry;
 217
 218                     struct pending_client *cleanup_cl =
 219                             (struct pending_client*) malloc(
 220                                     sizeof(struct pending_client));
 221                     cleanup_cl->b = cl->b;
 222                     cleanup_cl->type = ClientType_Cleanup;
 223                     cleanup_cl->st = cleanup;
 224
 225                     struct msg_queue_elem *msg = (struct msg_queue_elem*) malloc(
 226                             sizeof(struct msg_queue_elem));
 227                     msg->st = cleanup_cl;
 228                     msg->cont = cleanup_request_sender;
 229
 230                     err = spawnd_state_enqueue_send(entry->spawnds[i], msg);
 231
 232                     if (err_is_fail(err)) {
 233                         DEBUG_ERR(err, "enqueuing cleanup request");
 234                         free(cleanup);
 235                         free(cleanup_cl);
 236                         free(msg);
 237                     }
 238                 }
 239             }
 240
 241             free(kc);
 242             break;
 243
 244         default:
 245             USER_PANIC("Unknown client type in spawn_reply_handler: %u\n",
 246                        cl->type);
 247     }
 248
 249     free(cl);
 250 }
 251
 252 static bool spawn_request_sender(struct msg_queue_elem *m)
 253 {
 254     struct pending_client *cl = (struct pending_client*) m->st;
 255     struct pending_spawn *spawn = (struct pending_spawn*) cl->st;
 256     spawn->b->rx_vtbl.spawn_reply = spawn_reply_handler;
 257
 258     errval_t err;
 259     bool with_caps = !(capref_is_null(spawn->inheritcn_cap) &&
 260                        capref_is_null(spawn->argcn_cap));
 261     if (with_caps) {
 262         err = spawn->b->tx_vtbl.spawn_with_caps_request(spawn->b, NOP_CONT,
 263                                                         cap_procmng,
 264                                                         spawn->cap_node->domain_cap,
 265                                                         spawn->path,
 266                                                         spawn->argvbuf,
 267                                                         spawn->argvbytes,
 268                                                         spawn->envbuf,
 269                                                         spawn->envbytes,
 270                                                         spawn->inheritcn_cap,
 271                                                         spawn->argcn_cap,
 272                                                         spawn->flags);
 273     } else {
 274         err = spawn->b->tx_vtbl.spawn_request(spawn->b, NOP_CONT, cap_procmng,
 275                                               spawn->cap_node->domain_cap,
 276                                               spawn->path, spawn->argvbuf,
 277                                               spawn->argvbytes, spawn->envbuf,
 278                                               spawn->envbytes, spawn->flags);
 279     }
 280
 281     if (err_is_fail(err)) {
 282         if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
 283             return false;
 284         } else {
 285             USER_PANIC_ERR(err, "sending spawn request");
 286         }
 287     }
 288
 289     free(m);
 290
 291     return true;
 292 }
 293
 294 static bool span_request_sender(struct msg_queue_elem *m)
 295 {
 296     struct pending_client *cl = (struct pending_client*) m->st;
 297     struct pending_span *span = (struct pending_span*) cl->st;
 298
 299     errval_t err;
 300     span->b->rx_vtbl.spawn_reply = spawn_reply_handler;
 301     err = span->b->tx_vtbl.span_request(span->b, NOP_CONT, cap_procmng,
 302                                         span->domain_cap, span->vroot,
 303                                         span->dispframe);
 304
 305     if (err_is_fail(err)) {
 306         if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
 307             return false;
 308         } else {
 309             USER_PANIC_ERR(err, "sending span request");
 310         }
 311     }
 312
 313     free(m);
 314
 315     return true;
 316 }
 317
 318 static bool kill_request_sender(struct msg_queue_elem *m)
 319 {
 320     struct pending_client *cl = (struct pending_client*) m->st;
 321     struct pending_kill_cleanup *kill = (struct pending_kill_cleanup*) cl->st;
 322
 323     errval_t err;
 324     kill->b->rx_vtbl.spawn_reply = spawn_reply_handler;
 325     err = kill->b->tx_vtbl.kill_request(kill->b, NOP_CONT, cap_procmng,
 326                                         kill->domain_cap);
 327
 328     if (err_is_fail(err)) {
 329         if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
 330             return false;
 331         } else {
 332             USER_PANIC_ERR(err, "sending kill request");
 333         }
 334     }
 335
 336     free(m);
 337
 338     return true;
 339 }
 340
 341 static bool cleanup_request_sender(struct msg_queue_elem *m)
 342 {
 343     struct pending_client *cl = (struct pending_client*) m->st;
 344     struct pending_kill_cleanup *cleanup = (struct pending_kill_cleanup*) cl->st;
 345
 346     errval_t err;
 347     cleanup->b->rx_vtbl.spawn_reply = spawn_reply_handler;
 348     err = cleanup->b->tx_vtbl.cleanup_request(cleanup->b, NOP_CONT,
 349                                               cap_procmng,
 350                                               cleanup->domain_cap);
 351
 352     if (err_is_fail(err)) {
 353         if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
 354             return false;
 355         } else {
 356             USER_PANIC_ERR(err, "sending cleanup request");
 357         }
 358     }
 359
 360     free(m);
 361
 362     return true;
 363 }
 364
 365 static errval_t spawn_handler_common(struct proc_mgmt_binding *b,
 366                                      enum ClientType type,
 367                                      coreid_t core_id, const char *path,
 368                                      const char *argvbuf, size_t argvbytes,
 369                                      const char *envbuf, size_t envbytes,
 370                                      struct capref inheritcn_cap,
 371                                      struct capref argcn_cap, uint8_t flags)
 372 {
 373 #ifdef PROC_MGMT_BENCH
 374     cycles_t tsc_start, tsc_end;
 375     cycles_t result;
 376
 377     tsc_start = bench_tsc();
 378 #endif
 379     if (!spawnd_state_exists(core_id)) {
 380         return PROC_MGMT_ERR_INVALID_SPAWND;
 381     }
 382 #ifdef PROC_MGMT_BENCH
 383     tsc_end = bench_tsc();
 384     result = calculate_time(tsc_start, tsc_end);
 385
 386     if (req_ctl_1 != NULL && bench_ctl_add_run(req_ctl_1, &result)) {
 387         bench_ctl_dump_analysis(req_ctl_1, 0, "proc_mgmt_req 1", tscperus);
 388         bench_ctl_destroy(req_ctl_1);
 389         req_ctl_1 = NULL;
 390     }
 391 #endif
 392
 393 #ifdef PROC_MGMT_BENCH
 394     tsc_start = bench_tsc();
 395 #endif
 396     struct spawnd_state *spawnd = spawnd_state_get(core_id);
 397     assert(spawnd != NULL);
 398     struct spawn_binding *cl = spawnd->b;
 399     assert(cl != NULL);
 400 #ifdef PROC_MGMT_BENCH
 401     tsc_end = bench_tsc();
 402     result = calculate_time(tsc_start, tsc_end);
 403
 404     if (req_ctl_2 != NULL && bench_ctl_add_run(req_ctl_2, &result)) {
 405         bench_ctl_dump_analysis(req_ctl_2, 0, "proc_mgmt_req 2", tscperus);
 406         bench_ctl_destroy(req_ctl_2);
 407         req_ctl_2 = NULL;
 408     }
 409 #endif
 410
 411 #ifdef PROC_MGMT_BENCH
 412     tsc_start = bench_tsc();
 413 #endif
 414     errval_t err;
 415     if (domain_should_refill_caps()) {
 416         err = domain_prealloc_caps();
 417         if (err_is_fail(err)) {
 418             return err_push(err, PROC_MGMT_ERR_CREATE_DOMAIN_CAP);
 419         }
 420     }
 421
 422     struct domain_cap_node *cap_node = next_cap_node();
 423
 424     // struct capref domain_cap;
 425     // errval_t err = slot_alloc(&domain_cap);
 426     // if (err_is_fail(err)) {
 427     //     DEBUG_ERR(err, "slot_alloc domain_cap");
 428     //     return err_push(err, PROC_MGMT_ERR_CREATE_DOMAIN_CAP);
 429     // }
 430 #ifdef PROC_MGMT_BENCH
 431     tsc_end = bench_tsc();
 432     result = calculate_time(tsc_start, tsc_end);
 433
 434     if (req_ctl_3 != NULL && bench_ctl_add_run(req_ctl_3, &result)) {
 435         bench_ctl_dump_analysis(req_ctl_3, 0, "proc_mgmt_req 3", tscperus);
 436         bench_ctl_destroy(req_ctl_3);
 437         req_ctl_3 = NULL;
 438     }
 439 #endif
 440
 441 #ifdef PROC_MGMT_BENCH
 442     tsc_start = bench_tsc();
 443 #endif
 444     struct pending_spawn *spawn = (struct pending_spawn*) malloc(
 445             sizeof(struct pending_spawn));
 446     spawn->cap_node = cap_node;
 447     // spawn->domain_cap = domain_cap;
 448     spawn->b = cl;
 449     spawn->core_id = core_id;
 450     spawn->path = path;
 451     spawn->argvbuf = argvbuf;
 452     spawn->argvbytes = argvbytes;
 453     spawn->envbuf = envbuf;
 454     spawn->envbytes = envbytes;
 455     spawn->inheritcn_cap = inheritcn_cap;
 456     spawn->argcn_cap = argcn_cap;
 457     spawn->flags = flags;
 458
 459     struct pending_client *spawn_cl = (struct pending_client*) malloc(
 460             sizeof(struct pending_client));
 461     spawn_cl->b = b;
 462     spawn_cl->type = type;
 463     spawn_cl->st = spawn;
 464
 465     struct msg_queue_elem *msg = (struct msg_queue_elem*) malloc(
 466             sizeof(struct msg_queue_elem));
 467     msg->st = spawn_cl;
 468     msg->cont = spawn_request_sender;
 469
 470     err = spawnd_state_enqueue_send(spawnd, msg);
 471     if (err_is_fail(err)) {
 472         DEBUG_ERR(err, "enqueuing spawn request");
 473         free(spawn);
 474         free(spawn_cl);
 475         free(msg);
 476     }
 477
 478 #ifdef PROC_MGMT_BENCH
 479     tsc_end = bench_tsc();
 480     result = calculate_time(tsc_start, tsc_end);
 481
 482     if (req_ctl_5 != NULL && bench_ctl_add_run(req_ctl_5, &result)) {
 483         bench_ctl_dump_analysis(req_ctl_5, 0, "proc_mgmt_req 5", tscperus);
 484         bench_ctl_destroy(req_ctl_5);
 485         req_ctl_5 = NULL;
 486     }
 487 #endif
 488
 489     return SYS_ERR_OK;
 490 }
 491
 492 static void spawn_handler(struct proc_mgmt_binding *b, coreid_t core_id,
 493                           const char *path, const char *argvbuf,
 494                           size_t argvbytes, const char *envbuf, size_t envbytes,
 495                           uint8_t flags)
 496 {
 497     errval_t err, resp_err;
 498     err = spawn_handler_common(b, ClientType_Spawn, core_id, path, argvbuf,
 499                                argvbytes, envbuf, envbytes, NULL_CAP, NULL_CAP,
 500                                flags);
 501
 502     if (err_is_fail(err)) {
 503         resp_err = b->tx_vtbl.spawn_response(b, NOP_CONT, err, NULL_CAP);
 504         if (err_is_fail(resp_err)) {
 505             DEBUG_ERR(resp_err, "failed to send spawn_response");
 506         }
 507     }
 508 }
 509
 510 static void spawn_with_caps_handler(struct proc_mgmt_binding *b,
 511                                     coreid_t core_id, const char *path,
 512                                     const char *argvbuf, size_t argvbytes,
 513                                     const char *envbuf, size_t envbytes,
 514                                     struct capref inheritcn_cap,
 515                                     struct capref argcn_cap, uint8_t flags)
 516 {
 517     errval_t err, resp_err;
 518     err = spawn_handler_common(b, ClientType_SpawnWithCaps, core_id, path,
 519                                argvbuf, argvbytes, envbuf, envbytes,
 520                                inheritcn_cap, argcn_cap, flags);
 521     if (err_is_ok(err)) {
 522         // Will respond to client when we get the reply from spawnd.
 523         return;
 524     }
 525
 526     resp_err = b->tx_vtbl.spawn_with_caps_response(b, NOP_CONT, err,
 527                                                             NULL_CAP);
 528     if (err_is_fail(resp_err)) {
 529         DEBUG_ERR(resp_err, "failed to send spawn_with_caps_response");
 530     }
 531 }
 532
 533 static void span_handler(struct proc_mgmt_binding *b, struct capref domain_cap,
 534                          coreid_t core_id, struct capref vroot,
 535                          struct capref dispframe)
 536 {
 537     errval_t err, resp_err;
 538     struct domain_entry *entry = NULL;
 539     err = domain_get_by_cap(domain_cap, &entry);
 540     if (err_is_fail(err)) {
 541         goto respond_with_err;
 542     }
 543
 544     assert(entry != NULL);
 545     if (entry->status != DOMAIN_STATUS_RUNNING) {
 546         err = PROC_MGMT_ERR_DOMAIN_NOT_RUNNING;
 547         goto respond_with_err;
 548     }
 549
 550     if (entry->spawnds[core_id] != NULL) {
 551         // TODO(razvan): Maybe we want to allow the same domain to span multiple
 552         // dispatchers onto the same core?
 553         err = PROC_MGMT_ERR_ALREADY_SPANNED;
 554         goto respond_with_err;
 555     }
 556
 557     if (!spawnd_state_exists(core_id)) {
 558         err = PROC_MGMT_ERR_INVALID_SPAWND;
 559         goto respond_with_err;
 560     }
 561
 562     struct spawnd_state *spawnd = spawnd_state_get(core_id);
 563     assert(spawnd != NULL);
 564     struct spawn_binding *cl = spawnd->b;
 565     assert(cl != NULL);
 566
 567     struct pending_span *span = (struct pending_span*) malloc(
 568             sizeof(struct pending_span));
 569     span->domain_cap = domain_cap;
 570     span->entry = entry;
 571     span->b = cl;
 572     span->core_id = core_id;
 573     span->vroot = vroot;
 574     span->dispframe = dispframe;
 575
 576     struct pending_client *span_cl = (struct pending_client*) malloc(
 577             sizeof(struct pending_client));
 578     span_cl->b = b;
 579     span_cl->type = ClientType_Span;
 580     span_cl->st = span;
 581
 582     struct msg_queue_elem *msg = (struct msg_queue_elem*) malloc(
 583             sizeof(struct msg_queue_elem));
 584     msg->st = span_cl;
 585     msg->cont = span_request_sender;
 586
 587     err = spawnd_state_enqueue_send(spawnd, msg);
 588
 589     if (err_is_fail(err)) {
 590         DEBUG_ERR(err, "enqueuing span request");
 591         free(span);
 592         free(span_cl);
 593         free(msg);
 594     }
 595
 596 respond_with_err:
 597     resp_err = b->tx_vtbl.span_response(b, NOP_CONT, err);
 598     if (err_is_fail(resp_err)) {
 599         DEBUG_ERR(resp_err, "failed to send span_response");
 600     }
 601 }
 602
 603 static errval_t kill_handler_common(struct proc_mgmt_binding *b,
 604                                     struct capref domain_cap,
 605                                     enum ClientType type,
 606                                     uint8_t exit_status)
 607 {
 608     struct domain_entry *entry;
 609     errval_t err = domain_get_by_cap(domain_cap, &entry);
 610     if (err_is_fail(err)) {
 611         return err;
 612     }
 613
 614     entry->exit_status = exit_status;
 615     domain_stop_pending(entry);
 616
 617     for (coreid_t i = 0; i < MAX_COREID; ++i) {
 618         if (entry->spawnds[i] == NULL) {
 619             continue;
 620         }
 621
 622         struct spawn_binding *spb = entry->spawnds[i]->b;
 623
 624         struct pending_kill_cleanup *cmd = (struct pending_kill_cleanup*) malloc(
 625                 sizeof(struct pending_kill_cleanup));
 626         cmd->domain_cap = domain_cap;
 627         cmd->entry = entry;
 628         cmd->b = spb;
 629
 630         struct pending_client *cl = (struct pending_client*) malloc(
 631                 sizeof(struct pending_client));
 632         cl->b = b;
 633         cl->type = type;
 634         cl->st = cmd;
 635
 636         struct msg_queue_elem *msg = (struct msg_queue_elem*) malloc(
 637                 sizeof(struct msg_queue_elem));
 638         msg->st = cl;
 639         msg->cont = kill_request_sender;
 640
 641         err = spawnd_state_enqueue_send(entry->spawnds[i], msg);
 642         if (err_is_fail(err)) {
 643             DEBUG_ERR(err, "enqueuing kill request");
 644             free(cmd);
 645             free(cl);
 646             free(msg);
 647         }
 648     }
 649
 650     return SYS_ERR_OK;
 651 }
 652
 653 static void kill_handler(struct proc_mgmt_binding *b,
 654                          struct capref victim_domain_cap)
 655 {
 656     errval_t err = kill_handler_common(b, victim_domain_cap, ClientType_Kill,
 657                                        EXIT_STATUS_KILLED);
 658     if (err_is_fail(err)) {
 659         errval_t resp_err = b->tx_vtbl.kill_response(b, NOP_CONT, err);
 660         if (err_is_fail(resp_err)) {
 661             DEBUG_ERR(resp_err, "failed to send kill_response");
 662         }
 663     }
 664 }
 665
 666 static void exit_handler(struct proc_mgmt_binding *b, struct capref domain_cap,
 667                          uint8_t exit_status)
 668 {
 669     errval_t err = kill_handler_common(b, domain_cap, ClientType_Exit,
 670                                        exit_status);
 671     if (err_is_fail(err)) {
 672         DEBUG_ERR(err, "processing exit_handler for requesting domain, exit "
 673                   "code %u", exit_status);
 674     }
 675     // Error or not, there's no client to respond to anymore.
 676 }
 677
 678 static void wait_handler(struct proc_mgmt_binding *b, struct capref domain_cap)
 679 {
 680     errval_t err, resp_err;
 681     struct domain_entry *entry;
 682     err = domain_get_by_cap(domain_cap, &entry);
 683     if (err_is_fail(err)) {
 684         goto respond;
 685     }
 686
 687     if (entry->status == DOMAIN_STATUS_STOPPED) {
 688         // Domain has already been stopped, so just reply with exit status.
 689         goto respond;
 690     }
 691
 692     struct domain_waiter *waiter = (struct domain_waiter*) malloc(
 693             sizeof(struct domain_waiter));
 694     waiter->b = b;
 695     waiter->next = entry->waiters;
 696     entry->waiters = waiter;
 697     // Will respond when domain is stopped.
 698     return;
 699
 700 respond:
 701     resp_err = b->tx_vtbl.wait_response(b, NOP_CONT, err, entry->exit_status);
 702     if (err_is_fail(resp_err)) {
 703         DEBUG_ERR(resp_err, "failed to send wait_response");
 704     }
 705 }
 706
 707 static struct proc_mgmt_rx_vtbl monitor_vtbl = {
 708     .add_spawnd           = add_spawnd_handler,
 709     .spawn_call           = spawn_handler,
 710     .spawn_with_caps_call = spawn_with_caps_handler,
 711     .span_call            = span_handler,
 712     .kill_call            = kill_handler,
 713     // .exit_call            = exit_handler,
 714     .exit                 = exit_handler,
 715     .wait_call            = wait_handler
 716 };
 717
 718 static struct proc_mgmt_rx_vtbl non_monitor_vtbl = {
 719     .add_spawnd           = add_spawnd_handler_non_monitor,
 720     .spawn_call           = spawn_handler,
 721     .spawn_with_caps_call = spawn_with_caps_handler,
 722     .span_call            = span_handler,
 723     .kill_call            = kill_handler,
 724     // .exit_call            = exit_handler,
 725     .exit                 = exit_handler,
 726     .wait_call            = wait_handler
 727 };
 728
 729 static errval_t alloc_ep_for_monitor(struct capref *ep)
 730 {
 731     struct proc_mgmt_lmp_binding *lmpb =
 732         malloc(sizeof(struct proc_mgmt_lmp_binding));
 733     assert(lmpb != NULL);
 734
 735     // setup our end of the binding
 736     errval_t err = proc_mgmt_client_lmp_accept(lmpb, get_default_waitset(),
 737                                                DEFAULT_LMP_BUF_WORDS);
 738     if (err_is_fail(err)) {
 739         free(lmpb);
 740         return err_push(err, LIB_ERR_PROC_MGMT_CLIENT_ACCEPT);
 741     }
 742
 743     *ep = lmpb->chan.local_cap;
 744     lmpb->b.rx_vtbl = monitor_vtbl;
 745
 746     return SYS_ERR_OK;
 747 }
 748
 749 static void export_cb(void *st, errval_t err, iref_t iref)
 750 {
 751     if (err_is_fail(err)) {
 752         USER_PANIC_ERR(err, "export failed");
 753     }
 754
 755     // Allocate an endpoint for the local monitor, who will use it to inform
 756     // us about new spawnd irefs on behalf of other monitors.
 757     struct capref ep;
 758     err = alloc_ep_for_monitor(&ep);
 759     if (err_is_fail(err)) {
 760         USER_PANIC_ERR(err, "failed to allocate LMP EP for local monitor");
 761     }
 762
 763     // Send the endpoint to the monitor, so it can finish the handshake.
 764     struct monitor_binding *mb = get_monitor_binding();
 765     err = mb->tx_vtbl.set_proc_mgmt_ep_request(mb, NOP_CONT, ep);
 766     if (err_is_fail(err)) {
 767         USER_PANIC_ERR(err, "failed to send set_proc_mgmt_ep_request to "
 768                        "monitor");
 769     }
 770
 771     // Also register this iref with the name service, for arbitrary client
 772     // domains to use for spawn-related ops.
 773     err = nameservice_register(SERVICE_BASENAME, iref);
 774     if (err_is_fail(err)) {
 775         USER_PANIC_ERR(err, "nameservice_register failed");
 776     }
 777 }
 778
 779 static errval_t connect_cb(void *st, struct proc_mgmt_binding *b)
 780 {
 781     b->rx_vtbl = non_monitor_vtbl;
 782     return SYS_ERR_OK;
 783 }
 784
 785 errval_t start_service(void)
 786 {
 787     errval_t err;
 788 #ifdef PROC_MGMT_BENCH
 789     bench_init();
 790
 791     req_ctl_1 = calloc(1, sizeof(*req_ctl_1));
 792     req_ctl_1->mode = BENCH_MODE_FIXEDRUNS;
 793     req_ctl_1->result_dimensions = 1;
 794     req_ctl_1->min_runs = PROC_MGMT_BENCH_MIN_RUNS;
 795     req_ctl_1->data = calloc(req_ctl_1->min_runs * req_ctl_1->result_dimensions,
 796                            sizeof(*req_ctl_1->data));
 797
 798     req_ctl_2 = calloc(1, sizeof(*req_ctl_2));
 799     req_ctl_2->mode = BENCH_MODE_FIXEDRUNS;
 800     req_ctl_2->result_dimensions = 1;
 801     req_ctl_2->min_runs = PROC_MGMT_BENCH_MIN_RUNS;
 802     req_ctl_2->data = calloc(req_ctl_2->min_runs * req_ctl_2->result_dimensions,
 803                            sizeof(*req_ctl_2->data));
 804
 805     req_ctl_3 = calloc(1, sizeof(*req_ctl_3));
 806     req_ctl_3->mode = BENCH_MODE_FIXEDRUNS;
 807     req_ctl_3->result_dimensions = 1;
 808     req_ctl_3->min_runs = PROC_MGMT_BENCH_MIN_RUNS;
 809     req_ctl_3->data = calloc(req_ctl_3->min_runs * req_ctl_3->result_dimensions,
 810                            sizeof(*req_ctl_3->data));
 811
 812     req_ctl_4 = calloc(1, sizeof(*req_ctl_4));
 813     req_ctl_4->mode = BENCH_MODE_FIXEDRUNS;
 814     req_ctl_4->result_dimensions = 1;
 815     req_ctl_4->min_runs = PROC_MGMT_BENCH_MIN_RUNS;
 816     req_ctl_4->data = calloc(req_ctl_4->min_runs * req_ctl_4->result_dimensions,
 817                            sizeof(*req_ctl_4->data));
 818
 819     req_ctl_5 = calloc(1, sizeof(*req_ctl_5));
 820     req_ctl_5->mode = BENCH_MODE_FIXEDRUNS;
 821     req_ctl_5->result_dimensions = 1;
 822     req_ctl_5->min_runs = PROC_MGMT_BENCH_MIN_RUNS;
 823     req_ctl_5->data = calloc(req_ctl_5->min_runs * req_ctl_5->result_dimensions,
 824                            sizeof(*req_ctl_5->data));
 825
 826     resp_ctl = calloc(1, sizeof(*resp_ctl));
 827     resp_ctl->mode = BENCH_MODE_FIXEDRUNS;
 828     resp_ctl->result_dimensions = 1;
 829     resp_ctl->min_runs = PROC_MGMT_BENCH_MIN_RUNS;
 830     resp_ctl->data = calloc(resp_ctl->min_runs * resp_ctl->result_dimensions,
 831                             sizeof(*resp_ctl->data));
 832
 833     err = sys_debug_get_tsc_per_ms(&tscperus);
 834     assert(err_is_ok(err));
 835     tscperus /= 1000;
 836 #endif
 837
 838     err = domain_prealloc_caps();
 839     if (err_is_fail(err)) {
 840         USER_PANIC_ERR(err_push(err, PROC_MGMT_ERR_CREATE_DOMAIN_CAP),
 841                        "domain_prealloc_caps in start_service");
 842     }
 843
 844     return proc_mgmt_export(NULL, export_cb, connect_cb, get_default_waitset(),
 845             IDC_EXPORT_FLAGS_DEFAULT);
 846 }