Refactor process manager to enable spawnd discovery.
authorRazvan Damachi <razvan.damachi@gmail.com>
Fri, 9 Jun 2017 14:27:55 +0000 (16:27 +0200)
committerSimon Gerber <simon.gerber@inf.ethz.ch>
Thu, 31 Aug 2017 14:35:07 +0000 (16:35 +0200)
The process manager now allocates a special LMP endpoint for monitor.0 and
gives said endpoint to monitor.0 as part of the export_cb. After monitor.0
spawns spawnd, it uses that endpoint to send the new spawnd's iref to the
process manager. The latter then binds with spawnd, to keep track of its state.

For app cores, a similar workflow applies, except inter-monitor communication
is now performed. Namely, the new app core's monitor sends the new app core's
spawnd to monitor.0, which forwards it to the process manager. The latter then
binds with the app core's spawnd just the same.

Diagrams for the discovery protocol can be found at https://goo.gl/eJE37u.

Signed-off-by: Razvan Damachi <razvan.damachi@gmail.com>

25 files changed:
errors/errno.fugu
if/intermon.if
if/monitor.if
if/proc_mgmt.if
if/spawn.if
include/barrelfish/core_state.h
include/barrelfish/domain.h
include/barrelfish/proc_mgmt_client.h
include/barrelfish/spawn_client.h
lib/barrelfish/Hakefile
lib/barrelfish/domain.c
lib/barrelfish/proc_mgmt_client.c
lib/barrelfish/spawn_client.c
usr/monitor/include/monitor.h
usr/monitor/inter.c
usr/monitor/main.c
usr/monitor/monitor_server.c
usr/monitor/spawn.c
usr/proc_mgmt/Hakefile
usr/proc_mgmt/internal.h
usr/proc_mgmt/main.c
usr/proc_mgmt/service.c
usr/proc_mgmt/spawnd_state.c [new file with mode: 0644]
usr/proc_mgmt/spawnd_state.h [new file with mode: 0644]
usr/spawnd/service.c

index b0e9a8f..f245a5c 100755 (executable)
@@ -419,6 +419,9 @@ errors libbarrelfish LIB_ERR_ {
     failure SEGBASE_OVER_4G_LIMIT  "Segment base address is above 32-bit boundary",
     failure LDT_FULL               "LDT is out of space",
     failure LDT_SELECTOR_INVALID   "Segment selector is invalid for LDT",
+
+    // Process management client library
+    failure PROC_MGMT_CLIENT_ACCEPT "Error in proc_mgmt_client_lmp_accept()",
 };
 
 // errors in Flounder-generated bindings
@@ -660,6 +663,12 @@ errors spawn SPAWN_ERR_ {
     failure MALFORMED_SPAWND_RECORD "Spawn record without ID found?",
 };
 
+// errors related to the process manager
+errors proc_mgmt PROC_MGMT_ERR_ {
+    failure NOT_MONITOR         "Received monitor-only request from non-monitor domain",
+    failure SPAWND_EXISTS       "Requested spawnd slot already exists",
+};
+
 // errors from ELF library
 errors libelf ELF_ERR_ {
     failure FILESZ              "Invalid file size",
index 69a3e82..09b81c9 100644 (file)
@@ -99,6 +99,10 @@ interface intermon "The Interface between monitors" {
                                 uint64 framebytes);
     message span_domain_reply(state_id state_id, errval err);
 
+    // Inform the monitor connected to the process manager about the spawnd
+    // that the caller just spawned.
+    message add_spawnd(iref iref);
+
     // Resource control
     message rsrc_join(rsrcid id, coreid coreid);
     message rsrc_join_complete(rsrcid id);
index c3c9826..d0991f3 100644 (file)
@@ -110,6 +110,10 @@ interface monitor "The monitor to client Interface" {
     message get_ramfs_iref_reply(iref iref, uintptr st);
     message set_ramfs_iref_request(iref iref);
 
+    message set_proc_mgmt_ep_request(cap ep);
+
+    message set_spawn_iref_request(iref iref);
+
     message set_mem_iref_request(iref iref);
 
     message set_name_iref_request(iref iref);
index b957e34..07a73d3 100644 (file)
@@ -9,10 +9,11 @@
 
 interface proc_mgmt "Process management service" {
   
-  // Allocate a new endpoint for a future connecting client.
-  rpc alloc_ep(out errval err, out cap ep);
+  // Add a new spawnd to the process manager's list.
+  message add_spawnd(coreid core, iref iref);
 
   // Spawn a new domain, returning its domainid cap.
+  /*
   rpc spawn(in coreid core,
             in String path[2048],
             in char argvbuf[argvbytes, 2048],
@@ -30,6 +31,7 @@ interface proc_mgmt "Process management service" {
 
   // Kill a domain for which the caller has a domainid cap.
   rpc kill(in cap domainid_cap, out errval err);
+  */
 
   // TODO(razvan): exit, wait, status, some other calls from spawn.if?
 };
index 1a2f38b..59bba6c 100644 (file)
@@ -39,4 +39,6 @@ interface spawn "Interface to spawn domains" {
 
     // Capability debugging
     rpc dump_capabilities(in domainid domain_id, out errval err);
+
+    rpc echo(in String sender_name[2048], in coreid sender_core);
 };
index 2b4e32a..9b6cf16 100644 (file)
@@ -82,6 +82,7 @@ struct monitor_blocking_binding;
 struct mem_binding;
 struct spawn_binding;
 struct arrakis_binding;
+struct proc_mgmt_binding;
 
 struct core_state_generic {
     struct waitset default_waitset;
@@ -92,6 +93,7 @@ struct core_state_generic {
     struct ram_alloc_state ram_alloc_state;
     struct octopus_binding *octopus_binding;
     struct spawn_binding *spawn_bindings[MAX_CPUS];
+    struct proc_mgmt_binding *proc_mgmt_binding;
     struct arrakis_binding *arrakis_bindings[MAX_CPUS];
     struct terminal_state *terminal_state;
     struct domain_state *domain_state;
index ff7e9ff..5c3c119 100644 (file)
@@ -30,6 +30,7 @@ struct monitor_blocking_binding;
 struct waitset;
 struct spawn_binding;
 struct arrakis_binding;
+struct proc_mgmt_binding;
 
 struct waitset *get_default_waitset(void);
 void disp_set_core_id(coreid_t core_id);
@@ -65,6 +66,8 @@ struct spawn_state *get_spawn_state(void);
 void set_spawn_state(struct spawn_state *st);
 struct slot_alloc_state *get_slot_alloc_state(void);
 struct skb_state *get_skb_state(void);
+struct proc_mgmt_binding *get_proc_mgmt_binding(void);
+void set_proc_mgmt_binding(struct proc_mgmt_binding *st);
 
 errval_t domain_init(void);
 errval_t domain_new_dispatcher(coreid_t core_id,
index 508372d..a5d54da 100644 (file)
@@ -15,6 +15,7 @@
 #ifndef BARRELFISH_PROC_MGMT_CLIENT_H
 #define BARRELFISH_PROC_MGMT_CLIENT_H
 
+#include <if/proc_mgmt_defs.h>
 #include <sys/cdefs.h>
 
 __BEGIN_DECLS
@@ -22,7 +23,17 @@ __BEGIN_DECLS
 struct proc_mgmt_lmp_binding;
 
 errval_t proc_mgmt_client_lmp_accept(struct proc_mgmt_lmp_binding *lmpb,
-               struct waitset *ws, size_t lmp_buflen_words);
+                                     struct waitset *ws,
+                                     size_t lmp_buflen_words);
+errval_t proc_mgmt_client_lmp_bind(struct proc_mgmt_lmp_binding *lmpb,
+                                   struct capref ep,
+                                   proc_mgmt_bind_continuation_fn *cont,
+                                   void *st,
+                                   struct waitset *ws,
+                                   size_t lmp_buflen_words);
+errval_t proc_mgmt_bind_client(void);
+
+errval_t proc_mgmt_add_spawnd(iref_t iref, coreid_t core_id);
 
 __END_DECLS
 
index 9bb1d8f..4a7e4db 100644 (file)
@@ -50,6 +50,7 @@ errval_t spawn_wait(domainid_t domainid, uint8_t *exitcode, bool nohang);
 errval_t spawn_wait_core(coreid_t coreid, domainid_t domainid,
                          uint8_t *exitcode, bool nohang);
 errval_t spawn_binding(coreid_t coreid, struct spawn_binding **ret_client);
+errval_t spawn_bind_iref(iref_t iref, struct spawn_binding **ret_client);
 errval_t spawn_get_domain_list(uint8_t **domains, size_t *len);
 errval_t spawn_get_status(uint8_t domain, struct spawn_ps_entry *pse,
                           char **argbuf, size_t *arglen, errval_t *reterr);
index 43e2e78..bddb1a0 100644 (file)
@@ -112,7 +112,8 @@ in
                              ++ common_srcs ++ idc_srcs,
                     assemblyFiles = arch_assembly (archFamily arch),
                     flounderBindings = [ "mem", "octopus", "interdisp", "spawn",
-                                         "terminal", "arrakis", "terminal_config" ],
+                                         "proc_mgmt", "terminal", "arrakis",
+                                         "terminal_config" ],
                     -- only makes sense to compile monitor binding for lmp
                     flounderTHCStubs = [ "octopus" ],
                     flounderExtraBindings = [ ("monitor", ["lmp"]),
@@ -120,6 +121,7 @@ in
                                               ("mem", ["rpcclient"]),
                                               ("octopus", ["rpcclient"]),
                                               ("spawn", ["rpcclient"]),
+                                              ("proc_mgmt", ["rpcclient"]),
                                               ("arrakis", ["rpcclient"])],
                     addCFlags = [ "-DMORECORE_PAGESIZE="++(morecore_pagesize arch) ],
                     addIncludes = [ "include", "include" </> arch_dir, (arch_include arch) ],
@@ -198,7 +200,8 @@ in
                     assemblyFiles = arch_assembly (archFamily arch),
                     addCFlags = [ "-DARRAKIS", "-DMORECORE_PAGESIZE="++(morecore_pagesize arch) ],
                     flounderBindings = [ "mem", "octopus", "interdisp", "spawn", "arrakis",
-                                         "terminal", "terminal_config", "terminal_session" ],
+                                         "proc_mgmt", "terminal", "terminal_config",
+                                         "terminal_session" ],
                     -- only makes sense to compile monitor binding for lmp
                     flounderTHCStubs = [ "octopus" ],
                     flounderExtraBindings = [ ("monitor", ["lmp"]),
@@ -206,6 +209,7 @@ in
                                               ("mem", ["rpcclient"]),
                                               ("octopus", ["rpcclient"]),
                                               ("spawn", ["rpcclient"]),
+                                              ("proc_mgmt", ["rpcclient"]),
                                               ("arrakis", ["rpcclient"])],
                     addIncludes = [ "include", "include" </> arch_dir, (arch_include arch) ],
                     addGeneratedDependencies = [ "/include/asmoffsets.h" ]
index fd53c9c..93400a5 100644 (file)
@@ -1300,6 +1300,25 @@ void set_spawn_binding(coreid_t core, struct spawn_binding *c)
     assert(core < MAX_CPUS);
     disp->core_state.c.spawn_bindings[core] = c;
 }
+/**
+ * \brief Returns a pointer to the proc_mgmt rpc client on the dispatcher priv
+ */
+struct proc_mgmt_binding *get_proc_mgmt_binding(void)
+{
+    dispatcher_handle_t handle = curdispatcher();
+    struct dispatcher_generic* disp = get_dispatcher_generic(handle);
+    return disp->core_state.c.proc_mgmt_binding;
+}
+
+/**
+ * \brief Sets the prog_mgmt rpc client on the dispatcher priv
+ */
+void set_proc_mgmt_binding(struct proc_mgmt_binding *c)
+{
+    dispatcher_handle_t handle = curdispatcher();
+    struct dispatcher_generic* disp = get_dispatcher_generic(handle);
+    disp->core_state.c.proc_mgmt_binding = c;
+}
 
 struct arrakis_binding *get_arrakis_binding(coreid_t core)
 {
index 537e6dd..c12be3b 100644 (file)
  */
 
 #include <barrelfish/barrelfish.h>
+#include <barrelfish/nameservice_client.h>
 #include <barrelfish/proc_mgmt_client.h>
+#include <if/octopus_defs.h>
 #include <if/proc_mgmt_defs.h>
 
+struct proc_mgmt_bind_retst {
+    errval_t err;
+    struct proc_mgmt_binding *b;
+    bool present;
+};
+
 static void error_handler(struct proc_mgmt_binding *b, errval_t err)
 {
 #if defined(__x86_64__) || defined(__i386__)
-    debug_printf("%p %p %p %p\n", __builtin_return_address(0), __builtin_return_address(1),__builtin_return_address(2),__builtin_return_address(3));
+    debug_printf("%p %p %p %p\n",
+                 __builtin_return_address(0),
+                 __builtin_return_address(1),
+                 __builtin_return_address(2),
+                 __builtin_return_address(3));
 #endif
     debug_err(__FILE__, __func__, __LINE__, err,
-              "error in proc_mgmt binding");
+              "asynchronous error in proc_mgmt binding");
     abort();
 }
 
+static void proc_mgmt_bind_cont(void *st, errval_t err,
+        struct proc_mgmt_binding *b)
+{
+    struct proc_mgmt_bind_retst *retst = (struct proc_mgmt_bind_retst*) st;
+    assert(retst != NULL);
+    assert(!retst->present);
+    retst->err = err;
+    retst->b = b;
+    retst->present = true;
+}
+
 static void proc_mgmt_accept_recv_handler(void *arg)
 {
     struct proc_mgmt_lmp_binding *b = arg;
@@ -43,8 +66,7 @@ static void proc_mgmt_accept_recv_handler(void *arg)
                 .arg = b,
             };
             err = lmp_chan_register_recv(&b->chan, b->b.waitset, recv_handler);
-            b->b.error_handler(&b->b,
-                    err_push(err, LIB_ERR_CHAN_REGISTER_RECV));
+            b->b.error_handler(&b->b, err_push(err, LIB_ERR_CHAN_REGISTER_RECV));
         } else {
             // real error, report to user
             b->b.error_handler(&b->b, err_push(err, LIB_ERR_LMP_CHAN_RECV));
@@ -52,29 +74,26 @@ static void proc_mgmt_accept_recv_handler(void *arg)
         return;
     }
 
-    // if we're the monitor, we might be waiting for the other side's cap
-    // TODO(razvan): Or the process manager, which for uses LMP_MONITOR_ACCEPT
-    // as well.
+    // TODO(razvan): LMP_PROC_MGMT_ACCEPT ?
     assert(b->chan.connstate == LMP_MONITOR_ACCEPT);
     assert(!capref_is_null(cap));
     b->chan.remote_cap = cap;
     b->chan.connstate = LMP_CONNECTED;
 
-    // Allocate a new receive slot.
-    // TODO(razvan): Whom is the receive slot for? Is it one of the spawnds?
-    // Or is it some arbitrary non-spawnd client domain? Need a way to tell.
+    /* allocate a new receive slot */
     err = lmp_chan_alloc_recv_slot(&b->chan);
     if (err_is_fail(err)) {
         // XXX: report the error, but continue
         b->b.error_handler(&b->b, err_push(err, LIB_ERR_LMP_ALLOC_RECV_SLOT));
     }
 
-    // Run the RX handler; has a side-effect of registering for receive events.
+    /* Run the RX handler; has a side-effect of registering for receive events */
     proc_mgmt_lmp_rx_handler(b);
 }
 
 static errval_t init_lmp_binding(struct proc_mgmt_lmp_binding *lmpb,
-                                 struct waitset *ws, size_t buflen_words)
+                                 struct waitset *ws,
+                                 size_t buflen_words)
 {
     errval_t err;
 
@@ -88,9 +107,9 @@ static errval_t init_lmp_binding(struct proc_mgmt_lmp_binding *lmpb,
 
     /* allocate a local endpoint */
     err = lmp_endpoint_create_in_slot(buflen_words, lmpb->chan.local_cap,
-            &lmpb->chan.endpoint);
+                                      &lmpb->chan.endpoint);
     if (err_is_fail(err)) {
-        /* TODO: free cap slot */
+        // TODO(razvan): Free cap slot.
         return err_push(err, LIB_ERR_ENDPOINT_CREATE);
     }
 
@@ -104,7 +123,8 @@ static errval_t init_lmp_binding(struct proc_mgmt_lmp_binding *lmpb,
     lmpb->b.error_handler = error_handler;
 
     /* setup initial receive handlers */
-    // lmpb->b.rx_vtbl = proc_mgmt_rx_vtbl;
+    // TODO(razvan): Don't think this is needed, but dunno for sure yet.
+    // lmpb->b.rx_vtbl = monitor_rx_vtbl;
 
     // connect handlers
     lmpb->b.change_waitset(&lmpb->b, lmpb->b.waitset);
@@ -112,35 +132,146 @@ static errval_t init_lmp_binding(struct proc_mgmt_lmp_binding *lmpb,
 }
 
 /**
- * \brief Accept a new LMP binding in a client from the process manager.
+ * \brief Accept a new LMP binding to a proc mgmt client.
  *
  * Should only be used in the process manager.
  *
- * \param pmcb         Storage for binding state
+ * \param lmpb         Storage for binding state
  * \param ws           Waitset for handling incoming messages
  * \param buflen_words Size of incoming buffer, in number of words
  */
 errval_t proc_mgmt_client_lmp_accept(struct proc_mgmt_lmp_binding *lmpb,
-        struct waitset *ws, size_t lmp_buflen_words)
+                                     struct waitset *ws,
+                                     size_t lmp_buflen_words)
 {
     errval_t err = init_lmp_binding(lmpb, ws, lmp_buflen_words);
     if (err_is_fail(err)) {
         return err;
     }
 
-    // TODO(razvan): This should probably be something like LMP_PROC_MGMT_ACCEPT
-    lmpb->chan.connstate = LMP_MONITOR_ACCEPT;
+    lmpb->chan.connstate = LMP_MONITOR_ACCEPT;  // TODO(razvan): LMP_PROC_MGMT_ACCEPT?
     lmpb->chan.remote_cap = NULL_CAP; // will be sent to us by the client
 
-    // Register for receive notification on our special handler.
+    /* Register for receive notification on our special handler */
     struct event_closure receive_handler = {
         .handler = proc_mgmt_accept_recv_handler,
         .arg = lmpb,
     };
     err = lmp_chan_register_recv(&lmpb->chan, ws, receive_handler);
     if (err_is_fail(err)) {
-        return err; // TODO: cleanup?
+        return err;  // TODO(razvan): cleanup?
+    }
+
+    return SYS_ERR_OK;
+}
+
+
+/**
+ * \brief Initiate a new LMP binding to the process manager
+ *
+ * To be used by the monitor for setting up the privileged channel used for
+ * spawnd discovery.
+ * Requires an explicit remote endpoint cap allocated by the process manager.
+ *
+ * \param lmpb         Storage for binding state
+ * \param ep           Remote endpoint of the process manager
+ * \param ws           Waitset for handling incoming messages
+ * \param cont         Continuation for when binding completes or fails
+ * \param st           State passed to continuation function
+ * \param buflen_words Size of incoming buffer, in number of words
+ */
+errval_t proc_mgmt_client_lmp_bind(struct proc_mgmt_lmp_binding *lmpb,
+                                   struct capref ep,
+                                   proc_mgmt_bind_continuation_fn *cont,
+                                   void *st,
+                                   struct waitset *ws,
+                                   size_t lmp_buflen_words)
+{
+    errval_t err = init_lmp_binding(lmpb, ws, lmp_buflen_words);
+    if (err_is_fail(err)) {
+        return err;
+    }
+
+    lmpb->chan.remote_cap = ep;
+
+    // Send the local endpoint cap to the process manager.
+    lmpb->chan.connstate = LMP_CONNECTED; /* pre-established */
+    err = lmp_chan_send0(&lmpb->chan, 0, lmpb->chan.local_cap);
+    if (err_is_fail(err)) {
+        // TODO(razvan): This, below.
+        /* XXX: I'm lazily assuming this can never fail with a transient error,
+         * since we only do it once at dispatcher startup. If not, we need to
+         * register and retry here */
+        assert(!lmp_err_is_transient(err));
+        return err;
+    }
+
+    /* Run the RX handler; has a side-effect of registering for receive events */
+    proc_mgmt_lmp_rx_handler(lmpb);
+
+    /* Run the continuation */
+    cont(st, SYS_ERR_OK, &lmpb->b);
+
+    return SYS_ERR_OK;
+}
+
+errval_t proc_mgmt_bind_client(void)
+{
+    struct proc_mgmt_binding *b = get_proc_mgmt_binding();
+    if (b != NULL) {
+        return SYS_ERR_OK;
+    }
+
+    errval_t err;
+    iref_t iref;
+    // Try using nameserver to retrievew the proc mgmt iref.
+    err = nameservice_blocking_lookup("proc_mgmt", &iref);
+    if (err_is_fail(err)) {
+        return err;
+    }
+    
+    // Initiate bind.
+    struct proc_mgmt_bind_retst bindst = {
+        .present = false
+    };
+
+    err = proc_mgmt_bind(iref, proc_mgmt_bind_cont, &bindst,
+            get_default_waitset(), /*IDC_BIND_FLAG_RPC_CAP_TRANSFER*/IDC_BIND_FLAGS_DEFAULT);
+    if (err_is_fail(err)) {
+        USER_PANIC_ERR(err, "proc_mgmt_bind");
+    }
+
+    // Wait for bind completion.
+    while (!bindst.present) {
+        messages_wait_and_handle_next();
     }
 
+    if (err_is_fail(bindst.err)) {
+        return bindst.err;
+    }
+
+    proc_mgmt_rpc_client_init(bindst.b);
+
+    set_proc_mgmt_binding(bindst.b);
+
     return SYS_ERR_OK;
 }
+
+errval_t proc_mgmt_add_spawnd(iref_t iref, coreid_t core_id)
+{
+    errval_t err = proc_mgmt_bind_client();
+    if (err_is_fail(err)) {
+        DEBUG_ERR(err, "proc_mgmt_bind_client");
+        return err;
+    }
+
+    struct proc_mgmt_binding *b = get_proc_mgmt_binding();
+    assert(b != NULL);
+
+    err = b->tx_vtbl.add_spawnd(b, NOP_CONT, core_id, iref);
+    if (err_is_fail(err)) {
+        DEBUG_ERR(err, "add_spawnd");
+    }
+
+    return err;
+}
index 0350c5e..bb439c7 100644 (file)
@@ -115,6 +115,33 @@ static errval_t bind_client(coreid_t coreid)
     return err;
 }
 
+errval_t spawn_bind_iref(iref_t iref, struct spawn_binding **ret_client)
+{
+    assert(ret_client != NULL);
+
+    struct spawn_bind_retst bindst = { .present = false };
+    errval_t err = spawn_bind(iref, spawn_bind_cont, &bindst,
+                              get_default_waitset(), IDC_BIND_FLAGS_DEFAULT);
+    if (err_is_fail(err)) {
+        DEBUG_ERR(err, "spawn_bind failed");
+        return err;
+    }
+
+    // XXX: block for bind completion
+    while (!bindst.present) {
+        messages_wait_and_handle_next();
+    }
+
+    if (err_is_fail(bindst.err)) {
+        return bindst.err;
+    }
+
+    spawn_rpc_client_init(bindst.b);
+    *ret_client = bindst.b;
+    // set_spawn_binding(coreid, bindst.b);
+
+    return err;
+}
 
 /**
  * \brief Request the spawn daemon on a specific core to spawn a program
index 3f0f05f..1b1fadf 100644 (file)
@@ -64,6 +64,7 @@ struct monitor_state {
 extern iref_t mem_serv_iref;
 extern iref_t name_serv_iref;
 extern iref_t ramfs_serv_iref;
+extern iref_t spawn_iref;
 extern iref_t monitor_rpc_iref;
 extern iref_t monitor_mem_iref;
 extern coreid_t my_core_id;
index 2672c23..f7f013c 100644 (file)
@@ -17,6 +17,7 @@
 #include <inttypes.h>
 #include <monitor.h>
 #include <barrelfish/dispatch.h>
+#include <barrelfish/proc_mgmt_client.h>
 #include <trace/trace.h>
 #include "send_cap.h"
 #include "capops.h"
@@ -423,6 +424,15 @@ static void span_domain_reply(struct intermon_binding *b,
     }
 }
 
+static void add_spawnd(struct intermon_binding *b, iref_t iref)
+{
+    struct intermon_state *st = (struct intermon_state*) b->st;
+    errval_t err = proc_mgmt_add_spawnd(iref, st->core_id);
+    if (err_is_fail(err)) {
+        USER_PANIC_ERR(err, "Sending proc_mgmt_add_spawnd request failed");
+    }
+}
+
 static void trace_caps_request(struct intermon_binding *b)
 {
     errval_t err;
@@ -730,6 +740,8 @@ static struct intermon_rx_vtbl the_intermon_vtable = {
     .span_domain_request       = span_domain_request,
     .span_domain_reply         = span_domain_reply,
 
+    .add_spawnd                = add_spawnd,
+
     .rsrc_join                 = inter_rsrc_join,
     .rsrc_join_complete        = inter_rsrc_join_complete,
     .rsrc_timer_sync           = inter_rsrc_timer_sync,
index eff3b47..631b888 100644 (file)
 #include <barrelfish/dispatch.h>
 #include <barrelfish/deferred.h>
 #include <barrelfish/domain.h>
+#include <barrelfish/proc_mgmt_client.h>
 #include <trace/trace.h>
 
 #ifdef __k1om__
 extern char **environ;
 #endif
 
-/* irefs for mem server name service and ramfs */
+/* irefs for mem server, name service, ramfs and spawnd*/
 iref_t mem_serv_iref = 0;
 iref_t ramfs_serv_iref = 0;
 iref_t name_serv_iref = 0;
+iref_t spawn_iref = 0;
 iref_t monitor_rpc_iref = 0;
 
 // Capref to trace cap
@@ -130,6 +132,18 @@ static errval_t boot_bsp_core(int argc, char *argv[])
         return err;
     }
 
+    // Spawn process manager, to be used by further domains.
+    set_proc_mgmt_binding(NULL);
+    err = spawn_domain("proc_mgmt");
+    if (err_is_fail(err)) {
+        DEBUG_ERR(err, "failed spawning proc_mgmt");
+        return err;
+    }
+    // XXX: Wait for proc_mgmt to initialize
+    while (get_proc_mgmt_binding() == NULL) {
+        messages_wait_and_handle_next();
+    }
+
     /* Spawn boot domains in menu.lst */
     err = spawn_all_domains();
     if (err_is_fail(err)) {
@@ -137,6 +151,16 @@ static errval_t boot_bsp_core(int argc, char *argv[])
         return err;
     }
 
+    // XXX: Wait for spawnd to initialize
+    while (spawn_iref == 0) {
+        messages_wait_and_handle_next();
+    }
+    // Now tell process manager about our new spawnd.
+    err = proc_mgmt_add_spawnd(spawn_iref, disp_get_core_id());
+    if (err_is_fail(err)) {
+        USER_PANIC_ERR(err, "error sending spawnd iref to process manager");
+    }
+
     return SYS_ERR_OK;
 }
 
@@ -212,6 +236,20 @@ static errval_t boot_app_core(int argc, char *argv[])
     if (err_is_fail(err)) {
         USER_PANIC_ERR(err, "error spawning spawnd");
     }
+    // XXX: Wait for spawnd to initialize
+    while (spawn_iref == 0) {
+        messages_wait_and_handle_next();
+    }
+    // Use monitor.0 to tell the process manager about our new spawnd.
+    struct intermon_binding *ib0;
+    err = intermon_binding_get(0, &ib0);
+    if (err_is_fail(err)) {
+        USER_PANIC_ERR(err, "error retrieving intermon_binding for monitor 0");
+    }
+    err = ib0->tx_vtbl.add_spawnd(ib0, NOP_CONT, spawn_iref);
+    if (err_is_fail(err)) {
+        USER_PANIC_ERR(err, "error sending add_spawnd request to monitor 0");
+    }
 
     /* Signal the monitor that booted us that we have initialized */
     err = intermon_binding->tx_vtbl.monitor_initialized(intermon_binding, NOP_CONT);
index ec7724f..7757173 100644 (file)
@@ -19,6 +19,7 @@
 #include <trace_definitions/trace_defs.h>
 #include <if/mem_defs.h>
 #include <barrelfish/monitor_client.h>
+#include <barrelfish/proc_mgmt_client.h>
 #include <barrelfish/syscalls.h>
 #include <barrelfish_kpi/distcaps.h>
 #include <if/monitor_loopback_defs.h>
@@ -36,6 +37,11 @@ struct multiboot_cap_state {
     cslot_t slot;
 };
 
+struct proc_mgmt_bind_st {
+    errval_t err;
+    bool present;
+};
+
 static void ms_multiboot_cap_request(struct monitor_binding *b, cslot_t slot);
 
 static void ms_multiboot_cap_request_handler(struct monitor_binding *b,
@@ -704,6 +710,66 @@ static void set_ramfs_iref_request(struct monitor_binding *b,
     ramfs_serv_iref = iref;
 }
 
+static void proc_mgmt_bind_cont(void *st,
+                                errval_t err,
+                                struct proc_mgmt_binding *b)
+{
+    struct proc_mgmt_bind_st* bind_st = (struct proc_mgmt_bind_st*) st;
+    assert(!bind_st->present);
+    bind_st->err = err;
+    bind_st->present = true;
+}
+
+static void set_proc_mgmt_ep_request(struct monitor_binding *b,
+                                     struct capref ep)
+{
+    // We got the endpoint which the process manager has allocated for us.
+    // Time to set up our part of the LMP connection and finish the handshake.
+    struct proc_mgmt_lmp_binding *lmpb =
+        malloc(sizeof(struct proc_mgmt_lmp_binding));
+    assert(lmpb != NULL);
+
+    set_proc_mgmt_binding(&lmpb->b);
+
+    struct proc_mgmt_bind_st bind_st = {
+        .present = false
+    };
+    errval_t err = proc_mgmt_client_lmp_bind(lmpb,
+                                             ep,
+                                             proc_mgmt_bind_cont,
+                                             &bind_st,
+                                             get_default_waitset(),
+                                             DEFAULT_LMP_BUF_WORDS);
+    if (err_is_fail(err)) {
+        USER_PANIC_ERR(err, "proc_mgmt_client_lmp_bind");
+    }
+
+    // Dispatch events on the waitset until proc_mgmt binding completes.
+    while (!bind_st.present) {
+        err = event_dispatch(get_default_waitset());
+        if (err_is_fail(err)) {
+            USER_PANIC_ERR(err, "monitor event dispatch");
+        }
+    }
+
+    if(err_is_fail(bind_st.err)) {
+        USER_PANIC_ERR(err, "during proc_mgmt bind initialization");
+    }
+
+    proc_mgmt_rpc_client_init(&lmpb->b);
+}
+
+static void set_spawn_iref_request(struct monitor_binding *b, iref_t iref)
+{
+    if (spawn_iref != 0) {
+        // Called multiple times, return error
+        DEBUG_ERR(0, "Attempt to reset spawn IREF ignored");
+        return;
+    }
+
+    spawn_iref = iref;
+}
+
 struct send_cap_st {
     struct intermon_msg_queue_elem qe; // must be first
     uintptr_t my_mon_id;
@@ -908,6 +974,8 @@ struct monitor_rx_vtbl the_table = {
     .set_mem_iref_request  = set_mem_iref_request,
     .set_name_iref_request = set_name_iref_request,
     .set_ramfs_iref_request = set_ramfs_iref_request,
+    .set_proc_mgmt_ep_request = set_proc_mgmt_ep_request,
+    .set_spawn_iref_request = set_spawn_iref_request,
     .get_monitor_rpc_iref_request  = get_monitor_rpc_iref_request,
 
     .cap_send_request = cap_send_request,
index 7574388..ada09b6 100644 (file)
@@ -256,6 +256,7 @@ errval_t spawn_all_domains(void)
            || !strcmp(short_name, "monitor")
            || !strcmp(short_name, "mem_serv")
            || !strcmp(short_name, "xeon_phi")
+           || !strcmp(short_name, "proc_mgmt")
           )
         {
             continue;
index 48cb949..7b02d9a 100644 (file)
 --------------------------------------------------------------------------
 
 [ build application { target = "proc_mgmt",
-                      cFiles = [ "main.c", "service.c" ],
+                      cFiles = [ "main.c", "service.c", "spawnd_state.c" ],
                       addLibraries = libDeps [ "skb", "dist", "lwip" ],
-                      flounderDefs = [ "monitor", "monitor_blocking"],
-                      flounderExtraDefs = [ ("monitor_blocking",["rpcclient"]) ],
+                      flounderDefs = [ "monitor", "monitor_blocking" ],
+                      flounderExtraDefs = [ ("monitor_blocking",["rpcclient"]), 
+                                            ("spawn",["rpcclient"]) ],
                       flounderBindings = [ "proc_mgmt" ],
                       architectures = [ "x86_64", "x86_32" ]
                     },
   build application { target = "proc_mgmt",
-                      cFiles = [ "main.c", "service.c" ],
+                      cFiles = [ "main.c", "service.c", "spawnd_state.c" ],
                       addLibraries = libDeps [ "skb", "dist", "lwip" ],
                       flounderDefs = [ "monitor", "monitor_blocking"],
-                      flounderExtraDefs = [ ("monitor_blocking",["rpcclient"]) ],
+                      flounderExtraDefs = [ ("monitor_blocking",["rpcclient"]), 
+                                            ("spawn",["rpcclient"]) ],
                       flounderBindings = [ "proc_mgmt" ],
                       architectures = [ "k1om" ]
                     },
   build application { target = "proc_mgmt",
-                      cFiles = [ "main.c", "service.c" ],
+                      cFiles = [ "main.c", "service.c", "spawnd_state.c" ],
                       addLibraries = libDeps [ "skb", "dist", "lwip" ],
                       flounderDefs = [ "monitor", "monitor_blocking"],
-                      flounderExtraDefs = [ ("monitor_blocking",["rpcclient"]) ],
+                      flounderExtraDefs = [ ("monitor_blocking",["rpcclient"]), 
+                                            ("spawn",["rpcclient"]) ],
                       flounderBindings = [ "proc_mgmt" ],
                       architectures = [ "armv7", "armv8" ]
                     }
index 94fe5b7..351cf69 100644 (file)
@@ -15,7 +15,7 @@
 #ifndef PROC_MGMT_INTERNAL_H_
 #define PROC_MGMT_INTERNAL_H_
 
-#define SERVICE_BASENAME    "proc_mgmt" // the core ID is appended to this
+#define SERVICE_BASENAME    "proc_mgmt"
 
 extern coreid_t my_core_id;
 
index 37c8d3e..5c0628f 100644 (file)
  * ETH Zurich D-INFK, Universitaetstr. 6, CH-8092 Zurich. Attn: Systems Group.
  */
 
-#include <stdio.h>
-#include <string.h>
-
 #include <barrelfish/barrelfish.h>
-#include <barrelfish/dispatch.h>
 
 #include "internal.h"
 
@@ -26,14 +22,12 @@ int main(int argc, const char *argv[])
 {
     errval_t err;
 
-    my_core_id = disp_get_core_id();
-
-    printf("proc_mgmt.%u up.\n", my_core_id);
-
     err = start_service();
     if (err_is_fail(err)) {
         USER_PANIC_ERR(err, "failed to start proc_mgmt service loop");
     }
 
+    debug_printf("ready\n");
+
     messages_handler_loop();
 }
index 7c724d9..d338eb6 100644 (file)
 #include <barrelfish/barrelfish.h>
 #include <barrelfish/nameservice_client.h>
 #include <barrelfish/proc_mgmt_client.h>
+#include <barrelfish/spawn_client.h>
+#include <if/monitor_defs.h>
 #include <if/proc_mgmt_defs.h>
 
 #include "internal.h"
+#include "spawnd_state.h"
 
-static struct proc_mgmt_rpc_rx_vtbl rpc_rx_vtbl;
-
-static errval_t alloc_ep_handler(struct proc_mgmt_binding *b, errval_t *err,
-        struct capref *ep)
+static void add_spawnd_handler(struct proc_mgmt_binding *b, coreid_t core_id,
+                               iref_t iref)
 {
-    struct proc_mgmt_lmp_binding *lmpb = (struct proc_mgmt_lmp_binding*) malloc(
-            sizeof(struct proc_mgmt_lmp_binding));
-    assert(lmpb != NULL);
-
-    *err = proc_mgmt_client_lmp_accept(lmpb, get_default_waitset(),
-            DEFAULT_LMP_BUF_WORDS);
-    if (err_is_ok(*err)) {
-        *ep = lmpb->chan.local_cap;
+    if (spawnd_state_exists(core_id)) {
+        DEBUG_ERR(PROC_MGMT_ERR_SPAWND_EXISTS, "spawnd_state_exists");
+        return;
+    }
 
-        // struct proc_mgmt_state *st = (struct proc_mgmt_state*) malloc(
-        //         sizeof(struct proc_mgmt_state));
-        // assert(st != NULL);
-        // st->queue.head = st->queue.tail = NULL;
+    // Bind with the spawnd.
+    struct spawn_binding *spawnb;
+    errval_t err = spawn_bind_iref(iref, &spawnb);
+    if (err_is_fail(err)) {
+        DEBUG_ERR(err, "spawn_bind_iref");
+        return;
+    }
 
-        lmpb->b.rpc_rx_vtbl = rpc_rx_vtbl;
-        // lmpb->b.st = st;
-    } else {
-        free(lmpb);
+    err = spawnd_state_alloc(core_id, spawnb);
+    if (err_is_fail(err)) {
+        DEBUG_ERR(err, "spawnd_state_alloc");
     }
 
-    return SYS_ERR_OK;
-}
+    debug_printf("Process manager bound with spawnd.%u on iref %u\n", core_id,
+            iref);
 
-static errval_t spawn_handler(struct proc_mgmt_binding *b,
-                          coreid_t core,
-                          const char *path,
-                          const char *argvbuf,
-                          size_t argvbytes,
-                          const char *envbuf,
-                          size_t envbytes,
-                          uint8_t flags,
-                          errval_t *err,
-                          struct capref *domainid_cap)
-{
-    return LIB_ERR_NOT_IMPLEMENTED;
+    err = spawnd_state_get_binding(core_id)->rpc_tx_vtbl.echo(
+            spawnd_state_get_binding(core_id),
+            SERVICE_BASENAME,
+            disp_get_current_core_id());
+    if (err_is_fail(err)) {
+        USER_PANIC_ERR(err, "spawnd echo request failed");
+    }
 }
 
-static errval_t span_handler(struct proc_mgmt_binding *b,
-                         struct capref domainid_cap,
-                         coreid_t core,
-                         struct capref vroot,
-                         struct capref disp_mem,
-                         errval_t *err)
+static void add_spawnd_handler_non_monitor(struct proc_mgmt_binding *b,
+                                           coreid_t core_id, iref_t iref)
 {
-    return LIB_ERR_NOT_IMPLEMENTED;
+    debug_printf("Ignoring add_spawnd call: %s\n",
+                 err_getstring(PROC_MGMT_ERR_NOT_MONITOR));
 }
 
-static errval_t kill_handler(struct proc_mgmt_binding *b,
-                         struct capref domainid_cap,
-                         errval_t *err)
-{
-    return LIB_ERR_NOT_IMPLEMENTED;
-}
+// static errval_t spawn_handler(struct proc_mgmt_binding *b,
+//                           coreid_t core,
+//                           const char *path,
+//                           const char *argvbuf,
+//                           size_t argvbytes,
+//                           const char *envbuf,
+//                           size_t envbytes,
+//                           uint8_t flags,
+//                           errval_t *err,
+//                           struct capref *domainid_cap)
+// {
+//     return LIB_ERR_NOT_IMPLEMENTED;
+// }
+
+// static errval_t span_handler(struct proc_mgmt_binding *b,
+//                          struct capref domainid_cap,
+//                          coreid_t core,
+//                          struct capref vroot,
+//                          struct capref disp_mem,
+//                          errval_t *err)
+// {
+//     return LIB_ERR_NOT_IMPLEMENTED;
+// }
+
+// static errval_t kill_handler(struct proc_mgmt_binding *b,
+//                          struct capref domainid_cap,
+//                          errval_t *err)
+// {
+//     return LIB_ERR_NOT_IMPLEMENTED;
+// }
+
+static struct proc_mgmt_rx_vtbl monitor_vtbl = {
+    .add_spawnd = add_spawnd_handler
+};
 
-static struct proc_mgmt_rpc_rx_vtbl rpc_rx_vtbl = {
-    .alloc_ep_call = alloc_ep_handler,
-    .spawn_call = spawn_handler,
-    .span_call = span_handler,
-    .kill_call = kill_handler
+static struct proc_mgmt_rx_vtbl non_monitor_vtbl = {
+    .add_spawnd = add_spawnd_handler_non_monitor
 };
 
+static errval_t alloc_ep_for_monitor(struct capref *ep)
+{
+    struct proc_mgmt_lmp_binding *lmpb =
+        malloc(sizeof(struct proc_mgmt_lmp_binding));
+    assert(lmpb != NULL);
+
+    // setup our end of the binding
+    errval_t err = proc_mgmt_client_lmp_accept(lmpb, get_default_waitset(),
+                                               DEFAULT_LMP_BUF_WORDS);
+    if (err_is_fail(err)) {
+        free(lmpb);
+        return err_push(err, LIB_ERR_PROC_MGMT_CLIENT_ACCEPT);
+    }
+
+    *ep = lmpb->chan.local_cap;
+    lmpb->b.rx_vtbl = monitor_vtbl;
+
+    return SYS_ERR_OK;
+}
+
 static void export_cb(void *st, errval_t err, iref_t iref)
 {
     if (err_is_fail(err)) {
         USER_PANIC_ERR(err, "export failed");
     }
 
-    // construct name
-    char namebuf[32];
-    size_t len = snprintf(namebuf, sizeof(namebuf), "%s.%d", SERVICE_BASENAME,
-                          my_core_id);
-    assert(len < sizeof(namebuf));
-    namebuf[sizeof(namebuf) - 1] = '\0';
+    // Allocate an endpoint for the local monitor, who will use it to inform
+    // us about new spawnd irefs on behalf of other monitors.
+    struct capref ep;
+    err = alloc_ep_for_monitor(&ep);
+    if (err_is_fail(err)) {
+        USER_PANIC_ERR(err, "failed to allocate LMP EP for local monitor");
+    }
+
+    // Send the endpoint to the monitor, so it can finish the handshake.
+    struct monitor_binding *mb = get_monitor_binding();
+    err = mb->tx_vtbl.set_proc_mgmt_ep_request(mb, NOP_CONT, ep);
+    if (err_is_fail(err)) {
+        USER_PANIC_ERR(err, "failed to send set_proc_mgmt_ep_request to "
+                       "monitor");
+    }
 
-    // register this iref with the name service
-    err = nameservice_register(namebuf, iref);
+    // Also register this iref with the name service, for arbitrary client
+    // domains to use for spawn-related ops.
+    err = nameservice_register(SERVICE_BASENAME, iref);
     if (err_is_fail(err)) {
         USER_PANIC_ERR(err, "nameservice_register failed");
     }
@@ -108,7 +154,7 @@ static void export_cb(void *st, errval_t err, iref_t iref)
 
 static errval_t connect_cb(void *st, struct proc_mgmt_binding *b)
 {
-    b->rpc_rx_vtbl = rpc_rx_vtbl;
+    b->rx_vtbl = non_monitor_vtbl;
     return SYS_ERR_OK;
 }
 
diff --git a/usr/proc_mgmt/spawnd_state.c b/usr/proc_mgmt/spawnd_state.c
new file mode 100644 (file)
index 0000000..f07d196
--- /dev/null
@@ -0,0 +1,46 @@
+/*
+ * \brief Spawnd state internals for the process manager.
+ *
+ * Copyright (c) 2017, ETH Zurich.
+ * All rights reserved.
+ *
+ * This file is distributed under the terms in the attached LICENSE file.
+ * If you do not find this file, copies can be found by writing to:
+ * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
+ */
+
+#include <barrelfish/barrelfish.h>
+
+#include "spawnd_state.h"
+
+static struct spawnd_state *spawnds[MAX_SPAWNDS];
+
+errval_t spawnd_state_alloc(coreid_t core_id, struct spawn_binding *b)
+{
+    spawnds[core_id] = (struct spawnd_state*) malloc(
+            sizeof(struct spawnd_state));
+    if (spawnds[core_id] == NULL) {
+        return LIB_ERR_MALLOC_FAIL;
+    }
+
+    spawnds[core_id]->b = b;
+
+    return SYS_ERR_OK;
+}
+
+void spawnd_state_free(coreid_t core_id)
+{
+    if (spawnds[core_id] != NULL) {
+        free(spawnds[core_id]);
+    }
+}
+
+inline bool spawnd_state_exists(coreid_t core_id)
+{
+    return spawnds[core_id] != NULL;
+}
+
+inline struct spawn_binding *spawnd_state_get_binding(coreid_t core_id)
+{
+    return spawnds[core_id]->b;
+}
diff --git a/usr/proc_mgmt/spawnd_state.h b/usr/proc_mgmt/spawnd_state.h
new file mode 100644 (file)
index 0000000..a14fd0f
--- /dev/null
@@ -0,0 +1,33 @@
+/*
+ * \brief Spawnd state internals for the process manager.
+ *
+ * Copyright (c) 2017, ETH Zurich.
+ * All rights reserved.
+ *
+ * This file is distributed under the terms in the attached LICENSE file.
+ * If you do not find this file, copies can be found by writing to:
+ * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
+ */
+
+#ifndef SPAWND_STATE_H
+#define SPAWND_STATE_H
+
+#include <stdbool.h>
+
+#include <if/spawn_defs.h>
+#include <barrelfish/barrelfish.h>
+
+// TODO(razvan): Use a hash map instead.
+#define MAX_SPAWNDS 256
+
+struct spawnd_state {
+    struct spawn_binding *b;
+    // TODO(razvan): will need more state here.
+};
+
+errval_t spawnd_state_alloc(coreid_t core_id, struct spawn_binding *b);
+void spawnd_state_free(coreid_t core_id);
+bool spawnd_state_exists(coreid_t core_id);
+struct spawn_binding *spawnd_state_get_binding(coreid_t core_id);
+
+#endif  // SPAWND_STATE
index 7f1e8dc..0e9f700 100644 (file)
@@ -22,6 +22,7 @@
 #include <vfs/vfs_path.h>
 #include <dist/barrier.h>
 #include <if/spawn_defs.h>
+#include <if/monitor_defs.h>
 #include <if/monitor_blocking_defs.h>
 #include <barrelfish/dispatcher_arch.h>
 #include <barrelfish/invocations_arch.h>
@@ -529,6 +530,15 @@ static void dump_capabilities_handler(struct spawn_binding *b, domainid_t domain
     }
 }
 
+// TODO(razvan): This is just used to test that the process manager can connect
+// to arbitrary spawnds; maybe remove it soon.
+static errval_t echo_handler(struct spawn_binding *b, const char *sender_name,
+                             coreid_t sender_core)
+{
+    debug_printf("spawnd ECHO from: %s.%u\n", sender_name, sender_core);
+    return SYS_ERR_OK;
+}
+
 static struct spawn_rx_vtbl rx_vtbl = {
     // .spawn_domain_call = spawn_handler,
     // .spawn_domain_with_caps_call = spawn_with_caps_handler,
@@ -544,6 +554,7 @@ static struct spawn_rx_vtbl rx_vtbl = {
 static struct spawn_rpc_rx_vtbl rpc_rx_vtbl = {
     .spawn_domain_call = spawn_handler,
     .spawn_domain_with_caps_call = spawn_with_caps_handler,
+    .echo_call = echo_handler
     // .use_local_memserv_call = use_local_memserv_handler,
     // .kill_call = kill_handler,
     // .exit_call = exit_handler,
@@ -559,6 +570,14 @@ static void export_cb(void *st, errval_t err, iref_t iref)
         USER_PANIC_ERR(err, "export failed");
     }
 
+    // Send iref back to monitor, which will forward it to the process manager.
+    struct monitor_binding *mb = get_monitor_binding();
+    err = mb->tx_vtbl.set_spawn_iref_request(mb, NOP_CONT, iref);
+    if (err_is_fail(err)) {
+        USER_PANIC_ERR(err, "failed to send set_spawn_iref_request to "
+                "monitor");
+    }
+
     // construct name
     char namebuf[32];
     size_t len = snprintf(namebuf, sizeof(namebuf), "%s.%d", SERVICE_BASENAME,