Added: Xeon Phi Manager
[barrelfish] / usr / drivers / xeon_phi / service.c
1 /**
2  * \file
3  * \brief Driver for booting the Xeon Phi Coprocessor card on a Barrelfish Host
4  */
5
6 /*
7  * Copyright (c) 2014 ETH Zurich.
8  * All rights reserved.
9  *
10  * This file is distributed under the terms in the attached LICENSE file.
11  * If you do not find this file, copies can be found by writing to:
12  * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group.
13  */
14
15 #include <stdio.h>
16 #include <string.h>
17 #include <barrelfish/barrelfish.h>
18 #include <xeon_phi/xeon_phi.h>
19
20 #include <if/xeon_phi_defs.h>
21
22 #include "xeon_phi.h"
23 #include "service.h"
24
25 static uint32_t is_exported;
26
27 static iref_t svc_iref;
28
29 /*
30  * ---------------------------------------------------------------------------
31  * Intra Xeon Phi Driver Communication Regigistration
32  */
33
34 static void register_response_sent_cb(void *a)
35 {
36
37 }
38
39 static void register_response_send(void *a)
40 {
41     errval_t err;
42
43     struct xnode *topology = a;
44
45     struct event_closure txcont = MKCONT(register_response_sent_cb, a);
46
47     if (topology->state == XNODE_STATE_READY) {
48         err = SYS_ERR_OK;
49     } else {
50         err = -1;  // TODO> ERROR NUMBEr
51     }
52
53     err = xeon_phi_register_response__tx(topology->binding, txcont, err);
54     if (err_is_fail(err)) {
55         if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
56             struct waitset *ws = get_default_waitset();
57             txcont = MKCONT(register_response_send, a);
58             err = topology->binding->register_send(topology->binding, ws, txcont);
59             if (err_is_fail(err)) {
60                 topology->state = XNODE_STATE_FAILURE;
61             }
62         }
63     }
64 }
65
66 /**
67  *
68  */
69 static void register_call_recv(struct xeon_phi_binding *_binding,
70                                uint8_t id)
71 {
72
73     assert(((struct xnode * )(_binding->st))->binding != _binding);
74     struct xeon_phi *phi = _binding->st;
75
76     assert(id < XEON_PHI_NUM_MAX);
77     phi->topology[id].binding = _binding;
78     phi->topology[id].state = XNODE_STATE_READY;
79
80     phi->connected++;
81
82     XSERVICE_DEBUG("[%u] New register call: id=0x%x, num_connected=%i\n",
83                    phi->id,
84                    id,
85                    phi->connected);
86
87     _binding->st = &phi->topology[id];
88
89     register_response_send(&phi->topology[id]);
90 }
91
92 /**
93  *
94  */
95 static void register_response_recv(struct xeon_phi_binding *_binding,
96                                    xeon_phi_errval_t msgerr)
97 {
98     assert(((struct xnode * )(_binding->st))->binding == _binding);
99
100     struct xnode *topology = _binding->st;
101
102     if (err_is_fail(msgerr)) {
103         topology->state = XNODE_STATE_FAILURE;
104     } else {
105         topology->local->connected++;
106         topology->state = XNODE_STATE_READY;
107     }
108
109     XSERVICE_DEBUG("[%u] New register response: 0x%lx, num_connected=%u\n",
110                    topology->local->id,
111                    msgerr,
112                    topology->local->connected);
113 }
114
115 static void register_call_sent_cb(void *a)F
116 {
117
118 }
119
120 static void register_call_send(void *a)
121 {
122     errval_t err;
123
124     struct xnode *topology = a;
125
126     struct event_closure txcont = MKCONT(register_call_sent_cb, a);
127
128     topology->state = XNODE_STATE_REGISTERING;
129
130     err = xeon_phi_register_call__tx(topology->binding, txcont, topology->local->id);
131     if (err_is_fail(err)) {
132         if (err_no(err) == FLOUNDER_ERR_TX_BUSY) {
133             struct waitset *ws = get_default_waitset();
134             txcont = MKCONT(register_call_send, a);
135             err = topology->binding->register_send(topology->binding, ws, txcont);
136             if (err_is_fail(err)) {
137                 topology->state = XNODE_STATE_FAILURE;
138             }
139         }
140     }
141 }
142
143 /// Receive handler table
144 static struct xeon_phi_rx_vtbl xps_rx_vtbl = {
145     .register_call = register_call_recv,
146     .register_response = register_response_recv
147 };
148
149 /*
150  * ---------------------------------------------------------------------------
151  * Service Setup
152  */
153 static void svc_bind_cb(void *st,
154                         errval_t err,
155                         struct xeon_phi_binding *b)
156 {
157     struct xnode *node = st;
158
159     XSERVICE_DEBUG("binding done.\n");
160     b->rx_vtbl = xps_rx_vtbl;
161     node->binding = b;
162     b->st = node;
163     node->state = XNODE_STATE_REGISTERING;
164 }
165
166 static errval_t svc_register(struct xnode *node)
167 {
168     XSERVICE_DEBUG("binding to node %i (iref = 0x%x)\n", node->id, node->iref);
169     errval_t err;
170
171     err = xeon_phi_bind(node->iref,
172                         svc_bind_cb,
173                         node,
174                         get_default_waitset(),
175                         IDC_BIND_FLAGS_DEFAULT);
176     if (err_is_fail(err)) {
177         node->state = XNODE_STATE_FAILURE;
178         return err;
179     }
180
181     return SYS_ERR_OK;
182 }
183
184 static errval_t svc_connect_cb(void *st,
185                                struct xeon_phi_binding *b)
186 {
187     struct xeon_phi *phi = st;
188
189     XSERVICE_DEBUG("[%u] New connection\n", phi->id);
190
191     b->st = st;
192     b->rx_vtbl = xps_rx_vtbl;
193     return SYS_ERR_OK;
194 }
195
196 static void svc_export_cb(void *st,
197                           errval_t err,
198                           iref_t iref)
199 {
200     if (err_is_fail(err)) {
201         svc_iref = 0x0;
202         return;
203     }
204
205     svc_iref = iref;
206
207     struct xeon_phi *phi = st;
208     phi->iref = iref;
209
210     is_exported = 0x1;
211 }
212
213 /**
214  * \brief initializes the service
215  *
216  * \param iref  returns the iref of the initialized service
217  *
218  * \return SYS_ERR_OK on success
219  */
220 errval_t service_init(struct xeon_phi *phi)
221 {
222     errval_t err;
223
224     for (uint32_t i = 0; i < XEON_PHI_NUM_MAX; ++i) {
225         phi->topology[i].local = phi;
226     }
227
228     err = xeon_phi_export(phi,
229                           svc_export_cb,
230                           svc_connect_cb,
231                           get_default_waitset(),
232                           IDC_EXPORT_FLAGS_DEFAULT);
233     if (err_is_fail(err)) {
234         return err;
235     }
236
237     while (!is_exported) {
238         messages_wait_and_handle_next();
239     }
240
241     if (svc_iref == 0x0) {
242         return -1;
243     }
244
245     return SYS_ERR_OK;
246
247 }
248
249 /**
250  * \brief registers the local service with the other Xeon Phi drivers
251  *        in the topology
252  *
253  * \param phi   pointer to the local card structure
254  * \param irefs the irefs of the other cards
255  * \param num   the number of irefs in the array
256  */
257 errval_t service_register(struct xeon_phi *phi,
258                           iref_t *irefs,
259                           size_t num)
260 {
261     struct xnode *xnode;
262     XSERVICE_DEBUG("start binding to nodes\n");
263     for (uint32_t i = 0; i < num; ++i) {
264         xnode = &phi->topology[i];
265         if (i == phi->id) {
266             xnode->iref = phi->iref;
267             xnode->id = i;
268             xnode->state = XNODE_STATE_READY;
269             continue;
270         }
271
272         xnode->iref = irefs[i];
273         xnode->id = i;
274         xnode->state = XNODE_STATE_NONE;
275         svc_register(xnode);
276         while (xnode->state == XNODE_STATE_NONE) {
277             messages_wait_and_handle_next();
278         }
279     }
280
281     XSERVICE_DEBUG("Start registring\n");
282
283     for (uint32_t i = 0; i < num; ++i) {
284         if (i == phi->id) {
285             continue;
286         }
287         xnode = &phi->topology[i];
288         register_call_send(xnode);
289         while (xnode->state == XNODE_STATE_READY) {
290             messages_wait_and_handle_next();
291         }
292     }
293
294     return SYS_ERR_OK;
295 }
296
297 /**
298  * \brief starts the service request handling
299  */
300 errval_t service_start(void)
301 {
302     messages_handler_loop();
303
304     return SYS_ERR_OK;
305 }
306