3 * \brief Boot module for the Xeon Phi
5 * Loads the co processor OS onto the card and boots it
9 * Copyright (c) 2014 ETH Zurich.
10 * All rights reserved.
12 * This file is distributed under the terms in the attached LICENSE file.
13 * If you do not find this file, copies can be found by writing to:
14 * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group.
19 #include <barrelfish/barrelfish.h>
20 #include <spawndomain/spawndomain.h>
24 #include <xeon_phi/xeon_phi.h>
26 #include <dev/xeon_phi/xeon_phi_boot_dev.h>
27 #include <dev/xeon_phi/xeon_phi_apic_dev.h>
29 struct bootinfo *bi = NULL;
32 #include "messaging.h"
35 #define BOOT_TIMEOUT 3000
36 #define BOOT_COUNTER 0xFFFFF
39 * TODO: Verify these values if they are really needed
41 #define MEMORY_RESERVE_PERCENT 50
42 #define UOS_RESERVE_SIZE_MIN ((128) * 1024 * 1024)
43 #define UOS_RESERVE_SIZE_MAX (((4) * 1024 * 1024 * 1024ULL) - ((4) * 1024))
48 #define MAX(a, b) ( ((a) > (b)) ? (a) : (b) )
49 #define MIN(a, b) ( ((a) < (b)) ? (a) : (b) )
50 #define ALIGN(x) ((x + BASE_PAGE_SIZE-1) & ~(BASE_PAGE_SIZE-1))
52 static xeon_phi_boot_t boot_registers;
53 static xeon_phi_apic_t apic_registers;
56 * \brief get the load offset to where to place the bootloader
58 * The bootstrap on the card will write the offset into the SBOX_SCRATCH2
59 * register once the bootstrap is finished
61 static inline lvaddr_t get_load_offset(struct xeon_phi *phi)
63 return ((lvaddr_t) xeon_phi_boot_download_offset_rdf(&boot_registers)) << 12;
66 static uint64_t get_adapter_memsize(void)
68 xeon_phi_boot_meminfo_t meminfo = xeon_phi_boot_meminfo_rd(&boot_registers);
70 uint64_t memsize = xeon_phi_boot_meminfo_size_kb_extract(meminfo);
73 switch (xeon_phi_boot_meminfo_usage_extract(meminfo)) {
74 case xeon_phi_boot_mem_all:
76 case xeon_phi_boot_mem_half:
78 case xeon_phi_boot_mem_third:
81 case xeon_phi_boot_mem_fourth:
89 static errval_t load_module(char *path,
96 /* read file into memory */
98 err = vfs_open(path, &fh);
99 if (err_is_fail(err)) {
100 return err_push(err, SPAWN_ERR_LOAD);
103 struct vfs_fileinfo info;
104 err = vfs_stat(fh, &info);
105 if (err_is_fail(err)) {
107 return err_push(err, SPAWN_ERR_LOAD);
110 assert(info.type == VFS_FILE);
112 uint8_t *image = buf;
114 size_t pos = 0, readlen;
116 err = vfs_read(fh, &image[pos], info.size - pos, &readlen);
117 if (err_is_fail(err)) {
119 return err_push(err, SPAWN_ERR_LOAD);
120 } else if (readlen == 0) {
122 return SPAWN_ERR_LOAD; // XXX
126 } while (err_is_ok(err) && readlen > 0 && pos < info.size);
129 if (err_is_fail(err)) {
130 DEBUG_ERR(err, "failed to close file %s", path);
134 *ret_size = info.size;
142 * \brief Loads the bootloader image onto the card
144 * \param phi the xeon phi card information
145 * \param xloader_img name of the bootloader image
146 * \param ret_imgsize returned image size
148 * Note: it is important that the bootloader just uses statically allocated
149 * memory and does not exceed its image size with additional memory.
150 * Otherwise the CMD line or the multiboot image will be overwritten.
152 static errval_t load_bootloader(struct xeon_phi *phi,
158 * find the boot loader image in the host multiboot
160 struct mem_region *module = multiboot_find_module(bi, xloader_img);
161 if (module == NULL) {
162 return SPAWN_ERR_FIND_MODULE;
168 err = spawn_map_module(module, &imgsize, &binary, NULL);
169 if (err_is_fail(err)) {
170 return err_push(err, SPAWN_ERR_ELF_MAP);
173 imgsize = module->mrmod_size;
178 * get the load offset: we do not want to write into the boot loade
180 lvaddr_t loadoffset = get_load_offset(phi);
182 get_adapter_memsize();
184 phi->apicid = xeon_phi_boot_download_apicid_rdf(&boot_registers);
186 XBOOT_DEBUG("Loading xloader onto card... offset = 0x%lx\n", loadoffset);
188 memcpy((void *) (phi->apt.vbase + loadoffset), (void *) binary, imgsize);
191 lvaddr_t loadoffset = get_load_offset(phi);
194 void *buf = (void *) (phi->apt.vbase + loadoffset);
196 err = load_module(xloader_img, buf, &imgsize);
197 if (err_is_fail(err)) {
201 phi->apicid = xeon_phi_boot_download_apicid_rdf(&boot_registers);
203 phi->os_offset = loadoffset;
204 phi->os_size = imgsize;
212 static errval_t load_multiboot_image(struct xeon_phi *phi,
214 lvaddr_t load_offset)
218 assert(phi->os_offset != 0);
222 * find the boot loader image in the host multiboot
224 struct mem_region *module = multiboot_find_module(bi, multiboot_img);
225 if (module == NULL) {
226 return SPAWN_ERR_FIND_MODULE;
232 err = spawn_map_module(module, &imgsize, &image, NULL);
233 if (err_is_fail(err)) {
234 return err_push(err, SPAWN_ERR_ELF_MAP);
237 imgsize = module->mrmod_size;
239 XBOOT_DEBUG("loading multiboot image onto card... offset = 0x%lx\n",
242 memcpy((void *) (phi->apt.vbase + load_offset), (void *) image, imgsize);
247 void *buf = (void *) (phi->apt.vbase + load_offset);
249 err = load_module(multiboot_img, buf, &imgsize);
250 if (err_is_fail(err)) {
255 * we are using the Linux style way in booting. The following will update
256 * the corresponding fields in struct boot_param of the header.
258 struct xeon_phi_boot_params *bp;
259 bp = (struct xeon_phi_boot_params *)(phi->apt.vbase + phi->os_offset);
260 bp->ramdisk_image = (uint32_t)load_offset;
261 bp->ramdisk_size = (uint32_t)imgsize;
267 * \brief generates the cmdline supplied to the card kernel
269 * \param phi the card information structure
270 * \param load_offset offset where to load the cmdline
271 * \param ret_size size of the cmdline in bytes
273 static errval_t load_cmdline(struct xeon_phi *phi,
274 lvaddr_t load_offset)
278 struct xeon_phi_boot_params *bp;
279 bp = (struct xeon_phi_boot_params *)(phi->apt.vbase + phi->os_offset);
281 XBOOT_DEBUG("copying cmdline onto card, offset = 0x%lx\n", load_offset);
283 void *buf = (void *) (phi->apt.vbase + load_offset);
286 cmdlen += sprintf(buf + cmdlen,
287 "msg_base=%lx, msg_size=%lx",
293 cmdlen += sprintf(buf + cmdlen, "%s", phi->cmdline);
296 cmdlen += sprintf(buf + cmdlen, "card_id=%i", phi->id);
302 * TODO: Add multihop / communication information here..
305 printf("cmdline = %x, %s\n", (uint32_t)load_offset, (char*)buf);
308 phi->cmdlen = cmdlen;
311 bp->cmdline_ptr = (uint32_t)(load_offset);
312 bp->cmdline_size = (uint32_t)cmdlen;
317 static errval_t bootstrap_notify(struct xeon_phi *phi)
319 // set the bootimage size to tell the bootloader
320 xeon_phi_boot_os_size_rawwr(&boot_registers, phi->os_size);
322 uint64_t memsize = get_adapter_memsize();
324 uint64_t reserved = (memsize * MEMORY_RESERVE_PERCENT / 100);
326 // Keep in mind maximum uos reserve size is uint32_t, so we never overflow
327 reserved = MIN(reserved, UOS_RESERVE_SIZE_MAX);
328 reserved = MAX(reserved, UOS_RESERVE_SIZE_MIN);
330 // Always align uos reserve size to a page
331 reserved = (reserved & ~(BASE_PAGE_SIZE - 1));
333 xeon_phi_boot_res_size_rawwr(&boot_registers, (uint32_t) reserved);
335 // sending the bootstrap interrupt
336 xeon_phi_apic_icr_lo_t icr_lo = xeon_phi_apic_icr_lo_default;
337 icr_lo = xeon_phi_apic_icr_lo_vector_insert(icr_lo, xeon_phi_apic_vec_bsp);
338 icr_lo = xeon_phi_apic_icr_lo_boot_notify_insert(icr_lo, 0x1);
340 assert(icr_lo == (229 | (1 << 13)));
342 xeon_phi_apic_icr_hi_wr(&apic_registers, xeon_phi_apic_bootstrap, phi->apicid);
344 xeon_phi_apic_icr_lo_wr(&apic_registers, xeon_phi_apic_bootstrap, icr_lo);
350 * \brief boots the card with the given loader and multiboot image
352 * \param phi pointer to the card information
353 * \param xloader_img pointer to the card bootloader image
354 * \param multiboot_img pointer to the card multiboot image
356 errval_t xeon_phi_boot(struct xeon_phi *phi,
364 return SYS_ERR_ILLEGAL_INVOCATION;
367 xeon_phi_boot_initialize(&boot_registers,
368 XEON_PHI_MMIO_TO_SBOX(phi),
369 XEON_PHI_MMIO_TO_DBOX(phi));
370 xeon_phi_apic_initialize(&apic_registers, XEON_PHI_MMIO_TO_SBOX(phi));
372 phi->apicid = xeon_phi_boot_download_apicid_rdf(&boot_registers);
374 // load the coprocessor OS
375 err = load_bootloader(phi, xloader_img);
376 if (err_is_fail(err)) {
377 USER_PANIC_ERR(err, "Could not load bootloader image");
380 // round to next page
381 offset = ALIGN(phi->os_offset + phi->os_size);
383 err = messaging_init(phi, NULL_CAP);
384 if (err_is_fail(err)) {
385 USER_PANIC_ERR(err, "Could not initialize messagin");
389 err = load_cmdline(phi, offset);
390 if (err_is_fail(err)) {
391 USER_PANIC_ERR(err, "Could not load multiboot image");
394 // round to next page
395 offset = ALIGN(offset+phi->cmdlen);
397 // load multiboot image
398 err = load_multiboot_image(phi, multiboot_img, offset);
399 if (err_is_fail(err)) {
400 USER_PANIC_ERR(err, "Could not load multiboot image");
403 xeon_phi_boot_download_status_wrf(&boot_registers, 0x0);
405 phi->apicid = xeon_phi_boot_download_apicid_rdf(&boot_registers);
407 xeon_phi_serial_init(phi);
409 // notify the bootstrap
410 bootstrap_notify(phi);
412 xeon_phi_boot_postcode_t postcode;
413 xeon_phi_boot_postcodes_t pc, pc_prev = 0;
414 uint32_t counter = BOOT_COUNTER;
416 postcode = xeon_phi_boot_postcode_rd(&boot_registers);
417 pc = xeon_phi_boot_postcode_code_extract(postcode);
419 debug_printf("Xeon Phi Booting: %s\n",
420 xeon_phi_boot_postcodes_describe(pc));
422 if (postcode == xeon_phi_boot_postcode_done) {
428 XBOOT_DEBUG("Bootstrap has finished execution. Waiting for Firmware...\n");
430 uint32_t time = 0, time_steps = 0;
431 while (time < BOOT_TIMEOUT) {
432 /* read all the pending messages */
433 xeon_phi_serial_handle_recv();
435 if (xeon_phi_boot_download_status_rdf(&boot_registers)) {
436 XBOOT_DEBUG("Firmware signaled with ready bit. \n");
440 debug_printf("Xeon Phi Booting: Waiting for ready signal %u\n",
446 if (!xeon_phi_boot_download_status_rdf(&boot_registers)) {
447 USER_PANIC("Firmware not responding with ready bit");
448 // TODO return error code
451 // we don't need the aperture mapped anymore so unmap it
452 err = xeon_phi_unmap_aperture(phi);
453 if (err_is_fail(err)) {
454 USER_PANIC_ERR(err, "Failed to map aperture range");