Xeon Phi Host Driver
authorReto Achermann <acreto@student.ethz.ch>
Sat, 26 Apr 2014 20:11:24 +0000 (22:11 +0200)
committerStefan Kaestle <stefan.kaestle@inf.ethz.ch>
Wed, 20 Aug 2014 13:59:05 +0000 (15:59 +0200)
- Initialization of the PCI device
- Interrupt enable/disable
- SMPT management
- Reset sequence
- Diverse additions to the mackerel device specification

19 files changed:
devices/Hakefile
devices/xeon_phi/xeon_phi_boot.dev
devices/xeon_phi/xeon_phi_irq.dev
devices/xeon_phi/xeon_phi_smpt.dev
usr/drivers/xeon_phi/Hakefile
usr/drivers/xeon_phi/boot.c
usr/drivers/xeon_phi/dma.c [new file with mode: 0644]
usr/drivers/xeon_phi/dma.h [new file with mode: 0644]
usr/drivers/xeon_phi/interrupts.c [new file with mode: 0644]
usr/drivers/xeon_phi/interrupts.h [new file with mode: 0644]
usr/drivers/xeon_phi/main.c
usr/drivers/xeon_phi/serial.c
usr/drivers/xeon_phi/service.c [new file with mode: 0644]
usr/drivers/xeon_phi/sleep.c [new file with mode: 0644]
usr/drivers/xeon_phi/sleep.h [new file with mode: 0644]
usr/drivers/xeon_phi/smpt.c [new file with mode: 0644]
usr/drivers/xeon_phi/smpt.h [new file with mode: 0644]
usr/drivers/xeon_phi/xeon_phi.c [new file with mode: 0644]
usr/drivers/xeon_phi/xeon_phi.h

index f5e3846..8e4de3b 100644 (file)
@@ -91,7 +91,9 @@
            "virtio/virtio_mmio",
            "xeon_phi/xeon_phi_serial",
            "xeon_phi/xeon_phi_boot", 
-           "xeon_phi/xeon_phi_apic"
+           "xeon_phi/xeon_phi_apic",
+           "xeon_phi/xeon_phi_smpt",
+           "xeon_phi/xeon_phi_irq"
          ], arch <- allArchitectures
 ] ++
 
index 9ac1139..2008dd6 100644 (file)
@@ -12,7 +12,7 @@
  * description: register definitions for the Xeon Phi Boot Time Registers
  */
 
-device xeon_phi_boot lsbfirst ( addr base ) "Intel Xeon Phi Boot Registers" {
+device xeon_phi_boot lsbfirst ( addr sbox_base, addr dbox_base ) "Intel Xeon Phi Boot Registers" {
 
     /*
      * Protection Level: Ring 0
@@ -29,7 +29,7 @@ device xeon_phi_boot lsbfirst ( addr base ) "Intel Xeon Phi Boot Registers" {
     /*
      * alias for scratch register 2
      */
-    register download rw addr(base, 0xAB28) {
+    register download rw addr(sbox_base, 0xAB28) {
         status       1 "Download status bit";
         apicid       9 "APIC ID to send the boot interrupt";
         _            2 "unused";
@@ -39,14 +39,14 @@ device xeon_phi_boot lsbfirst ( addr base ) "Intel Xeon Phi Boot Registers" {
     /*
      * alias for scratch register 3
      */
-    register res_size rw addr(base, 0xAB2C) {
+    register res_size rw addr(sbox_base, 0xAB2C) {
         os 32 "Reserved size";
     };
     
     /*
      * alias for scratch register 5
      */
-    register os_size rw addr(base, 0xAB34) {
+    register os_size rw addr(sbox_base, 0xAB34) {
         size 32 "Size of the loaded image";
     };
     
@@ -57,12 +57,107 @@ device xeon_phi_boot lsbfirst ( addr base ) "Intel Xeon Phi Boot Registers" {
           mem_fourth = 0x3;
     };
     
-    register meminfo rw addr(base, 0xAB20) {
+    register meminfo rw addr(sbox_base, 0xAB20) {
           test_disable  1 "Memtest disable";
           usage         2 "Memory usage";
           size_kb      29 "Memory size in KB";
     };
     
+    register reset rw addr(sbox_base, 0x4010) {
+       reset  1 "Perform device rest";
+       _     31; 
+    };
+    /*
+    constants postcodes "" {
+       postcode_invalid = 0x0 "Invalid Postcode";
+               postcode_lidt = 0x01 "LIDT";
+               postcode_sboxinit = 0x02 "SBOX initialization";
+               postcode_gddrtop = 0x03 "Set GDDR Top";
+               postcode_memtest = 0x04 "Begin memory test";
+               postcode_e820 = 0x05 "Program E820 table";
+               postcode_dbox = 0x06 "Initialize DBOX";
+               postcode_cache = 0x09 "Enable Cache";    
+               postcode_initap = 0x0b "Pass initialization params to APs";
+               postcode_code = 0x0c "Cache C code";
+               postcode_mp = 0x0d "Program MP table";
+               postcode_apwkup = 0x0f "Wake up APs";
+               postcode_apboot = 0x10 "Wait for APs to boot";
+               postcode_sig = 0x11 "Signal host to download OS";
+               postcode_ready = 0x12 "Wait for download READY";
+               postcode_boot = 0x13 "Signal to boot received";
+               postcode_pinfo = 0x15 "Report platform information";
+               postcode_ptable = 0x17 "Page table setup";
+               postcode_memtrain = 0x30 ""Begin memory training;
+               postcode_gddrtrain = 0x31 ""GDDR Training to query memory modules;
+               postcode_findgddrtrain = 0x32 "Find GDDR training parameters in flash";
+               postcode_mmiotrain = 0x33 "MMIO training";
+               postcode_rcomptrain = 0x34 "RCOMP training";
+               postcode_dcctrain = 0x35 "DCC disable training";
+               postcode_hcktrain = 0x36 "HCK training";
+               postcode_ucodetrain = 0x37 "UCode Training";
+               postcode_vendortrain = 0x38 "Vendor specific training";
+               postcode_addrtrain = 0x39 "GDDR address training";
+               postcode_gddrident = 0x3A "GDDR memory module identification";
+               postcode_wcktrain = 0x3B "GDDR WCK training";
+               postcode_cdrdtrain = 0x3C "GDDR read training with CDR enabled";
+               postcode_cdretrain = 0x3D "GDDR Read Training with CDR disabled";
+               postcode_wrtrain = 0x3E "GDDR Write Training";
+               postcode_fintrain = 0x3F "Finalize GDDR Training";
+               postcode_osauth = 0x40 "Begin Coprocessor OS authentification";
+               postcode_loading0 = 0x50 "Coprocessor OS Loading 0";
+               postcode_loading1 = 0x51 "Coprocessor OS Loading 1";
+               postcode_loading2 = 0x52 "Coprocessor OS Loading 2";
+               postcode_loading3 = 0x53 "Coprocessor OS Loading 3";
+               postcode_loading4 = 0x54 "Coprocessor OS Loading 4";
+               postcode_loading5 = 0x55 "Coprocessor OS Loading 5";
+               postcode_loading6 = 0x56 "Coprocessor OS Loading 6";
+               postcode_loading7 = 0x57 "Coprocessor OS Loading 7";
+               postcode_loading8 = 0x58 "Coprocessor OS Loading 8";
+               postcode_loading9 = 0x59 "Coprocessor OS Loading 9";
+               postcode_loadingb = 0x5A "Coprocessor OS Loading A";
+               postcode_loadinga = 0x5B "Coprocessor OS Loading B";
+               postcode_loadingc = 0x5C "Coprocessor OS Loading C";
+               postcode_loadingd = 0x5D "Coprocessor OS Loading D";
+               postcode_loadinge = 0x5E "Coprocessor OS Loading E";
+               postcode_loadingf = 0x5F "Coprocessor OS Loading F";
+               postcode_gp = 0x6P "Int 13 - General Protection Fault";
+               postcode_invaltss = 0x75 "Int 10 - Invalid TSS";
+               postcode_fpu = 0x87 "Int 16 - x87 FPU Error";
+               postcode_algin = 0xac "";
+               postcode_bp = 0x "";
+               postcode_bound = 0x "";
+               postcode_mc = 0x "";
+               postcode_seg = 0x "";
+               postcode_dbg = 0x "";
+               postcode_div = 0x "";
+               postcode_df = 0x "";
+               postcode_memf = 0x "";
+               postcode_ = 0x "";
+               postcode_ = 0x "";
+               postcode_ = 0x "";
+               postcode_ = 0x "";
+               
+               postcode_ = 0x "";
+
+
+
+
+
+
+
+
+       postcode_invalid         = 0; 
+               postcode_reset_failed_f2 = 12870;
+               postcode_reset_failed_f4 = 13382;
+       postcode_fatal               = 0xffffffff;
+    };
+    */
     
+    register postcode rw addr(dbox_base, 0x242c) {
+       code0  8;
+       code1  8;
+       _     16;
+    };
+
+};
 
-};
\ No newline at end of file
index 055f9a6..ff2e31f 100644 (file)
@@ -87,7 +87,7 @@ device xeon_phi_irq lsbfirst ( addr base ) "Intel Xeon Phi Interrupts" {
     
     register int_status rw addr(base, 0x9004) "System Interrupt Status Register" {
        reg 32 "registr3";
-    }
+    };
     
     /*
      * Note: This register is used for SW testing and HW debug only. The intent 
@@ -123,7 +123,15 @@ device xeon_phi_irq lsbfirst ( addr base ) "Intel Xeon Phi Interrupts" {
      * to defeature ordering checks to avoid the hang condition.
      */
     register int_enable rw addr(base, 0x900C) "System Interrupt Enable Register" {
+       dbr 4 "";
+       _   4;  
+       dma 8 "";
+       _  16;  
+    };
     
+    constants int_en "" {
+       dbr_enable_all = 0xF; 
+       dma_enable_all = 0xFF;
     };
     
     /* 
@@ -136,7 +144,10 @@ device xeon_phi_irq lsbfirst ( addr base ) "Intel Xeon Phi Interrupts" {
      * separate Interrupt sources.
      */
     register int_disable rw addr(base, 0x9010) "System Interrupt Disable" {
-    
+       dbr 4 "";
+       _   4;  
+       dma 8 "";
+       _  16;    
     };
     
     /*
@@ -185,7 +196,10 @@ device xeon_phi_irq lsbfirst ( addr base ) "Intel Xeon Phi Interrupts" {
      * otherwise the behavior is undefined.
      */
     regarray msi_vector rw addr(base, 0x9044) [16] "MSI(-X) Vector Assignment Register 0-15" {
-       reg 32 "ff";
+               dbr 4 "";
+       _   4;  
+       dma 8 "";
+       _  16;  
     };
     
     
index 5867e94..318fa67 100644 (file)
 device xeon_phi_smpt lsbfirst ( addr base ) "Intel Xeon Phi System Memory Page Tables" {
 
     constants snooping "System Snooping Control" {
-        enabled  = 0  "Enable Snooping";
-        disabled = 1  "Disabled Snooping";
+        snooping_enabled  = 0  "Enable Snooping";
+        snooping_disabled = 1  "Disabled Snooping";
     };
+    
+    constants system_page "System Page Constants" {
+       system_page_size  = 0x400000000 "Size of a System Memory Page (16GB)";
+       system_page_num   = 16                  "The number of system memory pages";
+       system_page_shift = 34                  "The amount of bits to shift"; 
+       system_page_mask  = 0x1F        "System memory page mask";
+    };
+    
 
     /*
      * Protection Level: Ring 0
@@ -39,7 +47,7 @@ device xeon_phi_smpt lsbfirst ( addr base ) "Intel Xeon Phi System Memory Page T
      */
     regarray entry rw addr(base, 0x3100) [32] "System Memory Page Table Entry" {
         snoop_disabled 1 type(snooping) "Disable snooping";
-        _              1 "Reserved"
+        _              1 "Reserved";
         host_address  30 "High 32bit of the host address";
     };
 
index ec6ec58..b55fe4b 100644 (file)
 --------------------------------------------------------------------------
 
 [ build application { target = "xeon_phi",
-                      cFiles = [ "main.c", 
+                      cFiles = [ "main.c",
+                                        "xeon_phi.c", 
                                  "boot.c", 
                                  "serial.c", 
-                                 "host_bootstrap.c" ],
-                      addLibraries = libDeps ["skb", "pci", "spawndomain", "elf" ],
+                                 "host_bootstrap.c", 
+                                 "dma.c", 
+                                 "interrupts.c", 
+                                 "service.c", 
+                                 "smpt.c", 
+                                 "sleep.c" ],
+                      addLibraries = libDeps ["skb", 
+                                                                 "pci", 
+                                                                 "spawndomain", 
+                                                                 "elf", 
+                                                                 "bench" ],
                       flounderExtraDefs = [ ("monitor_blocking",["rpcclient"]) ],
                       flounderDefs = ["monitor"],
                       architectures= ["x86_64"],
                       mackerelDevices = [ "xeon_phi/xeon_phi_apic", 
                                           "xeon_phi/xeon_phi_boot", 
-                                          "xeon_phi/xeon_phi_serial"]
+                                          "xeon_phi/xeon_phi_serial",
+                                          "xeon_phi/xeon_phi_smpt", 
+                                          "xeon_phi/xeon_phi_irq"]
                     }
 ]
 
index 4d018ab..5d4aed6 100644 (file)
@@ -86,11 +86,11 @@ static errval_t load_os(struct xeon_phi *phi,
 
     printf("Loading xloader onto card...\n");
     XBOOT_DEBUG("aper_base=0x%lx, offset = 0x%lx, size=0x%lx\n",
-                phi->aper_base,
+                phi->apt.vbase,
                 loadoffset,
                 imgsize);
 
-    memcpy((void *) (phi->aper_base + loadoffset), (void *) binary, imgsize);
+    memcpy((void *) (phi->apt.vbase + loadoffset), (void *) binary, imgsize);
 
     if (ret_cmdoffset) {
         *ret_cmdoffset = loadoffset + imgsize;
@@ -132,19 +132,19 @@ static errval_t load_multiboot_image(struct xeon_phi *phi,
 
     printf("Loading multiboot image onto card...\n");
     XBOOT_DEBUG("aper_base=0x%lx, offset = 0x%lx, size=0x%lx\n",
-                phi->aper_base,
+                phi->apt.vbase,
                 load_offset,
                 imgsize);
 
-    memcpy((void *) (phi->aper_base + load_offset), (void *) image, imgsize);
+    memcpy((void *) (phi->apt.vbase + load_offset), (void *) image, imgsize);
 
     /*
      * we are using the Linux style way in booting. The following will update
      * the corresponding fields in struct boot_param of the header.
      */
-    uint32_t *ramfs_addr_ptr = (uint32_t *) (phi->aper_base + os_offset + 0x218);
+    uint32_t *ramfs_addr_ptr = (uint32_t *) (phi->apt.vbase + os_offset + 0x218);
     *ramfs_addr_ptr = load_offset;
-    ramfs_addr_ptr = (uint32_t *) (phi->aper_base + os_offset + 0x21c);
+    ramfs_addr_ptr = (uint32_t *) (phi->apt.vbase + os_offset + 0x21c);
     *ramfs_addr_ptr = imgsize;
 
     return SYS_ERR_OK;
@@ -162,7 +162,7 @@ static errval_t load_cmdline(struct xeon_phi *phi,
                              uint32_t *ret_size)
 {
     uint32_t cmdlen = 0;
-    void *buf = (void *) (phi->aper_base + load_offset);
+    void *buf = (void *) (phi->apt.vbase + load_offset);
 
     if (phi->cmdline) {
         cmdlen += sprintf(buf, "%s foobar=%i", phi->cmdline, 123);
@@ -256,8 +256,10 @@ errval_t xeon_phi_boot(struct xeon_phi *phi,
         return SYS_ERR_ILLEGAL_INVOCATION;
     }
 
-    xeon_phi_boot_initialize(&boot_registers, XEON_PHI_MMIO_TO_SBOX(phi->mmio_base));
-    xeon_phi_apic_initialize(&apic_registers, XEON_PHI_MMIO_TO_SBOX(phi->mmio_base));
+    xeon_phi_boot_initialize(&boot_registers,
+                             XEON_PHI_MMIO_TO_SBOX(phi),
+                             XEON_PHI_MMIO_TO_DBOX(phi));
+    xeon_phi_apic_initialize(&apic_registers, XEON_PHI_MMIO_TO_SBOX(phi));
 
     // load the coprocessor OS
     err = load_os(phi, xloader_img, &osimg_size, &offset);
@@ -288,13 +290,3 @@ errval_t xeon_phi_boot(struct xeon_phi *phi,
 
     return SYS_ERR_OK;
 }
-
-/**
- * \brief performs a soft reset of the card
- *
- * \param phi   pointer to the card information
- */
-errval_t xeon_phi_reset(struct xeon_phi *phi)
-{
-    return SYS_ERR_OK;
-}
diff --git a/usr/drivers/xeon_phi/dma.c b/usr/drivers/xeon_phi/dma.c
new file mode 100644 (file)
index 0000000..eec6807
--- /dev/null
@@ -0,0 +1,20 @@
+/**
+ * \file
+ * \brief Driver for booting the Xeon Phi Coprocessor card on a Barrelfish Host
+ */
+
+/*
+ * Copyright (c) 2014 ETH Zurich.
+ * All rights reserved.
+ *
+ * This file is distributed under the terms in the attached LICENSE file.
+ * If you do not find this file, copies can be found by writing to:
+ * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <barrelfish/barrelfish.h>
+
+#include "xeon_phi.h"
+
diff --git a/usr/drivers/xeon_phi/dma.h b/usr/drivers/xeon_phi/dma.h
new file mode 100644 (file)
index 0000000..4461815
--- /dev/null
@@ -0,0 +1,37 @@
+/**
+ * \file
+ * \brief Driver for booting the Xeon Phi Coprocessor card on a Barrelfish Host
+ */
+
+/*
+ * Copyright (c) 2014 ETH Zurich.
+ * All rights reserved.
+ *
+ * This file is distributed under the terms in the attached LICENSE file.
+ * If you do not find this file, copies can be found by writing to:
+ * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group.
+ */
+#ifndef XEON_PHI_SMPT_H
+#define XEON_PHI_SMPT_H
+
+#include <dev/xeon_phi/xeon_phi_smpt_dev.h>
+
+
+struct dma_info {
+
+};
+
+struct dma_channel {
+
+
+};
+
+errval_t dma_request_alloc(void);
+
+errval_t dma_request_exec(void);
+
+errval_t dma_request_free(void)
+
+errval_t dma_init(struct xeon_phi *phi);
+
+#endif /* XEON_PHI_SMPT_H */
diff --git a/usr/drivers/xeon_phi/interrupts.c b/usr/drivers/xeon_phi/interrupts.c
new file mode 100644 (file)
index 0000000..b37b241
--- /dev/null
@@ -0,0 +1,170 @@
+/**
+ * \file
+ * \brief Driver for booting the Xeon Phi Coprocessor card on a Barrelfish Host
+ */
+
+/*
+ * Copyright (c) 2014 ETH Zurich.
+ * All rights reserved.
+ *
+ * This file is distributed under the terms in the attached LICENSE file.
+ * If you do not find this file, copies can be found by writing to:
+ * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <barrelfish/barrelfish.h>
+#include <pci/pci.h>
+
+#include "xeon_phi.h"
+#include "interrupts.h"
+
+xeon_phi_irq_t irq_registers;
+
+/**
+ * \brief Initializes the MSI-X Interrupt allocator
+ *
+ * \param alloc pointer to the allocator state struct
+ * \param n     number of vectors to track
+ *
+ * \return true on SUCCESS, false on FAILURE
+ */
+bool msix_allocator_init(struct msix_allocator *alloc, size_t n)
+{
+    alloc->count = n;
+    alloc->state = calloc((n + MSIX_ALLOCATOR_BITS -1 ) / MSIX_ALLOCATOR_BITS,
+                          MSIX_ALLOCATOR_BITS / 8);
+    return (alloc->state != NULL);
+}
+
+
+/**
+ * \brief Destroys a previously allocated MSI-X Interrupt allocator
+ *
+ * \param alloc the allocator to destroy
+ */
+void msix_allocator_destory(struct msix_allocator *alloc)
+{
+    if (alloc->state) {
+        free(alloc->state);
+    }
+    alloc->state = NULL;
+    alloc->count = 0;
+}
+
+/**
+ * \brief Allocates a new MSI-X vector ID
+ *
+ * \param alloc  pointer to the allocator state struct
+ * \param ret_id returned MSI-X vector ID
+ *
+ * \return true on SUCCESS, false on FAILURE (no more vectors left)
+ */
+bool msix_allocator_new(struct msix_allocator *alloc, size_t *ret_id)
+{
+    size_t chunks = (alloc->count + MSIX_ALLOCATOR_BITS - 1) / MSIX_ALLOCATOR_BITS;
+
+    for (uint32_t i = 0; i < chunks; i++) {
+        if (alloc->state[i] != 0xFF) {
+
+            uint32_t idx = 0;
+            for (uint32_t j = i * MSIX_ALLOCATOR_BITS;  j < (i+1) * MSIX_ALLOCATOR_BITS; ++j) {
+                if (!(j < alloc->count)) {
+                    return false;
+                }
+
+                if (!(alloc->state[i] & (0x1 << idx))) {
+                    alloc->state[i] |= (0x1 << idx);
+                    *ret_id = j;
+                    return true;
+                }
+                idx++;
+            }
+        }
+    }
+    return false;
+}
+
+
+/**
+ * \brief Frees a previously allocated id
+ *
+ * \param alloc pointer to the allocator state struct
+ * \param n     number of vectors to track
+ *
+ * \return true on if it was previously allocated, false on if not
+ */
+bool msix_allocator_free(struct msix_allocator *alloc, size_t id)
+{
+    bool result;
+
+    if (id >= alloc->count) {
+        return false;
+    }
+
+    size_t chunk = (id  / MSIX_ALLOCATOR_BITS);
+    size_t shift = id - (chunk * MSIX_ALLOCATOR_BITS);
+    result = alloc->state[chunk] & (0x1 << shift);
+    alloc->state[chunk] &= ~(0x1 << shift);
+
+    return result;
+
+}
+
+/**
+ * \brief enables the MSI-X Interrupts
+ *
+ * \param phi   the xeon phi device data structure
+ *
+ * \return SYS_ERR_OK on success
+ */
+errval_t msix_enable(struct xeon_phi *phi)
+{
+    errval_t err;
+
+    if (!phi->irq->msix_enabled) {
+        return SYS_ERR_OK;
+    }
+    err = pci_msix_enable(&phi->irq->msix_count);
+    if (err_is_fail(err)) {
+        return err;
+    }
+
+    if (!msix_allocator_init(&phi->irq->msix_alloc, phi->irq->msix_count)) {
+        return LIB_ERR_MALLOC_FAIL;
+    }
+
+    // Only support single MSI interrupt for now
+    xeon_phi_irq_msi_vector_t val = xeon_phi_irq_msi_vector_default;
+    val = xeon_phi_irq_msi_vector_dbr_insert(val, xeon_phi_irq_dbr_enable_all);
+    val = xeon_phi_irq_msi_vector_dma_insert(val, xeon_phi_irq_dma_enable_all);
+    xeon_phi_irq_msi_vector_wr(&phi->irq->irq_registers, 0, val);
+
+    phi->irq->msix_enabled = 1;
+
+    return SYS_ERR_OK;
+}
+
+
+
+
+
+
+errval_t interrupts_init(struct xeon_phi *phi)
+{
+    phi->irq = calloc(1, sizeof(struct irq_info));
+    if (phi->irq == NULL) {
+        return LIB_ERR_MALLOC_FAIL;
+    }
+
+    xeon_phi_irq_initialize(&phi->irq->irq_registers, XEON_PHI_MMIO_TO_SBOX(phi));
+
+    return SYS_ERR_OK;
+}
+
+void interrupt_handler(void* arg)
+{
+
+}
+
diff --git a/usr/drivers/xeon_phi/interrupts.h b/usr/drivers/xeon_phi/interrupts.h
new file mode 100644 (file)
index 0000000..b14137b
--- /dev/null
@@ -0,0 +1,136 @@
+/*
+ * Copyright (c) 2014 ETH Zurich.
+ * All rights reserved.
+ *
+ * This file is distributed under the terms in the attached LICENSE file.
+ * If you do not find this file, copies can be found by writing to:
+ * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group.
+ */
+
+#ifndef XEON_PHI_INTERRUPTS_H_
+#define XEON_PHI_INTERRUPTS_H_
+
+#include <dev/xeon_phi/xeon_phi_irq_dev.h>
+
+// forward declaration
+struct xeon_phi;
+
+#define PASTER(x,y) uint ## x ## ## y
+#define EVALUATOR(x)  PASTER(x,_t)
+
+#define MSIX_ALLOCATOR_BITS 8
+#define MSIX_ALLOCATOR_TYPE EVALUATOR(MSIX_ALLOCATOR_BITS)
+
+#define XEON_PHI_NUM_MSIX_ENTRIES 1
+
+/**
+ *
+ */
+struct msix_allocator
+{
+    size_t   count;
+    MSIX_ALLOCATOR_TYPE *state;
+};
+
+struct irq_info
+{
+    uint8_t                 irq_enabled;
+    uint8_t                 msix_enabled;
+    uint16_t                msix_count;
+    xeon_phi_irq_t          irq_registers;
+    struct msix_allocator   msix_alloc;
+};
+
+
+/**
+ * \brief Initializes the MSI-X Interrupt allocator
+ *
+ * \param alloc pointer to the allocator state struct
+ * \param n     number of vectors to track
+ *
+ * \return true on SUCCESS, false on FAILURE
+ */
+bool msix_allocator_init(struct msix_allocator *alloc, size_t n);
+
+/**
+ * \brief Destroys a previously allocated MSI-X Interrupt allocator
+ *
+ * \param alloc the allocator to destroy
+ *
+ * \return true on SUCCESS, false on FAILURE
+ */
+void msix_allocator_destory(struct msix_allocator *alloc);
+
+/**
+ * \brief Allocates a new MSI-X vector ID
+ *
+ * \param alloc  pointer to the allocator state struct
+ * \param ret_id returned MSI-X vector ID
+ *
+ * \return true on SUCCESS, false on FAILURE (no more vectors left)
+ */
+bool msix_allocator_new(struct msix_allocator *alloc, size_t *ret_id);
+
+/**
+ * \brief Frees a previously allocated id
+ *
+ * \param alloc pointer to the allocator state struct
+ * \param n     number of vectors to track
+ *
+ * \return true on if it was previously allocated, false on if not
+ */
+bool msix_allocator_free(struct msix_allocator *alloc, size_t id);
+
+
+/**
+ * \brief Enables the MSI-X interrupts
+ *
+ * \param phi   the xeon phi structure
+ *
+ * \return SYS_ERR_OK on success
+ */
+errval_t msix_enable(struct xeon_phi *phi);
+
+/**
+ * \brief Initializes the interrupts data structures
+ *
+ * \param phi   the xeon phi structure
+ *
+ * \return SYS_ERR_OK on success
+ */
+errval_t interrupts_init(struct xeon_phi *phi);
+
+
+/**
+ * \brief Enables the Xeon Phi Interrupts
+ *
+ * \param phi   the xeon phi structure
+ */
+static inline void interrupts_enable(struct xeon_phi *phi)
+{
+    xeon_phi_irq_int_enable_t val = xeon_phi_irq_int_enable_default;
+    val = xeon_phi_irq_int_enable_dbr_insert(val, xeon_phi_irq_dbr_enable_all);
+    val = xeon_phi_irq_int_enable_dma_insert(val, xeon_phi_irq_dma_enable_all);
+    xeon_phi_irq_int_enable_wr(&phi->irq->irq_registers, val);
+
+    phi->irq->irq_enabled = 1;
+}
+
+/**
+ * \brief Disables the Xeon Phi Interrupts
+ *
+ * \param phi   the xeon phi structure
+ */
+static inline void interrupts_disable(struct xeon_phi *phi)
+{
+    xeon_phi_irq_int_enable_t val = xeon_phi_irq_int_enable_rd(&phi->irq->irq_registers);
+    xeon_phi_irq_int_disable_wr(&phi->irq->irq_registers, val);
+
+    phi->irq->irq_enabled = 0;
+}
+/**
+ *
+ */
+void interrupt_handler(void* arg);
+
+#endif /* XEON_PHI_INTERRUPTS_H_ */
index fa20eef..f23c2e3 100644 (file)
@@ -17,6 +17,8 @@
 #include <barrelfish/barrelfish.h>
 
 #include "xeon_phi.h"
+#include "smpt.h"
+
 
 volatile uint32_t bootstrap_done = 0;
 
@@ -25,21 +27,33 @@ struct xeon_phi xphi;
 int main(int argc,
          char *argv[])
 {
+    errval_t err;
     debug_printf("Xeon Phi host module started.\n");
 
-    host_bootstrap();
+    err = host_bootstrap();
+    if (err_is_fail(err)) {
+        USER_PANIC_ERR(err, "could not do the host bootstrap\n");
+    }
 
     while (bootstrap_done == 0) {
         messages_wait_and_handle_next();
     }
 
-    debug_printf("Host bootstrap done\n");
+    xphi.state = XEON_PHI_STATE_NULL;
+
+    err = xeon_phi_init(&xphi);
+    if (err_is_fail(err)) {
+        USER_PANIC_ERR(err, "could not do the card initialization\n");
+    }
 
-    xeon_phi_boot(&xphi,
+    err = xeon_phi_boot(&xphi,
                   XEON_PHI_BOOTLOADER,
                   XEON_PHI_MULTIBOOT);
+    if (err_is_fail(err)) {
+            DEBUG_ERR(err, "could not boot the card\n");
+    }
 
-
+    debug_printf("Xeon Phi host module terminated.\n");
 
     return 0;
 }
index f63ac54..259d53b 100644 (file)
@@ -151,7 +151,7 @@ errval_t serial_start_recv_thread(struct xeon_phi *phi)
 
     xarg->xid = phi->id;
 
-    xeon_phi_serial_initialize(&xarg->base, XEON_PHI_MMIO_TO_SBOX(phi->mmio_base));
+    xeon_phi_serial_initialize(&xarg->base, XEON_PHI_MMIO_TO_SBOX(phi));
 
     /*
      * xxx: maybe store the thread pointer
diff --git a/usr/drivers/xeon_phi/service.c b/usr/drivers/xeon_phi/service.c
new file mode 100644 (file)
index 0000000..eec6807
--- /dev/null
@@ -0,0 +1,20 @@
+/**
+ * \file
+ * \brief Driver for booting the Xeon Phi Coprocessor card on a Barrelfish Host
+ */
+
+/*
+ * Copyright (c) 2014 ETH Zurich.
+ * All rights reserved.
+ *
+ * This file is distributed under the terms in the attached LICENSE file.
+ * If you do not find this file, copies can be found by writing to:
+ * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <barrelfish/barrelfish.h>
+
+#include "xeon_phi.h"
+
diff --git a/usr/drivers/xeon_phi/sleep.c b/usr/drivers/xeon_phi/sleep.c
new file mode 100644 (file)
index 0000000..3037bca
--- /dev/null
@@ -0,0 +1,70 @@
+/** \file
+ *  \brief Simple sleep call
+ */
+
+/*
+ * Copyright (c) 2010-2011, ETH Zurich.
+ * All rights reserved.
+ *
+ * This file is distributed under the terms in the attached LICENSE file.
+ * If you do not find this file, copies can be found by writing to:
+ * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
+ */
+
+#include <barrelfish/barrelfish.h>
+#include <barrelfish/sys_debug.h>
+
+#include <bench/bench.h>
+
+#include "sleep.h"
+
+#if defined(__x86_64__)
+static uint64_t tscperms;
+#endif
+
+static bool initialised = false;
+
+void sleep_init(void)
+{
+    if (!initialised) {
+        bench_init();
+#if defined(__x86_64__)
+        errval_t err = sys_debug_get_tsc_per_ms(&tscperms);
+        assert(err_is_ok(err));
+#endif
+        initialised = true;
+    }
+}
+
+
+void cycle_sleep(uint64_t cycles)
+{
+    if (!initialised) {
+        sleep_init();
+    }
+    uint64_t start = bench_tsc();
+    uint64_t stop = bench_tsc();
+    while ((stop - start) < cycles) {
+        //        sys_yield(CPTR_NULL);
+        thread_yield_dispatcher(NULL_CAP);
+        stop = bench_tsc();
+    }
+}
+
+// FIXME: this should be determined at runtime from the kernel...
+// #define TSC_PER_MS 2513385
+
+void milli_sleep(uint64_t ms)
+{
+    if (!initialised) {
+        sleep_init();
+    }
+
+#if defined(__x86_64__)
+    uint64_t cycles = ms * tscperms;
+    cycle_sleep(cycles);
+#else
+    USER_PANIC("milli_sleep NYI for non-x86_64");
+#endif
+}
+
diff --git a/usr/drivers/xeon_phi/sleep.h b/usr/drivers/xeon_phi/sleep.h
new file mode 100644 (file)
index 0000000..8534115
--- /dev/null
@@ -0,0 +1,21 @@
+/** \file
+ *  \brief Simple sleep call
+ */
+
+/*
+ * Copyright (c) 2010-2011, ETH Zurich.
+ * All rights reserved.
+ *
+ * This file is distributed under the terms in the attached LICENSE file.
+ * If you do not find this file, copies can be found by writing to:
+ * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
+ */
+
+#ifndef __SLEEP_H__
+#define __SLEEP_H_
+
+void sleep_init(void);
+void cycle_sleep(uint64_t cycles);
+void milli_sleep(uint64_t ms);
+
+#endif
diff --git a/usr/drivers/xeon_phi/smpt.c b/usr/drivers/xeon_phi/smpt.c
new file mode 100644 (file)
index 0000000..afe52c4
--- /dev/null
@@ -0,0 +1,104 @@
+/**
+ * \file
+ * \brief Module to set the System Memory Page Tables of the Xeon Phi
+ */
+
+/*
+ * Copyright (c) 2014 ETH Zurich.
+ * All rights reserved.
+ *
+ * This file is distributed under the terms in the attached LICENSE file.
+ * If you do not find this file, copies can be found by writing to:
+ * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <barrelfish/barrelfish.h>
+
+#include <dev/xeon_phi/xeon_phi_smpt_dev.h>
+
+#include "xeon_phi.h"
+#include "smpt.h"
+
+/*
+ * XXX: Setting a system memory pagetable entry basically enables the coprocessor
+ *      to access host memory directly. This is an open issue and should be
+ *      restricted by capabilities somehow.
+ *
+ *      Anyhow, since the access to this memory range is controlled as normal
+ *      memory, we can trust the kernels on the card not to introduce potential
+ *      vulnerabilities...
+ *
+ *      Note: the system memory pagetables are rather coarse grained (16GB)
+ */
+
+/**
+ * \brief Sets an entry in the system memory pagetable to a give address
+ *
+ * \param phi      reference to the card structure
+ * \param slot     pagetable entry to set
+ * \param address  host address to set
+ * \param snooping enable snooping = 1, disable snooping = 0
+ *
+ * \return SYS_ERR_OK on success
+ */
+void smpt_set_address(struct xeon_phi *phi,
+                      uint8_t slot,
+                      lpaddr_t address,
+                      uint8_t snooping)
+{
+    uint32_t e_address = (uint32_t)(address >> xeon_phi_smpt_system_page_shift);
+    xeon_phi_smpt_entry_t e = xeon_phi_smpt_entry_default;
+    e=xeon_phi_smpt_entry_host_address_insert(e, e_address);
+    if (snooping) {
+        snooping = xeon_phi_smpt_snooping_enabled;
+    } else {
+        snooping = xeon_phi_smpt_snooping_disabled;
+    }
+    e=xeon_phi_smpt_entry_snoop_disabled_insert(e, snooping);
+
+    smpt_set_entry(phi, slot, e);
+}
+
+/**
+ * \brief Resets the system memory page tables
+ */
+void smpt_reset(struct xeon_phi *phi)
+{
+    for (uint32_t i = 0; i < xeon_phi_smpt_system_page_num; ++i) {
+        smpt_clear_entry(phi, i);
+    }
+}
+
+/**
+ * \brief initializes the system memory page tables with a
+ *        1:1 mapping
+ *
+ * \return SYS_ERR_OK on success
+ */
+errval_t smpt_init(struct xeon_phi *phi)
+{
+    if (phi->smpt->smpt_enabled) {
+        debug_printf("WARNING: SMPT already setup");
+        return SYS_ERR_OK;
+    }
+    phi->smpt = malloc(sizeof(struct smpt_info));
+    if (!phi->smpt) {
+        return LIB_ERR_MALLOC_FAIL;
+    }
+
+    xeon_phi_smpt_initialize(&phi->smpt->smpt_register,
+                             XEON_PHI_MMIO_TO_SBOX(phi));
+
+    lpaddr_t host_address = 0;
+
+    for (uint32_t i = 0; i < xeon_phi_smpt_system_page_num; ++i) {
+        smpt_set_address(phi, i, host_address, 1);
+        host_address += xeon_phi_smpt_system_page_size;
+    }
+
+    phi->smpt->smpt_enabled = 1;
+
+    return SYS_ERR_OK;
+}
diff --git a/usr/drivers/xeon_phi/smpt.h b/usr/drivers/xeon_phi/smpt.h
new file mode 100644 (file)
index 0000000..9c2f1e3
--- /dev/null
@@ -0,0 +1,87 @@
+/**
+ * \file
+ * \brief Driver for booting the Xeon Phi Coprocessor card on a Barrelfish Host
+ */
+
+/*
+ * Copyright (c) 2014 ETH Zurich.
+ * All rights reserved.
+ *
+ * This file is distributed under the terms in the attached LICENSE file.
+ * If you do not find this file, copies can be found by writing to:
+ * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group.
+ */
+#ifndef XEON_PHI_SMPT_H
+#define XEON_PHI_SMPT_H
+
+#include <dev/xeon_phi/xeon_phi_smpt_dev.h>
+
+
+struct smpt_info {
+    xeon_phi_smpt_t         smpt_register;
+    xeon_phi_smpt_entry_t   entries[xeon_phi_smpt_system_page_num];
+    uint8_t                 smpt_enabled;
+};
+
+/**
+ * \brief Sets an entry in the system memory pagetable to a give address
+ *
+ * \param phi      reference to the card structure
+ * \param slot     pagetable entry to set
+ * \param address  host address to set
+ * \param snooping enable snooping = 1, disable snooping = 0
+ *
+ * \return SYS_ERR_OK on success
+ */
+void smpt_set_address(struct xeon_phi *phi,
+                      uint8_t slot,
+                      lpaddr_t address,
+                      uint8_t snooping);
+
+/**
+ * \brief Sets an entry in the system memory pagetable
+ *
+ * \param phi   reference to the card structure
+ * \param entry the entry to set
+ * \param e     the information about the new entry to set
+ */
+static inline void smpt_set_entry(struct xeon_phi *phi,
+                                  uint8_t slot,
+                                  xeon_phi_smpt_entry_t e)
+{
+    assert(slot < xeon_phi_smpt_system_page_num);
+
+    xeon_phi_smpt_entry_wr(&phi->smpt->smpt_register, slot, e);
+    phi->smpt->entries[slot] = e;
+}
+
+/**
+ * \brief clears an entry in the system memory page table
+ *
+ * \param phi   reference to the card
+ * \param entry entry number to clear
+ *
+ * \return SYS_ERR_OK on success
+ */
+static inline void smpt_clear_entry(struct xeon_phi *phi, uint8_t slot)
+{
+    assert(slot < xeon_phi_smpt_system_page_num);
+
+    xeon_phi_smpt_entry_wr(&phi->smpt->smpt_register, slot, 0x0);
+    phi->smpt->entries[slot] = 0x0;
+}
+
+/**
+ * \brief Resets the system memory page tables
+ */
+void smpt_reset(struct xeon_phi *phi);
+
+/**
+ * \brief initializes the system memory page tables with a
+ *        1:1 mapping
+ *
+ * \return SYS_ERR_OK on success
+ */
+errval_t smpt_init(struct xeon_phi *phi);
+
+#endif /* XEON_PHI_SMPT_H */
diff --git a/usr/drivers/xeon_phi/xeon_phi.c b/usr/drivers/xeon_phi/xeon_phi.c
new file mode 100644 (file)
index 0000000..f64e1e4
--- /dev/null
@@ -0,0 +1,275 @@
+/**
+ * \file
+ * \brief Card Configuration
+ */
+
+/*
+ * Copyright (c) 2014 ETH Zurich.
+ * All rights reserved.
+ *
+ * This file is distributed under the terms in the attached LICENSE file.
+ * If you do not find this file, copies can be found by writing to:
+ * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <barrelfish/barrelfish.h>
+#include <pci/pci.h>
+
+#include <dev/xeon_phi/xeon_phi_boot_dev.h>
+
+#include "xeon_phi.h"
+#include "interrupts.h"
+#include "sleep.h"
+
+static uint32_t initialized = 0;
+
+#define XEON_PHI_RESET_TIME 300
+
+
+static struct xeon_phi *card;
+
+static uint32_t pci_bus = PCI_DONT_CARE;
+static uint32_t pci_device = PCI_DONT_CARE;
+static uint32_t pci_function = PCI_DONT_CARE;
+
+/// PCI Vendor ID of Intel
+#define PCI_VENDOR_ID_INTEL     0x8086
+
+#define PCI_SUBSYSTEM_DEVICE    0x2500
+
+#define PCI_SUBSYSTEM_VENDOR    0x8086
+/*
+ * These are the possible device IDs of the Xeon PHi according to the
+ * Intel MPSS implementation.
+ *
+ * Querying the host with lspci -nn | grep "8086:225" gives the actual
+ * device ID of the built in cards to be the last one.
+ *
+ * 07:00.0 Co-processor [0b40]: Intel Corporation Device [8086:225e] (rev 11)
+ * 82:00.0 Co-processor [0b40]: Intel Corporation Device [8086:225e] (rev 11)
+ */
+#define PCI_DEVICE_KNC_2250 0x2250
+#define PCI_DEVICE_KNC_2251 0x2251
+#define PCI_DEVICE_KNC_2252 0x2252
+#define PCI_DEVICE_KNC_2253 0x2253
+#define PCI_DEVICE_KNC_2254 0x2254
+#define PCI_DEVICE_KNC_2255 0x2255
+#define PCI_DEVICE_KNC_2256 0x2256
+#define PCI_DEVICE_KNC_2257 0x2257
+#define PCI_DEVICE_KNC_2258 0x2258
+#define PCI_DEVICE_KNC_2259 0x2259
+#define PCI_DEVICE_KNC_225a 0x225a
+#define PCI_DEVICE_KNC_225b 0x225b
+#define PCI_DEVICE_KNC_225c 0x225c
+#define PCI_DEVICE_KNC_225d 0x225d
+#define PCI_DEVICE_KNC_225e 0x225e
+
+
+
+#define XEON_PHI_APT_BAR 0
+#define XEON_PHI_MMIO_BAR 4
+
+
+static void device_init(struct xeon_phi *phi)
+{
+
+
+#if 0
+    scratch13 = SBOX_READ(mic_ctx->mmio.va, SBOX_SCRATCH13);
+        mic_ctx->bi_stepping = SCRATCH13_STEP_ID(scratch13);
+        mic_ctx->bi_substepping = SCRATCH13_SUB_STEP(scratch13);
+    #ifdef MIC_IS_EMULATION
+        mic_ctx->bi_platform = PLATFORM_EMULATOR;
+    #else
+        mic_ctx->bi_platform = SCRATCH13_PLATFORM_ID(scratch13);
+    #endif
+        mic_enable_msi_interrupts(mic_ctx);
+        mic_enable_interrupts(mic_ctx);
+
+        mic_reg_irqhandler(mic_ctx, 1, "MIC SHUTDOWN DoorBell 1",
+                    mic_shutdown_host_doorbell_intr_handler);
+
+
+#endif
+
+    initialized=true;
+}
+
+static void
+pci_init_card(struct device_mem* bar_info,
+              int bar_count)
+{
+    errval_t err;
+
+    if (initialized) {
+        debug_printf("WARNING> Device already initialized\n");
+        return;
+    }
+
+    // ok may be 5
+    if (bar_count != 5) {
+        USER_PANIC("There is something wrong. The Card should have 2 MBARs.");
+    }
+
+    err = map_device(&bar_info[XEON_PHI_APT_BAR]);
+    if (err_is_fail(err)) {
+        USER_PANIC_ERR(err, "Failed to map aperture range");
+    }
+
+    err = map_device(&bar_info[XEON_PHI_MMIO_BAR]);
+    if (err_is_fail(err)) {
+        USER_PANIC_ERR(err, "Failed to map MMIO range");
+    }
+
+    card->apt.vbase = (lvaddr_t)bar_info[XEON_PHI_APT_BAR].vaddr;
+    card->apt.length = bar_info[XEON_PHI_APT_BAR].bytes;
+    card->apt.pbase = (lpaddr_t)bar_info[XEON_PHI_APT_BAR].paddr;
+    card->mmio.vbase = (lvaddr_t)bar_info[XEON_PHI_MMIO_BAR].vaddr;
+    card->mmio.length = bar_info[XEON_PHI_MMIO_BAR].bytes;
+    card->mmio.pbase = (lpaddr_t)bar_info[XEON_PHI_MMIO_BAR].paddr;
+
+    card->state = XEON_PHI_STATE_PCI_OK;
+}
+
+static void pci_register(struct xeon_phi *phi)
+{
+    errval_t err;
+
+    err = pci_client_connect();
+    if (err_is_fail(err)) {
+        USER_PANIC_ERR(err, "Could not connect to PCI\n");
+    }
+
+    err = pci_register_driver_irq(pci_init_card, PCI_DONT_CARE,
+                                  PCI_DONT_CARE,
+                                  PCI_DONT_CARE,
+                                  PCI_VENDOR_ID_INTEL,
+                                  PCI_DEVICE_KNC_225e, pci_bus, pci_device,
+                                  pci_function, interrupt_handler, phi);
+
+    if (err_is_fail(err)) {
+        USER_PANIC_ERR(err, "Could not register the PCI device");
+    }
+}
+
+/**
+ * \brief initializes the coprocessor card
+ *
+ * \param phi pointer to the information structure
+ */
+errval_t xeon_phi_init(struct xeon_phi *phi)
+{
+    card = phi;
+
+    pci_register(phi);
+
+    xeon_phi_reset(phi);
+
+    interrupts_init(phi);
+
+    device_init(phi);
+
+    return SYS_ERR_OK;
+}
+
+/**
+ * \brief performs a soft reset of the card
+ *
+ * \param phi   pointer to the card information
+ */
+errval_t xeon_phi_reset(struct xeon_phi *phi)
+{
+    if (phi->state == XEON_PHI_STATE_NULL) {
+        return SYS_ERR_ILLEGAL_INVOCATION;
+    }
+
+    phi->state = XEON_PHI_STATE_RESET;
+
+    xeon_phi_boot_t boot_registers;
+
+    xeon_phi_boot_initialize(&boot_registers,
+                             XEON_PHI_MMIO_TO_SBOX(phi),
+                             XEON_PHI_MMIO_TO_DBOX(phi));
+
+    // clearing the download status register before rest.
+    xeon_phi_boot_download_wr(&boot_registers, 0x0);
+
+    // perform the actual reset sequence
+    xeon_phi_boot_reset_t res = xeon_phi_boot_reset_rd(&boot_registers);
+    res = xeon_phi_boot_reset_reset_insert(res, 0x1);
+    xeon_phi_boot_reset_wr(&boot_registers, res);
+
+    // wait a bit to prevent potential problems
+    milli_sleep(1000);
+
+    xeon_phi_boot_postcode_t postcode;
+    for (uint32_t time = 0; time < XEON_PHI_RESET_TIME; ++time) {
+        postcode = xeon_phi_boot_postcode_rd(&boot_registers);
+
+        debug_printf("Resetting (Post Code %c%c)\n",
+                     xeon_phi_boot_postcode_code0_extract(postcode),
+                     xeon_phi_boot_postcode_code1_extract(postcode));
+
+
+
+        if (xeon_phi_boot_download_status_rdf(&boot_registers)) {
+            phi->state = XEON_PHI_STATE_READY;
+            debug_printf("Reset successful\n");
+            /*
+             * if (mic_ctx->msie)
+                    mic_enable_msi_interrupts(mic_ctx);
+                mic_enable_interrupts(mic_ctx);
+                mic_smpt_restore(mic_ctx);
+                micscif_start(mic_ctx);
+             */
+            return SYS_ERR_OK;
+        }
+
+        milli_sleep(1000);
+    }
+
+    if (phi->state != XEON_PHI_STATE_READY) {
+        debug_printf("Reset Failed (Post Code %c%c)\n",
+                             xeon_phi_boot_postcode_code0_extract(postcode),
+                             xeon_phi_boot_postcode_code1_extract(postcode));
+    }
+#if 0
+
+
+
+        void
+        reset_timer(unsigned long arg)
+        {
+            if (mic_ctx->reset_count == RESET_FAIL_TIME ||
+                !postcode || 0xffffffff == postcode || mic_ctx->state == MIC_RESETFAIL) {
+                mic_ctx->reset_count = 0;
+                mic_setstate(mic_ctx, MIC_RESETFAIL);
+                wake_up(&mic_ctx->resetwq);
+                printk("MIC %d RESETFAIL postcode %c%c %d\n", mic_ctx->bi_id,
+                    postcode & 0xff, (postcode >> 8) & 0xff, postcode);
+                return;
+            }
+
+            /* check for F2 or F4 error codes from bootstrap */
+            if ((postcode == RESET_FAILED_F2) || (postcode == RESET_FAILED_F4)) {
+                if (mic_ctx->resetworkq) {
+                    queue_work(mic_ctx->resetworkq, &mic_ctx->resetwork);
+                } else {
+                    mic_ctx->reset_count = 0;
+                    mic_setstate(mic_ctx, MIC_RESETFAIL);
+                    wake_up(&mic_ctx->resetwq);
+                    return;
+                }
+            }
+
+
+
+
+        }
+#endif
+    return SYS_ERR_OK;
+}
+
+
index eb363f2..190945e 100644 (file)
 #define XEON_PHI_BOOTLOADER "xeon_phi_bootloader"
 #define XEON_PHI_MULTIBOOT "xeon_phi_multiboot"
 
+#define XEON_PHI_MSIX_ENABLED 1
+
 #define HOST_DBOX_OFFSET      0x00000000
 #define HOST_SBOX_OFFSET      0x00010000
 
-#define XEON_PHI_MMIO_TO_SBOX(x) \
-    ((mackerel_addr_t)((lvaddr_t)(x)+HOST_SBOX_OFFSET))
+#define XEON_PHI_MMIO_TO_SBOX(phi) \
+    ((mackerel_addr_t)((lvaddr_t)(phi->mmio.vbase)+HOST_SBOX_OFFSET))
+#define XEON_PHI_MMIO_TO_DBOX(phi) \
+    ((mackerel_addr_t)((lvaddr_t)(phi->mmio.vbase)+HOST_DBOX_OFFSET))
 
 #define XEON_PHI_NUM_CARDS 8
 
 #define MAX(a, b)   ( ((a) > (b)) ? (a) : (b) )
 #define MIN(a, b)   ( ((a) < (b)) ? (a) : (b) )
 
+typedef enum xeon_phi_state {
+    XEON_PHI_STATE_NULL,
+    XEON_PHI_STATE_PCI_OK,
+    XEON_PHI_STATE_RESET,
+    XEON_PHI_STATE_READY,
+    XEON_PHI_STATE_BOOTING,
+    XEON_PHI_STATE_ONLINE
+} xeon_phi_state_t;
+
 /*
  * TODO: Verify these values if they are really needed
  */
 #define UOS_RESERVE_SIZE_MIN    ((128) * 1024 * 1024)
 #define UOS_RESERVE_SIZE_MAX    (((4) * 1024 * 1024 * 1024ULL) - ((4) * 1024))
 
+struct mbar {
+    lvaddr_t vbase;
+    lpaddr_t pbase;
+    size_t   length;
+};
+
 struct xeon_phi
 {
-    lvaddr_t mmio_base;     ///< base address of the MMIO register space
-    lvaddr_t mmio_lengh;    ///< length of the MMIO register space
-    lvaddr_t aper_base;     ///< base address of the cards GDDR
-    lvaddr_t aper_length;   ///< length of the cards GDDR
+    xeon_phi_state_t state;
+    struct mbar mmio;
+    struct mbar apt;
 
+    uint8_t pci_init;
     uint8_t id;             ///< card id for identifying the card
 
     uint32_t apicid;        ///< APIC id used for sending the boot interrupt
 
     char *cmdline;          ///< pointer to the bootloader cmdline
+
+    struct smpt_info       *smpt;
+
+
+    struct irq_info         *irq;
 };
 
 /**
@@ -64,8 +88,8 @@ errval_t serial_start_recv_thread(struct xeon_phi *phi);
  * \param multiboot_img pointer to the card multiboot image
  */
 errval_t xeon_phi_boot(struct xeon_phi *phi,
-              char *xloader_img,
-              char *multiboot_img);
+                       char *xloader_img,
+                       char *multiboot_img);
 
 /**
  * \brief performs a soft reset of the card
@@ -75,9 +99,18 @@ errval_t xeon_phi_boot(struct xeon_phi *phi,
 errval_t xeon_phi_reset(struct xeon_phi *phi);
 
 /**
+ * \brief initializes the coprocessor card
+ *
+ * \param phi pointer to the information structure
+ */
+errval_t xeon_phi_init(struct xeon_phi *phi);
+
+/**
  * \brief Bootstraps the host driver to get the multiboot images of the
  *        xeon phi loader and the xeon phi multiboot image
  */
 errval_t host_bootstrap(void);
 
+
+
 #endif /* XEON_PHI_H_ */