libdma: enabling msix support for IOAT. [Closes T101]
authorReto Achermann <reto.achermann@inf.ethz.ch>
Tue, 19 May 2015 14:06:23 +0000 (16:06 +0200)
committerReto Achermann <reto.achermann@inf.ethz.ch>
Tue, 19 May 2015 14:09:08 +0000 (16:09 +0200)
- enabling of MSI-X interrupt support for IOAT DMA
- movied definition of pci_address into pci.h
- interface of pci service takes pci address now

Signed-off-by: Reto Achermann <reto.achermann@inf.ethz.ch>

if/pci.if
include/dma/ioat/ioat_dma_device.h
include/pci/pci.h
lib/dma/include/debug.h
lib/dma/include/ioat/ioat_dma_device_internal.h
lib/dma/ioat/ioat_dma_channel.c
lib/dma/ioat/ioat_dma_device.c
lib/pci/pci_client.c
usr/drivers/ioat_dma/device.c
usr/pci/pci_service.c
usr/vmkitmon/pci_ethernet.c

index 5e522bf..c5496d2 100644 (file)
--- a/if/pci.if
+++ b/if/pci.if
@@ -63,10 +63,18 @@ interface pci "The PCI Interface" {
     rpc write_conf_header(in uint32 dword, in uint32 val, out errval err);
 
     /* Enable MSI-X for the specified PCI device. */
+    rpc msix_enable_addr(in uint8 bus, in uint8 dev, in uint8 fn,
+                         out errval err,
+                         out uint16 vec_count);
     rpc msix_enable(out errval err,
                     out uint16 vec_count);
 
     /* Configure specified MSI-X vector */
+    rpc msix_vector_init_addr(in uint8 bus, in uint8 dev, in uint8 fn,
+                              in uint16 idx,        /* Index of MSI-X vector */
+                              in uint8 destination, /* Interrupt Destination */
+                              in uint8 vector,      /* Interrupt Vector */
+                              out errval err);
     rpc msix_vector_init(in uint16 idx,        /* Index of MSI-X vector */
                          in uint8 destination, /* Interrupt Destination */
                          in uint8 vector,      /* Interrupt Vector */
index cd6844b..1b97155 100644 (file)
@@ -33,13 +33,15 @@ static inline struct ioat_dma_device *dma_device_to_ioat(struct dma_device *dev)
 /**
  * \brief initializes a IOAT DMA device with the giving capability
  *
- * \param mmio capability representing the device's MMIO registers
- * \param dev  returns a pointer to the device structure
+ * \param mmio     capability representing the device's MMIO registers
+ * \param pci_addr the PCI address of this device
+ * \param dev      returns a pointer to the device structure
  *
  * \returns SYS_ERR_OK on success
  *          errval on error
  */
 errval_t ioat_dma_device_init(struct capref mmio,
+                              struct pci_address *pci_addr,
                               struct ioat_dma_device **dev);
 
 /**
index f8c63c6..172c2f2 100644 (file)
 #include <pci/mem.h>
 #include <pci/devids.h>
 
+struct pci_address {
+    uint8_t bus;
+    uint8_t device;
+    uint8_t function;
+};
+
 typedef void (*pci_driver_init_fn)(struct device_mem *bar_info,
                                    int nr_mapped_bars);
 typedef void (*legacy_driver_init_fn)(void);
@@ -74,6 +80,17 @@ errval_t pci_client_connect(void);
 errval_t pci_msix_enable(uint16_t *count);
 
 /**
+ * \brief enables MSI-X interupts for a given device
+ *
+ * \param addr  PCI address of the device to activate or NULL if don't care
+ * \param count returns the number of supported MSI-X interrupts
+ *
+ * \returns SYS_ERR_OK on success
+ *          errval on FAILURE
+ */
+errval_t pci_msix_enable_addr(struct pci_address *addr, uint16_t *count);
+
+/**
  * Configure an MSI-X vector
  * @param index       MSI-X Vector index
  * @param destination Destination APIC where the interrupt should be sent
@@ -81,5 +98,14 @@ errval_t pci_msix_enable(uint16_t *count);
  */
 errval_t pci_msix_vector_init(uint16_t index, uint8_t destination,
                               uint8_t vector);
+/**
+ * Configure an MSI-X vector
+ * \param addr  PCI address of the device to activate or NULL if don't care
+ * \param index       MSI-X Vector index
+ * \param destination Destination APIC where the interrupt should be sent
+ * \param vector      Interrupt vector to send
+ */
+errval_t pci_msix_vector_init_addr(struct pci_address *addr, uint16_t index,
+                                   uint8_t destination, uint8_t vector);
 
 #endif
index 1b61cb2..e820b46 100644 (file)
 #define DMAMGR_DEBUG(x...)
 #endif
 #if DMA_DEBUG_DESC_ENABLED
-#define DMADESC_DEBUG(x...) IOAT_DEBUG_PRINT("[dma desc] " x)
+#define DMADESC_DEBUG(x...) DMA_DEBUG_PRINT("[dma desc] " x)
 #else
 #define DMADESC_DEBUG(x...)
 #endif
index a202425..0fe7f3b 100644 (file)
@@ -10,6 +10,8 @@
 #ifndef IOAT_DMA_DEVICE_INTERNAL_H
 #define IOAT_DMA_DEVICE_INTERNAL_H
 
+#include <pci/pci.h>
+
 #include <dma_mem_utils.h>
 
 #include <dma_device_internal.h>
index ec86508..bfcc1a3 100644 (file)
@@ -31,8 +31,6 @@ struct ioat_dma_channel
     struct dma_mem completion;
     struct dma_ring *ring;          ///< Descriptor ring
     uint64_t status;                 ///< channel status
-    uint8_t irq_vector;
-    size_t irq_msix;
 };
 
 /**
index f3591ab..29288e5 100644 (file)
@@ -7,6 +7,7 @@
  */
 #include <string.h>
 #include <barrelfish/barrelfish.h>
+#include <barrelfish/waitset.h>
 #include <bench/bench.h>
 #include <dev/ioat_dma_dev.h>
 
@@ -30,6 +31,10 @@ struct ioat_dma_device
     ioat_dma_cbver_t version;           ///< Crystal Beach version number
 
     struct dma_mem complstatus;         ///< memory region for channels CHANSTS
+    struct pci_address pci_addr;        ///< the PCI address of the device
+
+    uint8_t irq_msix_vector;
+    uint16_t irq_msix_count;
 
     uint32_t flags;
 };
@@ -149,7 +154,15 @@ static errval_t device_init_ioat_v3(struct ioat_dma_device *dev)
     }
 
     if (dev->flags & IOAT_DMA_DEV_F_DCA) {
-        /*TODO: DCA initialization device->dca = ioat3_dca_init(pdev, device->reg_base);*/
+        /*
+         * TODO: DCA initialization
+         * device->dca = ioat3_dca_init(pdev, device->reg_base);
+         */
+    }
+
+    err = ioat_dma_device_irq_setup(dev, DMA_IRQ_MSIX);
+    if (err_is_fail(err)) {
+        return err;
     }
 
     return SYS_ERR_OK;
@@ -173,10 +186,48 @@ void ioat_dma_device_get_complsts_addr(struct ioat_dma_device *dev,
     *mem = dev->complstatus;
     mem->bytes = IOAT_DMA_COMPLSTATUS_SIZE;
     mem->paddr += (IOAT_DMA_COMPLSTATUS_SIZE * dev->common.channels.next);
-    mem->frame = NULL_CAP
-    ;
+    mem->frame = NULL_CAP;
     mem->vaddr += (IOAT_DMA_COMPLSTATUS_SIZE * dev->common.channels.next++);
+}
+
+#if IOAT_DEBUG_INTR_ENABLED
+///< flag indicating that the interrupt has happened for debugging purposes
+static uint32_t msix_intr_happened = 0;
+#include <dma/ioat/ioat_dma_request.h>
+#endif
+
+static void ioat_dma_device_irq_handler(void* arg)
+{
+    errval_t err;
+    struct dma_device *dev = arg;
+
+    IOATDEV_DEBUG("############ MSIX INTERRUPT HAPPENED.\n", dev->id);
 
+#if IOAT_DEBUG_INTR_ENABLED
+    msix_intr_happened=1;
+#endif
+
+    err = ioat_dma_device_poll_channels(dev);
+    if (err_is_fail(err)) {
+        if (err_no(err) == DMA_ERR_DEVICE_IDLE) {
+            IOATDEV_DEBUG("WARNING: MSI-X interrupt on idle device\n", dev->id);
+            return;
+        }
+        USER_PANIC_ERR(err, "dma poll device returned an error\n");
+    }
+}
+
+/**
+ * \brief gets the local apic ID from the CPU id
+ *
+ * \return local apic ID
+ */
+static inline uint8_t get_local_apic_id(void)
+{
+    uint32_t eax, ebx;
+
+    cpuid(1, &eax, &ebx, NULL, NULL);
+    return  ebx >> 24;
 }
 
 /**
@@ -188,14 +239,45 @@ void ioat_dma_device_get_complsts_addr(struct ioat_dma_device *dev,
 errval_t ioat_dma_device_irq_setup(struct ioat_dma_device *dev,
                                    dma_irq_t type)
 {
+    errval_t err;
+
     ioat_dma_intrctrl_t intcrtl = 0;
     intcrtl = ioat_dma_intrctrl_intp_en_insert(intcrtl, 1);
 
     dev->common.irq_type = type;
     switch (type) {
         case DMA_IRQ_MSIX:
-            IOATDEV_DEBUG("Initializing MSI-X interrupts \n", dev->common.id);
-            assert(!"NYI");
+            /* The number of MSI-X vectors should equal the number of channels */
+            IOATDEV_DEBUG("MSI-X interrupt setup for device (%u, %u, %u)\n",
+                          dev->common.id, dev->pci_addr.bus, dev->pci_addr.device,
+                          dev->pci_addr.function);
+
+            err = pci_msix_enable_addr(&dev->pci_addr, &dev->irq_msix_count);
+            if (err_is_fail(err)) {
+                return err;
+            }
+
+            assert(dev->irq_msix_count > 0);
+
+            IOATDEV_DEBUG("MSI-X enabled #vecs=%d\n", dev->common.id,
+                          dev->irq_msix_count);
+
+            err = pci_setup_inthandler(ioat_dma_device_irq_handler, dev,
+                                       &dev->irq_msix_vector);
+            assert(err_is_ok(err));
+
+            uint8_t dest = get_local_apic_id();
+
+            IOATDEV_DEBUG("MSI-X routing to apic=%u\n", dev->common.id,
+                          dest);
+
+            err = pci_msix_vector_init_addr(&dev->pci_addr, 0, dest,
+                                            dev->irq_msix_vector);
+            assert(err_is_ok(err));
+
+            /* enable the interrupts */
+            intcrtl = ioat_dma_intrctrl_msix_vec_insert(intcrtl, 1);
+            intcrtl = ioat_dma_intrctrl_intp_en_insert(intcrtl, 1);
             break;
         case DMA_IRQ_MSI:
             IOATDEV_DEBUG("Initializing MSI interrupts \n", dev->common.id);
@@ -214,6 +296,42 @@ errval_t ioat_dma_device_irq_setup(struct ioat_dma_device *dev,
 
     ioat_dma_intrctrl_wr(&dev->device, intcrtl);
 
+
+#if IOAT_DEBUG_INTR_ENABLED
+    /*
+     * check if interrupts are working.
+     */
+    msix_intr_happened = 0;
+
+    struct ioat_dma_channel *chan;
+    chan = (struct ioat_dma_channel *)dev->common.channels.c[0];
+
+    ioat_dma_request_nop_chan(chan);
+    err = ioat_dma_channel_issue_pending(chan);
+    if (err_is_fail(err)) {
+        return err;
+    }
+
+    while(msix_intr_happened == 0) {
+        uint64_t status = ioat_dma_channel_get_status(chan);
+        err = event_dispatch_non_block(get_default_waitset());
+
+        if (!ioat_dma_channel_is_active(status) && !ioat_dma_channel_is_idle(status)) {
+            USER_PANIC("DMA request turned channel into erroneous state.")
+        }
+
+        switch(err_no(err)) {
+            case LIB_ERR_NO_EVENT:
+                thread_yield();
+                break;
+            case SYS_ERR_OK:
+                continue;
+            default:
+                USER_PANIC_ERR(err, "dispatching event");
+        }
+    }
+#endif
+
     return SYS_ERR_OK;
 }
 
@@ -232,13 +350,15 @@ errval_t ioat_dma_device_irq_setup(struct ioat_dma_device *dev,
 /**
  * \brief initializes a IOAT DMA device with the giving capability
  *
- * \param mmio capability representing the device's MMIO registers
- * \param dev  returns a pointer to the device structure
+ * \param mmio     capability representing the device's MMIO registers
+ * \param pci_addr the PCI address of this device
+ * \param dev      returns a pointer to the device structure
  *
  * \returns SYS_ERR_OK on success
  *          errval on error
  */
 errval_t ioat_dma_device_init(struct capref mmio,
+                              struct pci_address *pci_addr,
                               struct ioat_dma_device **dev)
 {
     errval_t err;
@@ -265,6 +385,7 @@ errval_t ioat_dma_device_init(struct capref mmio,
     dma_dev->mmio.paddr = mmio_id.base;
     dma_dev->mmio.bytes = (1UL << mmio_id.bits);
     dma_dev->mmio.frame = mmio;
+    ioat_device->pci_addr = *pci_addr;
 
     IOATDEV_DEBUG("init device with mmio range: {paddr=0x%016lx, size=%u kB}\n",
                   dma_dev->id, mmio_id.base, 1 << mmio_id.bits);
@@ -272,8 +393,7 @@ errval_t ioat_dma_device_init(struct capref mmio,
     err = vspace_map_one_frame_attr((void**) &dma_dev->mmio.vaddr,
                                     dma_dev->mmio.bytes, dma_dev->mmio.frame,
                                     VREGION_FLAGS_READ_WRITE_NOCACHE,
-                                    NULL,
-                                    NULL);
+                                    NULL, NULL);
     if (err_is_fail(err)) {
         free(ioat_device);
         return err;
index d547d3d..78f21be 100644 (file)
@@ -263,21 +263,44 @@ errval_t pci_write_conf_header(uint32_t dword, uint32_t val)
     return err_is_fail(err) ? err : msgerr;
 }
 
-errval_t pci_msix_enable(uint16_t *count)
+errval_t pci_msix_enable_addr(struct pci_address *addr, uint16_t *count)
 {
     errval_t err, msgerr;
-    err = pci_client->vtbl.msix_enable(pci_client, &msgerr, count);
+    if (addr == NULL) {
+        err = pci_client->vtbl.msix_enable(pci_client, &msgerr, count);
+    } else {
+        err = pci_client->vtbl.msix_enable_addr(pci_client, addr->bus, addr->device,
+                                                addr->function, &msgerr, count);
+    }
     return err_is_fail(err) ? err : msgerr;
 }
 
-errval_t pci_msix_vector_init(uint16_t idx, uint8_t destination,
-                              uint8_t vector)
+errval_t pci_msix_enable(uint16_t *count)
+{
+    return pci_msix_enable_addr(NULL, count);
+}
+
+errval_t pci_msix_vector_init_addr(struct pci_address *addr, uint16_t idx,
+                                   uint8_t destination, uint8_t vector)
 {
     errval_t err, msgerr;
-    err = pci_client->vtbl.msix_vector_init(pci_client, idx, destination,
-                                            vector, &msgerr);
+    if (addr == NULL) {
+        err = pci_client->vtbl.msix_vector_init(pci_client, idx, destination,
+                                                    vector, &msgerr);
+    } else {
+        err = pci_client->vtbl.msix_vector_init_addr(pci_client, addr->bus,
+                                                     addr->device, addr->function,
+                                                     idx, destination,
+                                                     vector, &msgerr);
+    }
+
     return err_is_fail(err) ? err : msgerr;
+}
 
+errval_t pci_msix_vector_init(uint16_t idx, uint8_t destination,
+                              uint8_t vector)
+{
+    return pci_msix_vector_init_addr(NULL, idx, destination, vector);
 }
 
 static void bind_cont(void *st, errval_t err, struct pci_binding *b)
index 23c2835..e1810e9 100644 (file)
@@ -23,6 +23,7 @@
 static uint8_t device_count = 0;
 struct ioat_dma_device **devices;
 static uint8_t device_next = 0;
+static struct pci_address pci_addr;
 
 static void handle_device_interrupt(void *arg)
 {
@@ -48,7 +49,8 @@ static void pci_dev_init_service(struct device_mem *bar_info,
     }
 
     /* initialize the device */
-    err = ioat_dma_device_init(*bar_info->frame_cap, &devices[device_count]);
+    err = ioat_dma_device_init(*bar_info->frame_cap, &pci_addr,
+                               &devices[device_count]);
     if (err_is_fail(err)) {
         DEV_ERR("Could not initialize the device: %s\n", err_getstring(err));
         return;
@@ -165,15 +167,17 @@ errval_t ioat_device_discovery(struct pci_addr addr,
      * Bus x, Device 4, Function 0..7
      */
     for (uint8_t i = 0; i < dev_cnt; ++i) {
+        pci_addr.bus = addr.bus;
+        pci_addr.device = addr.device;
+        pci_addr.function = i;
+
         if (is_dev_mgr == IOAT_DMA_OPERATION_LIBRARY) {
             /*
              * discover devices as manager i.e. don't initialize them as they
              * are handed over to the domains upon request
              */
-            err = pci_register_driver_noirq(pci_dev_init_manager,
-            PCI_DONT_CARE,
-                                            PCI_DONT_CARE,
-                                            PCI_DONT_CARE,
+            err = pci_register_driver_noirq(pci_dev_init_manager, PCI_DONT_CARE,
+                                            PCI_DONT_CARE, PCI_DONT_CARE,
                                             PCI_VENDOR_INTEL, dev_ids[i], addr.bus,
                                             addr.device, addr.function + i);
         } else {
@@ -181,8 +185,7 @@ errval_t ioat_device_discovery(struct pci_addr addr,
              * discover devices as a service i.e. initialize and map devices
              */
             err = pci_register_driver_irq(pci_dev_init_service, PCI_DONT_CARE,
-            PCI_DONT_CARE,
-                                          PCI_DONT_CARE,
+                                          PCI_DONT_CARE, PCI_DONT_CARE,
                                           PCI_VENDOR_INTEL,
                                           dev_ids[i], addr.bus, addr.device,
                                           addr.function + i, handle_device_interrupt,
index 70ac1ed..69c242a 100644 (file)
@@ -261,31 +261,63 @@ static void write_conf_header_handler(struct pci_binding *b, uint32_t dword, uin
     assert(err_is_ok(err));
 }
 
-static void msix_enable_handler(struct pci_binding *b)
+static void msix_enable_addr_handler(struct pci_binding *b, uint8_t bus,
+                                      uint8_t dev, uint8_t fun)
 {
     struct client_state *cc = (struct client_state *) b->st;
-    struct pci_address addr = {
-        .bus= cc->bus,
-        .device=cc->dev,
-        .function=cc->fun,
-    };
+    struct pci_address addr;
+
+    /* XXX: find another way to do this */
+
+    if (bus == cc->bus && dev == cc->dev) {
+        addr.bus= bus;
+        addr.device=dev;
+        addr.function=fun;
+    } else {
+        addr.bus= cc->bus;
+        addr.device=cc->dev;
+        addr.function=fun;
+    }
+
     errval_t err;
     uint16_t count;
 
+    debug_printf("enabling MSI-X for device (%u, %u, %u)\n", addr.bus,
+                 addr.device, addr.function);
+
     err = pci_msix_enable(&addr, &count);
     err = b->tx_vtbl.msix_enable_response(b, NOP_CONT, err, count);
     assert(err_is_ok(err));
 }
 
-static void msix_vector_init_handler(struct pci_binding *b, uint16_t idx,
-                                     uint8_t destination, uint8_t vector)
+static void msix_enable_handler(struct pci_binding *b)
 {
     struct client_state *cc = (struct client_state *) b->st;
-    struct pci_address addr = {
-        .bus= cc->bus,
-        .device=cc->dev,
-        .function=cc->fun,
-    };
+    msix_enable_addr_handler(b, cc->bus, cc->dev, cc->fun);
+}
+
+static void msix_vector_init_addr_handler(struct pci_binding *b, uint8_t bus,
+                                          uint8_t dev, uint8_t fun, uint16_t idx,
+                                          uint8_t destination, uint8_t vector)
+{
+    struct client_state *cc = (struct client_state *) b->st;
+    struct pci_address addr;
+
+    /* XXX: find another way to do this */
+
+    if (bus == cc->bus && dev == cc->dev) {
+        addr.bus= bus;
+        addr.device=dev;
+        addr.function=fun;
+    } else {
+        addr.bus= cc->bus;
+        addr.device=cc->dev;
+        addr.function=fun;
+    }
+
+    debug_printf("initialize MSI-X vector for device (%u, %u, %u)\n", addr.bus,
+                     addr.device, addr.function);
+
     errval_t err;
 
     err = pci_msix_vector_init(&addr, idx, destination, vector);
@@ -293,6 +325,15 @@ static void msix_vector_init_handler(struct pci_binding *b, uint16_t idx,
     assert(err_is_ok(err));
 }
 
+static void msix_vector_init_handler(struct pci_binding *b, uint16_t idx,
+                                     uint8_t destination, uint8_t vector)
+{
+    struct client_state *cc = (struct client_state *) b->st;
+
+    msix_vector_init_addr_handler(b, cc->bus, cc->dev, cc->fun, idx, destination,
+                                  vector);
+}
+
 struct pci_rx_vtbl pci_rx_vtbl = {
     .init_pci_device_call = init_pci_device_handler,
     .init_legacy_device_call = init_legacy_device_handler,
@@ -303,7 +344,9 @@ struct pci_rx_vtbl pci_rx_vtbl = {
     .write_conf_header_call = write_conf_header_handler,
 
     .msix_enable_call = msix_enable_handler,
+    .msix_enable_addr_call = msix_enable_addr_handler,
     .msix_vector_init_call = msix_vector_init_handler,
+    .msix_vector_init_addr_call = msix_vector_init_addr_handler,
 };
 
 static void export_callback(void *st, errval_t err, iref_t iref)
index 636666a..ded9ad9 100644 (file)
 #define PCI_CONFIG_DATA_PORT    0x0cfc
 #define PCI_ETHERNET_IRQ 11
 
-struct pci_address {
-    uint8_t bus;
-    uint8_t device;
-    uint8_t function;
-};
 
 static struct guest *guest_info;
 static struct pci_ethernet *pci_ethernet_unique;
@@ -73,7 +68,7 @@ static void confspace_write(struct pci_device *dev,
     if(addr.d.fnct_nr != 0) {
         return;
     }
-    
+
     if(addr.d.doubleword < 0x40) {
         errval_t r = pci_write_conf_header(addr.d.doubleword, val);
         if(err_is_fail(r)) {
@@ -110,7 +105,7 @@ static void pci_ethernet_init(void *bar_info, int nr_allocated_bars)
 {
        VMKIT_PCI_DEBUG("pci_ethernet_init. nr_allocated_bars: %d!\n", nr_allocated_bars);
     struct device_mem *bar = (struct device_mem *)bar_info;
-    
+
     eth_base_paddr = bar[0].paddr;
 
     struct pci_ethernet * eth = pci_ethernet_unique;
@@ -264,7 +259,7 @@ struct pci_device *pci_ethernet_new(struct lpc *lpc, struct guest *g)
 {
     struct pci_device *dev = calloc(1, sizeof(struct pci_device));
     struct pci_ethernet *host = calloc(1, sizeof(struct pci_ethernet));
-    
+
     pci_ethernet_unique = host;
     host->pci_device = dev;
     guest_info = g;
@@ -277,12 +272,12 @@ struct pci_device *pci_ethernet_new(struct lpc *lpc, struct guest *g)
     dev->state = host;
     dev->irq = PCI_ETHERNET_IRQ;
     dev->lpc = lpc;
-    
+
     //Connect to pci server
        errval_t r = pci_client_connect();
        assert(err_is_ok(r));
        VMKIT_PCI_DEBUG("connected to pci\n");
-    
+
     //Register as driver
        r = pci_register_driver_irq((pci_driver_init_fn)pci_ethernet_init,
                                 PCI_CLASS_ETHERNET,