rpc write_conf_header(in uint32 dword, in uint32 val, out errval err);
/* Enable MSI-X for the specified PCI device. */
+ rpc msix_enable_addr(in uint8 bus, in uint8 dev, in uint8 fn,
+ out errval err,
+ out uint16 vec_count);
rpc msix_enable(out errval err,
out uint16 vec_count);
/* Configure specified MSI-X vector */
+ rpc msix_vector_init_addr(in uint8 bus, in uint8 dev, in uint8 fn,
+ in uint16 idx, /* Index of MSI-X vector */
+ in uint8 destination, /* Interrupt Destination */
+ in uint8 vector, /* Interrupt Vector */
+ out errval err);
rpc msix_vector_init(in uint16 idx, /* Index of MSI-X vector */
in uint8 destination, /* Interrupt Destination */
in uint8 vector, /* Interrupt Vector */
/**
* \brief initializes a IOAT DMA device with the giving capability
*
- * \param mmio capability representing the device's MMIO registers
- * \param dev returns a pointer to the device structure
+ * \param mmio capability representing the device's MMIO registers
+ * \param pci_addr the PCI address of this device
+ * \param dev returns a pointer to the device structure
*
* \returns SYS_ERR_OK on success
* errval on error
*/
errval_t ioat_dma_device_init(struct capref mmio,
+ struct pci_address *pci_addr,
struct ioat_dma_device **dev);
/**
#include <pci/mem.h>
#include <pci/devids.h>
+struct pci_address {
+ uint8_t bus;
+ uint8_t device;
+ uint8_t function;
+};
+
typedef void (*pci_driver_init_fn)(struct device_mem *bar_info,
int nr_mapped_bars);
typedef void (*legacy_driver_init_fn)(void);
errval_t pci_msix_enable(uint16_t *count);
/**
+ * \brief enables MSI-X interupts for a given device
+ *
+ * \param addr PCI address of the device to activate or NULL if don't care
+ * \param count returns the number of supported MSI-X interrupts
+ *
+ * \returns SYS_ERR_OK on success
+ * errval on FAILURE
+ */
+errval_t pci_msix_enable_addr(struct pci_address *addr, uint16_t *count);
+
+/**
* Configure an MSI-X vector
* @param index MSI-X Vector index
* @param destination Destination APIC where the interrupt should be sent
*/
errval_t pci_msix_vector_init(uint16_t index, uint8_t destination,
uint8_t vector);
+/**
+ * Configure an MSI-X vector
+ * \param addr PCI address of the device to activate or NULL if don't care
+ * \param index MSI-X Vector index
+ * \param destination Destination APIC where the interrupt should be sent
+ * \param vector Interrupt vector to send
+ */
+errval_t pci_msix_vector_init_addr(struct pci_address *addr, uint16_t index,
+ uint8_t destination, uint8_t vector);
#endif
#define DMAMGR_DEBUG(x...)
#endif
#if DMA_DEBUG_DESC_ENABLED
-#define DMADESC_DEBUG(x...) IOAT_DEBUG_PRINT("[dma desc] " x)
+#define DMADESC_DEBUG(x...) DMA_DEBUG_PRINT("[dma desc] " x)
#else
#define DMADESC_DEBUG(x...)
#endif
#ifndef IOAT_DMA_DEVICE_INTERNAL_H
#define IOAT_DMA_DEVICE_INTERNAL_H
+#include <pci/pci.h>
+
#include <dma_mem_utils.h>
#include <dma_device_internal.h>
struct dma_mem completion;
struct dma_ring *ring; ///< Descriptor ring
uint64_t status; ///< channel status
- uint8_t irq_vector;
- size_t irq_msix;
};
/**
*/
#include <string.h>
#include <barrelfish/barrelfish.h>
+#include <barrelfish/waitset.h>
#include <bench/bench.h>
#include <dev/ioat_dma_dev.h>
ioat_dma_cbver_t version; ///< Crystal Beach version number
struct dma_mem complstatus; ///< memory region for channels CHANSTS
+ struct pci_address pci_addr; ///< the PCI address of the device
+
+ uint8_t irq_msix_vector;
+ uint16_t irq_msix_count;
uint32_t flags;
};
}
if (dev->flags & IOAT_DMA_DEV_F_DCA) {
- /*TODO: DCA initialization device->dca = ioat3_dca_init(pdev, device->reg_base);*/
+ /*
+ * TODO: DCA initialization
+ * device->dca = ioat3_dca_init(pdev, device->reg_base);
+ */
+ }
+
+ err = ioat_dma_device_irq_setup(dev, DMA_IRQ_MSIX);
+ if (err_is_fail(err)) {
+ return err;
}
return SYS_ERR_OK;
*mem = dev->complstatus;
mem->bytes = IOAT_DMA_COMPLSTATUS_SIZE;
mem->paddr += (IOAT_DMA_COMPLSTATUS_SIZE * dev->common.channels.next);
- mem->frame = NULL_CAP
- ;
+ mem->frame = NULL_CAP;
mem->vaddr += (IOAT_DMA_COMPLSTATUS_SIZE * dev->common.channels.next++);
+}
+
+#if IOAT_DEBUG_INTR_ENABLED
+///< flag indicating that the interrupt has happened for debugging purposes
+static uint32_t msix_intr_happened = 0;
+#include <dma/ioat/ioat_dma_request.h>
+#endif
+
+static void ioat_dma_device_irq_handler(void* arg)
+{
+ errval_t err;
+ struct dma_device *dev = arg;
+
+ IOATDEV_DEBUG("############ MSIX INTERRUPT HAPPENED.\n", dev->id);
+#if IOAT_DEBUG_INTR_ENABLED
+ msix_intr_happened=1;
+#endif
+
+ err = ioat_dma_device_poll_channels(dev);
+ if (err_is_fail(err)) {
+ if (err_no(err) == DMA_ERR_DEVICE_IDLE) {
+ IOATDEV_DEBUG("WARNING: MSI-X interrupt on idle device\n", dev->id);
+ return;
+ }
+ USER_PANIC_ERR(err, "dma poll device returned an error\n");
+ }
+}
+
+/**
+ * \brief gets the local apic ID from the CPU id
+ *
+ * \return local apic ID
+ */
+static inline uint8_t get_local_apic_id(void)
+{
+ uint32_t eax, ebx;
+
+ cpuid(1, &eax, &ebx, NULL, NULL);
+ return ebx >> 24;
}
/**
errval_t ioat_dma_device_irq_setup(struct ioat_dma_device *dev,
dma_irq_t type)
{
+ errval_t err;
+
ioat_dma_intrctrl_t intcrtl = 0;
intcrtl = ioat_dma_intrctrl_intp_en_insert(intcrtl, 1);
dev->common.irq_type = type;
switch (type) {
case DMA_IRQ_MSIX:
- IOATDEV_DEBUG("Initializing MSI-X interrupts \n", dev->common.id);
- assert(!"NYI");
+ /* The number of MSI-X vectors should equal the number of channels */
+ IOATDEV_DEBUG("MSI-X interrupt setup for device (%u, %u, %u)\n",
+ dev->common.id, dev->pci_addr.bus, dev->pci_addr.device,
+ dev->pci_addr.function);
+
+ err = pci_msix_enable_addr(&dev->pci_addr, &dev->irq_msix_count);
+ if (err_is_fail(err)) {
+ return err;
+ }
+
+ assert(dev->irq_msix_count > 0);
+
+ IOATDEV_DEBUG("MSI-X enabled #vecs=%d\n", dev->common.id,
+ dev->irq_msix_count);
+
+ err = pci_setup_inthandler(ioat_dma_device_irq_handler, dev,
+ &dev->irq_msix_vector);
+ assert(err_is_ok(err));
+
+ uint8_t dest = get_local_apic_id();
+
+ IOATDEV_DEBUG("MSI-X routing to apic=%u\n", dev->common.id,
+ dest);
+
+ err = pci_msix_vector_init_addr(&dev->pci_addr, 0, dest,
+ dev->irq_msix_vector);
+ assert(err_is_ok(err));
+
+ /* enable the interrupts */
+ intcrtl = ioat_dma_intrctrl_msix_vec_insert(intcrtl, 1);
+ intcrtl = ioat_dma_intrctrl_intp_en_insert(intcrtl, 1);
break;
case DMA_IRQ_MSI:
IOATDEV_DEBUG("Initializing MSI interrupts \n", dev->common.id);
ioat_dma_intrctrl_wr(&dev->device, intcrtl);
+
+#if IOAT_DEBUG_INTR_ENABLED
+ /*
+ * check if interrupts are working.
+ */
+ msix_intr_happened = 0;
+
+ struct ioat_dma_channel *chan;
+ chan = (struct ioat_dma_channel *)dev->common.channels.c[0];
+
+ ioat_dma_request_nop_chan(chan);
+ err = ioat_dma_channel_issue_pending(chan);
+ if (err_is_fail(err)) {
+ return err;
+ }
+
+ while(msix_intr_happened == 0) {
+ uint64_t status = ioat_dma_channel_get_status(chan);
+ err = event_dispatch_non_block(get_default_waitset());
+
+ if (!ioat_dma_channel_is_active(status) && !ioat_dma_channel_is_idle(status)) {
+ USER_PANIC("DMA request turned channel into erroneous state.")
+ }
+
+ switch(err_no(err)) {
+ case LIB_ERR_NO_EVENT:
+ thread_yield();
+ break;
+ case SYS_ERR_OK:
+ continue;
+ default:
+ USER_PANIC_ERR(err, "dispatching event");
+ }
+ }
+#endif
+
return SYS_ERR_OK;
}
/**
* \brief initializes a IOAT DMA device with the giving capability
*
- * \param mmio capability representing the device's MMIO registers
- * \param dev returns a pointer to the device structure
+ * \param mmio capability representing the device's MMIO registers
+ * \param pci_addr the PCI address of this device
+ * \param dev returns a pointer to the device structure
*
* \returns SYS_ERR_OK on success
* errval on error
*/
errval_t ioat_dma_device_init(struct capref mmio,
+ struct pci_address *pci_addr,
struct ioat_dma_device **dev)
{
errval_t err;
dma_dev->mmio.paddr = mmio_id.base;
dma_dev->mmio.bytes = (1UL << mmio_id.bits);
dma_dev->mmio.frame = mmio;
+ ioat_device->pci_addr = *pci_addr;
IOATDEV_DEBUG("init device with mmio range: {paddr=0x%016lx, size=%u kB}\n",
dma_dev->id, mmio_id.base, 1 << mmio_id.bits);
err = vspace_map_one_frame_attr((void**) &dma_dev->mmio.vaddr,
dma_dev->mmio.bytes, dma_dev->mmio.frame,
VREGION_FLAGS_READ_WRITE_NOCACHE,
- NULL,
- NULL);
+ NULL, NULL);
if (err_is_fail(err)) {
free(ioat_device);
return err;
return err_is_fail(err) ? err : msgerr;
}
-errval_t pci_msix_enable(uint16_t *count)
+errval_t pci_msix_enable_addr(struct pci_address *addr, uint16_t *count)
{
errval_t err, msgerr;
- err = pci_client->vtbl.msix_enable(pci_client, &msgerr, count);
+ if (addr == NULL) {
+ err = pci_client->vtbl.msix_enable(pci_client, &msgerr, count);
+ } else {
+ err = pci_client->vtbl.msix_enable_addr(pci_client, addr->bus, addr->device,
+ addr->function, &msgerr, count);
+ }
return err_is_fail(err) ? err : msgerr;
}
-errval_t pci_msix_vector_init(uint16_t idx, uint8_t destination,
- uint8_t vector)
+errval_t pci_msix_enable(uint16_t *count)
+{
+ return pci_msix_enable_addr(NULL, count);
+}
+
+errval_t pci_msix_vector_init_addr(struct pci_address *addr, uint16_t idx,
+ uint8_t destination, uint8_t vector)
{
errval_t err, msgerr;
- err = pci_client->vtbl.msix_vector_init(pci_client, idx, destination,
- vector, &msgerr);
+ if (addr == NULL) {
+ err = pci_client->vtbl.msix_vector_init(pci_client, idx, destination,
+ vector, &msgerr);
+ } else {
+ err = pci_client->vtbl.msix_vector_init_addr(pci_client, addr->bus,
+ addr->device, addr->function,
+ idx, destination,
+ vector, &msgerr);
+ }
+
return err_is_fail(err) ? err : msgerr;
+}
+errval_t pci_msix_vector_init(uint16_t idx, uint8_t destination,
+ uint8_t vector)
+{
+ return pci_msix_vector_init_addr(NULL, idx, destination, vector);
}
static void bind_cont(void *st, errval_t err, struct pci_binding *b)
static uint8_t device_count = 0;
struct ioat_dma_device **devices;
static uint8_t device_next = 0;
+static struct pci_address pci_addr;
static void handle_device_interrupt(void *arg)
{
}
/* initialize the device */
- err = ioat_dma_device_init(*bar_info->frame_cap, &devices[device_count]);
+ err = ioat_dma_device_init(*bar_info->frame_cap, &pci_addr,
+ &devices[device_count]);
if (err_is_fail(err)) {
DEV_ERR("Could not initialize the device: %s\n", err_getstring(err));
return;
* Bus x, Device 4, Function 0..7
*/
for (uint8_t i = 0; i < dev_cnt; ++i) {
+ pci_addr.bus = addr.bus;
+ pci_addr.device = addr.device;
+ pci_addr.function = i;
+
if (is_dev_mgr == IOAT_DMA_OPERATION_LIBRARY) {
/*
* discover devices as manager i.e. don't initialize them as they
* are handed over to the domains upon request
*/
- err = pci_register_driver_noirq(pci_dev_init_manager,
- PCI_DONT_CARE,
- PCI_DONT_CARE,
- PCI_DONT_CARE,
+ err = pci_register_driver_noirq(pci_dev_init_manager, PCI_DONT_CARE,
+ PCI_DONT_CARE, PCI_DONT_CARE,
PCI_VENDOR_INTEL, dev_ids[i], addr.bus,
addr.device, addr.function + i);
} else {
* discover devices as a service i.e. initialize and map devices
*/
err = pci_register_driver_irq(pci_dev_init_service, PCI_DONT_CARE,
- PCI_DONT_CARE,
- PCI_DONT_CARE,
+ PCI_DONT_CARE, PCI_DONT_CARE,
PCI_VENDOR_INTEL,
dev_ids[i], addr.bus, addr.device,
addr.function + i, handle_device_interrupt,
assert(err_is_ok(err));
}
-static void msix_enable_handler(struct pci_binding *b)
+static void msix_enable_addr_handler(struct pci_binding *b, uint8_t bus,
+ uint8_t dev, uint8_t fun)
{
struct client_state *cc = (struct client_state *) b->st;
- struct pci_address addr = {
- .bus= cc->bus,
- .device=cc->dev,
- .function=cc->fun,
- };
+ struct pci_address addr;
+
+ /* XXX: find another way to do this */
+
+ if (bus == cc->bus && dev == cc->dev) {
+ addr.bus= bus;
+ addr.device=dev;
+ addr.function=fun;
+ } else {
+ addr.bus= cc->bus;
+ addr.device=cc->dev;
+ addr.function=fun;
+ }
+
errval_t err;
uint16_t count;
+ debug_printf("enabling MSI-X for device (%u, %u, %u)\n", addr.bus,
+ addr.device, addr.function);
+
err = pci_msix_enable(&addr, &count);
err = b->tx_vtbl.msix_enable_response(b, NOP_CONT, err, count);
assert(err_is_ok(err));
}
-static void msix_vector_init_handler(struct pci_binding *b, uint16_t idx,
- uint8_t destination, uint8_t vector)
+static void msix_enable_handler(struct pci_binding *b)
{
struct client_state *cc = (struct client_state *) b->st;
- struct pci_address addr = {
- .bus= cc->bus,
- .device=cc->dev,
- .function=cc->fun,
- };
+ msix_enable_addr_handler(b, cc->bus, cc->dev, cc->fun);
+}
+
+static void msix_vector_init_addr_handler(struct pci_binding *b, uint8_t bus,
+ uint8_t dev, uint8_t fun, uint16_t idx,
+ uint8_t destination, uint8_t vector)
+{
+ struct client_state *cc = (struct client_state *) b->st;
+ struct pci_address addr;
+
+ /* XXX: find another way to do this */
+
+ if (bus == cc->bus && dev == cc->dev) {
+ addr.bus= bus;
+ addr.device=dev;
+ addr.function=fun;
+ } else {
+ addr.bus= cc->bus;
+ addr.device=cc->dev;
+ addr.function=fun;
+ }
+
+ debug_printf("initialize MSI-X vector for device (%u, %u, %u)\n", addr.bus,
+ addr.device, addr.function);
+
errval_t err;
err = pci_msix_vector_init(&addr, idx, destination, vector);
assert(err_is_ok(err));
}
+static void msix_vector_init_handler(struct pci_binding *b, uint16_t idx,
+ uint8_t destination, uint8_t vector)
+{
+ struct client_state *cc = (struct client_state *) b->st;
+
+ msix_vector_init_addr_handler(b, cc->bus, cc->dev, cc->fun, idx, destination,
+ vector);
+}
+
struct pci_rx_vtbl pci_rx_vtbl = {
.init_pci_device_call = init_pci_device_handler,
.init_legacy_device_call = init_legacy_device_handler,
.write_conf_header_call = write_conf_header_handler,
.msix_enable_call = msix_enable_handler,
+ .msix_enable_addr_call = msix_enable_addr_handler,
.msix_vector_init_call = msix_vector_init_handler,
+ .msix_vector_init_addr_call = msix_vector_init_addr_handler,
};
static void export_callback(void *st, errval_t err, iref_t iref)
#define PCI_CONFIG_DATA_PORT 0x0cfc
#define PCI_ETHERNET_IRQ 11
-struct pci_address {
- uint8_t bus;
- uint8_t device;
- uint8_t function;
-};
static struct guest *guest_info;
static struct pci_ethernet *pci_ethernet_unique;
if(addr.d.fnct_nr != 0) {
return;
}
-
+
if(addr.d.doubleword < 0x40) {
errval_t r = pci_write_conf_header(addr.d.doubleword, val);
if(err_is_fail(r)) {
{
VMKIT_PCI_DEBUG("pci_ethernet_init. nr_allocated_bars: %d!\n", nr_allocated_bars);
struct device_mem *bar = (struct device_mem *)bar_info;
-
+
eth_base_paddr = bar[0].paddr;
struct pci_ethernet * eth = pci_ethernet_unique;
{
struct pci_device *dev = calloc(1, sizeof(struct pci_device));
struct pci_ethernet *host = calloc(1, sizeof(struct pci_ethernet));
-
+
pci_ethernet_unique = host;
host->pci_device = dev;
guest_info = g;
dev->state = host;
dev->irq = PCI_ETHERNET_IRQ;
dev->lpc = lpc;
-
+
//Connect to pci server
errval_t r = pci_client_connect();
assert(err_is_ok(r));
VMKIT_PCI_DEBUG("connected to pci\n");
-
+
//Register as driver
r = pci_register_driver_irq((pci_driver_init_fn)pci_ethernet_init,
PCI_CLASS_ETHERNET,
projects / barrelfish / commitdiff