IOAT Driver: initial working implementation. (to be cleaned up...)
authorReto Achermann <acreto@student.ethz.ch>
Thu, 17 Jul 2014 17:13:03 +0000 (19:13 +0200)
committerStefan Kaestle <stefan.kaestle@inf.ethz.ch>
Wed, 20 Aug 2014 21:39:53 +0000 (23:39 +0200)
12 files changed:
devices/ioat_dma_chan.dev
errors/errno.fugu
usr/drivers/ioat_dma/debug.h
usr/drivers/ioat_dma/ioat_dma_channel.c
usr/drivers/ioat_dma/ioat_dma_channel.h
usr/drivers/ioat_dma/ioat_dma_device.c
usr/drivers/ioat_dma/ioat_dma_device.h
usr/drivers/ioat_dma/ioat_dma_request.c
usr/drivers/ioat_dma/ioat_dma_request.h
usr/drivers/ioat_dma/ioat_dma_ring.c
usr/drivers/ioat_dma/ioat_dma_ring.h
usr/drivers/ioat_dma/main.c

index d3facee..f6fa9b2 100644 (file)
 
 device ioat_dma_chan msbfirst ( addr chan_base ) "IOAT DMA (Crystal Beach) Channel registers" {
 
+    constants cmpl_status_shift "Completion Status Shift Value" {
+        compl_addr_shift = 0x6 "Completion addrss shift";
+    };
+    
+    constants cmpl_status_mask width(64) "Completion Status Mask Value" {
+        status_mask = 0x3f "Mask for the status fields";
+    };
+    
+    datatype compl_status msbfirst (64) "Completion writeback data area" {
+        compl_desc 58 "Completed Descriptor Address";
+        _           1 "Reserved";
+        soft_err    1 "Software Error";
+        unaff_err   1 "Unaffiliated error";
+        status      3 "Channel status field";
+    };
+    
     constants chanctrl_snoop "Field values for Snoop Control" {
         chanctrl_snoop_disabled = 0x1 "Disabled snooping";
         chanctrl_snoop_enabled  = 0x0 "Enable snooping";
index e31fb15..2d2db50 100755 (executable)
@@ -1064,6 +1064,7 @@ errors xeon_phi XEON_PHI_ERR_ {
 errors ioat IOAT_ERR_ {
     failure PCI_ADDRESS           "The PCI address of the device is not as expected",
     failure DEVICE_UNSUPPORTED    "Device ID not supported /  wrong configuration",
+    failure DEVICE_IDLE           "The device is idle, no transfers finished",
     failure ARG_INVALID           "Supplied argument was not valid",
     failure CHAN_ERROR            "Hardware channel error",
     failure NO_DESCRIPTORS        "There are too less descriptors available",
index 74dd279..4ba402f 100644 (file)
@@ -37,7 +37,7 @@
 #define IOINT_DEBUG(x...)
 #endif
 #if IODEBUG_CHAN
-#define IOCHAN_DEBUG(x...) IODEBUG("[chan] " x)
+#define IOCHAN_DEBUG(x...) IODEBUG("[chan.%04x] " x)
 #else
 #define IOCHAN_DEBUG(x...)
 #endif
index 1004aff..a72689e 100644 (file)
 #include "ioat_dma_channel.h"
 #include "ioat_dma_ring.h"
 #include "ioat_dma_request.h"
+#include "ioat_dma_descriptors.h"
 
 #include "debug.h"
 
-enum ioat_dma_chan_st
-{
-    IOAT_DMA_CHAN_ST_INVALID,
-    IOAT_DMA_CHAN_ST_COMPL_PENDING,
-    IOAT_DMA_CHAN_ST_COMPL_ACK,
-    IOAT_DMA_CHAN_ST_RESETTING,
-    IOAT_DMA_CHAN_ST_INIT_FAIL,
-    IOAT_DMA_CHAN_ST_RESHAPING,
-    IOAT_DMA_CHAN_ST_RUNNING,
-    IOAT_DMA_CHAN_ST_ACTIVE
-};
-
 struct ioat_dma_channel
 {
-    ioat_dma_chan_id_t chan_id;      ///< unique channel id
+    ioat_dma_chan_id_t id;           ///< unique channel id
     ioat_dma_chan_t channel;         ///< Mackerel address
     struct ioat_dma_device *dev;     ///< the DMA device this channel belongs to
     size_t max_xfer_size;            ///< maximum transfer size of this channel
@@ -72,11 +61,13 @@ errval_t ioat_dma_channel_irq_setup_msix(struct ioat_dma_channel *chan)
 
 errval_t ioat_dma_channel_reset(struct ioat_dma_channel *chan)
 {
+    chan->state = IOAT_DMA_CHAN_ST_RESETTING;
+
     /* mask possible errors */
     ioat_dma_chan_err_t chanerr = ioat_dma_chan_err_rd(&chan->channel);
     ioat_dma_chan_err_wr(&chan->channel, chanerr);
 
-    IOCHAN_DEBUG("Reseting channel %x. Chanerrors=[%08x]\n", chan->chan_id, chanerr);
+    IOCHAN_DEBUG("Reseting channel. Chanerrors=[%08x]\n", chan->id, chanerr);
 
     /* TODO: Clear any pending errors in pci config space */
 #if 0
@@ -100,6 +91,8 @@ errval_t ioat_dma_channel_reset(struct ioat_dma_channel *chan)
         thread_yield();
     }
 
+    chan->state = IOAT_DMA_CHAN_ST_UNINITIALEZED;
+
     /* broadwell may need some additional work here */
     return SYS_ERR_OK;
 }
@@ -117,7 +110,7 @@ errval_t ioat_dma_channel_init(struct ioat_dma_device *dev)
     errval_t err;
 
     IOCHAN_DEBUG("Initializing %u channels max. xfer size is %u bytes\n",
-                 dev->chan_num, dev->xfer_size_max);
+                 0, dev->chan_num, dev->xfer_size_max);
 
     dev->channels = calloc(dev->chan_num, sizeof(struct ioat_dma_channel));
     if (dev->channels == NULL) {
@@ -130,7 +123,7 @@ errval_t ioat_dma_channel_init(struct ioat_dma_device *dev)
 
     for (uint8_t i = 0; i < dev->chan_num; ++i) {
         struct ioat_dma_channel *chan = dev->channels + i;
-        chan->chan_id = (((uint16_t) dev->devid + 1) << 8) | i;
+        chan->id = (((uint16_t) dev->devid + 1) << 8) | i;
         chan->dev = dev;
         chan->max_xfer_size = dev->xfer_size_max;
 
@@ -146,13 +139,14 @@ errval_t ioat_dma_channel_init(struct ioat_dma_device *dev)
         }
 
         err = ioat_dma_mem_alloc(IOAT_DMA_CHANNEL_COMPL_SIZE,
-        IOAT_DMA_CHANNEL_COMPL_FLAGS,
+                                 IOAT_DMA_CHANNEL_COMPL_FLAGS,
                                  &chan->completion);
         if (err_is_fail(err)) {
             return err;
         }
 
         memset(chan->completion.addr, 0, IOAT_DMA_CHANNEL_COMPL_SIZE);
+
         /* write the completion address */
         ioat_dma_chan_cmpl_lo_wr(&chan->channel, chan->completion.paddr);
         ioat_dma_chan_cmpl_hi_wr(&chan->channel, chan->completion.paddr >> 32);
@@ -170,11 +164,13 @@ errval_t ioat_dma_channel_init(struct ioat_dma_device *dev)
         chan_ctrl = ioat_dma_chan_ctrl_intp_dis_insert(chan_ctrl, 0x1);
         ioat_dma_chan_ctrl_wr(&chan->channel, chan_ctrl);
 
+        chan->state = IOAT_DMA_CHAN_ST_PREPARED;
+
         /*
          * do a check if the channel operates correctly by issuing a NOP
          */
 
-        IOCHAN_DEBUG("Performing selftest on channel with NOP\n");
+        IOCHAN_DEBUG("performing selftest on channel with NOP\n", chan->id);
 
         ioat_dma_request_nop(chan);
         err = ioat_dma_channel_submit_pending(chan);
@@ -190,12 +186,12 @@ errval_t ioat_dma_channel_init(struct ioat_dma_device *dev)
                  && !ioat_dma_channel_is_idle(chan));
 
         if (ioat_dma_channel_is_active(chan) || ioat_dma_channel_is_idle(chan)) {
-            IOCHAN_DEBUG("Channel worked properly: %016lx\n",
+            IOCHAN_DEBUG("channel worked properly: %016lx\n", chan->id,
                          *(uint64_t* )chan->completion.addr);
             return SYS_ERR_OK;
         } else {
             uint32_t error = ioat_dma_chan_err_rd(&chan->channel);
-            IOCHAN_DEBUG("Channel error ERROR: %08x\n", error);
+            IOCHAN_DEBUG(" channel error ERROR: %08x\n", chan->id, error);
             ioat_dma_mem_free(&chan->completion);
             return IOAT_ERR_CHAN_ERROR;
         }
@@ -206,7 +202,7 @@ errval_t ioat_dma_channel_init(struct ioat_dma_device *dev)
 
 ioat_dma_chan_id_t ioat_dma_channel_get_id(struct ioat_dma_channel *chan)
 {
-    return chan->chan_id;
+    return chan->id;
 }
 
 /**
@@ -224,15 +220,36 @@ struct ioat_dma_channel *ioat_dma_channel_get(struct ioat_dma_device *dev)
     return &dev->channels[dev->chan_next++];
 }
 
+/**
+ * \brief returns a channel to be used form the give device based on the channel
+ *        index
+ *
+ * \param dev IOAT DMA device
+ * \param idx channel index
+ *
+ * \returns IOAT DMA Channel
+ *          NULL if the index exceeds the number of channels
+ */
+struct ioat_dma_channel *ioat_dma_channel_get_by_idx(struct ioat_dma_device *dev,
+                                                     uint8_t idx)
+{
+    if (idx < dev->chan_num) {
+        return &dev->channels[idx];
+    }
+    return NULL;
+}
+
 struct ioat_dma_desc_alloc *ioat_dma_channel_get_desc_alloc(struct ioat_dma_channel *chan)
 {
     return chan->dev->dma_ctrl->alloc;
 }
 
-static inline void channel_set_chain_addr(struct ioat_dma_channel *chan,
-                                          lpaddr_t chain_addr)
+static inline void channel_set_chain_addr(struct ioat_dma_channel *chan)
 {
-    IOCHAN_DEBUG("  setting chain addr to [%016lx]\n", chain_addr);
+    lpaddr_t chain_addr = ioat_dma_ring_get_chain_addr(chan->ring);
+
+    IOCHAN_DEBUG(" setting chain addr to [%016lx]\n", chan->id, chain_addr);
+
     ioat_dma_chan_chainaddr_lo_wr(&chan->channel, (uint32_t) chain_addr);
     ioat_dma_chan_chainaddr_hi_wr(&chan->channel, chain_addr >> 32);
 }
@@ -277,20 +294,59 @@ inline bool ioat_dma_channel_is_suspended(struct ioat_dma_channel *chan)
     return tr_st == ioat_dma_chan_trans_state_susp;
 }
 
+static inline lpaddr_t channel_get_completion_addr(struct ioat_dma_channel *chan)
+{
+    lpaddr_t compl_addr = ioat_dma_chan_sts_hi_rd(&chan->channel);
+    compl_addr <<= 32;
+    compl_addr |= (ioat_dma_chan_sts_lo_rd(&chan->channel));
+
+    return (compl_addr & (~ioat_dma_chan_status_mask));
+}
+
+static inline lpaddr_t channel_has_completed_descr(struct ioat_dma_channel *chan)
+{
+    lpaddr_t curr_compl = channel_get_completion_addr(chan);
+    if (curr_compl != chan->last_completion) {
+        return curr_compl;
+    } else {
+        return 0;
+    }
+
+}
+
+errval_t ioat_dma_channel_start(struct ioat_dma_channel *chan)
+{
+    if (chan->state == IOAT_DMA_CHAN_ST_ERROR) {
+        return ioat_dma_channel_restart(chan);
+    }
+    IOCHAN_DEBUG(" starting channel.\n", chan->id);
+    chan->state = IOAT_DMA_CHAN_ST_RUNNING;
+    channel_set_chain_addr(chan);
+
+    return SYS_ERR_OK;
+}
+
+errval_t ioat_dma_channel_restart(struct ioat_dma_channel *chan)
+{
+    return SYS_ERR_OK;
+}
+
 uint16_t ioat_dma_channel_submit_pending(struct ioat_dma_channel *chan)
 {
+    errval_t err;
+
     uint16_t pending = ioat_dma_ring_get_pendig(chan->ring);
 
-    IOCHAN_DEBUG("Submitting %u pending descriptors to channel 0x%04x\n", pending,
-                 chan->chan_id);
-    if (pending > 0) {
-        lpaddr_t chain_addr = ioat_dma_ring_get_chain_addr(chan->ring);
-        channel_set_chain_addr(chan, chain_addr);
+    IOCHAN_DEBUG(" submitting %u pending desc\n", chan->id, pending);
 
-        uint16_t head = ioat_dma_ring_submit_pending(chan->ring);
+    if (chan->state != IOAT_DMA_CHAN_ST_RUNNING) {
+        err = ioat_dma_channel_start(chan);
+    }
+    if (pending > 0) {
+        uint16_t dmacnt = ioat_dma_ring_submit_pending(chan->ring);
+        ioat_dma_chan_dmacount_wr(&chan->channel, dmacnt);
 
-        IOCHAN_DEBUG("  setting dma_count to [%u]\n", head);
-        ioat_dma_chan_dmacount_wr(&chan->channel, head);
+        IOCHAN_DEBUG(" setting dma_count to [%u]\n", chan->id, dmacnt);
     }
 
     return pending;
@@ -309,8 +365,8 @@ inline uint32_t ioat_dma_channel_get_max_xfer_size(struct ioat_dma_channel *chan
 void ioat_dma_channel_enq_request(struct ioat_dma_channel *chan,
                                   struct ioat_dma_request *req)
 {
-    IOCHAN_DEBUG("Enqueuing request [%016lx] on channel [%04x]\n", req->id,
-                 chan->chan_id);
+    IOCHAN_DEBUG(" request : enq tail [%016lx]\n",chan->id,  req->id);
+
     req->next = NULL;
     if (chan->req_head == NULL) {
         chan->req_head = req;
@@ -319,6 +375,8 @@ void ioat_dma_channel_enq_request(struct ioat_dma_channel *chan,
         chan->req_tail->next = req;
         chan->req_tail = req;
     }
+
+    ioat_dma_channel_submit_pending(chan);
 }
 
 static inline struct ioat_dma_request *channel_deq_request_head(struct ioat_dma_channel *chan)
@@ -331,6 +389,9 @@ static inline struct ioat_dma_request *channel_deq_request_head(struct ioat_dma_
     if (chan->req_head == NULL) {
         chan->req_tail = NULL;
     }
+
+    IOCHAN_DEBUG(" request : deq head [%016lx]\n",chan->id,  req->id);
+
     return req;
 }
 
@@ -342,38 +403,97 @@ static inline void channel_enq_request_head(struct ioat_dma_channel *chan,
     if (chan->req_tail == NULL) {
         chan->req_tail = req;
     }
+
+    IOCHAN_DEBUG(" request : enq head [%016lx]\n",chan->id,  req->id);
 }
 
-errval_t ioat_dma_channel_poll(struct ioat_dma_channel *chan)
+static errval_t channel_process_descriptors(struct ioat_dma_channel *chan,
+                                            lpaddr_t compl_addr_phys)
 {
     errval_t err;
 
-    /* process finished descriptors */
-    if (chan->req_head == NULL) {
+    if (!compl_addr_phys) {
         return IOAT_ERR_CHAN_IDLE;
     }
 
-    uint32_t compl_count = 0;
+    IOCHAN_DEBUG(" processing [%016lx] head: %u, tail: %u, issued: %u\n",
+                 chan->id, compl_addr_phys, ioat_dma_ring_get_head(chan->ring),
+                 ioat_dma_ring_get_tail(chan->ring),
+                 ioat_dma_ring_get_issued(chan->ring));
 
-    do {
-        struct ioat_dma_request *req = channel_deq_request_head(chan);
-        err = ioat_dma_request_process(req);
-        if (err_is_fail(err)) {
-            channel_enq_request_head(chan, req);
-            if (err_no(err) != IOAT_ERR_REQUEST_UNFINISHED) {
+    uint16_t active_count = ioat_dma_ring_get_active(chan->ring);
+
+    struct ioat_dma_descriptor *desc;
+    struct ioat_dma_request *req, *req_head;
+
+    uint16_t processed = 0;
+    uint8_t request_done = 0;
+
+    for (uint16_t i = 0; i < active_count; i++) {
+        desc = ioat_dma_ring_get_tail_desc(chan->ring);
+
+        /*
+         * check if there is a request associated with the descriptor
+         * this indicates the last descriptor of a request
+         */
+        req = ioat_dma_desc_get_request(desc);
+        if (req) {
+            req_head = channel_deq_request_head(chan);
+            assert(req_head == req);
+            err = ioat_dma_request_process(req);
+            if (err_is_fail(err)) {
+                channel_enq_request_head(chan, req_head);
                 return err;
             }
+            request_done = 1;
+        }
+
+        /* this was the last completed descriptor */
+        if (ioat_dma_desc_get_paddr(desc) == compl_addr_phys) {
+            processed = i;
             break;
         }
-        compl_count++;
-    } while(chan->req_head == NULL);
+    }
 
-    if (compl_count) {
+    chan->last_completion = compl_addr_phys;
+
+    /* do a 5us delay per pending descriptor */
+    ioat_dma_device_set_intr_delay(chan->dev, (5 * active_count - processed));
+
+    if (request_done) {
         return SYS_ERR_OK;
-    } else {
+    }
+
+    return IOAT_ERR_REQUEST_UNFINISHED;
+}
+
+errval_t ioat_dma_channel_poll(struct ioat_dma_channel *chan)
+{
+    errval_t err;
+
+    if (ioat_dma_channel_is_halted(chan)) {
+        IOCHAN_DEBUG("channel is in error state\n", chan->id);
+    }
+
+    /* process finished descriptors */
+    if (chan->req_head == NULL) {
         return IOAT_ERR_CHAN_IDLE;
     }
 
+    lpaddr_t compl_addr_phys = channel_has_completed_descr(chan);
+    if (!compl_addr_phys) {
+        return IOAT_ERR_CHAN_IDLE;
+    }
 
+    err = channel_process_descriptors(chan, compl_addr_phys);
+    switch (err_no(err)) {
+        case SYS_ERR_OK:
+            /* this means we processed a descriptor request */
+            return SYS_ERR_OK;
+        case IOAT_ERR_REQUEST_UNFINISHED:
+            return IOAT_ERR_CHAN_IDLE;
+        default:
+            return err;
+    }
 }
 
index 0c75599..ac1ea12 100644 (file)
@@ -19,6 +19,19 @@ struct ioat_dma_request;
 
 typedef uint16_t ioat_dma_chan_id_t;
 
+enum ioat_dma_chan_st
+{
+    IOAT_DMA_CHAN_ST_INVALID,
+    IOAT_DMA_CHAN_ST_RESETTING,
+    IOAT_DMA_CHAN_ST_UNINITIALEZED,
+    IOAT_DMA_CHAN_ST_PREPARED,
+    IOAT_DMA_CHAN_ST_RUNNING,
+    IOAT_DMA_CHAN_ST_ERROR,
+    IOAT_DMA_CHAN_ST_SUSPENDED,
+
+    IOAT_DMA_CHAN_ST_INIT_FAIL,
+};
+
 /**
  * \brief initializes a new DMA channel and allocates the channel resources
  *
@@ -36,6 +49,18 @@ void ioat_dma_chan_free(struct ioat_dma_channel *chan);
 errval_t ioat_dma_channel_irq_setup_msix(struct ioat_dma_channel *chan);
 
 
+/**
+ * \brief returns a channel to be used form the give device based on the channel
+ *        index
+ *
+ * \param dev IOAT DMA device
+ * \param idx channel index
+ *
+ * \returns IOAT DMA Channel
+ *          NULL if the index exceeds the number of channels
+ */
+struct ioat_dma_channel *ioat_dma_channel_get_by_idx(struct ioat_dma_device *dev,
+                                                     uint8_t idx);
 
 /*
  * ---
@@ -56,6 +81,10 @@ struct ioat_dma_desc_alloc *ioat_dma_channel_get_desc_alloc(struct ioat_dma_chan
 
 errval_t ioat_dma_channel_poll(struct ioat_dma_channel *chan);
 
+errval_t ioat_dma_channel_start(struct ioat_dma_channel *chan);
+
+errval_t ioat_dma_channel_restart(struct ioat_dma_channel *chan);
+
 uint16_t ioat_dma_channel_submit_pending(struct ioat_dma_channel *chan);
 
 bool ioat_dma_channel_is_active(struct ioat_dma_channel *chan);
index 30bde95..a213025 100644 (file)
@@ -216,7 +216,7 @@ static void device_init(struct device_mem* bar_info,
             curr_err = IOAT_ERR_DEVICE_UNSUPPORTED;
     }
 
-    if (err_is_fail(err)) {
+    if (err_is_fail(curr_err)) {
         curr_dev->state = IOAT_DMA_DEV_ST_ERR;
         return;
     }
@@ -376,3 +376,38 @@ errval_t ioat_dma_device_discovery(struct pci_addr addr,
     return SYS_ERR_OK;
 }
 
+inline void ioat_dma_device_set_intr_delay(struct ioat_dma_device *dev,
+                                           uint16_t usec)
+{
+    ioat_dma_intrdelay_delay_us_wrf(&dev->device, usec);
+}
+
+errval_t ioat_dma_device_poll_channels(struct ioat_dma_device *dev)
+{
+    errval_t err;
+
+    uint8_t idle = 0x1;
+
+    for (uint8_t i = 0; i < dev->chan_num; ++i) {
+        struct ioat_dma_channel *chan;
+        chan = ioat_dma_channel_get_by_idx(dev, i);
+        assert(chan);
+        err = ioat_dma_channel_poll(chan);
+        switch(err_no(err)) {
+            case IOAT_ERR_CHAN_IDLE:
+                idle = idle && 0x1;
+                break;
+            case SYS_ERR_OK:
+                idle = 0;
+                break;
+            default:
+                return err;
+        }
+    }
+
+    if (idle) {
+        return IOAT_ERR_DEVICE_IDLE;
+    }
+
+    return SYS_ERR_OK;
+}
index 4afe2b8..e09709b 100644 (file)
@@ -119,4 +119,7 @@ errval_t ioat_dma_device_discovery(struct pci_addr addr,
 
 errval_t ioat_dma_device_poll_channels(struct ioat_dma_device *dev);
 
+void ioat_dma_device_set_intr_delay(struct ioat_dma_device *dev,
+                                    uint16_t usec);
+
 #endif /* IOAT_DMA_CHANNEL_H */
index f321e84..96f4c86 100644 (file)
@@ -23,7 +23,6 @@
 
 #include "debug.h"
 
-
 /*
  * ---------------------------------------------------------------------------
  * Request Management
@@ -39,21 +38,20 @@ static struct ioat_dma_request *request_alloc(void)
     if (req_free_list) {
         ret = req_free_list;
         req_free_list = ret->next;
+
+        IOREQ_DEBUG("meta: reusing request %p. freelist:%p\n", ret, req_free_list);
+
         return ret;
     }
-    ret = calloc(1, sizeof(*ret));
-
-    return ret;
+    return calloc(1, sizeof(*ret));
 }
 
-/*
 static void request_free(struct ioat_dma_request *req)
 {
+    IOREQ_DEBUG("meta: freeing request %p.\n", req);
     req->next = req_free_list;
     req_free_list = req;
 }
-*/
-
 
 /*
  * ---------------------------------------------------------------------------
@@ -69,7 +67,6 @@ static ioat_dma_req_id_t generate_req_id(struct ioat_dma_channel *chan)
     return (id << 48) | (req_counter++);
 }
 
-
 /*
  * ---------------------------------------------------------------------------
  * Helper Functions
@@ -93,18 +90,22 @@ inline static uint32_t req_num_desc_needed(struct ioat_dma_channel *chan,
 errval_t ioat_dma_request_memcpy_channel(struct ioat_dma_channel *chan,
                                          struct ioat_dma_req_setup *setup)
 {
-       uint32_t num_desc = req_num_desc_needed(chan, setup->bytes);
+    uint32_t num_desc = req_num_desc_needed(chan, setup->bytes);
 
     IOREQ_DEBUG("DMA Memcpy request: [0x%016lx]->[0x%016lx] of %lu bytes (%u desc)\n",
                 setup->src, setup->dst, setup->bytes, num_desc);
 
     struct ioat_dma_ring *ring = ioat_dma_channel_get_ring(chan);
+
     if (num_desc > ioat_dma_ring_get_space(ring)) {
+        IOREQ_DEBUG("Too less space in ring: %u / %u\n", num_desc,
+                    ioat_dma_ring_get_space(ring));
         return IOAT_ERR_NO_DESCRIPTORS;
     }
 
     struct ioat_dma_request *req = request_alloc();
     if (req == NULL) {
+        IOREQ_DEBUG("No request descriptors for holding request data\n");
         return IOAT_ERR_NO_REQUESTS;
     }
 
@@ -123,7 +124,7 @@ errval_t ioat_dma_request_memcpy_channel(struct ioat_dma_channel *chan,
         if (!req->desc_head) {
             req->desc_head = desc;
         }
-        if (length < max_xfer_size) {
+        if (length <= max_xfer_size) {
             /* the last one */
             bytes = length;
             req->desc_tail = desc;
@@ -136,16 +137,22 @@ errval_t ioat_dma_request_memcpy_channel(struct ioat_dma_channel *chan,
         }
 
         ioat_dma_desc_fill_memcpy(desc, src, dst, bytes, ctrl);
-        ioat_dma_desc_set_request(desc, req);
+        ioat_dma_desc_set_request(desc, NULL);
 
         length -= bytes;
         src += bytes;
         dst += bytes;
-    } while(length > 0);
+    } while (length > 0);
 
     req->setup = *setup;
     req->id = generate_req_id(chan);
 
+    /* set the request pointer in the last descriptor */
+    ioat_dma_desc_set_request(desc, req);
+
+    assert(req->desc_tail);
+    assert(ioat_dma_desc_get_request(req->desc_tail));
+
     ioat_dma_channel_enq_request(chan, req);
 
     return SYS_ERR_OK;
@@ -173,5 +180,24 @@ void ioat_dma_request_nop(struct ioat_dma_channel *chan)
 
 errval_t ioat_dma_request_process(struct ioat_dma_request *req)
 {
+    IOREQ_DEBUG("Processing done request [%016lx]:\n", req->id);
+
+    errval_t err;
+
+    switch (req->state) {
+        case IOAT_DMA_REQ_ST_DONE:
+            err = SYS_ERR_OK;
+            break;
+        default:
+            err = -1;  // todo: error code
+            break;
+    }
+
+    if (req->setup.done_cb) {
+        req->setup.done_cb(err, req->id, req->setup.arg);
+    }
+
+    request_free(req);
+
     return SYS_ERR_OK;
 }
index 0e7c5c8..a6c44ce 100644 (file)
 
 struct ioat_dma_request;
 
+typedef uint64_t ioat_dma_req_id_t;
+
 /// callback to be called when the request is finished
-typedef void (*ioat_dma_cb_t)(void);
+typedef void (*ioat_dma_cb_t)(errval_t, ioat_dma_req_id_t, void *);
+
 
-typedef uint64_t ioat_dma_req_id_t;
 
 enum ioat_dma_req_type
 {
@@ -49,6 +51,7 @@ struct ioat_dma_req_setup
 struct ioat_dma_request
 {
     ioat_dma_req_id_t id;
+    enum ioat_dma_req_state state;
     struct ioat_dma_req_setup setup;
     struct ioat_dma_descriptor *desc_head;
     struct ioat_dma_descriptor *desc_tail;
@@ -69,4 +72,12 @@ errval_t ioat_dma_request_memcpy(struct ioat_dma_device *dev,
 
 void ioat_dma_request_nop(struct ioat_dma_channel *chan);
 
+static inline bool ioat_dma_request_is_last(struct ioat_dma_request *req,
+                                            struct ioat_dma_descriptor *desc)
+{
+    return (req->desc_tail == desc);
+}
+
+
+
 #endif /* IOAT_DMA_REQUEST_H */
index ec72fe3..702379e 100644 (file)
@@ -29,7 +29,7 @@ struct ioat_dma_ring
     uint16_t head;          ///< allocated index
     uint16_t issued;        ///< hardware notification point
     uint16_t tail;          ///< cleanup index
-    uint16_t dmacount;      ///< identical to 'head' except for occasionally resetting to zero
+    uint16_t dmacount;  ///< identical to 'head' except for occasionally resetting to zero
     uint16_t alloc_order;   ///< log2 of the number of allocated descriptors
     uint16_t produce;       ///< number of descriptors to produce at submit time
     struct ioat_dma_descriptor **desc;  ///< descriptor pointer array
@@ -73,7 +73,7 @@ errval_t ioat_dma_ring_alloc(uint8_t size_bits,
 
     ring->chan = chan;
     ring->size = ndesc;
-    ring->desc = (void *)(ring + 1);
+    ring->desc = (void *) (ring + 1);
 
     err = ioat_dma_desc_alloc(IOAT_DMA_DESC_SIZE, IOAT_DMA_DESC_ALIGN, ndesc,
                               ring->desc);
@@ -113,6 +113,21 @@ inline uint16_t ioat_dma_ring_get_pendig(struct ioat_dma_ring *ring)
     return (ring->head - ring->issued) & (ring->size - 1);
 }
 
+inline uint16_t ioat_dma_ring_get_head(struct ioat_dma_ring *ring)
+{
+    return ring->head;
+}
+
+inline uint16_t ioat_dma_ring_get_tail(struct ioat_dma_ring *ring)
+{
+    return ring->tail;
+}
+
+inline uint16_t ioat_dma_ring_get_issued(struct ioat_dma_ring *ring)
+{
+    return ring->issued;
+}
+
 /*
  * ----------------------------------------------------------------------------
  * Ring Manipulation
@@ -120,7 +135,8 @@ inline uint16_t ioat_dma_ring_get_pendig(struct ioat_dma_ring *ring)
  */
 
 /**
- * \brief gets the next descriptor based on the head pointer
+ * \brief gets the next descriptor based on the head pointer used for issuing
+ *        new DMA requests
  *
  * \param ring the DMA ring
  *
@@ -128,7 +144,29 @@ inline uint16_t ioat_dma_ring_get_pendig(struct ioat_dma_ring *ring)
  */
 inline struct ioat_dma_descriptor *ioat_dma_ring_get_next_desc(struct ioat_dma_ring *ring)
 {
-    return ioat_dma_ring_get_desc(ring, ring->head++);
+    struct ioat_dma_descriptor *desc = ioat_dma_ring_get_desc(ring, ring->head++);
+
+    IODESC_DEBUG("ring getting next head desc:%p @ [%016lx], new head:%u\n", desc,
+                 ioat_dma_desc_get_paddr(desc), ring->head);
+
+    return desc;
+}
+
+/**
+ * \brief gets a pointer to the tail request used for processing the done queue
+ *
+ * \param ring the DMA ring
+ *
+ * \returns pointer to a DMA descriptor
+ */
+inline struct ioat_dma_descriptor *ioat_dma_ring_get_tail_desc(struct ioat_dma_ring *ring)
+{
+    struct ioat_dma_descriptor *desc = ioat_dma_ring_get_desc(ring, ring->tail++);
+
+    IODESC_DEBUG("ring getting tail desc:%p @ [%016lx], new tail: %u\n", desc,
+                 ioat_dma_desc_get_paddr(desc), ring->tail);
+
+    return desc;
 }
 
 /**
index 9b43b94..773bc5f 100644 (file)
@@ -71,6 +71,12 @@ static inline uint16_t ioat_dma_ring_get_space(struct ioat_dma_ring *ring)
     return ioat_dma_ring_get_size(ring) - ioat_dma_ring_get_active(ring);
 }
 
+uint16_t ioat_dma_ring_get_head(struct ioat_dma_ring *ring);
+
+uint16_t ioat_dma_ring_get_tail(struct ioat_dma_ring *ring);
+
+uint16_t ioat_dma_ring_get_issued(struct ioat_dma_ring *ring);
+
 
 /*
  * ----------------------------------------------------------------------------
@@ -98,6 +104,8 @@ struct ioat_dma_descriptor *ioat_dma_ring_get_next_desc(struct ioat_dma_ring *ri
 struct ioat_dma_descriptor *ioat_dma_ring_get_desc(struct ioat_dma_ring *ring,
                                                    uint16_t index);
 
+struct ioat_dma_descriptor *ioat_dma_ring_get_tail_desc(struct ioat_dma_ring *ring);
+
 /**
  * \brief submits the pending descriptors to the hardware
  *
index 4ee0e5d..90783c1 100644 (file)
 
 struct ioat_dma_ctrl dma_ctrl;
 
-#define BUFFER_SIZE (1<<21)
+#define BUFFER_SIZE (1<<22)
+
+static void impl_test_cb(errval_t err, ioat_dma_req_id_t id, void *arg)
+{
+    debug_printf("impl_test_cb\n");
+    assert(memcmp(arg, arg + BUFFER_SIZE, BUFFER_SIZE) == 0);
+    debug_printf("test ok\n");
+}
 
 static void impl_test(void)
 {
@@ -41,16 +48,32 @@ static void impl_test(void)
     err = invoke_frame_identify(frame, &id);
     assert(err_is_ok(err));
 
+    void *buf;
+    err = vspace_map_one_frame(&buf, 1UL << id.bits, frame, NULL, NULL);
+    assert(err_is_ok(err));
+
+    memset(buf, 0, 1UL << id.bits);
+    memset(buf, 0xA5, BUFFER_SIZE);
+
     struct ioat_dma_req_setup setup = {
         .type = IOAT_DMA_REQ_TYPE_MEMCPY,
         .src = id.base,
         .dst = id.base + BUFFER_SIZE,
         .bytes = BUFFER_SIZE,
+        .done_cb = impl_test_cb,
+        .arg = buf
     };
-
-    err = ioat_dma_request_memcpy(dma_ctrl.devices, &setup);
-    assert(err_is_ok(err));
-
+    int reps = 10;
+    do {
+        debug_printf("!!!!!! NEW ROUND\n");
+        err = ioat_dma_request_memcpy(dma_ctrl.devices, &setup);
+        assert(err_is_ok(err));
+
+        uint32_t i = 10;
+        while(i--) {
+            ioat_dma_device_poll_channels(dma_ctrl.devices);
+        }
+    }while(reps--);
 }
 
 int main(int argc,