DeviceQueue: Direct Solarflare queue implementation
authorRoni Häcki <roni.haecki@inf.ethz.ch>
Thu, 8 Sep 2016 18:09:59 +0000 (20:09 +0200)
committerRoni Häcki <roni.haecki@inf.ethz.ch>
Thu, 8 Sep 2016 18:09:59 +0000 (20:09 +0200)
Signed-off-by: Roni Häcki <roni.haecki@inf.ethz.ch>

errors/errno.fugu
if/Hakefile
if/sfn5122f_devif.if [new file with mode: 0644]
include/devif/queue_interface.h
include/devif/sfn5122f_devif_direct.h [new file with mode: 0644]
usr/drivers/solarflare/Hakefile
usr/drivers/solarflare/sfn5122f.h
usr/drivers/solarflare/sfn5122f_cdriver.c
usr/drivers/solarflare/sfn5122f_devif_direct.c [new file with mode: 0644]
usr/tests/devif/Hakefile
usr/tests/devif/sfn5122f_device.c [new file with mode: 0644]

index c2b4953..1b0067d 100755 (executable)
@@ -1241,5 +1241,7 @@ errors sfn SFN_ERR_{
     failure RX_PKT               "Error receiving packet",
     failure RX_DISCARD           "Error, packet needs to be discared",
     failure ALLOC_BUF            "Error allocating buffer",
+    failure REGISTER_REGION      "Error registering a region",
+    failure ALLOC_QUEUE          "Failure allocating queue",
 };
 
index b6fcf21..951c3bd 100644 (file)
@@ -92,7 +92,8 @@
                "devif_ctrl",
                "devif_data",
                "xomp_gateway",
-               "sfn5122f"
+               "sfn5122f",
+               "sfn5122f_devif"
            ],
              arch <- allArchitectures
 ] ++
diff --git a/if/sfn5122f_devif.if b/if/sfn5122f_devif.if
new file mode 100644 (file)
index 0000000..4f5cbed
--- /dev/null
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2007-2011, ETH Zurich.
+ * All rights reserved.
+ *
+ * This file is distributed under the terms in the attached LICENSE file.
+ * If you do not find this file, copies can be found by writing to:
+ * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
+ */
+
+/*
+ * This interface is used in the sfn5122f driver for the device specific
+ * communication between the device manager and the setup for the devif device
+ * interface. The driver exposes the interface and the devif solarflare device
+ * specific part connects to it
+ */
+interface sfn5122f_devif "sfn5122f devif communication interface" {
+    
+    // create and destroy a queue. Only the device driver itself should
+    // access configuration registers
+    rpc create_queue(in cap rx, in cap tx, in cap ev, 
+                     out uint16 qid, out cap regs, out errval err);
+    rpc destroy_queue(in uint16 qid, out errval err);
+
+    // add a memory region to the buffer table
+    rpc register_region(in uint16 qid, in cap reg, out uint64 buftbl_id, out errval err);
+
+};
index dcb1061..69bde31 100644 (file)
 // only internal?
 #define ENDPOINT_TYPE_FORWARD_TX 0x11
 
+
+#define DEVQ_BUF_FLAG_TX 0x1
+#define DEVQ_BUF_FLAG_RX 0x2
+
+
 typedef uint32_t regionid_t;
 typedef uint32_t bufferid_t;
 
diff --git a/include/devif/sfn5122f_devif_direct.h b/include/devif/sfn5122f_devif_direct.h
new file mode 100644 (file)
index 0000000..fa79b48
--- /dev/null
@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2016 ETH Zurich.
+ * All rights reserved.
+ *
+ * This file is distributed under the terms in the attached LICENSE file.
+ * If you do not find this file, copies can be found by writing to:
+ * ETH Zurich D-INFK, Universitaetstr. 6, CH-8092 Zurich. Attn: Systems Group.
+ */
+#ifndef SFN5122F_DEVIF_DIRECT_H_
+#define SFN5122F_DEVIF_DIRECT_H_ 1
+
+#include <devif/queue_interface.h>
+
+errval_t sfn5122f_create_direct(struct devq* q, uint64_t flags);
+errval_t sfn5122f_register_direct(struct devq* q, struct capref cap,
+                                  regionid_t rid);
+errval_t sfn5122f_deregister_direct(struct devq* q, regionid_t rid);
+errval_t sfn5122f_control_direct(struct devq* q, uint64_t cmd, uint64_t value);
+errval_t sfn5122f_destroy_direct(struct devq* q);
+errval_t sfn5122f_notify_direct(struct devq* q, uint8_t num_slots);
+errval_t sfn5122f_enqueue_direct(struct devq* q, regionid_t rid, bufferid_t bid, 
+                                 lpaddr_t base, size_t len, uint64_t flags);
+errval_t sfn5122f_dequeue_direct(struct devq* q, regionid_t* rid, bufferid_t* bid, 
+                                 lpaddr_t* base, size_t* len, uint64_t* flags);
+#endif
index e00c4a9..ae412fb 100644 (file)
   build application { target = "sfn5122f",
                       cFiles = [ "sfn5122f_cdriver.c", "sfn5122f_qdriver.c", "mcdi_rpc.c", 
                                  "helper.c", "buffer_tbl.c"],
-                      flounderBindings = [ "sfn5122f", "net_ARP" ],
+                      flounderBindings = [ "sfn5122f", "net_ARP", "sfn5122f_devif"],
+                      flounderExtraBindings = [ ("sfn5122f_devif", ["rpcclient"]) ],
+                      flounderDefs = [ "sfn5122f", "sfn5122f_devif" ],
+                      flounderExtraDefs = [ ("sfn5122f_devif",["rpcclient"]) ],
                       mackerelDevices = [ "sfn5122f"],
                       addLibraries = libDeps["netQmng", "pci", "contmng", 
                                              "net_device_manager", "bench", "trace", "skb" ]
+                    }, 
+
+  build library { target = "sfn5122f_devif_direct",
+                      cFiles = [ "sfn5122f_devif_direct.c"],
+                      flounderBindings = [ "sfn5122f"],
+                      mackerelDevices = [ "sfn5122f", "sfn5122f_q"],
+                      addLibraries = libDeps ["netQmng", "pci", "net_device_manager",
+                                              "skb"]
                     }
   ]
 
index c7bef82..8365499 100644 (file)
@@ -22,6 +22,8 @@
 #include "mcdi_rpc.h"
 #include "helper.h"
 
+#define NUM_QUEUES 1024
+
 #define BUF_SIZE 4096
 #define DEVICE_ID 0x803
 // TX Queue
index ee1bf99..6f4013b 100644 (file)
@@ -17,6 +17,8 @@
 #include <ipv4/lwip/inet.h>
 #include <barrelfish/debug.h>
 #include <if/sfn5122f_defs.h>
+#include <if/sfn5122f_devif_defs.h>
+#include <if/sfn5122f_devif_rpcclient_defs.h>
 #include <if/net_ARP_rpcclient_defs.h>
 #include <if/net_ARP_defs.h>
 
@@ -31,6 +33,7 @@ struct queue_state {
     bool use_irq;
 
     struct sfn5122f_binding *binding;
+    struct sfn5122f_devif_binding *devif;
     struct capref tx_frame;
     struct capref rx_frame;
     struct capref ev_frame;
@@ -1371,6 +1374,95 @@ static struct sfn5122f_rx_vtbl rx_vtbl = {
 };
 
 
+static void cd_create_queue(struct sfn5122f_devif_binding *b, struct capref rx, struct capref tx,
+                            struct capref ev) 
+{
+    // Save state so we can restore the configuration in case we need to do a
+    // reset
+    errval_t err;
+    int n = -1;
+    for (int i = 0; i < NUM_QUEUES; i++) {
+        if (queues[i].enabled == false) {
+            n = i;
+            break;
+        }
+    }
+
+    if (n == -1) {
+        err = SFN_ERR_ALLOC_QUEUE;
+        err = b->tx_vtbl.create_queue_response(b, NOP_CONT, 0, NULL_CAP, err);
+        assert(err_is_ok(err));
+    }
+    
+    queues[n].enabled = false;
+    queues[n].tx_frame = tx;
+    queues[n].rx_frame = rx;
+    queues[n].ev_frame = ev;
+    queues[n].tx_head = 0;
+    queues[n].rx_head = 0;
+    queues[n].ev_head = 0;
+    queues[n].rxbufsz = MTU_MAX;
+    queues[n].devif = b;
+    queues[n].use_irq = false;
+    queues[n].userspace = true;
+    queues[n].msix_index = -1;
+
+    queues[n].ev_buf_tbl = init_evq(n);
+    // enable checksums
+    queues[n].tx_buf_tbl = init_txq(n, csum_offload, true);
+    queues[n].rx_buf_tbl = init_rxq(n, true);
+
+    if(queues[n].ev_buf_tbl == -1 ||
+       queues[n].tx_buf_tbl == -1 ||
+       queues[n].rx_buf_tbl == -1){
+        err = SFN_ERR_ALLOC_QUEUE;
+        err = b->tx_vtbl.create_queue_response(b, NOP_CONT, 0, NULL_CAP, err);
+        assert(err_is_ok(err));
+    }      
+
+    queues[n].enabled = true;
+    err = b->tx_vtbl.create_queue_response(b, NOP_CONT, n, *regframe, SYS_ERR_OK);
+    assert(err_is_ok(err));
+}
+
+static void cd_register_region(struct sfn5122f_devif_binding *b, uint16_t qid, struct capref region) 
+{
+    errval_t err;
+    struct frame_identity id;
+    uint64_t buffer_offset = 0;    
+
+    err = invoke_frame_identify(region, &id);
+    if (err_is_fail(err)) {
+        err = b->tx_vtbl.register_region_response(b, NOP_CONT, 0, SFN_ERR_REGISTER_REGION);
+        assert(err_is_ok(err));
+    }
+
+    size_t size = id.bytes;
+    lpaddr_t addr = id.base;
+
+    // TODO unsigned/signed not nice ...
+    buffer_offset = alloc_buf_tbl_entries(addr, size/BUF_SIZE, qid, true, d);
+    if (buffer_offset == -1) {
+        err = b->tx_vtbl.register_region_response(b, NOP_CONT, 0, SFN_ERR_REGISTER_REGION);
+        assert(err_is_ok(err));
+    }
+    
+    err = b->tx_vtbl.register_region_response(b, NOP_CONT, buffer_offset, SYS_ERR_OK);
+    assert(err_is_ok(err));
+}
+
+static void cd_destroy_queue(struct sfn5122f_devif_binding *b, uint16_t qid)
+{
+    USER_PANIC("NIY \n");
+} 
+
+
+static struct sfn5122f_devif_rx_vtbl rx_vtbl_devif = {
+    .create_queue_call = cd_create_queue,
+    .destroy_queue_call = cd_destroy_queue,
+    .register_region_call = cd_register_region,
+};
+
 static void export_cb(void *st, errval_t err, iref_t iref)
 {
     const char *suffix = "_sfn5122fmng";
@@ -1394,6 +1486,29 @@ static errval_t connect_cb(void *st, struct sfn5122f_binding *b)
     return SYS_ERR_OK;
 }
 
+static void export_devif_cb(void *st, errval_t err, iref_t iref)
+{
+    const char *suffix = "_sfn5122fmng_devif";
+    char name[strlen(service_name) + strlen(suffix) + 1];
+
+    assert(err_is_ok(err));
+
+    // Build label for interal management service
+    sprintf(name, "%s%s", service_name, suffix);
+
+    err = nameservice_register(name, iref);
+    assert(err_is_ok(err));
+    DEBUG("Devif Management interface exported\n");
+}
+
+
+static errval_t connect_devif_cb(void *st, struct sfn5122f_devif_binding *b)
+{
+    DEBUG("New connection on devif management interface\n");
+    b->rx_vtbl = rx_vtbl_devif;
+    return SYS_ERR_OK;
+}
+
 /**
  * Initialize management interface for queue drivers.
  * This has to be done _after_ the hardware is initialized.
@@ -1405,6 +1520,10 @@ static void initialize_mngif(void)
     r = sfn5122f_export(NULL, export_cb, connect_cb, get_default_waitset(),
                     IDC_BIND_FLAGS_DEFAULT);
     assert(err_is_ok(r));
+    
+    r = sfn5122f_devif_export(NULL, export_devif_cb, connect_devif_cb, get_default_waitset(),
+                       IDC_BIND_FLAGS_DEFAULT);
+    assert(err_is_ok(r));
 
 }
 
diff --git a/usr/drivers/solarflare/sfn5122f_devif_direct.c b/usr/drivers/solarflare/sfn5122f_devif_direct.c
new file mode 100644 (file)
index 0000000..8b72436
--- /dev/null
@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2007, 2008, 2009, 2010, 2011, 2012, ETH Zurich.
+ * All rights reserved.
+ *
+ * This file is distributed under the terms in the attached LICENSE file.
+ * If you do not find this file, copies can be found by writing to:
+ * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <time.h>
+#include <barrelfish/barrelfish.h>
+#include <barrelfish/waitset.h>
+#include <barrelfish/deferred.h>
+#include <devif/queue_interface.h>
+#include <devif/sfn5122f_devif_direct.h>
+#include "sfn5122f.h"
+#include "sfn5122f_queue.h"
+
+errval_t sfn5122f_create_direct(struct devq* q, uint64_t flags)
+{
+    struct capref tx_frame, rx_frame, ev_frame;
+    size_t tx_size, rx_size, ev_size;
+    void *tx_virt, *rx_virt, *ev_virt;
+    struct sfn5122f_queue* queue;
+
+    struct sfn5122f_queue_ops ops = {
+        .update_txtail = NULL,
+        .update_rxtail = NULL
+     };
+   
+    /* Allocate memory for descriptor rings  
+       No difference for userspace networking*/
+    tx_size = sfn5122f_q_tx_ker_desc_size * TX_ENTRIES;
+    tx_virt = alloc_map_frame(VREGION_FLAGS_READ_WRITE, tx_size, &tx_frame);
+    if (tx_virt == NULL) {
+        return SFN_ERR_ALLOC_QUEUE;
+    }
+
+    rx_size = sfn5122f_q_rx_user_desc_size * RX_ENTRIES;
+    rx_virt = alloc_map_frame(VREGION_FLAGS_READ_WRITE, rx_size, &rx_frame);
+    if (rx_virt == NULL) {
+        return SFN_ERR_ALLOC_QUEUE;
+    }
+
+    ev_size = sfn5122f_q_event_entry_size * EV_ENTRIES;
+    ev_virt = alloc_map_frame(VREGION_FLAGS_READ_WRITE, ev_size, &ev_frame);
+    if (ev_virt == NULL) {
+        return SFN_ERR_ALLOC_QUEUE;
+    }
+
+    queue = sfn5122f_queue_init(tx_virt, TX_ENTRIES, rx_virt, RX_ENTRIES,
+                                ev_virt, EV_ENTRIES, &ops,  NULL, true);
+
+ //   q->q = queue;
+    // TODO set queue state
+    return SYS_ERR_OK;
+}
+
+errval_t sfn5122f_register_direct(struct devq* q, struct capref cap,
+                                  regionid_t rid) 
+{
+   return SYS_ERR_OK;
+}
+
+errval_t sfn5122f_deregister_direct(struct devq* q, regionid_t rid) 
+{
+    return SYS_ERR_OK;
+}
+
+
+errval_t sfn5122f_control_direct(struct devq* q, uint64_t cmd, uint64_t value)
+{
+    return SYS_ERR_OK;
+}
+
+
+errval_t sfn5122f_notify_direct(struct devq* q, uint8_t num_slots)
+{
+    return SYS_ERR_OK;
+}
+
+errval_t sfn5122f_destroy_direct(struct devq* q)
+{
+    return SYS_ERR_OK;
+}
+
+
+errval_t sfn5122f_enqueue_direct(struct devq* q, regionid_t rid, bufferid_t bid, 
+                                 lpaddr_t base, size_t len, uint64_t flags)
+{
+    return SYS_ERR_OK;
+}
+
+errval_t sfn5122f_dequeue_direct(struct devq* q, regionid_t* rid, bufferid_t* bid, 
+                                 lpaddr_t* base, size_t* len, uint64_t* flags)
+{
+    return SYS_ERR_OK;
+}
+
index b43aa79..e9f2661 100644 (file)
@@ -20,5 +20,9 @@
 
   build application { target = "devif_forward_device", 
                       cFiles = [ "forward_device.c" ],
-                      addLibraries = [ "devif" ] }
+                      addLibraries = [ "devif" ] },
+
+  build application { target = "devif_test_sfn5122f", 
+                      cFiles = [ "sfn5122f_device.c" ],
+                      addLibraries = [ "devif" , "sfn5122f_devif_direct"] }
 ]
diff --git a/usr/tests/devif/sfn5122f_device.c b/usr/tests/devif/sfn5122f_device.c
new file mode 100644 (file)
index 0000000..96166a5
--- /dev/null
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2007, 2008, 2009, 2010, 2011, 2012, ETH Zurich.
+ * All rights reserved.
+ *
+ * This file is distributed under the terms in the attached LICENSE file.
+ * If you do not find this file, copies can be found by writing to:
+ * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <time.h>
+#include <barrelfish/barrelfish.h>
+#include <barrelfish/waitset.h>
+#include <barrelfish/deferred.h>
+#include <devif/queue_interface.h>
+#include <devif/sfn5122f_devif_direct.h>
+
+int main(int argc, char *argv[])
+{
+
+    errval_t err;
+    struct devq* q;   
+
+    struct endpoint_state my_state = {
+        .endpoint_type = ENDPOINT_TYPE_FORWARD,
+        .device_name = "", // name will be assigned 
+        .features = 0,
+        // TODO .f
+    };
+
+    printf("Forward queue created\n");
+    err = devq_create(&q, &my_state, "sfn5122f", 1);
+    if (err_is_fail(err)){
+        printf("%s \n", err_getstring(err));
+        USER_PANIC("Allocating devq failed \n");
+    }    
+
+    devq_event_loop(&my_state);
+    //messages_handler_loop();
+}
+