2 * Copyright (c) 2007-2011, 2013, 2014, ETH Zurich.
5 * This file is distributed under the terms in the attached LICENSE file.
6 * If you do not find this file, copies can be found by writing to:
7 * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
16 #include <net_device_manager/net_device_manager.h>
18 #include <barrelfish/nameservice_client.h>
19 #include <barrelfish/debug.h>
20 #include <barrelfish/deferred.h>
23 # include <netif/e1000.h>
25 #include <arpa/inet.h>
27 #include <if/e10k_defs.h>
28 #include <if/e10k_vf_defs.h>
29 #include <if/net_filter_defs.h>
30 #include <dev/e10k_dev.h>
36 //#define VTON_DCBOFF TODO use if VFs are enabled
39 //#define DEBUG(x...) printf("e10k: " x)
40 #define DEBUG(x...) do {} while (0)
47 struct e10k_binding *binding;
48 struct e10k_vf_binding *devif;
50 struct capref tx_frame;
51 struct capref txhwb_frame;
52 struct capref rx_frame;
77 MASK_L4PROTO = (1 << 0),
78 MASK_SRCIP = (1 << 1),
79 MASK_DSTIP = (1 << 2),
80 MASK_SRCPORT = (1 << 3),
81 MASK_DSTPORT = (1 << 4),
103 static union macentry mactable[128] = {
104 { .as8 = "\x0\x0\x0\x0\x0\x0" }, // First MAC is never set (loaded from card EEPROM)
106 { .as8 = "\x22\xc9\xfc\x96\x83\xfc" },
107 { .as8 = "\xce\x43\x5b\xf7\x3e\x60" },
108 { .as8 = "\x6a\xb0\x62\xf6\xa7\x21" },
109 { .as8 = "\xb2\xdf\xf9\x39\xc6\x10" },
110 { .as8 = "\x92\x77\xe7\x3f\x80\x30" },
111 { .as8 = "\xd6\x88\xd6\x86\x4a\x22" },
112 { .as8 = "\x7e\x64\xe9\x2e\xbe\x4b" },
113 { .as8 = "\xba\xac\x49\xd6\x3c\x77" },
115 // We set the rest to all zeroes
117 // Last MAC (127) never set (loaded from card EEPROM ... at least, it's already there)
120 static uint16_t credit_refill[128];
121 static uint32_t tx_rate[128];
123 // Hack for monolithic driver
124 void qd_main(void) __attribute__((weak));
125 void qd_argument(const char *arg) __attribute__((weak));
126 void qd_interrupt(bool is_rx, bool is_tx) __attribute__((weak));
127 void qd_queue_init_data(struct e10k_binding *b, struct capref registers,
128 uint64_t macaddr) __attribute__((weak));
129 void qd_queue_memory_registered(struct e10k_binding *b) __attribute__((weak));
130 void qd_write_queue_tails(struct e10k_binding *b) __attribute__((weak));
133 void cd_request_device_info(struct e10k_binding *b);
134 void cd_register_queue_memory(struct e10k_binding *b,
136 struct capref tx_frame,
137 struct capref txhwb_frame,
138 struct capref rx_frame,
142 uint8_t msix_intdest,
148 void cd_set_interrupt_rate(struct e10k_binding *b,
153 static void idc_write_queue_tails(struct e10k_binding *b);
154 static void stop_device(void);
156 static void device_init(void);
157 static void queue_hw_init(uint8_t n, bool set_tail);
158 //static void queue_hw_stop(uint8_t n);
159 static void interrupt_handler_msix(void* arg);
160 //static void interrupt_handler_msix_b(void* arg);
162 static void e10k_flt_ftqf_setup(int index, struct e10k_filter *filter);
163 //static void e10k_flt_etype_setup(int filter, int queue, uint16_t etype);
167 static const char *service_name = "e10k";
168 static int initialized = 0;
169 static bool exported = false;
170 static e10k_t *d = NULL;
171 static struct capref *regframe;
173 static bool msix = false;
175 /** Specifies if RX/TX is currently enabled on the device. */
176 static bool rxtx_enabled = false;
178 // Management of MSI-X vectors
179 static struct bmallocator msix_alloc;
180 /** MSI-X vector used by cdriver */
181 static size_t cdriver_msix = -1;
182 static uint8_t cdriver_vector;
185 // State of queues and filters
186 static struct queue_state queues[128];
187 static struct e10k_filter filters[128];
189 static char buf[4096];
191 /* PCI device address passed on command line */
192 static uint32_t pci_bus = PCI_DONT_CARE;
193 static uint32_t pci_device = PCI_DONT_CARE;
194 static uint32_t pci_function = 0;
195 static uint32_t pci_deviceid = E10K_PCI_DEVID;
197 /* VFs alloacation data*/
198 static bool vf_used[63];
200 static void e10k_flt_ftqf_setup(int idx, struct e10k_filter* filter)
202 uint16_t m = filter->mask;
204 e10k_ftqf_t ftqf = 0;
205 e10k_l34timir_t timir = 0;
206 e10k_sdpqf_t sdpqf = 0;
210 if (!(m & MASK_SRCIP)) {
211 DEBUG("src_ip=%"PRIx32" ", filter->src_ip);
212 e10k_saqf_wr(d, idx, htonl(filter->src_ip));
215 if (!(m & MASK_DSTIP)) {
216 DEBUG("dst_ip=%"PRIx32" ", filter->dst_ip);
217 e10k_daqf_wr(d, idx, htonl(filter->dst_ip));
220 if (!(m & MASK_SRCPORT)) {
221 DEBUG("src_port=%d ", filter->src_port);
222 sdpqf = e10k_sdpqf_src_port_insert(sdpqf, htons(filter->src_port));
225 if (!(m & MASK_DSTPORT)) {
226 DEBUG("dst_port=%d ", filter->dst_port);
227 sdpqf = e10k_sdpqf_dst_port_insert(sdpqf, htons(filter->dst_port));
229 e10k_sdpqf_wr(d, idx, sdpqf);
230 DEBUG("queue_id=%d \n", filter->queue);
232 if (!(m & MASK_L4PROTO)) {
233 switch (filter->l4_type) {
234 case L4_OTHER: p = e10k_l4other; break;
235 case L4_UDP: p = e10k_l4udp; break;
236 case L4_TCP: p = e10k_l4tcp; break;
237 case L4_SCTP: p = e10k_l4sctp; break;
238 default: assert(0); return;
240 ftqf = e10k_ftqf_protocol_insert(ftqf, p);
244 ftqf = e10k_ftqf_m_srcaddr_insert(ftqf, !!(m & MASK_SRCIP));
245 ftqf = e10k_ftqf_m_dstaddr_insert(ftqf, !!(m & MASK_DSTIP));
246 ftqf = e10k_ftqf_m_srcport_insert(ftqf, !!(m & MASK_SRCPORT));
247 ftqf = e10k_ftqf_m_dstport_insert(ftqf, !!(m & MASK_DSTPORT));
248 ftqf = e10k_ftqf_m_protocol_insert(ftqf, !!(m & MASK_L4PROTO));
251 // Configure destination queue and enable filter
252 timir = e10k_l34timir_rx_queue_insert(timir, filter->queue);
253 e10k_l34timir_wr(d, idx, timir);
255 ftqf = e10k_ftqf_priority_insert(ftqf, filter->priority);
256 ftqf = e10k_ftqf_pool_mask_insert(ftqf, 1);
257 ftqf = e10k_ftqf_queue_en_insert(ftqf, 1);
258 e10k_ftqf_wr(d, idx, ftqf);
262 static int ftqf_index = 0;
263 static int ftqf_alloc(void)
265 // FIXME: Do this reasonably
269 static errval_t reg_ftfq_filter(struct e10k_filter* f, uint64_t* fid)
273 DEBUG("reg_ftfq_filter: called\n");
275 if ((i = ftqf_alloc()) < 0) {
276 return FILTER_ERR_NOT_ENOUGH_MEMORY;
281 filters[i].enabled = true;
283 e10k_flt_ftqf_setup(i, f);
292 /****************************************************************************/
293 /* Net filter interface implementation */
294 /****************************************************************************/
297 static errval_t cb_install_filter(struct net_filter_binding *b,
298 net_filter_filter_type_t type,
308 struct e10k_filter f = {
309 .dst_port = dst_port,
310 .src_port = src_port,
313 .l4_type = (type == net_filter_PORT_TCP ? L4_TCP : L4_UDP),
319 f.mask = f.mask | MASK_SRCIP;
323 f.mask = f.mask | MASK_DSTIP;
326 f.mask = f.mask | MASK_DSTPORT;
330 f.mask = f.mask | MASK_SRCPORT;
335 err = reg_ftfq_filter(&f, fid);
336 DEBUG("filter registered: err=%s, fid=%"PRIu64"\n", err_getstring(err), *fid);
341 static errval_t cb_remove_filter(struct net_filter_binding *b,
342 net_filter_filter_type_t type,
346 if ((type == net_filter_PORT_UDP || type == net_filter_PORT_TCP)){
350 *err = NET_FILTER_ERR_NOT_FOUND;
353 DEBUG("unregister_filter: called (%"PRIx64")\n", filter_id);
357 static struct net_filter_rpc_rx_vtbl net_filter_rpc_rx_vtbl = {
358 .install_filter_ip_call = cb_install_filter,
359 .remove_filter_call = cb_remove_filter,
360 .install_filter_mac_call = NULL,
363 static void net_filter_export_cb(void *st, errval_t err, iref_t iref)
366 printf("exported net filter interface\n");
367 err = nameservice_register("net_filter_e10k", iref);
368 assert(err_is_ok(err));
369 DEBUG("Net filter interface exported\n");
373 static errval_t net_filter_connect_cb(void *st, struct net_filter_binding *b)
375 printf("New connection on net filter interface\n");
376 b->rpc_rx_vtbl = net_filter_rpc_rx_vtbl;
382 static void e10k_flt_etype_setup(int filter, int queue, uint16_t etype)
384 // Clear existing values
385 e10k_etqf_wr(d, filter, 0x0);
386 e10k_etqs_wr(d, filter, 0x0);
388 e10k_etqs_rx_queue_wrf(d, filter, queue);
389 e10k_etqs_queue_en_wrf(d, filter, 1);
391 e10k_etqf_etype_wrf(d, filter, etype);
392 e10k_etqf_filter_en_wrf(d, filter, 1);
396 static errval_t arp_filter(uint64_t qid, uint64_t* fid)
398 e10k_flt_etype_setup(0, (int) qid, 0x0806);
400 DEBUG("reg_arp_filter: called\n");
404 static errval_t reg_ftfq_filter(struct e10k_filter* f, uint64_t* fid)
408 DEBUG("reg_ftfq_filter: called\n");
410 if ((i = ftqf_alloc()) < 0) {
411 return ETHERSRV_ERR_NOT_ENOUGH_MEM;
416 filters[i].enabled = true;
418 e10k_flt_ftqf_setup(i, f);
425 static errval_t ipv4_tcp_port(uint64_t qid, uint16_t port, uint64_t* fid)
427 struct e10k_filter f = {
429 .mask = MASK_SRCIP | MASK_DSTIP | MASK_SRCPORT,
435 DEBUG("ipv4_tcp_port: called\n");
436 return reg_ftfq_filter(&f, fid);
439 static errval_t ipv4_udp_port(uint64_t qid, uint16_t port, uint64_t* fid)
441 struct e10k_filter f = {
443 .mask = MASK_SRCIP | MASK_DSTIP | MASK_SRCPORT,
449 DEBUG("ipv4_udp_port: called\n");
450 return reg_ftfq_filter( &f, fid);
453 static errval_t ipv4_tcp_conn(uint64_t qid,
454 uint32_t l_ip, uint16_t l_port,
455 uint32_t r_ip, uint16_t r_port,
458 struct e10k_filter f = {
469 DEBUG("ipv4_tcp_conn: called\n");
470 return reg_ftfq_filter(&f, fid);
473 static errval_t deregister_filter(uint64_t fid)
475 DEBUG("deregister_filter: called\n");
476 return LIB_ERR_NOT_IMPLEMENTED;
482 /** Enable RX operation for whole card. */
483 static void rx_enable(void)
485 e10k_secrxctrl_rx_dis_wrf(d, 1);
486 while (e10k_secrxstat_sr_rdy_rdf(d) == 0); // TODO: Timeout
487 e10k_rxctrl_rxen_wrf(d, 1);
488 e10k_secrxctrl_rx_dis_wrf(d, 0);
491 /** Disable RX operation for whole card. */
492 static void rx_disable(void)
494 e10k_secrxctrl_rx_dis_wrf(d, 1);
495 while (e10k_secrxstat_sr_rdy_rdf(d) == 0); // TODO: Timeout
496 e10k_rxctrl_rxen_wrf(d, 0);
497 e10k_secrxctrl_rx_dis_wrf(d, 0);
500 /** Enable TX operation for whole card. */
501 static void tx_enable(void)
503 e10k_dmatxctl_txen_wrf(d, 1);
506 /** Disable TX operation for whole card. */
507 static void tx_disable(void)
509 e10k_dmatxctl_txen_wrf(d, 0);
510 while (e10k_dmatxctl_txen_rdf(d) != 0); // TODO: timeout
514 static void setup_interrupt(size_t *msix_index, uint8_t core, uint8_t vector)
520 res = bmallocator_alloc(&msix_alloc, msix_index);
523 err = get_apicid_from_core(core, &dest);
524 assert(err_is_ok(err));
526 err = pci_msix_vector_init(*msix_index, dest, vector);
527 assert(err_is_ok(err));
529 DEBUG("e10k: MSI-X vector setup index=%"PRIx64", core=%d apic=%d swvec=%x\n",
530 *msix_index, core, dest, vector);
534 * Initialize hardware registers.
535 * Is also called after a reset of the device.
537 static void device_init(void)
543 bool initialized_before = initialized;
549 if (initialized_before) {
550 // Save queue heads and tails
551 for (i = 0; i < 128; i++) {
552 if (queues[i].enabled) {
553 queues[i].tx_head = e10k_tdh_rd(d, i);
555 queues[i].rx_head = e10k_rdh_1_rd(d, i);
557 queues[i].rx_head = e10k_rdh_2_rd(d, i - 64);
563 // Make a double reset to be sure
564 for (i = 0; i < 2; i++) {
565 // Issue Global reset
566 ctrl = e10k_ctrl_rd(d);
567 ctrl = e10k_ctrl_lrst_insert(ctrl, 1);
568 ctrl = e10k_ctrl_rst_insert(ctrl, 1);
569 e10k_ctrl_wr(d, ctrl);
570 while ((e10k_ctrl_rst_rdf(d) != 0) ||
571 (e10k_ctrl_lrst_rdf(d) != 0)); // TODO: Timeout
573 // Spec says 10, fbsd driver 50
576 DEBUG("Global reset done\n");
578 // Disable interrupts
579 e10k_eimc_cause_wrf(d, 0x7FFFFFFF);
582 // Let firmware know that we have taken over
583 e10k_ctrl_ext_drv_load_wrf(d, 1);
585 // NO Snoop disable (from FBSD)
586 // Without this, the driver only works on sbrinz1 if the receive buffers are
587 // mapped non cacheable. If the buffers are mapped cacheable, sometimes we
588 // seem to read old buffer contents, not sure exactly why, as far as
589 // understood this, No snoop should only be enabled by the device if it is
591 // TODO: Also check performance implications of this on gottardo and other
592 // machnies where it works without this.
593 e10k_ctrl_ext_ns_dis_wrf(d, 1);
595 // Initialize flow-control registers
596 for (i = 0; i < 8; i++) {
597 if (i < 4) e10k_fcttv_wr(d, i, 0x0);
598 e10k_fcrtl_wr(d, i, 0x0);
599 e10k_fcrth_wr(d, i, 0x0);
601 e10k_fcrtv_wr(d, 0x0);
602 e10k_fccfg_wr(d, 0x0);
607 // Wait for EEPROM auto read
608 while (e10k_eec_auto_rd_rdf(d) == 0); // TODO: Timeout
609 DEBUG("EEPROM auto read done\n");
611 // Wait for DMA initialization
612 while (e10k_rdrxctl_dma_initok_rdf(d) == 0); // TODO: Timeout
614 // Wait for link to come up
615 while (e10k_links_lnk_up_rdf(d) == 0); // TODO: Timeout
619 // Initialize interrupts
620 e10k_eicr_wr(d, 0xffffffff);
622 // Switch to MSI-X mode
623 e10k_gpie_msix_wrf(d, 1);
624 e10k_gpie_pba_sup_wrf(d, 1);
625 e10k_gpie_ocd_wrf(d, 1);
627 // Allocate msix vector for cdriver and set up handler
628 if (cdriver_msix == -1) {
629 err = pci_setup_inthandler(interrupt_handler_msix, NULL, &cdriver_vector);
630 assert(err_is_ok(err));
632 setup_interrupt(&cdriver_msix, disp_get_core_id(), cdriver_vector);
635 // Map management interrupts to our vector
636 e10k_ivar_misc_i_alloc0_wrf(d, cdriver_msix);
637 e10k_ivar_misc_i_alloc1_wrf(d, cdriver_msix);
638 e10k_ivar_misc_i_allocval0_wrf(d, 1);
639 e10k_ivar_misc_i_allocval1_wrf(d, 1);
641 // Enable auto masking of interrupt
642 e10k_gpie_eiame_wrf(d, 1);
643 e10k_eiamn_wr(d, cdriver_msix / 32, (1 << (cdriver_msix % 32)));
645 // Set no interrupt delay
646 e10k_eitr_l_wr(d, cdriver_msix, 0);
647 e10k_gpie_eimen_wrf(d, 1);
650 e10k_eimsn_wr(d, cdriver_msix / 32, (1 << (cdriver_msix % 32)));
652 e10k_gpie_msix_wrf(d, 0);
653 // Set no Interrupt delay
654 e10k_eitr_l_wr(d, 0, 0);
655 e10k_gpie_eimen_wrf(d, 1);
657 // Enable all interrupts
658 e10k_eimc_wr(d, e10k_eims_rd(d));
659 e10k_eims_cause_wrf(d, 0x7fffffff);
662 // Just a guess for RSC delay
663 e10k_gpie_rsc_delay_wrf(d, 2);
665 // Initialize multiple register tables (MAC 0 and 127 are not set)
666 for (i = 0; i < 128; i++) {
667 /* uint64_t mac = e10k_ral_ral_rdf(d, i) | ((uint64_t) e10k_rah_rah_rdf(d, i) << 32); */
668 /* uint8_t *m = (uint8_t *)&mac; */
669 /* DEBUG("Old MAC %d: %02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx ... mac valid = %x\n", */
670 /* i, m[0], m[1], m[2], m[3], m[4], m[5], e10k_rah_av_rdf(d, 0)); */
672 if(i > 0 && i < 127) {
673 e10k_ral_wr(d, i, mactable[i].as64 & 0xffffffff);
674 e10k_rah_wr(d, i, mactable[i].as64 >> 32);
675 e10k_rah_av_wrf(d, i, 1);
677 /* mac = e10k_ral_ral_rdf(d, i) | ((uint64_t) e10k_rah_rah_rdf(d, i) << 32); */
678 /* m = (uint8_t *)&mac; */
679 /* DEBUG("New MAC %d: %02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx ... mac valid = %x\n", */
680 /* i, m[0], m[1], m[2], m[3], m[4], m[5], e10k_rah_av_rdf(d, 0)); */
683 for (i = 0; i < 128; i++)
684 e10k_mta_bit_vec_wrf(d, i, 0);
685 for (i = 0; i < 128; i++)
686 e10k_vfta_vlan_flt_wrf(d, i, 0);
687 for (i = 0; i < 128; i++)
688 e10k_pfvlvfb_wr(d, i, 0);
690 for (i = 0; i < 64; i++) {
692 e10k_pfvlvf_vi_en_wrf(d, i, 1);
694 e10k_pfvlvf_vi_en_wrf(d, i, 0);
696 e10k_psrtype_wr(d, i, 0);
698 for (i = 0; i < 128; i++)
699 e10k_pfuta_wr(d, i, 0);
700 for (i = 0; i < 256; i++)
701 e10k_mpsar_pool_ena_wrf(d, i, 0);
703 // Program direct match MAC forwarding rules
704 // This setup will assign the first 64 MAC addresses each to a different
705 // RX pool. This assumes we have 64 VFs. The rest is set to filtered.
706 for(i = 0; i < 128; i++) {
708 // Pools < 32 (low bits)
709 e10k_mpsar_pool_ena_wrf(d, 2 * i, 1 << i);
710 e10k_mpsar_pool_ena_wrf(d, 2 * i + 1, 0);
712 // Pools >= 32 and < 64 (high bits)
713 e10k_mpsar_pool_ena_wrf(d, 2 * i, 0);
714 e10k_mpsar_pool_ena_wrf(d, 2 * i + 1, 1 << (i - 32));
716 // Pools >= 64 -> DROP
717 e10k_mpsar_pool_ena_wrf(d, 2 * i, 0);
718 e10k_mpsar_pool_ena_wrf(d, 2 * i + 1, 0);
722 for (i = 0; i < 128; i++) {
723 e10k_fhft_1_wr(d, i, 0);
725 e10k_fhft_2_wr(d, i, 0);
730 // Disallow per-queue RSC (not supported in SR-IOV mode)
731 e10k_rfctl_rsc_dis_wrf(d, 1);
733 // Allow for per-queue RSC
734 e10k_rfctl_rsc_dis_wrf(d, 0);
737 // Initialize RX filters
738 for (i = 0; i < 128; i++) {
739 e10k_ftqf_wr(d, i, 0);
740 e10k_saqf_wr(d, i, 0);
741 e10k_daqf_wr(d, i, 0);
742 e10k_sdpqf_wr(d, i, 0);
744 for (i = 0; i < 32; i++)
745 e10k_reta_wr(d, i, 0);
746 e10k_mcstctrl_mfe_wrf(d, 0);
749 e10k_fctrl_bam_wrf(d, 1);
751 // Enable Jumbo frames
752 e10k_hlreg0_jumboen_wrf(d, 1);
753 e10k_maxfrs_mfs_wrf(d, 15872);
755 // Make sure Rx CRC strip is consistently enabled in HLREG0 and RDRXCTL
756 e10k_hlreg0_rxcrcstrp_wrf(d, 1);
757 // Note: rscfrstsz has to be set to 0 (is mbz)
758 e10k_rdrxctl_t rdrxctl = e10k_rdrxctl_rd(d);
759 rdrxctl = e10k_rdrxctl_crcstrip_insert(rdrxctl, 1);
760 e10k_rdrxctl_wr(d, rdrxctl);
763 // Configure buffers etc. according to specification
764 // Section 4.6.11.3.4 (DCB, virtualization, no RSS)
765 // 1:1 from spec, though not sure if everything is necessary, but since
766 // initialization is still buggy, I'd rather be conservative and set some
767 // additional flags, even if they aren't strictly necessary.
768 e10k_rttdcs_arbdis_wrf(d, 1);
771 e10k_rxpbsize_size_wrf(d, 0, 0x200);
772 e10k_txpbsize_size_wrf(d, 0, 0xA0);
773 e10k_txpbthresh_thresh_wrf(d, 0, 0xA0);
774 for (i = 1; i < 8; i++) {
775 e10k_rxpbsize_size_wrf(d, i, 0x0);
776 e10k_txpbsize_size_wrf(d, i, 0x0);
777 e10k_txpbthresh_thresh_wrf(d, i, 0x0);
780 e10k_mrqc_mrque_wrf(d, e10k_vrt_only);
781 e10k_mtqc_rt_en_wrf(d, 0);
782 e10k_mtqc_vt_en_wrf(d, 1);
783 e10k_mtqc_num_tc_wrf(d, 1);
784 e10k_pfvtctl_vt_en_wrf(d, 1);
786 e10k_rxpbsize_size_wrf(d, 0, 0x200);
787 e10k_txpbsize_size_wrf(d, 0, 0xA0);
788 e10k_txpbthresh_thresh_wrf(d, 0, 0xA0);
789 for (i = 1; i < 8; i++) {
790 e10k_rxpbsize_size_wrf(d, i, 0x0);
791 e10k_txpbsize_size_wrf(d, i, 0x0);
792 e10k_txpbthresh_thresh_wrf(d, i, 0x0);
795 e10k_mrqc_mrque_wrf(d, e10k_no_rss);
796 e10k_mtqc_rt_en_wrf(d, 0);
797 e10k_mtqc_vt_en_wrf(d, 0);
798 e10k_mtqc_num_tc_wrf(d, 0);
799 e10k_pfvtctl_vt_en_wrf(d, 0);
801 e10k_rtrup2tc_wr(d, 0);
802 e10k_rttup2tc_wr(d, 0);
805 e10k_dtxmxszrq_max_bytes_wrf(d, 0xFFF);
807 e10k_dtxmxszrq_max_bytes_wrf(d, 0x010);
810 e10k_rttdcs_arbdis_wrf(d, 0);
812 for (i = 0; i < 128; i++) {
813 pfqde = e10k_pfqde_queue_idx_insert(0x0, i);
814 pfqde = e10k_pfqde_we_insert(pfqde, 1);
815 // XXX: Might want to set drop enable here
816 /* pfqde = e10k_pfqde_qde_insert(pfqde, 1); */
817 e10k_pfqde_wr(d, pfqde);
821 e10k_mflcn_rpfce_wrf(d, 0);
822 e10k_mflcn_rfce_wrf(d, 0);
823 e10k_fccfg_tfce_wrf(d, e10k_lfc_en);
825 e10k_mflcn_rpfce_wrf(d, 1);
826 e10k_mflcn_rfce_wrf(d, 0);
827 e10k_fccfg_tfce_wrf(d, e10k_pfc_en);
830 /* Causes ECC error (could be same problem as with l34timir (see e10k.dev) */
831 for (i = 0; i < 128; i++) {
832 e10k_rttdqsel_txdq_idx_wrf(d, i);
833 e10k_rttdt1c_wr(d, credit_refill[i]); // Credit refill x 64 bytes
834 e10k_rttbcnrc_wr(d, 0);
835 if(tx_rate[i] != 0) {
836 // Turn on rate scheduler for this queue and set rate factor
837 e10k_rttbcnrc_t rttbcnrc = 0;
838 // XXX: Assuming 10Gb/s link speed. Change if that's not correct.
839 uint32_t tx_factor = (10000 << 14) / tx_rate[i];
841 rttbcnrc = e10k_rttbcnrc_rf_dec_insert(rttbcnrc, tx_factor & 0x3fff);
842 rttbcnrc = e10k_rttbcnrc_rf_int_insert(rttbcnrc, tx_factor >> 14);
843 rttbcnrc = e10k_rttbcnrc_rs_ena_insert(rttbcnrc, 1);
844 e10k_rttbcnrc_wr(d, rttbcnrc);
846 printf("Setting rate for queue %d to %u\n", i, tx_rate[i]);
850 for (i = 0; i < 8; i++) {
851 e10k_rttdt2c_wr(d, i, 0);
852 e10k_rttpt2c_wr(d, i, 0);
853 e10k_rtrpt4c_wr(d, i, 0);
857 e10k_rttdcs_tdpac_wrf(d, 0);
858 e10k_rttdcs_vmpac_wrf(d, 1); // Remember to set RTTDT1C >= MTU when this is 1
860 e10k_rttdcs_tdrm_wrf(d, 0);
861 e10k_rttdcs_bdpm_wrf(d, 1);
862 e10k_rttdcs_bpbfsm_wrf(d, 0);
863 e10k_rttpcs_tppac_wrf(d, 0);
864 e10k_rttpcs_tprm_wrf(d, 0);
865 e10k_rttpcs_arbd_wrf(d, 0x224);
866 e10k_rtrpcs_rac_wrf(d, 0);
867 e10k_rtrpcs_rrm_wrf(d, 0);
870 e10k_rttdcs_tdpac_wrf(d, 0);
871 e10k_rttdcs_vmpac_wrf(d, 0);
872 e10k_rttdcs_tdrm_wrf(d, 0);
873 e10k_rttdcs_bdpm_wrf(d, 1);
874 e10k_rttdcs_bpbfsm_wrf(d, 1);
875 e10k_rttpcs_tppac_wrf(d, 0);
876 e10k_rttpcs_tprm_wrf(d, 0);
877 e10k_rttpcs_arbd_wrf(d, 0x224);
878 e10k_rtrpcs_rac_wrf(d, 0);
879 e10k_rtrpcs_rrm_wrf(d, 0);
882 // disable relaxed ordering
883 for (i = 0; i < 128; i++) {
884 e10k_dca_txctrl_txdesc_wbro_wrf(d, i, 0);
886 e10k_dca_rxctrl_1_rxhdr_ro_wrf(d, i, 0);
887 e10k_dca_rxctrl_1_rxdata_wrro_wrf(d, i, 0);
889 e10k_dca_rxctrl_2_rxhdr_ro_wrf(d, i - 64, 0);
890 e10k_dca_rxctrl_2_rxdata_wrro_wrf(d, i - 64, 0);
894 // disable all queues
895 for (i = 0; i < 128; i++) {
896 e10k_txdctl_enable_wrf(d, i, 0);
898 e10k_rxdctl_1_enable_wrf(d, i, 0);
900 e10k_rxdctl_2_enable_wrf(d, i - 64, 0);
904 for(i = 0; i < 64; i++) {
905 e10k_pfvml2flt_mpe_wrf(d, i, 1);
906 e10k_pfvml2flt_bam_wrf(d, i, 1);
907 e10k_pfvml2flt_aupe_wrf(d, i, 1);
911 // Enable DCA (Direct Cache Access)
913 e10k_dca_ctrl_t dca_ctrl = 0;
914 dca_ctrl = e10k_dca_ctrl_dca_mode_insert(dca_ctrl, e10k_dca10);
915 e10k_dca_ctrl_wr(d, dca_ctrl);
918 printf("DCA globally enabled\n");
921 DEBUG("Card initialized (%d)\n", initialized_before);
924 // Restore configuration
925 if (initialized_before) {
927 for (i = 0; i < 128; i++) {
928 if (filters[i].enabled) {
929 e10k_flt_ftqf_setup(i, filters + i);
934 for (i = 0; i < 128; i++) {
935 if (queues[i].enabled) {
936 queue_hw_init(i, true);
940 DEBUG("Configuration restored\n");
946 /** Initialize hardware queue n. */
947 static void queue_hw_init(uint8_t n, bool set_tail)
950 struct frame_identity frameid = { .base = 0, .bytes = 0 };
951 uint64_t tx_phys, txhwb_phys, rx_phys;
952 size_t tx_size, rx_size;
953 bool enable_global = !rxtx_enabled;
955 // Get physical addresses for rx/tx rings
956 r = invoke_frame_identify(queues[n].tx_frame, &frameid);
957 assert(err_is_ok(r));
958 tx_phys = frameid.base;
959 tx_size = frameid.bytes;
961 r = invoke_frame_identify(queues[n].rx_frame, &frameid);
962 assert(err_is_ok(r));
963 rx_phys = frameid.base;
964 rx_size = frameid.bytes;
966 DEBUG("tx.phys=%"PRIx64" tx.size=%"PRIu64"\n", tx_phys, tx_size);
967 DEBUG("rx.phys=%"PRIx64" rx.size=%"PRIu64"\n", rx_phys, rx_size);
970 // Initialize RX queue in HW
971 if (queues[n].rx_va) {
972 e10k_rdbal_1_wr(d, n, queues[n].rx_va);
973 e10k_rdbah_1_wr(d, n, (queues[n].rx_va) >> 32);
975 e10k_rdbal_1_wr(d, n, rx_phys);
976 e10k_rdbah_1_wr(d, n, rx_phys >> 32);
978 e10k_rdlen_1_wr(d, n, rx_size);
980 e10k_srrctl_1_bsz_pkt_wrf(d, n, queues[n].rxbufsz / 1024);
981 uint32_t hdrsz = queues[n].rxhdrsz;
985 assert(hdrsz % 64 == 0);
986 assert(hdrsz >= 128 && hdrsz <= 1024);
988 e10k_srrctl_1_bsz_hdr_wrf(d, n, hdrsz / 64);
989 // Enable header split if desired
990 if (queues[n].rxhdrsz != 0) {
991 e10k_srrctl_1_desctype_wrf(d, n, e10k_adv_hdrsp);
992 // Split packets after TCP, UDP, IP4, IP6 and L2 headers if we enable
994 e10k_psrtype_split_tcp_wrf(d, n, 1);
995 e10k_psrtype_split_udp_wrf(d, n, 1);
996 e10k_psrtype_split_ip4_wrf(d, n, 1);
997 e10k_psrtype_split_ip6_wrf(d, n, 1);
998 e10k_psrtype_split_l2_wrf(d, n, 1);
1000 //e10k_srrctl_1_desctype_wrf(d, n, e10k_adv_1buf);
1001 e10k_srrctl_1_desctype_wrf(d, n, e10k_legacy);
1003 e10k_srrctl_1_bsz_hdr_wrf(d, n, 128 / 64); // TODO: Do 128 bytes suffice in
1005 e10k_srrctl_1_drop_en_wrf(d, n, 1);
1008 if (queues[n].use_rsc) {
1009 USER_PANIC("RSC not supported in SR-IOV mode!\n");
1010 e10k_rscctl_1_maxdesc_wrf(d, n, 3);
1011 e10k_rscctl_1_rsc_en_wrf(d, n, 1);
1012 // TODO: (how) does this work for queues >=64?
1013 e10k_psrtype_split_tcp_wrf(d, n, 1); // needed for RSC
1015 e10k_rscctl_1_maxdesc_wrf(d, n, 0);
1016 e10k_rscctl_1_rsc_en_wrf(d, n, 0);
1019 // Initialize queue pointers (empty)
1020 e10k_rdt_1_wr(d, n, queues[n].rx_head);
1021 e10k_rdh_1_wr(d, n, queues[n].rx_head);
1024 // Open virtualization pool gate (assumes 64 VF mapping)
1025 e10k_pfvfre_wr(d, n / 64, e10k_pfvfre_rd(d, n / 64) | (1 << ((n / 2) % 32)));
1028 e10k_rxdctl_1_enable_wrf(d, n, 1);
1029 while (e10k_rxdctl_1_enable_rdf(d, n) == 0); // TODO: Timeout
1030 DEBUG("[%x] RX queue enabled\n", n);
1032 // Setup Interrupts for this queue
1033 if (queues[n].use_irq) {
1035 // Look for interrupt vector
1036 if (queues[n].msix_intvec != 0) {
1037 if (queues[n].msix_index == -1) {
1038 setup_interrupt(&queues[n].msix_index, queues[n].msix_intdest,
1039 queues[n].msix_intvec);
1041 rxv = txv = queues[n].msix_index;
1043 //rxv = QUEUE_INTRX;
1044 //txv = QUEUE_INTTX;
1048 DEBUG("rxv=%d txv=%d\n", rxv, txv);
1050 // Setup mapping queue Rx/Tx -> interrupt
1053 e10k_ivar_i_alloc0_wrf(d, i, rxv);
1054 e10k_ivar_i_allocval0_wrf(d, i, 1);
1055 e10k_ivar_i_alloc1_wrf(d, i, txv);
1056 e10k_ivar_i_allocval1_wrf(d, i, 1);
1058 e10k_ivar_i_alloc2_wrf(d, i, rxv);
1059 e10k_ivar_i_allocval2_wrf(d, i, 1);
1060 e10k_ivar_i_alloc3_wrf(d, i, txv);
1061 e10k_ivar_i_allocval3_wrf(d, i, 1);
1063 if (queues[n].msix_intvec != 0) {
1064 e10k_eitr_l_wr(d, rxv, 0);
1066 // Enable autoclear (higher ones are always auto cleared)
1068 e10k_eiac_rtxq_wrf(d, e10k_eiac_rtxq_rdf(d) | (1 << rxv));
1073 // Make sure interrupt is cleared
1074 e10k_eicr_wr(d, 1 << rxv);
1078 e10k_eimsn_wr(d, rxv / 32, (1 << (rxv % 32)));
1082 if (enable_global) {
1083 DEBUG("[%x] Enabling RX globally...\n", n);
1085 DEBUG("[%x] RX globally enabled\n", n);
1090 // Enable DCA for this queue
1091 e10k_dca_rxctrl_t dca_rxctrl = 0;
1093 dca_rxctrl = e10k_dca_rxctrl_rxdca_desc_insert(dca_rxctrl, 1);
1094 dca_rxctrl = e10k_dca_rxctrl_rxdca_hdr_insert(dca_rxctrl, 1);
1095 dca_rxctrl = e10k_dca_rxctrl_rxdca_payl_insert(dca_rxctrl, 1);
1098 errval_t err = sys_debug_get_apic_id(&my_apic_id);
1099 assert(err_is_ok(err));
1101 dca_rxctrl = e10k_dca_rxctrl_cpuid_insert(dca_rxctrl, my_apic_id);
1104 e10k_dca_rxctrl_1_wr(d, n, dca_rxctrl);
1106 e10k_dca_rxctrl_2_wr(d, n - 64, dca_rxctrl);
1109 printf("DCA enabled on queue %d with APIC ID %d\n", n, my_apic_id);
1113 // Initialize TX queue in HW
1114 if (queues[n].rx_va) {
1115 e10k_tdbal_wr(d, n, queues[n].tx_va);
1116 e10k_tdbah_wr(d, n, (queues[n].tx_va) >> 32);
1118 e10k_tdbal_wr(d, n, tx_phys);
1119 e10k_tdbah_wr(d, n, tx_phys >> 32);
1121 e10k_tdlen_wr(d, n, tx_size);
1123 // Initialize TX head index write back
1124 if (!capref_is_null(queues[n].txhwb_frame)) {
1125 r = invoke_frame_identify(queues[n].txhwb_frame, &frameid);
1126 assert(err_is_ok(r));
1127 txhwb_phys = frameid.base;
1128 if (queues[n].rx_va) {
1129 e10k_tdwbal_headwb_low_wrf(d, n, (queues[n].txhwb_va) >> 2);
1130 e10k_tdwbah_headwb_high_wrf(d, n, (queues[n].txhwb_va) >> 32);
1132 e10k_tdwbal_headwb_low_wrf(d, n, txhwb_phys >> 2);
1133 e10k_tdwbah_headwb_high_wrf(d, n, txhwb_phys >> 32);
1135 e10k_tdwbal_headwb_en_wrf(d, n, 1);
1138 // Initialized by queue driver to avoid race conditions
1139 // Initialize queue pointers
1140 e10k_tdh_wr(d, n, queues[n].tx_head);
1141 e10k_tdt_wr(d, n, queues[n].tx_head);
1143 // Configure prefetch and writeback threshhold
1144 e10k_txdctl_pthresh_wrf(d, n, 8); // FIXME: Figure out what the right number
1146 e10k_txdctl_hthresh_wrf(d, n, 0);
1147 e10k_txdctl_wthresh_wrf(d, n, 0);
1149 if (enable_global) {
1150 DEBUG("[%x] Enabling TX globally...\n", n);
1152 rxtx_enabled = true;
1153 DEBUG("[%x] TX globally enabled\n", n);
1157 // Open virtualization pool gate (assumes 64 VF mapping)
1158 e10k_pfvfte_wr(d, n / 64, e10k_pfvfte_rd(d, n / 64) | (1 << ((n / 2) % 32)));
1161 e10k_txdctl_enable_wrf(d, n, 1);
1162 while (e10k_txdctl_enable_rdf(d, n) == 0); // TODO: Timeout
1163 DEBUG("[%x] TX queue enabled\n", n);
1165 // Some initialization stuff from BSD driver
1166 e10k_dca_txctrl_txdesc_wbro_wrf(d, n, 0);
1169 idc_write_queue_tails(queues[n].binding);
1175 static void queue_hw_stop(uint8_t n)
1177 // This process is described in 4.6.7.1.2
1179 // Disable TX for this queue
1180 e10k_txdctl_enable_wrf(d, n, 0);
1182 // TODO: Flush packet buffers
1183 // TODO: Remove all filters
1184 // TODO: With RSC we have to wait here (see spec), not used atm
1186 // Disable RX for this queue
1187 e10k_rxdctl_1_enable_wrf(d, n, 0);
1188 while (e10k_rxdctl_1_enable_rdf(d, n) != 0); // TODO: Timeout
1190 // A bit too much, but make sure memory is not used anymore
1196 /** Stop whole device. */
1197 static void stop_device(void)
1201 DEBUG("Stopping device\n");
1203 // Disable RX and TX
1206 rxtx_enabled = false;
1208 // Disable interrupts
1209 e10k_eimc_cause_wrf(d, 0x7FFFFFFF);
1212 // Disable each RX and TX queue
1213 for (i = 0; i < 128; i++) {
1214 e10k_txdctl_wr(d, i, e10k_txdctl_swflsh_insert(0x0, 1));
1217 e10k_rxdctl_1_wr(d, i, 0x0);
1219 e10k_rxdctl_2_wr(d, i - 64, 0x0);
1224 // From BSD driver (not in spec)
1227 // Master disable procedure
1228 e10k_ctrl_pcie_md_wrf(d, 1);
1229 while (e10k_status_pcie_mes_rdf(d) != 0); // TODO: Timeout
1230 DEBUG("Stopping device done\n");
1233 static void management_interrupt(e10k_eicr_t eicr)
1235 if (e10k_eicr_ecc_extract(eicr)) {
1236 DEBUG("##########################################\n");
1237 DEBUG("ECC Error, resetting device :-/\n");
1238 DEBUG("##########################################\n");
1240 } else if (eicr >> 16) {
1241 DEBUG("Interrupt: %x\n", eicr);
1242 e10k_eicr_prtval(buf, sizeof(buf), eicr);
1245 DEBUG("Weird management interrupt without cause: eicr=%x\n", eicr);
1249 static void interrupt_handler_msix(void* arg)
1251 DEBUG("e10k: MSI-X management interrupt\n");
1252 e10k_eicr_t eicr = e10k_eicr_rd(d);
1254 eicr &= ~(1 << cdriver_msix);
1255 management_interrupt(eicr);
1257 // Ensure management MSI-X vector is cleared
1258 e10k_eicr_wr(d, (1 << cdriver_msix));
1260 // Reenable interrupt
1261 e10k_eimsn_cause_wrf(d, cdriver_msix / 32, (1 << (cdriver_msix % 32)));
1265 static void resend_interrupt(void* arg)
1268 uint64_t i = (uint64_t) arg;
1269 err = queues[i].devif->tx_vtbl.interrupt(queues[i].devif, NOP_CONT, i);
1270 // If the queue is busy, there is already an oustanding message
1271 if (err_is_fail(err) && err != FLOUNDER_ERR_TX_BUSY) {
1272 USER_PANIC("Error when sending interrupt %s \n", err_getstring(err));
1276 /** Here are the global interrupts handled. */
1277 static void interrupt_handler(void* arg)
1280 e10k_eicr_t eicr = e10k_eicr_rd(d);
1283 management_interrupt(eicr);
1285 e10k_eicr_wr(d, eicr);
1287 for (uint64_t i = 0; i < 16; i++) {
1288 if ((eicr >> i) & 0x1) {
1289 DEBUG("Interrupt eicr=%"PRIx32" \n", eicr);
1290 if (queues[i].use_irq && queues[i].devif != NULL) {
1291 err = queues[i].devif->tx_vtbl.interrupt(queues[i].devif, NOP_CONT, i);
1292 if (err_is_fail(err)) {
1293 err = queues[i].devif->register_send(queues[i].devif,
1294 get_default_waitset(),
1295 MKCONT(resend_interrupt,
1303 /******************************************************************************/
1304 /* Management interface implemetation */
1306 /** Send register cap and mac address to queue driver. */
1307 static void idc_queue_init_data(struct e10k_binding *b,
1308 struct capref registers,
1312 r = e10k_queue_init_data__tx(b, NOP_CONT, registers, macaddr);
1313 // TODO: handle busy
1314 assert(err_is_ok(r));
1317 /** Tell queue driver that we are done initializing the queue. */
1318 static void idc_queue_memory_registered(struct e10k_binding *b)
1321 r = e10k_queue_memory_registered__tx(b, NOP_CONT);
1322 // TODO: handle busy
1323 assert(err_is_ok(r));
1326 /** Send request to queue driver to rewrite the tail pointers of its queues. */
1327 static void idc_write_queue_tails(struct e10k_binding *b)
1331 qd_write_queue_tails(b);
1335 r = e10k_write_queue_tails__tx(b, NOP_CONT);
1336 // TODO: handle busy
1337 assert(err_is_ok(r));
1340 /** Request from queue driver for register memory cap */
1341 void cd_request_device_info(struct e10k_binding *b)
1343 assert(initialized);
1345 uint64_t d_mac = e10k_ral_ral_rdf(d, qi) | ((uint64_t) e10k_rah_rah_rdf(d, qi) << 32);
1346 DEBUG("mac valid = %x\n", e10k_rah_av_rdf(d, qi));
1348 uint64_t d_mac = e10k_ral_ral_rdf(d, 0) | ((uint64_t) e10k_rah_rah_rdf(d, 0) << 32);
1349 DEBUG("mac valid = %x\n", e10k_rah_av_rdf(d, 0));
1354 errval_t err = slot_alloc(&cr);
1355 assert(err_is_ok(err));
1356 err = cap_copy(cr, *regframe);
1357 assert(err_is_ok(err));
1358 qd_queue_init_data(b, cr, d_mac);
1361 idc_queue_init_data(b, *regframe, d_mac);
1364 /** Request from queue driver to initialize hardware queue. */
1365 void cd_register_queue_memory(struct e10k_binding *b,
1367 struct capref tx_frame,
1368 struct capref txhwb_frame,
1369 struct capref rx_frame,
1372 int16_t msix_intvec,
1373 uint8_t msix_intdest,
1380 DEBUG("register_queue_memory(%"PRIu8")\n", n);
1381 // TODO: Make sure that rxbufsz is a power of 2 >= 1024
1383 if (use_irq && msix_intvec != 0 && !msix) {
1384 printf("e10k: Queue %d requests MSI-X, but MSI-X is not enabled "
1385 " card driver. Ignoring queue\n", n);
1388 // Save state so we can restore the configuration in case we need to do a
1390 queues[n].enabled = true;
1391 queues[n].tx_frame = tx_frame;
1392 queues[n].txhwb_frame = txhwb_frame;
1393 queues[n].rx_frame = rx_frame;
1394 queues[n].tx_head = 0;
1395 queues[n].rx_head = 0;
1396 queues[n].rxbufsz = rxbufsz;
1397 queues[n].rxhdrsz = rxhdrsz;
1398 queues[n].msix_index = -1;
1399 queues[n].msix_intvec = msix_intvec;
1400 queues[n].msix_intdest = msix_intdest;
1401 queues[n].binding = b;
1402 queues[n].use_irq = use_irq;
1403 queues[n].use_rsc = use_rsc;
1404 queues[n].tx_va = tx_va;
1405 queues[n].rx_va = rx_va;
1406 queues[n].txhwb_va = txhwb_va;
1408 queue_hw_init(n, true);
1411 qd_queue_memory_registered(b);
1414 idc_queue_memory_registered(b);
1418 /** Request from queue driver to initialize hardware queue. */
1419 void cd_set_interrupt_rate(struct e10k_binding *b,
1423 DEBUG("set_interrupt_rate(%"PRIu8")\n", n);
1426 e10k_eitrn_t eitr = 0;
1427 eitr = e10k_eitrn_itr_int_insert(eitr, rate);
1429 i = (queues[n].msix_index == -1 ? 0 : queues[n].msix_index);
1431 e10k_eitr_l_wr(d, i, eitr);
1433 e10k_eitr_h_wr(d, i - 24, eitr);
1439 * Request from queue driver to stop hardware queue and free everything
1440 * associated with that queue.
1442 static errval_t idc_terminate_queue(struct e10k_binding *b, uint8_t n)
1444 DEBUG("idc_terminate_queue(q=%d)\n", n);
1448 queues[n].enabled = false;
1449 queues[n].binding = NULL;
1451 // TODO: Do we have to free the frame caps, or destroy the binding?
1455 static errval_t idc_register_port_filter(struct e10k_binding *b,
1459 e10k_port_type_t type,
1464 struct e10k_filter f = {
1466 .mask = MASK_SRCIP | MASK_DSTIP | MASK_SRCPORT,
1467 .l4_type = (type == e10k_PORT_TCP ? L4_TCP : L4_UDP),
1473 DEBUG("idc_register_port_filter: called (q=%d t=%d p=%d)\n",
1476 *err = reg_ftfq_filter(&f, filter);
1477 DEBUG("filter registered: err=%"PRIu64", fid=%"PRIu64"\n", *err, *filter);
1481 static errval_t idc_unregister_filter(struct e10k_binding *b,
1482 uint64_t filter, errval_t *err)
1484 DEBUG("unregister_filter: called (%"PRIx64")\n", filter);
1485 *err = LIB_ERR_NOT_IMPLEMENTED;
1489 static struct e10k_rx_vtbl rx_vtbl = {
1490 .request_device_info = cd_request_device_info,
1491 .register_queue_memory = cd_register_queue_memory,
1492 .set_interrupt_rate = cd_set_interrupt_rate,
1495 static struct e10k_rpc_rx_vtbl rpc_rx_vtbl = {
1496 .terminate_queue_call = idc_terminate_queue,
1497 .register_port_filter_call = idc_register_port_filter,
1498 .unregister_filter_call = idc_unregister_filter,
1502 static void export_cb(void *st, errval_t err, iref_t iref)
1504 const char *suffix = "_e10kmng";
1505 char name[strlen(service_name) + strlen(suffix) + 1];
1507 assert(err_is_ok(err));
1509 // Build label for interal management service
1510 sprintf(name, "%s%s", service_name, suffix);
1512 err = nameservice_register(name, iref);
1513 assert(err_is_ok(err));
1514 DEBUG("Management interface exported\n");
1517 static errval_t connect_cb(void *st, struct e10k_binding *b)
1519 DEBUG("New connection on management interface\n");
1520 b->rx_vtbl = rx_vtbl;
1521 b->rpc_rx_vtbl = rpc_rx_vtbl;
1526 * Initialize management interface for queue drivers.
1527 * This has to be done _after_ the hardware is initialized.
1529 static void initialize_mngif(void)
1533 r = e10k_export(NULL, export_cb, connect_cb, get_default_waitset(),
1534 IDC_BIND_FLAGS_DEFAULT);
1535 assert(err_is_ok(r));
1539 /****** VF/PF server interface *******/
1541 static void init_done_vf(struct e10k_vf_binding *b, uint8_t vfn)
1545 DEBUG("VF %d init done\n", vfn);
1547 // Enable correct pool for VF
1548 e10k_pfvfre_wr(d, vfn / 32, e10k_pfvfre_rd(d, vfn / 32) | (1 << (vfn % 32)));
1549 e10k_pfvfte_wr(d, vfn / 32, e10k_pfvfte_rd(d, vfn / 32) | (1 << (vfn % 32)));
1552 e10k_pfvflrec_wr(d, 0, 1 << vfn);
1554 e10k_pfvflrec_wr(d, 1, 1 << (vfn - 32));
1557 errval_t err = b->tx_vtbl.init_done_response(b, NOP_CONT);
1558 assert(err_is_ok(err));
1561 static void get_mac_address_vf(struct e10k_vf_binding *b, uint8_t vfn)
1563 assert(initialized);
1564 uint64_t d_mac = e10k_ral_ral_rdf(d, vfn) | ((uint64_t) e10k_rah_rah_rdf(d, vfn) << 32);
1565 errval_t err = b->tx_vtbl.get_mac_address_response(b, NOP_CONT, d_mac);
1566 assert(err_is_ok(err));
1569 static void request_vf_number(struct e10k_vf_binding *b)
1571 DEBUG("VF allocated\n");
1573 uint8_t vf_num = 255;
1574 for (int i = 0; i < 64; i++) {
1583 err = NIC_ERR_ALLOC_QUEUE;
1588 err = b->tx_vtbl.request_vf_number_response(b, NOP_CONT, vf_num, err);
1589 assert(err_is_ok(err));
1593 static errval_t cd_create_queue_rpc(struct e10k_vf_binding *b,
1594 struct capref tx_frame, struct capref txhwb_frame,
1595 struct capref rx_frame, uint32_t rxbufsz,
1596 int16_t msix_intvec, uint8_t msix_intdest,
1597 bool use_irq, bool use_rsc, bool default_q,
1598 uint64_t *mac, int32_t *qid, struct capref *regs,
1601 // TODO: Make sure that rxbufsz is a power of 2 >= 1024
1603 if (use_irq && msix_intvec != 0 && !msix) {
1604 printf("e10k: Queue requests MSI-X, but MSI-X is not enabled "
1605 " card driver. Ignoring queue\n");
1606 *ret_err = NIC_ERR_ALLOC_QUEUE;
1607 return NIC_ERR_ALLOC_QUEUE;
1612 for (int i = 1; i < 128; i++) {
1613 if (!queues[i].enabled) {
1620 if (queues[0].enabled == false) {
1623 printf("Default queue already initalized \n");
1624 return NIC_ERR_ALLOC_QUEUE;
1628 DEBUG("create queue(%"PRIu8": interrupt %d )\n", n, use_irq);
1631 *ret_err = NIC_ERR_ALLOC_QUEUE;
1632 return NIC_ERR_ALLOC_QUEUE;
1635 // Save state so we can restore the configuration in case we need to do a
1638 queues[n].tx_frame = tx_frame;
1639 queues[n].txhwb_frame = txhwb_frame;
1640 queues[n].rx_frame = rx_frame;
1641 queues[n].tx_head = 0;
1642 queues[n].rx_head = 0;
1643 queues[n].devif = b;
1644 queues[n].rxbufsz = rxbufsz;
1645 queues[n].msix_index = -1;
1646 queues[n].msix_intvec = msix_intvec;
1647 queues[n].msix_intdest = msix_intdest;
1648 queues[n].use_irq = use_irq;
1649 queues[n].use_rsc = use_rsc;
1650 queues[n].enabled = true;
1653 queue_hw_init(n, false);
1655 // TODO for now vfn = 0
1656 uint64_t d_mac = e10k_ral_ral_rdf(d, 0) | ((uint64_t) e10k_rah_rah_rdf(d, 0) << 32);
1662 DEBUG("[%d] Queue int done\n", n);
1663 *ret_err = SYS_ERR_OK;
1667 static void cd_create_queue(struct e10k_vf_binding *b,
1668 struct capref tx_frame, struct capref txhwb_frame,
1669 struct capref rx_frame, uint32_t rxbufsz,
1670 int16_t msix_intvec, uint8_t msix_intdest,
1671 bool use_irq, bool use_rsc, bool default_q)
1680 err = cd_create_queue_rpc(b, tx_frame, txhwb_frame, rx_frame,
1681 rxbufsz, msix_intvec, msix_intdest, use_irq, use_rsc,
1682 default_q, &mac, &queueid, ®s, &err);
1684 err = b->tx_vtbl.create_queue_response(b, NOP_CONT, mac, queueid, regs, err);
1685 assert(err_is_ok(err));
1686 DEBUG("cd_create_queue end\n");
1689 static void vf_export_cb(void *st, errval_t err, iref_t iref)
1691 const char *suffix = "_vf";
1692 char name[strlen(service_name) + strlen(suffix) + 100];
1694 assert(err_is_ok(err));
1696 // Build label for interal management service
1697 sprintf(name, "%s%s%u", service_name, suffix, pci_function);
1699 err = nameservice_register(name, iref);
1700 assert(err_is_ok(err));
1701 DEBUG("VF/PF interface [%s] exported\n", name);
1705 static errval_t vf_connect_cb(void *st, struct e10k_vf_binding *b)
1707 DEBUG("New connection on VF/PF interface\n");
1709 b->rx_vtbl.create_queue_call = cd_create_queue;
1710 b->rx_vtbl.request_vf_number_call = request_vf_number;
1711 b->rx_vtbl.init_done_call = init_done_vf;
1712 b->rx_vtbl.get_mac_address_call = get_mac_address_vf;
1714 b->rpc_rx_vtbl.create_queue_call = cd_create_queue_rpc;
1721 * Initialize management interface for queue drivers.
1722 * This has to be done _after_ the hardware is initialized.
1724 static void initialize_vfif(void)
1728 r = e10k_vf_export(NULL, vf_export_cb, vf_connect_cb, get_default_waitset(),
1729 IDC_BIND_FLAGS_DEFAULT);
1730 assert(err_is_ok(r));
1732 r = net_filter_export(NULL, net_filter_export_cb, net_filter_connect_cb,
1733 get_default_waitset(), IDC_BIND_FLAGS_DEFAULT);
1734 assert(err_is_ok(r));
1737 /******************************************************************************/
1738 /* Initialization code for driver */
1740 /** Callback from pci to initialize a specific PCI device. */
1741 static void pci_init_card(void *arg, struct device_mem* bar_info, int bar_count)
1746 assert(!initialized);
1748 d = malloc(sizeof(*d));
1750 // Map first BAR for register access
1751 assert(bar_count >= 1);
1752 map_device(&bar_info[0]);
1753 regframe = bar_info[0].frame_cap;
1754 DEBUG("BAR[0] mapped (v=%llx p=%llx l=%llx)\n",
1755 (unsigned long long) bar_info[0].vaddr,
1756 (unsigned long long) bar_info[0].paddr,
1757 (unsigned long long) bar_info[0].bytes);
1759 // Initialize Mackerel binding
1760 e10k_initialize(d, (void*) bar_info[0].vaddr);
1762 DEBUG("STATUS = %x\n", e10k_status_rd(d));
1764 // Initialize manager for MSI-X vectors
1766 DEBUG("Enabling MSI-X interrupts\n");
1767 uint16_t msix_count = 0;
1768 err = pci_msix_enable(&msix_count);
1769 assert(err_is_ok(err));
1770 assert(msix_count > 0);
1771 DEBUG("MSI-X #vecs=%d\n", msix_count);
1773 res = bmallocator_init(&msix_alloc, msix_count);
1776 DEBUG("Using legacy interrupts\n");
1779 // Initialize hardware registers etc.
1780 DEBUG("Initializing hardware\n");
1783 assert(initialized);
1786 DEBUG("SR-IOV device up routine\n");
1788 // Setup support for 64 VFs
1789 e10k_gcr_ext_vtmode_wrf(d, e10k_vt_64);
1790 e10k_gpie_vtmode_wrf(d, e10k_vt_64);
1792 // Enable virtualization, disable default pool, replication enable
1793 e10k_pfvtctl_t pfvtctl = e10k_pfvtctl_rd(d);
1794 pfvtctl = e10k_pfvtctl_vt_en_insert(pfvtctl, 1);
1795 pfvtctl = e10k_pfvtctl_def_pl_insert(pfvtctl, 0);
1796 pfvtctl = e10k_pfvtctl_dis_def_pl_insert(pfvtctl, 1);
1797 pfvtctl = e10k_pfvtctl_rpl_en_insert(pfvtctl, 1);
1798 e10k_pfvtctl_wr(d, pfvtctl);
1800 // Enable L2 loopback
1801 e10k_pfdtxgswc_lbe_wrf(d, 1);
1803 // TODO: Accept untagged packets in all VMDQ pools
1804 // TODO: Broadcast accept mode
1805 // TODO: Accept packets matching PFUTA table
1806 // TODO: Accept packets matching MTA table
1807 // TODO: Accept untagged packets enable
1808 // TODO: Strip VLAN tag for incoming packets
1810 DEBUG("STATUS = %x\n", e10k_status_rd(d));
1812 e10k_ctrl_ext_pfrstd_wrf(d, 1);
1816 // Now we initialize the management interface
1817 DEBUG("Initializing management interface\n");
1821 DEBUG("Initializing VF/PF interface\n");
1823 DEBUG("Done with initialization\n");
1827 /** Register with PCI */
1828 static void pci_register(void)
1832 r = pci_client_connect();
1833 assert(err_is_ok(r));
1834 DEBUG("connected to pci\n");
1836 r = pci_register_driver_irq(pci_init_card, NULL, PCI_CLASS_ETHERNET,
1837 PCI_DONT_CARE, PCI_DONT_CARE,
1838 PCI_VENDOR_INTEL, pci_deviceid,
1839 pci_bus, pci_device, pci_function,
1840 interrupt_handler, NULL);
1841 assert(err_is_ok(r));
1844 static void parse_cmdline(int argc, char **argv)
1848 for (i = 1; i < argc; i++) {
1849 if (strncmp(argv[i], "cardname=", strlen("cardname=")) == 0) {
1850 service_name = argv[i] + strlen("cardname=");
1851 } else if (strncmp(argv[i], "bus=", strlen("bus=")) == 0) {
1852 pci_bus = atol(argv[i] + strlen("bus="));
1853 } else if (strncmp(argv[i], "device=", strlen("device=")) == 0) {
1854 pci_device = atol(argv[i] + strlen("device="));
1855 } else if (strncmp(argv[i], "function=", strlen("function=")) == 0) {
1856 pci_function = atol(argv[i] + strlen("function="));
1857 } else if (strncmp(argv[i], "deviceid=", strlen("deviceid=")) == 0) {
1858 pci_deviceid = strtoul(argv[i] + strlen("deviceid="), NULL, 0);
1859 } else if (strncmp(argv[i], "msix=", strlen("msix=")) == 0) {
1860 msix = !!atol(argv[i] + strlen("msix="));
1861 // also pass this to queue driver
1862 qd_argument(argv[i]);
1863 } else if (strncmp(argv[i], "credit_refill[", strlen("credit_refill[") - 1) == 0) {
1864 // Controls the WRR (weighted round-robin) scheduler's credit refill rate
1865 // This seems to be per VM pool
1866 unsigned int entry, val;
1867 int r = sscanf(argv[i], "credit_refill[%u]=%u", &entry, &val);
1869 assert(entry < 128);
1870 assert(val < 0x3fff);
1871 credit_refill[entry] = val;
1872 } else if (strncmp(argv[i], "tx_rate[", strlen("tx_rate[") - 1) == 0) {
1873 // This is specified in Mbits/s and must be >= 10 and <= link speed (typically 10,000)
1874 // This seems to be per Tx queue
1875 unsigned int entry, val;
1876 int r = sscanf(argv[i], "tx_rate[%u]=%u", &entry, &val);
1878 assert(entry < 128);
1879 assert(val >= 10 && val <= 10000);
1880 tx_rate[entry] = val;
1882 qd_argument(argv[i]);
1888 static void eventloop(void)
1892 printf("Entering polling loop\n");
1893 ws = get_default_waitset();
1904 void qd_argument(const char *arg) { }
1905 void qd_interrupt(bool is_rx, bool is_tx) { }
1906 void qd_queue_init_data(struct e10k_binding *b, struct capref registers,
1907 uint64_t macaddr) { }
1908 void qd_queue_memory_registered(struct e10k_binding *b) { }
1909 void qd_write_queue_tails(struct e10k_binding *b) { }
1911 int main(int argc, char **argv)
1913 int e1000n_driver_init(int argc, char *argv[])
1916 //barrelfish_usleep(10*1000*1000);
1917 DEBUG("PF driver started\n");
1918 // credit_refill value must be >= 1 for a queue to be able to send.
1919 // Set them all to 1 here. May be overridden via commandline.
1920 for(int i = 0; i < 128; i++) {
1921 credit_refill[i] = 1;
1924 memset(tx_rate, 0, sizeof(tx_rate));
1926 parse_cmdline(argc, argv);
1929 while (!initialized || !exported) {
1930 event_dispatch(get_default_waitset());