[lwip-devel] [PATCH 2/3] Add support for scheduling disciplines to LwIP

[Russ Dill]

Currently if a network device cannot send a packet, it is dropped. If
applications have very bursty behavior, it is often desirable to queue
packets for a short time rather than dropping them.

A scheduling discipline not only allows packets to be queued for a short
time rather than dropped, it also is able to intelligently select which
packets to drop when the queue fills or if packets have been in the queue
for too long. Algorithms can even categorize traffic by flow and attempt to
fairly treat each flow. For a good example of a scheduling algorithm, see
https://en.wikipedia.org/wiki/CoDel

Scheduling disciplines are enabled by adding them to a network interface.
The discipline replaces the output callbacks of the network interface with
it's own callbacks allowing it to intercept traffic bound for the interface.
So long as the driver does not return ERR_WOULDBLOCK no packets are queued
and functionality is identical to without a scheduling discipline. When
ERR_WOULDBLOCK is returned, the packet is added to a queue. When the driver
can again accept packets, it should call the write_ready function at which
point the scheduling API will send queued packets to the network driver.

In order to support queuing packets when the network device is busy, a
network driver that supports nonblocking write support is required.

Signed-off-by: Russ Dill <address@hidden>
---
 src/core/sched.c         | 416 +++++++++++++++++++++++++++++++++++++++++++++++
 src/include/lwip/netif.h |   6 +
 src/include/lwip/opt.h   |  11 ++
 src/include/lwip/sched.h | 178 ++++++++++++++++++++
 4 files changed, 611 insertions(+)
 create mode 100644 src/core/sched.c
 create mode 100644 src/include/lwip/sched.h

diff --git a/src/core/sched.c b/src/core/sched.c
new file mode 100644
index 0000000..c5652f9
--- /dev/null
+++ b/src/core/sched.c
@@ -0,0 +1,416 @@
+/**
+ * @file
+ * lwIP network interface abstraction
+ *
+ */
+
+/*
+ * Copyright (c) 2015 Russ Dill <address@hidden>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without 
modification,
+ * are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 
EVENT
+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
PROCUREMENT
+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
POSSIBILITY
+ * OF SUCH DAMAGE.
+ *
+ * This file is part of the lwIP TCP/IP stack.
+ *
+ * Author: Russ Dill <address@hidden>
+ *
+ */
+
+#include "lwip/opt.h"
+
+#if LWIP_SCHED
+
+#if !LWIP_NETIF_NONBLOCKING
+#error "netif non-blocking support is required for scheduling discipline 
support"
+#endif
+
+#include "lwip/def.h"
+#include "lwip/sched.h"
+#include "lwip/pbuf.h"
+#include "lwip/ip_addr.h"
+
+/** Indicate that the address information is invalid. This is for signalling 
that
+ * the associated packet data is raw */
+#define IPADDR_TYPE_INVALID 1
+
+/** Allocate a packet queue entry from the free list and assign packet and 
address
+ * data.
+ *
+ * @param sq The associated scheduling queue
+ * @param p The packet data to assign, the pbuf reference count is incremented
+ *          if a free queue entry is available
+ * @param ipaddr The destination address of this packet, a NULL parameter 
indicates
+ *               a raw packet.
+ *
+ * @return The allocated packet queue entry on success, NULL if no free queue
+ *         entries are available.
+ */
+static struct sched_packet *
+sched_packet_alloc(struct sched_queue *sq, struct pbuf *p, const ip_addr_t 
*ipaddr)
+{
+  struct sched_packet *ret = sq->free_head;
+
+  if (ret) {
+    sq->nr_free--;
+    sq->free_head = ret->next;
+
+    /* Free queue is now empty */
+    if (!sq->free_head)
+      sq->free_tail = NULL;
+
+    pbuf_ref(p);
+    ret->p = p;
+
+#if LWIP_IPV4 && LWIP_IPV6
+    if (!ipaddr)
+      ret->dest.type = IPADDR_TYPE_INVALID;
+    else
+      ip_addr_copy(ret->dest, *ipaddr);
+#else /* LWIP_IPV4 && LWIP_IPV6 */
+    ret->raw = ipaddr == NULL;
+    if (ipaddr)
+      ip_addr_copy(ret->dest, *ipaddr);
+#endif /* LWIP_IPV4 && LWIP_IPV6 */
+
+    ret->next = NULL;
+  }
+
+  return ret;
+}
+
+/** Return a packet queue entry to the free list and decrement the use count on
+ * the packet data pbuf
+ *
+ * @param sq The associated scheduling queue.
+ * @param packet The packet queue entry to free.
+ */
+static void
+sched_packet_free(struct sched_queue *sq, struct sched_packet *packet)
+{
+  sq->nr_free++;
+  pbuf_free(packet->p);
+
+  if (sq->free_tail)
+    sq->free_tail->next = packet;
+  else
+    /* Free queue was empty */
+    sq->free_head = packet;
+
+  sq->free_tail = packet;
+}
+
+/** Helper function for transmitting a packet on the scheduling queue's 
associated
+ * network interface. This uses the appropriate callback based on the address 
type.
+ * If the address type is set to IPADDR_TYPE_INVALID, the packet is assumed to 
be
+ * raw.
+ *
+ * @param sq The associated scheduling queue.
+ * @param p The packet data.
+ * @param ipaddr The destination address of this packet, a NULL parameter 
indicates
+ *               a raw packet.
+ *
+ * @return The value returned by the associated netif callback.
+ */
+static err_t
+sched_transmit(struct sched_queue *sq, struct pbuf *p, const ip_addr_t *ipaddr)
+{
+  if (ipaddr) {
+#if LWIP_IPV6
+    if (IP_IS_V6_VAL(ipaddr))
+      return sq->output_ip6(sq->netif, p, ip_2_ip6(ipaddr));
+#endif /* LWIP_IPV6 */
+#if LWIP_IPV4
+    return sq->output(sq->netif, p, ip_2_ip4(ipaddr));
+#endif /* LWIP_IPV4 */
+  }
+  return sq->linkoutput(sq->netif, p);
+}
+
+/** Function for the scheduling discipline to indicate it would like to drop
+ * a previously queued packet. The scheduling API will free the associated
+ * pbuf, increment the link.drop stat, and add the packet back to the queue's
+ * free list
+ *
+ * @param sq The associated scheduling queue
+ * @param packet The packet to drop
+ */
+void
+sched_drop(struct sched_queue *sq, struct sched_packet *packet)
+{
+  LINK_STATS_INC(link.drop);
+  sched_packet_free(sq, packet);
+}
+
+/** Callback for netif nonblocking API. Used by the network device driver to
+ * indicate that it has newly available space for transmitting packets after
+ * previously returning ERR_WOULDBLOCK from one of it's output functions.
+ *
+ * @param netif The associated netif.
+ */
+static void
+sched_write_ready(struct netif *netif)
+{
+  struct sched_packet *packet;
+  struct sched_queue *sq = netif->sq;
+
+  /* Keep transmitting packets until the interface once again returns
+   * ERR_WOULDBLOCK or our queue is empty */
+  while ((packet = sq->peek(sq))) {
+    err_t err;
+    const ip_addr_t *dest = NULL;
+
+#if LWIP_IPV4 && LWIP_IPV6
+    if (packet->dest.type != IPADDR_TYPE_INVALID)
+#else /* LWIP_IPV4 && LWIP_IPV6 */
+    if (!packet->raw)
+#endif /* LWIP_IPV4 && LWIP_IPV6 */
+      dest = &packet->dest;
+
+    err = sched_transmit(sq, packet->p, dest);
+
+    if (err == ERR_WOULDBLOCK)
+      break;
+
+    sq->dequeue(sq, packet);
+    sched_packet_free(sq, packet);
+  }
+}
+
+/** Function to queue a new outbound packet. This bypasses the scheduling
+ * discipline if the queue is currently empty and the networking device
+ * accepts the packet. If the link is found to be down, the packet is dropped
+ * along with all other currently queued packets.
+ *
+ * @param netif The associated netif
+ * @param p The packet data. If the packet data gets queued, the reference 
count
+ *          is incremented
+ * @param ipaddr The destination address of this packet, a NULL parameter 
indicates
+ *               a raw packet.
+ *
+ * @return ERR_OK unless the underlying network device returns an error other
+ *         than ERR_OK or ERR_WOULDBLOCK. Note that for queued packets that 
later
+ *         receive an error other than ERR_OK or ERR_WOULDBLOCK the error 
cannot
+ *         be returned to higher layers via this API.
+ */
+static err_t
+sched_queue(struct netif *netif, struct pbuf *p, const ip_addr_t *ipaddr)
+{
+  struct sched_packet *packet;
+  struct sched_queue *sq = netif->sq;
+
+  /* If the link is down, dump our current queue and reject any packets */
+  if (!(netif->flags & NETIF_FLAG_UP)) {
+    if (!sq->empty(sq))
+      sq->drop_all(sq);
+    LINK_STATS_INC(link.drop);
+    return ERR_OK;
+  }
+
+  /* Try a direct transmit first */
+  if (sq->empty(sq)) {
+    err_t err = sched_transmit(sq, p, ipaddr);
+    if (err != ERR_WOULDBLOCK)
+      return err;
+  }
+
+  /* If a free queue entry is not available, drop the packet. If the scheduling
+   * discipline wants to control which packet is dropped in the case of a full
+   * queue, it should always keep one free entry in the free queue */
+  packet = sched_packet_alloc(sq, p, ipaddr);
+  if (!packet) {
+    LINK_STATS_INC(link.drop);
+    return ERR_OK;
+  }
+
+  return sq->queue(sq, packet);
+}
+
+/** Intercepting callback for the netif linkoutput function. Handles raw 
packets */
+static err_t
+sched_queue_linkoutput(struct netif *netif, struct pbuf *p)
+{
+  return sched_queue(netif, p, NULL);
+}
+
+#if LWIP_IPV6
+/** Intercepting callback for the netif output_ip6 function. Handles IPv6 
packets */
+static err_t
+sched_queue_ip6(struct netif *netif, struct pbuf *p, const ip6_addr_t *ip6addr)
+{
+  ip_addr_t ipaddr;
+  ip_addr_copy_from_ip6(ipaddr, *ip6addr);
+  return sched_queue(netif, p, &ipaddr);
+}
+#endif /* LWIP_IPV6 */
+
+#if LWIP_IPV4
+/** Intercepting callback for the netif output function. Handles IPv4 packets 
*/
+static err_t
+sched_queue_ip4(struct netif *netif, struct pbuf *p, const ip4_addr_t *ip4addr)
+{
+  ip_addr_t ipaddr;
+  ip_addr_copy_from_ip4(ipaddr, *ip4addr);
+  return sched_queue(netif, p, &ipaddr);
+}
+#endif /* LWIP_IPV4 */
+
+#if LWIP_NETIF_REMOVE_CALLBACK
+/** Intercepting callback for the netif remove_callback. This allows the
+ * scheduling API to gracefully handle the situation where a netif is removed
+ * while it still contains a scheduling discipline.
+ */
+static void
+sched_queue_remove_callback(struct netif *netif)
+{
+  netif->sq->drop_all(netif->sq);
+  netif->sq->netif = NULL;
+
+  netif->sq->remove_callback(netif);
+}
+#endif /* LWIP_NETIF_REMOVE_CALLBACK */
+
+/** Function to add a scheduling discipline to a netif. The netif must support
+ * nonblocking mode and not already have a scheduling discipline applied and/or
+ * a nonblocking API user. This function intercepts the output calls (output,
+ * output_ip6, linkoutput) of the netif with scheduling API calls that instead
+ * queue the outbound packets.
+ *
+ * @param sq Preallocated sched_queue data structure
+ * @param netif Initialized networking interface to add a scheduling discipline
+ *              to
+ * @param pool Memory pool for the packet queue. The packet queue only stores
+ *             metadata associated with the packet itself, not the actual
+ *             packet. Different queueing disciplines require different queue
+ *             entry sizes depending on required metadata.
+ * @param pool_size The size of the passed memory pool in bytes. The number of
+ *                  entries generated out of this pool will depend on the
+ *                  queue entry size of the selected discipline.
+ * @param state opaque data passed to the scheduling discipline init function.
+ * @param init The scheduling discipline init function, this selects the
+ *             desired discipline.
+ *
+ * @return ERR_OK on success, any other value indicates failure.
+ */
+err_t
+sched_queue_add(struct sched_queue *sq, struct netif *netif, void *pool,
+                mem_size_t pool_size, void *state, sched_queue_init_fn init)
+{
+  err_t err;
+  mem_size_t align_size;
+
+  if (!(netif->flags & NETIF_FLAG_NONBLOCK) || netif->write_ready)
+    return ERR_ARG;
+
+  sq->state = state;
+  sq->netif = netif;
+  sq->packet_size = sizeof(struct sched_packet);
+
+  netif->sq = sq;
+
+  err = init(sq);
+  if (err != ERR_OK)
+    return err;
+
+  align_size = LWIP_MEM_ALIGN_SIZE(sq->packet_size);
+  sq->nr_free = sq->nr_packets = pool_size / align_size;
+
+  if (sq->nr_packets) {
+    struct sched_packet *curr, *last;
+    int i;
+
+    sq->free_head = pool;
+    sq->free_tail = pool + align_size * (sq->nr_packets - 1);
+    curr = sq->free_tail;
+    last = NULL;
+    for (i = sq->nr_packets - 1; i >= 0; i--) {
+      curr->next = last;
+      last = curr;
+      curr = ((void *) curr) - align_size;
+    }
+  } else
+    sq->free_head = sq->free_tail = NULL;
+
+  sq->linkoutput = netif->linkoutput;
+  if (netif->linkoutput)
+    netif->linkoutput = sched_queue_linkoutput;
+
+#if LWIP_IPV4
+  sq->output = netif->output;
+  if (netif->output)
+    netif->output = sched_queue_ip4;
+#endif /* LWIP_IPV4 */
+
+#if LWIP_IPV6
+  sq->output_ip6 = netif->output_ip6;
+  if (netif->output_ip6)
+    netif->output_ip6 = sched_queue_ip6;
+#endif /* LWIP_IPV6 */
+
+#if LWIP_NETIF_REMOVE_CALLBACK
+  sq->remove_callback = netif->remove_callback;
+  netif->remove_callback = sched_queue_remove_callback;
+#endif /* LWIP_NETIF_REMOVE_CALLBACK */
+
+  netif->write_ready = sched_write_ready;
+
+  return ERR_OK;
+}
+
+/** Remove a scheduling discipline from a netif and drop all queued packets.
+ * Once called, the pool passed to sched_queue_add can be freed.
+ *
+ * @param sq The schedule discipline to remove
+ */
+void
+sched_queue_remove(struct sched_queue *sq)
+{
+  struct netif *netif;
+
+  netif = sq->netif;
+
+  if (netif) {
+    sq->drop_all(sq);
+
+    /* Restore callbacks */
+    netif->linkoutput = sq->linkoutput;
+
+#if LWIP_IPV4
+    netif->output = sq->output;
+#endif /* LWIP_IPV4 */
+
+#if LWIP_IPV6
+    netif->output_ip6 = sq->output_ip6;
+#endif /* LWIP_IPV6 */
+
+#if LWIP_NETIF_REMOVE_CALLBACK
+    netif->remove_callback = sq->remove_callback;
+#endif /* LWIP_NETIF_REMOVE_CALLBACK */
+
+    netif->write_ready = NULL;
+
+    sq->netif = NULL;
+    netif->sq = NULL;
+  }
+}
+
+#endif /* LWIP_SCHED */
diff --git a/src/include/lwip/netif.h b/src/include/lwip/netif.h
index e7ad81b..24a2121 100644
--- a/src/include/lwip/netif.h
+++ b/src/include/lwip/netif.h
@@ -53,6 +53,9 @@ struct autoip;
 #if LWIP_IPV6_DHCP6
 struct dhcp6;
 #endif /* LWIP_IPV6_DHCP6 */
+#if LWIP_SCHED
+struct sched_queue;
+#endif /* LWIP_SCHED */
 
 #ifdef __cplusplus
 extern "C" {
@@ -322,6 +325,9 @@ struct netif {
   u16_t loop_cnt_current;
 #endif /* LWIP_LOOPBACK_MAX_PBUFS */
 #endif /* ENABLE_LOOPBACK */
+#if LWIP_SCHED
+  struct sched_queue *sq;
+#endif /* LWIP_SCHED */
 };
 
 #if LWIP_CHECKSUM_CTRL_PER_NETIF
diff --git a/src/include/lwip/opt.h b/src/include/lwip/opt.h
index 926c2ab..f87defa 100644
--- a/src/include/lwip/opt.h
+++ b/src/include/lwip/opt.h
@@ -1480,6 +1480,17 @@
 #endif
 
 /*
+   ---------------------------------------------------
+   ---------- Scheduling Discipline options ----------
+   ---------------------------------------------------
+*/
+/**
+ * LWIP_SCHED==1: support the scheduling discipline API
+ */
+#ifndef LWIP_SCHED
+#define LWIP_SCHED                      0
+#endif
+
    ------------------------------------
    ---------- Thread options ----------
    ------------------------------------
diff --git a/src/include/lwip/sched.h b/src/include/lwip/sched.h
new file mode 100644
index 0000000..6d0b3a3
--- /dev/null
+++ b/src/include/lwip/sched.h
@@ -0,0 +1,178 @@
+/*
+ * Copyright (c) 2015 Russ Dill <address@hidden>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without 
modification,
+ * are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 
EVENT
+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
PROCUREMENT
+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
POSSIBILITY
+ * OF SUCH DAMAGE.
+ *
+ * This file is part of the lwIP TCP/IP stack.
+ *
+ * Author: Russ Dill <address@hidden>
+ *
+ */
+#ifndef LWIP_HDR_SCHED_H
+#define LWIP_HDR_SCHED_H
+
+#include "lwip/opt.h"
+
+#if LWIP_SCHED
+
+#include "lwip/err.h"
+#include "lwip/def.h"
+#include "lwip/netif.h"
+#include "lwip/ip_addr.h"
+
+struct sched_queue;
+struct sched_packet;
+struct pbuf;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Function prototype for sched init functions. This function initializes a
+ * schedule queue including setting the remaining callbacks as well as the
+ * packet_size if it differs from the default. The state member will be set
+ * for private storage of the scheduling discipline and/or parameters for the
+ * scheduling discipline.
+ *
+ * @param sq The scheduler queue
+ */
+typedef err_t (*sched_queue_init_fn)(struct sched_queue *sq);
+/** Function prototype for sched queue_empty functions. This should return
+ * true if the queue is empty.
+ *
+ * @param sq The scheduler queue
+ */
+typedef int (*sched_queue_empty_fn)(struct sched_queue *sq);
+/** Function prototype for sched peek functions. This function should return
+ * the packet that should be passed to the network interface next.
+ *
+ * @param sq The scheduler queue
+ */
+typedef struct sched_packet *(*sched_queue_peek_fn)(struct sched_queue *sq);
+/** Function prototype for sched queue functions. This function is called to
+ * enqueue a packet into the scheduling discipline.
+ *
+ * @param sq The scheduler queue
+ * @param packet The packet to enqueue
+ */
+typedef err_t (*sched_queue_queue_fn)(struct sched_queue *sq, struct 
sched_packet *packet);
+/** Function prototype for sched dequeue functions. This function dequeues a
+ * packet from the queueing discipline. The packet is being removed because it
+ * was either transmitted successfully, or the network device had an error 
other
+ * than ERR_WOULDBLOCK when transmitting the packet. The packet passed is the
+ * packet that was last returned from peek. No modifications will be made to
+ * the queue between a call to peek and a call to dequeue.
+ *
+ * @param sq The scheduler queue
+ * @param packet The packet to remove from the queue
+ */
+typedef void (*sched_queue_dequeue_fn)(struct sched_queue *sq, struct 
sched_packet *packet);
+/** Function prototype for sched drop_all functions. Called when the scheduling
+ * discipline should dump the contents of its queues. sched_drop should be
+ * called for each dumped packet.
+ *
+ * @param sq The scheduler queue
+ */
+typedef void (*sched_queue_drop_all_fn)(struct sched_queue *sq);
+
+/** Generic data structure used for all lwIP scheduling disciplines.
+ *  The following fields should be filled in by the initialization
+ *  function for the scheduling discipline: empty, peek, queue, dequeue, 
drop_all,
+ *  and packet_size.
+ */
+struct sched_queue {
+  /** Pointer to the next free packet */
+  struct sched_packet *free_head;
+  /** Pointer to the last free packet in the packet list, new free packets
+   * are enqueued here. */
+  struct sched_packet *free_tail;
+  /** Total number of packets available to the queueing discipline */
+  int nr_packets;
+  /** Total number of free packet queue entries */
+  int nr_free;
+
+  /** state pointed passed into the sched_add function. To be used to pass
+   * discipline specific parameters to the scheduling discipline and for
+   * the discipline to store private state data */
+  void *state;
+  /** Storage required for a queued packet entry, defaults to sizeof(struct
+   * packet) */
+  mem_size_t packet_size;
+  /** Scheduling discipline function to test for empty queue */
+  sched_queue_empty_fn empty;
+  /** Scheduling discipline function to get next packet to transmit */
+  sched_queue_peek_fn peek;
+  /** Scheduling discipline function to enqueue new packet */
+  sched_queue_queue_fn queue;
+  /** Scheduling discipline function to dequeue previously peeked packet */
+  sched_queue_dequeue_fn dequeue;
+  /** Scheduling discipline function to drop all queued packets */
+  sched_queue_drop_all_fn drop_all;
+
+  /** Associated netif */
+  struct netif *netif;
+  /** Original linkoutput function of netif */
+  netif_linkoutput_fn linkoutput;
+#if LWIP_IPV4
+  /** Original output function of netif */
+  netif_output_fn output;
+#endif /* LWIP_IPV4 */
+#if LWIP_IPV6
+  /** Original output_ip6 function of netif */
+  netif_output_ip6_fn output_ip6;
+#endif /* LWIP_IPV6 */
+#if LWIP_NETIF_REMOVE_CALLBACK
+  /** Original remove_callback function of netif */
+  netif_status_callback_fn remove_callback;
+#endif /* LWIP_NETIF_REMOVE_CALLBACK */
+};
+
+/** Queue entry structure. Stores information about a queued packet contained
+ * in the pbuf in a simple queue structure implemented by the next pointer */
+struct sched_packet {
+  /** Contains the data for this packet */
+  struct pbuf *p;
+  /** Next entry in queue */
+  struct sched_packet *next;
+  /** Destination address for this packet, special address type of 1 indicates
+   * the packet is raw for dual stack*/
+  ip_addr_t dest;
+#if !(LWIP_IPV4 && LWIP_IPV6)
+  /** For single stack use a separate variable for storing if the packet is 
raw */
+  u8_t raw;
+#endif /* !(LWIP_IPV4 && LWIP_IPV6) */
+};
+
+void sched_drop(struct sched_queue *sq, struct sched_packet *packet);
+err_t sched_queue_add(struct sched_queue *sq, struct netif *netif, void *pool,
+                      mem_size_t pool_size, void *state, sched_queue_init_fn 
init);
+void sched_queue_remove(struct sched_queue *sq);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* LWIP_SCHED */
+
+#endif /* LWIP_HDR_SCHED_H */
-- 
2.5.0