Added sysctl's for ixgbe all tunables

Added back in hw.ixgbe.flow_control which enable the default flow_control
of all ixgbe interfaces to be set in loader.conf

Added hw.ixgbe.advertise_speed which allows /boot/loader.conf to configure
advertised speed as sysctl.conf is too late for driver changes.

Made enable_aim device independent based on hw.ixgbe.enable_aim default.
--- sys/dev/ixgbe/ixgbe.c.orig	2012-11-06 12:28:35.607778862 +0000
+++ sys/dev/ixgbe/ixgbe.c	2012-11-06 14:20:02.774297585 +0000
@@ -157,8 +157,10 @@
 static void     ixgbe_update_link_status(struct adapter *);
 static void	ixgbe_refresh_mbufs(struct rx_ring *, int);
 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
-static int	ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
-static int	ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
+static int	ixgbe_set_flowcntl_sysctl(SYSCTL_HANDLER_ARGS);
+static int	ixgbe_set_flowcntl(struct adapter *);
+static int	ixgbe_set_advertise_sysctl(SYSCTL_HANDLER_ARGS);
+static int	ixgbe_set_advertise(struct adapter *);
 static int	ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 static int	ixgbe_dma_malloc(struct adapter *, bus_size_t,
 		    struct ixgbe_dma_alloc *, int);
@@ -230,32 +232,52 @@
 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 
 /*
-** TUNEABLE PARAMETERS:
-*/
+ * TUNEABLE PARAMETERS:
+ */
+static SYSCTL_NODE(_hw, OID_AUTO, ixgbe, CTLFLAG_RD, 0, "IXGBE driver parameters");
 
 /*
-** AIM: Adaptive Interrupt Moderation
-** which means that the interrupt rate
-** is varied over time based on the
-** traffic for that interrupt vector
-*/
+ * AIM: Adaptive Interrupt Moderation
+ * which means that the interrupt rate
+ * is varied over time based on the
+ * traffic for that interrupt vector
+ */
 static int ixgbe_enable_aim = TRUE;
 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
+SYSCTL_INT(_hw_ixgbe, OID_AUTO, enable_aim, CTLFLAG_RDTUN,
+    &ixgbe_enable_aim, 0, "Enable adaptive interrupt moderation");
 
 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
+SYSCTL_INT(_hw_ixgbe, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
+    &ixgbe_max_interrupt_rate, 0, "Maximum interrupt rate used in AIM");
 
 /* How many packets rxeof tries to clean at a time */
 static int ixgbe_rx_process_limit = 128;
 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
+SYSCTL_INT(_hw_ixgbe, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
+    &ixgbe_rx_process_limit, 0, "Maximum number of received packets to process"
+    " at a time, -1 means unlimited");
+
+/* Flow control setting, default to full */
+static int ixgbe_flow_control = ixgbe_fc_full;
+TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control);
+SYSCTL_INT(_hw_ixgbe, OID_AUTO, flow_control, CTLFLAG_RDTUN,
+    &ixgbe_flow_control, 0, "Default flow control used for all adapters");
+
+/* Advertise Speed, default to 0 (10G) */
+static int ixgbe_advertise_speed = 0;
+TUNABLE_INT("hw.ixgbe.advertise_speed", &ixgbe_advertise_speed);
+SYSCTL_INT(_hw_ixgbe, OID_AUTO, advertise_speed, CTLFLAG_RDTUN,
+    &ixgbe_advertise_speed, 0, "Default advertised speed for all adapters");
 
 /*
-** Smart speed setting, default to on
-** this only works as a compile option
-** right now as its during attach, set
-** this to 'ixgbe_smart_speed_off' to
-** disable.
-*/
+ * Smart speed setting, default to on
+ * this only works as a compile option
+ * right now as its during attach, set
+ * this to 'ixgbe_smart_speed_off' to
+ * disable.
+ */
 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 
 /*
@@ -264,6 +286,8 @@
  */
 static int ixgbe_enable_msix = 1;
 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
+SYSCTL_INT(_hw_ixgbe, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixgbe_enable_msix,
+    0, "Enable/disble MSIX for ixgbe adapters");
 
 /*
  * Header split: this causes the hardware to DMA
@@ -273,6 +297,8 @@
  */
 static int ixgbe_header_split = FALSE;
 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
+SYSCTL_INT(_hw_ixgbe, OID_AUTO, hdr_split, CTLFLAG_RDTUN, &ixgbe_header_split,
+    0, "Enable receive mbuf header split");
 
 /*
  * Number of Queues, can be set to 0,
@@ -282,6 +308,8 @@
  */
 static int ixgbe_num_queues = 0;
 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
+SYSCTL_INT(_hw_ixgbe, OID_AUTO, num_queues, CTLFLAG_RDTUN, &ixgbe_num_queues, 0,
+    "Number of queues to configure, 0 indicates autoconfigure");
 
 /*
 ** Number of TX descriptors per ring,
@@ -290,10 +318,14 @@
 */
 static int ixgbe_txd = PERFORM_TXD;
 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
+SYSCTL_INT(_hw_ixgbe, OID_AUTO, txd, CTLFLAG_RDTUN, &ixgbe_txd, 0,
+    "Number of receive descriptors per queue");
 
 /* Number of RX descriptors per ring */
 static int ixgbe_rxd = PERFORM_RXD;
 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
+SYSCTL_INT(_hw_ixgbe, OID_AUTO, rxd, CTLFLAG_RDTUN, &ixgbe_rxd, 0,
+    "Number of transmit descriptors per queue");
 
 /*
 ** HW RSC control: 
@@ -426,12 +458,19 @@
 	/* Core Lock Init*/
 	IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 
+	/* hw.ixgbe defaults init */
+	adapter->advertise = ixgbe_advertise_speed;
+	adapter->fc = ixgbe_flow_control;
+	adapter->enable_aim = ixgbe_enable_aim;
+	ixgbe_set_advertise(adapter);
+	ixgbe_set_flowcntl(adapter);
+
 	/* SYSCTL APIs */
 
 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 			SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 			OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
-			adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
+			adapter, 0, ixgbe_set_flowcntl_sysctl, "I", "Flow Control");
 
         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 			SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
@@ -446,7 +485,7 @@
 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 			SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 			OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
-			adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
+			adapter, 0, ixgbe_set_advertise_sysctl, "I", "Link Speed");
 
 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 			SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
@@ -1528,7 +1567,7 @@
 
 	/* Do AIM now? */
 
-	if (ixgbe_enable_aim == FALSE)
+	if (adapter->enable_aim == FALSE)
 		goto no_calc;
 	/*
 	** Do Adaptive Interrupt Moderation:
@@ -5744,7 +5787,7 @@
 **	3 - full
 */
 static int
-ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
+ixgbe_set_flowcntl_sysctl(SYSCTL_HANDLER_ARGS)
 {
 	int error, last;
 	struct adapter *adapter = (struct adapter *) arg1;
@@ -5758,6 +5801,13 @@
 	if (adapter->fc == last)
 		return (0);
 
+	return ixgbe_set_flowcntl(adapter);
+}
+
+static int
+ixgbe_set_flowcntl(struct adapter *adapter)
+{
+
 	switch (adapter->fc) {
 		case ixgbe_fc_rx_pause:
 		case ixgbe_fc_tx_pause:
@@ -5775,7 +5825,7 @@
 	/* Don't autoneg if forcing a value */
 	adapter->hw.fc.disable_fc_autoneg = TRUE;
 	ixgbe_fc_enable(&adapter->hw);
-	return error;
+	return 0;
 }
 
 static void
@@ -5795,23 +5845,32 @@
 **	3 - advertise normal
 */
 static int
-ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
+ixgbe_set_advertise_sysctl(SYSCTL_HANDLER_ARGS)
 {
 	int			error = 0;
 	struct adapter		*adapter;
-	device_t		dev;
-	struct ixgbe_hw		*hw;
-	ixgbe_link_speed	speed, last;
 
 	adapter = (struct adapter *) arg1;
-	dev = adapter->dev;
-	hw = &adapter->hw;
-	last = adapter->advertise;
 
 	error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
 	if ((error) || (adapter->advertise == -1))
 		return (error);
 
+	return ixgbe_set_advertise(adapter);
+}
+ 
+static int
+ixgbe_set_advertise(struct adapter *adapter)
+{
+	int			error = 0;
+	device_t		dev;
+	struct ixgbe_hw		*hw;
+	ixgbe_link_speed	speed, last;
+
+	hw = &adapter->hw;
+	dev = adapter->dev;
+	last = hw->phy.autoneg_advertised;
+
 	if (adapter->advertise == last) /* no change */
 		return (0);
 
--- sys/dev/ixgbe/ixgbe.h.orig	2012-11-06 12:28:48.969656479 +0000
+++ sys/dev/ixgbe/ixgbe.h	2012-11-06 12:48:39.784556819 +0000
@@ -410,6 +410,7 @@
 	u32			optics;
 	u32			fc; /* local flow ctrl setting */
 	int			advertise;  /* link speeds */
+	bool			enable_aim; /* adaptive interrupt moderation */
 	bool			link_active;
 	u16			max_frame_size;
 	u16			num_segs;
=