SCTP: 'multi-home' support

This patch addresses the SCTP requirement for multiple sub-connections
to implement the so called 'multi-homed' scenario.

Change-Id: Ibce18f216e9d2bebe318992c441bf278e16aad17
Signed-off-by: Marco Varlese <marco.varlese@suse.com>

6 files changed, 218 insertions, 186 deletions

diff --git a/src/vnet/buffer.h b/src/vnet/buffer.h
index 807cd28f9e4..4e01a069bf3 100644
--- a/src/vnet/buffer.h
+++ b/src/vnet/buffer.h
@@ -333,6 +333,7 @@ typedef struct
       u16 hdr_offset;		/**< offset relative to ip hdr */
       u16 data_offset;		/**< offset relative to ip hdr */
       u16 data_len;		/**< data len */
+      u8 conn_idx;
       u8 flags;
     } sctp;
 
diff --git a/src/vnet/sctp/sctp.c b/src/vnet/sctp/sctp.c
index 529e4089644..224c97d419e 100644
--- a/src/vnet/sctp/sctp.c
+++ b/src/vnet/sctp/sctp.c
@@ -360,7 +360,7 @@ sctp_connection_open (transport_endpoint_t * rmt)
   uword thread_id;
   int rv;
 
-  u8 idx = sctp_pick_conn_idx_on_state (SCTP_STATE_CLOSED);
+  u8 idx = MAIN_SCTP_SUB_CONN_IDX;
 
   /*
    * Allocate local endpoint
@@ -452,6 +452,21 @@ sctp_session_open (transport_endpoint_t * tep)
 u16
 sctp_check_outstanding_data_chunks (sctp_connection_t * sctp_conn)
 {
+  u8 i;
+  for (i = 0; i < MAX_SCTP_CONNECTIONS; i++)
+    {
+      if (sctp_conn->sub_conn[i].state == SCTP_SUBCONN_STATE_DOWN)
+	continue;
+
+      if (sctp_conn->sub_conn[i].is_retransmitting == 1 ||
+	  sctp_conn->sub_conn[i].enqueue_state != SCTP_ERROR_ENQUEUED)
+	{
+	  SCTP_DBG_OUTPUT
+	    ("Connection %u has still DATA to be enqueued inboud / outboud");
+	  return 1;
+	}
+
+    }
   return 0;			/* Indicates no more data to be read/sent */
 }
 
@@ -622,8 +637,6 @@ sctp_expired_timers_cb (u32 conn_index, u32 timer_id)
     case SCTP_TIMER_T1_COOKIE:
     case SCTP_TIMER_T2_SHUTDOWN:
     case SCTP_TIMER_T3_RXTX:
-      clib_smp_atomic_add (&sctp_conn->sub_conn[conn_index].unacknowledged_hb,
-			   1);
       sctp_timer_reset (sctp_conn, conn_index, timer_id);
       break;
     case SCTP_TIMER_T4_HEARTBEAT:
@@ -632,18 +645,34 @@ sctp_expired_timers_cb (u32 conn_index, u32 timer_id)
     }
 
   if (sctp_conn->sub_conn[conn_index].unacknowledged_hb >
-      SCTP_ASSOCIATION_MAX_RETRANS)
+      SCTP_PATH_MAX_RETRANS)
     {
       // The remote-peer is considered to be unreachable hence shutting down
+      u8 i, total_subs_down = 1;
+      for (i = 0; i < MAX_SCTP_CONNECTIONS; i++)
+	{
+	  if (sctp_conn->sub_conn[i].state == SCTP_SUBCONN_STATE_DOWN)
+	    continue;
+
+	  u32 now = sctp_time_now ();
+	  if (now > (sctp_conn->sub_conn[i].last_seen + SCTP_HB_INTERVAL))
+	    {
+	      total_subs_down += 1;
+	      sctp_conn->sub_conn[i].state = SCTP_SUBCONN_STATE_DOWN;
+	    }
+	}
 
-      /* Start cleanup. App wasn't notified yet so use delete notify as
-       * opposed to delete to cleanup session layer state. */
-      stream_session_delete_notify (&sctp_conn->sub_conn
-				    [MAIN_SCTP_SUB_CONN_IDX].connection);
+      if (total_subs_down == MAX_SCTP_CONNECTIONS)
+	{
+	  /* Start cleanup. App wasn't notified yet so use delete notify as
+	   * opposed to delete to cleanup session layer state. */
+	  stream_session_delete_notify (&sctp_conn->sub_conn
+					[MAIN_SCTP_SUB_CONN_IDX].connection);
 
-      sctp_connection_timers_reset (sctp_conn);
+	  sctp_connection_timers_reset (sctp_conn);
 
-      sctp_connection_cleanup (sctp_conn);
+	  sctp_connection_cleanup (sctp_conn);
+	}
     }
   return;
 
diff --git a/src/vnet/sctp/sctp.h b/src/vnet/sctp/sctp.h
index 25fae37b6f7..af652dc06dc 100644
--- a/src/vnet/sctp/sctp.h
+++ b/src/vnet/sctp/sctp.h
@@ -89,6 +89,13 @@ typedef enum _sctp_error
 #define sctp_trajectory_add_start(b, start)
 #endif
 
+enum _sctp_subconn_state
+{
+  SCTP_SUBCONN_STATE_DOWN = 0,
+  SCTP_SUBCONN_STATE_UP,
+  SCTP_SUBCONN_STATE_ALLOW_HB
+};
+
 typedef struct _sctp_sub_connection
 {
   transport_connection_t connection;	      /**< Common transport data. First! */
@@ -122,14 +129,16 @@ typedef struct _sctp_sub_connection
   	  	  	  	  Every time the RTT calculation completes (i.e., the DATA chunk is SACK'd),
   	  	  	  	  clear this flag. */
 
-  u32 last_time; /**< The time to which this destination was last sent a packet to.
+  u32 last_seen; /**< The time to which this destination was last sent a packet to.
   	  	  	  	  This can be used to determine if a HEARTBEAT is needed. */
 
   u8 unacknowledged_hb;	/**< Used to track how many unacknowledged heartbeats we had;
-  	  	  	  	  If more than Max.Retransmit then connetion is considered unreachable. */
+  	  	  	  	  If more than SCTP_PATH_MAX_RETRANS then connection is considered unreachable. */
 
   u8 is_retransmitting;	/**< A flag (0 = no, 1 = yes) indicating whether the connection is retransmitting a previous packet */
 
+  u8 enqueue_state;
+
 } sctp_sub_connection_t;
 
 typedef struct
@@ -249,9 +258,9 @@ void sctp_connection_del (sctp_connection_t * sctp_conn);
 u32 sctp_push_header (transport_connection_t * tconn, vlib_buffer_t * b);
 void sctp_send_init (sctp_connection_t * sctp_conn);
 void sctp_send_shutdown (sctp_connection_t * sctp_conn);
-void sctp_send_shutdown_ack (sctp_connection_t * sctp_conn,
+void sctp_send_shutdown_ack (sctp_connection_t * sctp_conn, u8 idx,
 			     vlib_buffer_t * b);
-void sctp_send_shutdown_complete (sctp_connection_t * sctp_conn,
+void sctp_send_shutdown_complete (sctp_connection_t * sctp_conn, u8 idx,
 				  vlib_buffer_t * b0);
 void sctp_send_heartbeat (sctp_connection_t * sctp_conn);
 void sctp_flush_frame_to_output (vlib_main_t * vm, u8 thread_index,
@@ -268,24 +277,25 @@ u8 *format_sctp_header (u8 * s, va_list * args);
 u8 *format_sctp_tx_trace (u8 * s, va_list * args);
 
 clib_error_t *sctp_init (vlib_main_t * vm);
-void sctp_connection_timers_init (sctp_connection_t * tc);
-void sctp_connection_timers_reset (sctp_connection_t * tc);
-void sctp_init_snd_vars (sctp_connection_t * tc);
-void sctp_init_mss (sctp_connection_t * tc);
+void sctp_connection_timers_init (sctp_connection_t * sctp_conn);
+void sctp_connection_timers_reset (sctp_connection_t * sctp_conn);
+void sctp_init_snd_vars (sctp_connection_t * sctp_conn);
+void sctp_init_mss (sctp_connection_t * sctp_conn);
 
-void sctp_prepare_initack_chunk (sctp_connection_t * ts, vlib_buffer_t * b,
-				 ip4_address_t * ip4_addr,
+void sctp_prepare_initack_chunk (sctp_connection_t * sctp_conn, u8 idx,
+				 vlib_buffer_t * b, ip4_address_t * ip4_addr,
 				 ip6_address_t * ip6_addr);
-void sctp_prepare_cookie_echo_chunk (sctp_connection_t * tc,
+void sctp_prepare_cookie_echo_chunk (sctp_connection_t * sctp_conn, u8 idx,
 				     vlib_buffer_t * b,
 				     sctp_state_cookie_param_t * sc);
-void sctp_prepare_cookie_ack_chunk (sctp_connection_t * tc,
+void sctp_prepare_cookie_ack_chunk (sctp_connection_t * sctp_conn, u8 idx,
 				    vlib_buffer_t * b);
-void sctp_prepare_sack_chunk (sctp_connection_t * tc, vlib_buffer_t * b);
-void sctp_prepare_heartbeat_ack_chunk (sctp_connection_t * sctp_conn,
+void sctp_prepare_sack_chunk (sctp_connection_t * sctp_conn, u8 idx,
+			      vlib_buffer_t * b);
+void sctp_prepare_heartbeat_ack_chunk (sctp_connection_t * sctp_conn, u8 idx,
 				       vlib_buffer_t * b);
 
-u16 sctp_check_outstanding_data_chunks (sctp_connection_t * tc);
+u16 sctp_check_outstanding_data_chunks (sctp_connection_t * sctp_conn);
 
 #define IP_PROTOCOL_SCTP	132
 
@@ -407,7 +417,11 @@ sctp_optparam_type_to_string (u8 type)
 #define SCTP_RTO_ALPHA 1/8
 #define SCTP_RTO_BETA 1/4
 #define SCTP_VALID_COOKIE_LIFE 60 * SHZ	/* 60 seconds */
-#define SCTP_ASSOCIATION_MAX_RETRANS 10
+#define SCTP_ASSOCIATION_MAX_RETRANS 10	// the overall connection
+#define SCTP_PATH_MAX_RETRANS 5	// number of attempts per destination address
+#define SCTP_MAX_INIT_RETRANS 8	// number of attempts
+#define SCTP_HB_INTERVAL 30 * SHZ
+#define SCTP_HB_MAX_BURST 1
 
 #define SCTP_TO_TIMER_TICK       SCTP_TICK*10	/* Period for converting from SCTP_TICK */
 
@@ -696,56 +710,64 @@ sctp_connection_get (u32 conn_index, u32 thread_index)
   return pool_elt_at_index (sctp_main.connections[thread_index], conn_index);
 }
 
+#define SELECT_MAX_RETRIES 8
+
 always_inline u8
-sctp_pick_conn_idx_on_chunk (sctp_chunk_type chunk_type)
+sctp_data_subconn_select (sctp_connection_t * sctp_conn)
 {
-  u8 idx = MAIN_SCTP_SUB_CONN_IDX;
+  u8 i = 0;
+  u8 state = SCTP_SUBCONN_STATE_DOWN;
+  u32 sub = MAIN_SCTP_SUB_CONN_IDX;
+  u32 data_subconn_seed = random_default_seed ();
 
-  switch (chunk_type)
+  while (state == SCTP_SUBCONN_STATE_DOWN && i < SELECT_MAX_RETRIES)
     {
-    case DATA:
-    case INIT:
-    case INIT_ACK:
-    case SACK:
-    case HEARTBEAT:
-    case HEARTBEAT_ACK:
-    case ABORT:
-    case SHUTDOWN:
-    case SHUTDOWN_ACK:
-    case OPERATION_ERROR:
-    case COOKIE_ECHO:
-    case COOKIE_ACK:
-    case ECNE:
-    case CWR:
-    case SHUTDOWN_COMPLETE:
-      idx = MAIN_SCTP_SUB_CONN_IDX;
-      break;
-    default:
-      idx = 0;
+      u32 sub = random_u32 (&data_subconn_seed) % MAX_SCTP_CONNECTIONS;
+      if (sctp_conn->sub_conn[sub].state == SCTP_SUBCONN_STATE_UP)
+	break;
+      i++;
     }
-  return idx;
+  return sub;
 }
 
 always_inline u8
-sctp_pick_conn_idx_on_state (sctp_state_t state)
+sctp_sub_conn_id_via_ip6h (sctp_connection_t * sctp_conn, ip6_header_t * ip6h)
 {
-  u8 idx = MAIN_SCTP_SUB_CONN_IDX;
+  u8 i;
 
-  switch (state)
+  for (i = 0; i < MAX_SCTP_CONNECTIONS; i++)
     {
-    case SCTP_STATE_CLOSED:
-    case SCTP_STATE_COOKIE_WAIT:
-    case SCTP_STATE_COOKIE_ECHOED:
-    case SCTP_STATE_ESTABLISHED:
-    case SCTP_STATE_SHUTDOWN_PENDING:
-    case SCTP_STATE_SHUTDOWN_SENT:
-    case SCTP_STATE_SHUTDOWN_RECEIVED:
-    case SCTP_STATE_SHUTDOWN_ACK_SENT:
-      idx = MAIN_SCTP_SUB_CONN_IDX;
-    default:
-      idx = MAIN_SCTP_SUB_CONN_IDX;
+      if (sctp_conn->sub_conn[i].connection.lcl_ip.ip6.as_u64[0] ==
+	  ip6h->dst_address.as_u64[0] &&
+	  sctp_conn->sub_conn[i].connection.lcl_ip.ip6.as_u64[1] ==
+	  ip6h->dst_address.as_u64[1] &&
+	  sctp_conn->sub_conn[i].connection.rmt_ip.ip6.as_u64[0] ==
+	  ip6h->src_address.as_u64[0] &&
+	  sctp_conn->sub_conn[i].connection.rmt_ip.ip6.as_u64[1] ==
+	  ip6h->src_address.as_u64[1])
+	return i;
+    }
+  clib_warning ("Did not find a sub-connection; defaulting to %u",
+		MAIN_SCTP_SUB_CONN_IDX);
+  return MAIN_SCTP_SUB_CONN_IDX;
+}
+
+always_inline u8
+sctp_sub_conn_id_via_ip4h (sctp_connection_t * sctp_conn, ip4_header_t * ip4h)
+{
+  u8 i;
+
+  for (i = 0; i < MAX_SCTP_CONNECTIONS; i++)
+    {
+      if (sctp_conn->sub_conn[i].connection.lcl_ip.ip4.as_u32 ==
+	  ip4h->dst_address.as_u32
+	  && sctp_conn->sub_conn[i].connection.rmt_ip.ip4.as_u32 ==
+	  ip4h->src_address.as_u32)
+	return i;
     }
-  return idx;
+  clib_warning ("Did not find a sub-connection; defaulting to %u",
+		MAIN_SCTP_SUB_CONN_IDX);
+  return MAIN_SCTP_SUB_CONN_IDX;
 }
 
 /**
diff --git a/src/vnet/sctp/sctp_input.c b/src/vnet/sctp/sctp_input.c
index 8f4b043f297..6c804880d7e 100644
--- a/src/vnet/sctp/sctp_input.c
+++ b/src/vnet/sctp/sctp_input.c
@@ -399,7 +399,8 @@ sctp_handle_init (sctp_header_t * sctp_hdr,
     }
 
   /* Reuse buffer to make init-ack and send */
-  sctp_prepare_initack_chunk (sctp_conn, b0, ip4_addr, ip6_addr);
+  sctp_prepare_initack_chunk (sctp_conn, MAIN_SCTP_SUB_CONN_IDX, b0, ip4_addr,
+			      ip6_addr);
   return SCTP_ERROR_NONE;
 }
 
@@ -530,10 +531,10 @@ sctp_handle_init_ack (sctp_header_t * sctp_hdr,
 	}
     }
 
-  sctp_prepare_cookie_echo_chunk (sctp_conn, b0, &state_cookie);
+  sctp_prepare_cookie_echo_chunk (sctp_conn, idx, b0, &state_cookie);
 
   /* Start the T1_COOKIE timer */
-  sctp_timer_set (sctp_conn, sctp_pick_conn_idx_on_chunk (COOKIE_ECHO),
+  sctp_timer_set (sctp_conn, idx,
 		  SCTP_TIMER_T1_COOKIE, sctp_conn->sub_conn[idx].RTO);
 
   return SCTP_ERROR_NONE;
@@ -648,7 +649,7 @@ sctp_session_enqueue_data (sctp_connection_t * sctp_conn, vlib_buffer_t * b,
 }
 
 always_inline u8
-sctp_is_sack_delayable (sctp_connection_t * sctp_conn, u8 is_gapping)
+sctp_is_sack_delayable (sctp_connection_t * sctp_conn, u8 idx, u8 is_gapping)
 {
   if (is_gapping != 0)
     {
@@ -679,7 +680,7 @@ sctp_is_connection_gapping (sctp_connection_t * sctp_conn, u32 tsn,
     {
       SCTP_CONN_TRACKING_DBG
 	("GAPPING: CONN_INDEX = %u, sctp_conn->next_tsn_expected = %u, tsn = %u, diff = %u",
-	 sctp_conn->sub_conn[idx].connection.c_index,
+	 sctp_conn->sub_conn[MAIN_SCTP_SUB_CONN_IDX].connection.c_index,
 	 sctp_conn->next_tsn_expected, tsn,
 	 sctp_conn->next_tsn_expected - tsn);
 
@@ -723,7 +724,11 @@ sctp_handle_data (sctp_payload_data_chunk_t * sctp_data_chunk,
     {
       /* In order data, enqueue. Fifo figures out by itself if any out-of-order
        * segments can be enqueued after fifo tail offset changes. */
-      error = sctp_session_enqueue_data (sctp_conn, b, n_data_bytes, idx);
+      if (PREDICT_FALSE (is_gapping == 1))
+	error =
+	  sctp_session_enqueue_data_ooo (sctp_conn, b, n_data_bytes, idx);
+      else
+	error = sctp_session_enqueue_data (sctp_conn, b, n_data_bytes, idx);
     }
   else if (bbit == 1 && ebit == 0)	/* First piece of a fragmented user message */
     {
@@ -751,8 +756,10 @@ sctp_handle_data (sctp_payload_data_chunk_t * sctp_data_chunk,
 
   SCTP_ADV_DBG ("POINTER_WITH_DATA = %p", b->data);
 
-  if (!sctp_is_sack_delayable (sctp_conn, is_gapping))
-    sctp_prepare_sack_chunk (sctp_conn, b);
+  if (!sctp_is_sack_delayable (sctp_conn, idx, is_gapping))
+    sctp_prepare_sack_chunk (sctp_conn, idx, b);
+
+  sctp_conn->sub_conn[idx].enqueue_state = error;
 
   return error;
 }
@@ -787,10 +794,11 @@ sctp_handle_cookie_echo (sctp_header_t * sctp_hdr,
       return SCTP_ERROR_COOKIE_ECHO_VIOLATION;
     }
 
-  sctp_prepare_cookie_ack_chunk (sctp_conn, b0);
+  sctp_prepare_cookie_ack_chunk (sctp_conn, idx, b0);
 
   /* Change state */
   sctp_conn->state = SCTP_STATE_ESTABLISHED;
+  sctp_conn->sub_conn[idx].state = SCTP_SUBCONN_STATE_UP;
   *next0 = sctp_next_output (sctp_conn->sub_conn[idx].c_is_ip4);
 
   sctp_timer_set (sctp_conn, idx, SCTP_TIMER_T4_HEARTBEAT,
@@ -819,6 +827,8 @@ sctp_handle_cookie_ack (sctp_header_t * sctp_hdr,
   sctp_timer_reset (sctp_conn, idx, SCTP_TIMER_T1_COOKIE);
   /* Change state */
   sctp_conn->state = SCTP_STATE_ESTABLISHED;
+  sctp_conn->sub_conn[idx].state = SCTP_SUBCONN_STATE_UP;
+
   *next0 = sctp_next_drop (sctp_conn->sub_conn[idx].c_is_ip4);
 
   sctp_timer_set (sctp_conn, idx, SCTP_TIMER_T4_HEARTBEAT,
@@ -890,36 +900,18 @@ sctp46_rcv_phase_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	    {
 	      ip4_hdr = vlib_buffer_get_current (b0);
 	      sctp_hdr = ip4_next_header (ip4_hdr);
+	      idx = sctp_sub_conn_id_via_ip4h (sctp_conn, ip4_hdr);
 	    }
 	  else
 	    {
 	      ip6_hdr = vlib_buffer_get_current (b0);
 	      sctp_hdr = ip6_next_header (ip6_hdr);
+	      idx = sctp_sub_conn_id_via_ip6h (sctp_conn, ip6_hdr);
 	    }
-	  idx = sctp_pick_conn_idx_on_state (sctp_conn->state);
 
+	  sctp_conn->sub_conn[idx].parent = sctp_conn;
 	  sctp_full_hdr_t *full_hdr = (sctp_full_hdr_t *) sctp_hdr;
 
-	  transport_connection_t *trans_conn =
-	    &sctp_conn->sub_conn[idx].connection;
-
-	  trans_conn->lcl_port = sctp_hdr->dst_port;
-	  trans_conn->rmt_port = sctp_hdr->src_port;
-	  trans_conn->is_ip4 = is_ip4;
-
-	  if (is_ip4)
-	    {
-	      trans_conn->lcl_ip.ip4.as_u32 = ip4_hdr->dst_address.as_u32;
-	      trans_conn->rmt_ip.ip4.as_u32 = ip4_hdr->src_address.as_u32;
-	    }
-	  else
-	    {
-	      clib_memcpy (&trans_conn->lcl_ip.ip6, &ip6_hdr->dst_address,
-			   sizeof (ip6_address_t));
-	      clib_memcpy (&trans_conn->rmt_ip.ip6, &ip6_hdr->src_address,
-			   sizeof (ip6_address_t));
-	    }
-
 	  sctp_chunk_hdr =
 	    (sctp_chunks_common_hdr_t *) (&full_hdr->common_hdr);
 
@@ -1113,11 +1105,11 @@ sctp_handle_shutdown (sctp_header_t * sctp_hdr,
     case SCTP_STATE_ESTABLISHED:
       if (sctp_check_outstanding_data_chunks (sctp_conn) == 0)
 	sctp_conn->state = SCTP_STATE_SHUTDOWN_RECEIVED;
-      sctp_send_shutdown_ack (sctp_conn, b0);
+      sctp_send_shutdown_ack (sctp_conn, idx, b0);
       break;
 
     case SCTP_STATE_SHUTDOWN_SENT:
-      sctp_send_shutdown_ack (sctp_conn, b0);
+      sctp_send_shutdown_ack (sctp_conn, idx, b0);
       break;
     }
 
@@ -1156,7 +1148,7 @@ sctp_handle_shutdown_ack (sctp_header_t * sctp_hdr,
   sctp_timer_reset (sctp_conn, MAIN_SCTP_SUB_CONN_IDX,
 		    SCTP_TIMER_T2_SHUTDOWN);
 
-  sctp_send_shutdown_complete (sctp_conn, b0);
+  sctp_send_shutdown_complete (sctp_conn, idx, b0);
 
   *next0 = sctp_next_output (sctp_conn->sub_conn[idx].c_is_ip4);
 
@@ -1229,6 +1221,7 @@ sctp46_shutdown_phase_inline (vlib_main_t * vm,
 	  sctp_connection_t *sctp_conn;
 	  u16 sctp_implied_length = 0;
 	  u16 error0 = SCTP_ERROR_NONE, next0 = SCTP_RCV_PHASE_N_NEXT;
+	  u8 idx = 0;
 
 	  bi0 = from[0];
 	  to_next[0] = bi0;
@@ -1254,11 +1247,13 @@ sctp46_shutdown_phase_inline (vlib_main_t * vm,
 	    {
 	      ip4_hdr = vlib_buffer_get_current (b0);
 	      sctp_hdr = ip4_next_header (ip4_hdr);
+	      idx = sctp_sub_conn_id_via_ip4h (sctp_conn, ip4_hdr);
 	    }
 	  else
 	    {
 	      ip6_hdr = vlib_buffer_get_current (b0);
 	      sctp_hdr = ip6_next_header (ip6_hdr);
+	      idx = sctp_sub_conn_id_via_ip6h (sctp_conn, ip6_hdr);
 	    }
 
 	  sctp_full_hdr_t *full_hdr = (sctp_full_hdr_t *) sctp_hdr;
@@ -1267,8 +1262,6 @@ sctp46_shutdown_phase_inline (vlib_main_t * vm,
 	  sctp_implied_length =
 	    sctp_calculate_implied_length (ip4_hdr, ip6_hdr, is_ip4);
 
-	  u8 idx = sctp_pick_conn_idx_on_state (sctp_conn->state);
-
 	  u8 chunk_type = vnet_sctp_get_chunk_type (sctp_chunk_hdr);
 	  switch (chunk_type)
 	    {
@@ -1427,6 +1420,8 @@ sctp_handle_sack (sctp_selective_ack_chunk_t * sack_chunk,
       return SCTP_ERROR_INVALID_TAG;
     }
 
+  sctp_conn->sub_conn[idx].last_seen = sctp_time_now ();
+
   sctp_calculate_rto (sctp_conn, idx);
 
   sctp_timer_update (sctp_conn, idx, SCTP_TIMER_T3_RXTX,
@@ -1450,7 +1445,7 @@ sctp_handle_heartbeat (sctp_hb_req_chunk_t * sctp_hb_chunk,
       return SCTP_ERROR_INVALID_TAG;
     }
 
-  sctp_prepare_heartbeat_ack_chunk (sctp_conn, b0);
+  sctp_prepare_heartbeat_ack_chunk (sctp_conn, idx, b0);
 
   *next0 = sctp_next_output (sctp_conn->sub_conn[idx].connection.is_ip4);
 
@@ -1462,6 +1457,8 @@ sctp_handle_heartbeat_ack (sctp_hb_ack_chunk_t * sctp_hb_ack_chunk,
 			   sctp_connection_t * sctp_conn, u8 idx,
 			   vlib_buffer_t * b0, u16 * next0)
 {
+  sctp_conn->sub_conn[idx].last_seen = sctp_time_now ();
+
   sctp_conn->sub_conn[idx].unacknowledged_hb -= 1;
 
   sctp_timer_update (sctp_conn, idx, SCTP_TIMER_T4_HEARTBEAT,
@@ -1729,39 +1726,18 @@ sctp46_established_phase_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	    {
 	      ip4_hdr = vlib_buffer_get_current (b0);
 	      sctp_hdr = ip4_next_header (ip4_hdr);
+	      idx = sctp_sub_conn_id_via_ip4h (sctp_conn, ip4_hdr);
 	    }
 	  else
 	    {
 	      ip6_hdr = vlib_buffer_get_current (b0);
 	      sctp_hdr = ip6_next_header (ip6_hdr);
+	      idx = sctp_sub_conn_id_via_ip6h (sctp_conn, ip6_hdr);
 	    }
 
-	  idx = sctp_pick_conn_idx_on_state (sctp_conn->state);
-
-	  sctp_full_hdr_t *full_hdr = (sctp_full_hdr_t *) sctp_hdr;
-
-	  transport_connection_t *trans_conn =
-	    &sctp_conn->sub_conn[idx].connection;
-
-	  trans_conn->lcl_port = sctp_hdr->dst_port;
-	  trans_conn->rmt_port = sctp_hdr->src_port;
-	  trans_conn->is_ip4 = is_ip4;
-
 	  sctp_conn->sub_conn[idx].parent = sctp_conn;
 
-	  if (is_ip4)
-	    {
-	      trans_conn->lcl_ip.ip4.as_u32 = ip4_hdr->dst_address.as_u32;
-	      trans_conn->rmt_ip.ip4.as_u32 = ip4_hdr->src_address.as_u32;
-	    }
-	  else
-	    {
-	      clib_memcpy (&trans_conn->lcl_ip.ip6, &ip6_hdr->dst_address,
-			   sizeof (ip6_address_t));
-	      clib_memcpy (&trans_conn->rmt_ip.ip6, &ip6_hdr->src_address,
-			   sizeof (ip6_address_t));
-	    }
-
+	  sctp_full_hdr_t *full_hdr = (sctp_full_hdr_t *) sctp_hdr;
 	  sctp_chunk_hdr =
 	    (sctp_chunks_common_hdr_t *) (&full_hdr->common_hdr);
 
@@ -2106,7 +2082,7 @@ sctp46_input_dispatcher (vlib_main_t * vm, vlib_node_runtime_t * node,
 				      sctp_conn->sub_conn
 				      [idx].connection.c_index,
 				      sctp_state_to_string (sctp_conn->state),
-				      sctp_chunk_to_string (type),
+				      sctp_chunk_to_string (chunk_type),
 				      phase_to_string (next0));
 
 	      if (chunk_type == DATA)
diff --git a/src/vnet/sctp/sctp_output.c b/src/vnet/sctp/sctp_output.c
index 6012e50b908..459b33d46bc 100644
--- a/src/vnet/sctp/sctp_output.c
+++ b/src/vnet/sctp/sctp_output.c
@@ -261,6 +261,7 @@ sctp_reuse_buffer (vlib_main_t * vm, vlib_buffer_t * b)
   b->current_length = 0;
   b->total_length_not_including_first_buffer = 0;
   vnet_buffer (b)->sctp.flags = 0;
+  vnet_buffer (b)->sctp.conn_idx = MAX_SCTP_CONNECTIONS;
 
   /* Leave enough space for headers */
   return vlib_buffer_make_headroom (b, MAX_HDRS_LEN);
@@ -274,6 +275,7 @@ sctp_init_buffer (vlib_main_t * vm, vlib_buffer_t * b)
   b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
   b->total_length_not_including_first_buffer = 0;
   vnet_buffer (b)->sctp.flags = 0;
+  vnet_buffer (b)->sctp.conn_idx = MAX_SCTP_CONNECTIONS;
   VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b);
   /* Leave enough space for headers */
   return vlib_buffer_make_headroom (b, MAX_HDRS_LEN);
@@ -410,12 +412,12 @@ sctp_enqueue_to_ip_lookup (vlib_main_t * vm, vlib_buffer_t * b, u32 bi,
  * Convert buffer to INIT
  */
 void
-sctp_prepare_init_chunk (sctp_connection_t * sctp_conn, vlib_buffer_t * b)
+sctp_prepare_init_chunk (sctp_connection_t * sctp_conn, u8 idx,
+			 vlib_buffer_t * b)
 {
   u32 random_seed = random_default_seed ();
   u16 alloc_bytes = sizeof (sctp_init_chunk_t);
-  sctp_sub_connection_t *sub_conn =
-    &sctp_conn->sub_conn[sctp_pick_conn_idx_on_chunk (INIT)];
+  sctp_sub_connection_t *sub_conn = &sctp_conn->sub_conn[idx];
 
   sctp_ipv4_addr_param_t *ip4_param = 0;
   sctp_ipv6_addr_param_t *ip6_param = 0;
@@ -476,6 +478,7 @@ sctp_prepare_init_chunk (sctp_connection_t * sctp_conn, vlib_buffer_t * b)
   sctp_conn->local_tag = init_chunk->initiate_tag;
 
   vnet_buffer (b)->sctp.connection_index = sub_conn->c_c_index;
+  vnet_buffer (b)->sctp.conn_idx = idx;
 
   SCTP_DBG_STATE_MACHINE ("CONN_INDEX = %u, CURR_CONN_STATE = %u (%s), "
 			  "CHUNK_TYPE = %s, "
@@ -518,11 +521,10 @@ sctp_compute_mac (sctp_connection_t * sctp_conn,
 }
 
 void
-sctp_prepare_cookie_ack_chunk (sctp_connection_t * sctp_conn,
+sctp_prepare_cookie_ack_chunk (sctp_connection_t * sctp_conn, u8 idx,
 			       vlib_buffer_t * b)
 {
   vlib_main_t *vm = vlib_get_main ();
-  u8 idx = sctp_pick_conn_idx_on_chunk (COOKIE_ACK);
 
   sctp_reuse_buffer (vm, b);
 
@@ -549,15 +551,15 @@ sctp_prepare_cookie_ack_chunk (sctp_connection_t * sctp_conn,
 
   vnet_buffer (b)->sctp.connection_index =
     sctp_conn->sub_conn[idx].connection.c_index;
+  vnet_buffer (b)->sctp.conn_idx = idx;
 }
 
 void
-sctp_prepare_cookie_echo_chunk (sctp_connection_t * sctp_conn,
+sctp_prepare_cookie_echo_chunk (sctp_connection_t * sctp_conn, u8 idx,
 				vlib_buffer_t * b,
 				sctp_state_cookie_param_t * sc)
 {
   vlib_main_t *vm = vlib_get_main ();
-  u8 idx = sctp_pick_conn_idx_on_chunk (COOKIE_ECHO);
 
   sctp_reuse_buffer (vm, b);
 
@@ -583,20 +585,20 @@ sctp_prepare_cookie_echo_chunk (sctp_connection_t * sctp_conn,
 
   vnet_buffer (b)->sctp.connection_index =
     sctp_conn->sub_conn[idx].connection.c_index;
+  vnet_buffer (b)->sctp.conn_idx = idx;
 }
 
 /**
  * Convert buffer to INIT-ACK
  */
 void
-sctp_prepare_initack_chunk (sctp_connection_t * sctp_conn, vlib_buffer_t * b,
-			    ip4_address_t * ip4_addr,
+sctp_prepare_initack_chunk (sctp_connection_t * sctp_conn, u8 idx,
+			    vlib_buffer_t * b, ip4_address_t * ip4_addr,
 			    ip6_address_t * ip6_addr)
 {
   vlib_main_t *vm = vlib_get_main ();
   sctp_ipv4_addr_param_t *ip4_param = 0;
   sctp_ipv6_addr_param_t *ip6_param = 0;
-  u8 idx = sctp_pick_conn_idx_on_chunk (INIT_ACK);
   u32 random_seed = random_default_seed ();
 
   sctp_reuse_buffer (vm, b);
@@ -725,15 +727,16 @@ sctp_prepare_initack_chunk (sctp_connection_t * sctp_conn, vlib_buffer_t * b,
 
   vnet_buffer (b)->sctp.connection_index =
     sctp_conn->sub_conn[idx].connection.c_index;
+  vnet_buffer (b)->sctp.conn_idx = idx;
 }
 
 /**
  * Convert buffer to SHUTDOWN
  */
 void
-sctp_prepare_shutdown_chunk (sctp_connection_t * sctp_conn, vlib_buffer_t * b)
+sctp_prepare_shutdown_chunk (sctp_connection_t * sctp_conn, u8 idx,
+			     vlib_buffer_t * b)
 {
-  u8 idx = sctp_pick_conn_idx_on_chunk (SHUTDOWN);
   u16 alloc_bytes = sizeof (sctp_shutdown_association_chunk_t);
 
   /* As per RFC 4960 the chunk_length value does NOT contemplate
@@ -760,6 +763,7 @@ sctp_prepare_shutdown_chunk (sctp_connection_t * sctp_conn, vlib_buffer_t * b)
 
   vnet_buffer (b)->sctp.connection_index =
     sctp_conn->sub_conn[idx].connection.c_index;
+  vnet_buffer (b)->sctp.conn_idx = idx;
 }
 
 /*
@@ -779,11 +783,12 @@ sctp_send_shutdown (sctp_connection_t * sctp_conn)
   if (PREDICT_FALSE (sctp_get_free_buffer_index (tm, &bi)))
     return;
 
+  u8 idx = MAIN_SCTP_SUB_CONN_IDX;
+
   b = vlib_get_buffer (vm, bi);
   sctp_init_buffer (vm, b);
-  sctp_prepare_shutdown_chunk (sctp_conn, b);
+  sctp_prepare_shutdown_chunk (sctp_conn, idx, b);
 
-  u8 idx = sctp_pick_conn_idx_on_chunk (SHUTDOWN);
   sctp_enqueue_to_output_now (vm, b, bi,
 			      sctp_conn->sub_conn[idx].connection.is_ip4);
 }
@@ -792,10 +797,9 @@ sctp_send_shutdown (sctp_connection_t * sctp_conn)
  * Convert buffer to SHUTDOWN_ACK
  */
 void
-sctp_prepare_shutdown_ack_chunk (sctp_connection_t * sctp_conn,
+sctp_prepare_shutdown_ack_chunk (sctp_connection_t * sctp_conn, u8 idx,
 				 vlib_buffer_t * b)
 {
-  u8 idx = sctp_pick_conn_idx_on_chunk (SHUTDOWN_ACK);
   u16 alloc_bytes = sizeof (sctp_shutdown_association_chunk_t);
   alloc_bytes += vnet_sctp_calculate_padding (alloc_bytes);
 
@@ -817,13 +821,15 @@ sctp_prepare_shutdown_ack_chunk (sctp_connection_t * sctp_conn,
 
   vnet_buffer (b)->sctp.connection_index =
     sctp_conn->sub_conn[idx].connection.c_index;
+  vnet_buffer (b)->sctp.conn_idx = idx;
 }
 
 /*
  * Send SHUTDOWN_ACK
  */
 void
-sctp_send_shutdown_ack (sctp_connection_t * sctp_conn, vlib_buffer_t * b)
+sctp_send_shutdown_ack (sctp_connection_t * sctp_conn, u8 idx,
+			vlib_buffer_t * b)
 {
   vlib_main_t *vm = vlib_get_main ();
 
@@ -832,17 +838,17 @@ sctp_send_shutdown_ack (sctp_connection_t * sctp_conn, vlib_buffer_t * b)
 
   sctp_reuse_buffer (vm, b);
 
-  sctp_prepare_shutdown_ack_chunk (sctp_conn, b);
+  sctp_prepare_shutdown_ack_chunk (sctp_conn, idx, b);
 }
 
 /**
  * Convert buffer to SACK
  */
 void
-sctp_prepare_sack_chunk (sctp_connection_t * sctp_conn, vlib_buffer_t * b)
+sctp_prepare_sack_chunk (sctp_connection_t * sctp_conn, u8 idx,
+			 vlib_buffer_t * b)
 {
   vlib_main_t *vm = vlib_get_main ();
-  u8 idx = sctp_pick_conn_idx_on_chunk (SACK);
 
   sctp_reuse_buffer (vm, b);
 
@@ -870,18 +876,18 @@ sctp_prepare_sack_chunk (sctp_connection_t * sctp_conn, vlib_buffer_t * b)
 
   vnet_buffer (b)->sctp.connection_index =
     sctp_conn->sub_conn[idx].connection.c_index;
+  vnet_buffer (b)->sctp.conn_idx = idx;
 }
 
 /**
  * Convert buffer to HEARTBEAT_ACK
  */
 void
-sctp_prepare_heartbeat_ack_chunk (sctp_connection_t * sctp_conn,
+sctp_prepare_heartbeat_ack_chunk (sctp_connection_t * sctp_conn, u8 idx,
 				  vlib_buffer_t * b)
 {
   vlib_main_t *vm = vlib_get_main ();
 
-  u8 idx = sctp_pick_conn_idx_on_chunk (HEARTBEAT_ACK);
   u16 alloc_bytes = sizeof (sctp_hb_ack_chunk_t);
 
   sctp_reuse_buffer (vm, b);
@@ -909,16 +915,16 @@ sctp_prepare_heartbeat_ack_chunk (sctp_connection_t * sctp_conn,
 
   vnet_buffer (b)->sctp.connection_index =
     sctp_conn->sub_conn[idx].connection.c_index;
+  vnet_buffer (b)->sctp.conn_idx = idx;
 }
 
 /**
  * Convert buffer to HEARTBEAT
  */
 void
-sctp_prepare_heartbeat_chunk (sctp_connection_t * sctp_conn,
+sctp_prepare_heartbeat_chunk (sctp_connection_t * sctp_conn, u8 idx,
 			      vlib_buffer_t * b)
 {
-  u8 idx = sctp_pick_conn_idx_on_chunk (HEARTBEAT);
   u16 alloc_bytes = sizeof (sctp_hb_req_chunk_t);
 
   /* As per RFC 4960 the chunk_length value does NOT contemplate
@@ -944,6 +950,7 @@ sctp_prepare_heartbeat_chunk (sctp_connection_t * sctp_conn,
 
   vnet_buffer (b)->sctp.connection_index =
     sctp_conn->sub_conn[idx].connection.c_index;
+  vnet_buffer (b)->sctp.conn_idx = idx;
 }
 
 void
@@ -954,28 +961,39 @@ sctp_send_heartbeat (sctp_connection_t * sctp_conn)
   sctp_main_t *tm = vnet_get_sctp_main ();
   vlib_main_t *vm = vlib_get_main ();
 
-  if (PREDICT_FALSE (sctp_get_free_buffer_index (tm, &bi)))
-    return;
+  u8 i;
+  u32 now = sctp_time_now ();
 
-  b = vlib_get_buffer (vm, bi);
-  sctp_init_buffer (vm, b);
-  sctp_prepare_heartbeat_chunk (sctp_conn, b);
+  for (i = 0; i < MAX_SCTP_CONNECTIONS; i++)
+    {
+      if (sctp_conn->sub_conn[i].state == SCTP_SUBCONN_STATE_DOWN)
+	continue;
 
-  u8 idx = sctp_pick_conn_idx_on_state (SCTP_STATE_ESTABLISHED);
-  sctp_enqueue_to_output_now (vm, b, bi,
-			      sctp_conn->sub_conn[idx].connection.is_ip4);
+      if (now > (sctp_conn->sub_conn[i].last_seen + SCTP_HB_INTERVAL))
+	{
+	  if (PREDICT_FALSE (sctp_get_free_buffer_index (tm, &bi)))
+	    return;
+
+	  b = vlib_get_buffer (vm, bi);
+	  sctp_init_buffer (vm, b);
+	  sctp_prepare_heartbeat_chunk (sctp_conn, i, b);
+
+	  sctp_enqueue_to_output_now (vm, b, bi,
+				      sctp_conn->sub_conn[i].
+				      connection.is_ip4);
 
-  sctp_conn->sub_conn[idx].unacknowledged_hb += 1;
+	  sctp_conn->sub_conn[i].unacknowledged_hb += 1;
+	}
+    }
 }
 
 /**
  * Convert buffer to SHUTDOWN_COMPLETE
  */
 void
-sctp_prepare_shutdown_complete_chunk (sctp_connection_t * sctp_conn,
+sctp_prepare_shutdown_complete_chunk (sctp_connection_t * sctp_conn, u8 idx,
 				      vlib_buffer_t * b)
 {
-  u8 idx = sctp_pick_conn_idx_on_chunk (SHUTDOWN_COMPLETE);
   u16 alloc_bytes = sizeof (sctp_shutdown_association_chunk_t);
   alloc_bytes += vnet_sctp_calculate_padding (alloc_bytes);
 
@@ -997,10 +1015,11 @@ sctp_prepare_shutdown_complete_chunk (sctp_connection_t * sctp_conn,
 
   vnet_buffer (b)->sctp.connection_index =
     sctp_conn->sub_conn[idx].connection.c_index;
+  vnet_buffer (b)->sctp.conn_idx = idx;
 }
 
 void
-sctp_send_shutdown_complete (sctp_connection_t * sctp_conn,
+sctp_send_shutdown_complete (sctp_connection_t * sctp_conn, u8 idx,
 			     vlib_buffer_t * b0)
 {
   vlib_main_t *vm = vlib_get_main ();
@@ -1010,12 +1029,11 @@ sctp_send_shutdown_complete (sctp_connection_t * sctp_conn,
 
   sctp_reuse_buffer (vm, b0);
 
-  sctp_prepare_shutdown_complete_chunk (sctp_conn, b0);
+  sctp_prepare_shutdown_complete_chunk (sctp_conn, idx, b0);
 
   sctp_conn->state = SCTP_STATE_CLOSED;
 }
 
-
 /*
  *  Send INIT
  */
@@ -1031,10 +1049,10 @@ sctp_send_init (sctp_connection_t * sctp_conn)
     return;
 
   b = vlib_get_buffer (vm, bi);
-  u8 idx = sctp_pick_conn_idx_on_chunk (INIT);
+  u8 idx = MAIN_SCTP_SUB_CONN_IDX;
 
   sctp_init_buffer (vm, b);
-  sctp_prepare_init_chunk (sctp_conn, b);
+  sctp_prepare_init_chunk (sctp_conn, idx, b);
 
   sctp_push_ip_hdr (tm, &sctp_conn->sub_conn[idx], b);
   sctp_enqueue_to_ip_lookup (vm, b, bi, sctp_conn->sub_conn[idx].c_is_ip4);
@@ -1099,6 +1117,8 @@ sctp_push_hdr_i (sctp_connection_t * sctp_conn, u8 idx, vlib_buffer_t * b,
 
   vnet_buffer (b)->sctp.connection_index =
     sctp_conn->sub_conn[idx].connection.c_index;
+
+  vnet_buffer (b)->sctp.conn_idx = idx;
 }
 
 u32
@@ -1107,29 +1127,13 @@ sctp_push_header (transport_connection_t * trans_conn, vlib_buffer_t * b)
   sctp_connection_t *sctp_conn =
     sctp_get_connection_from_transport (trans_conn);
 
-  u8 idx = sctp_pick_conn_idx_on_chunk (DATA);
-
-  if (sctp_conn->sub_conn[idx].unacknowledged_hb >
-      SCTP_ASSOCIATION_MAX_RETRANS)
-    {
-      // The remote-peer is considered to be unreachable hence shutting down
-
-      /* Start cleanup. App wasn't notified yet so use delete notify as
-       * opposed to delete to cleanup session layer state. */
-      stream_session_delete_notify (&sctp_conn->sub_conn
-				    [MAIN_SCTP_SUB_CONN_IDX].connection);
-
-      sctp_connection_timers_reset (sctp_conn);
-
-      sctp_connection_cleanup (sctp_conn);
-    }
+  u8 idx = sctp_data_subconn_select (sctp_conn);
 
   sctp_push_hdr_i (sctp_conn, idx, b, SCTP_STATE_ESTABLISHED);
 
   sctp_trajectory_add_start (b0, 3);
 
   return 0;
-
 }
 
 #if SCTP_DEBUG_STATE_MACHINE
@@ -1227,7 +1231,7 @@ sctp46_output_inline (vlib_main_t * vm,
 	      goto done;
 	    }
 
-	  u8 idx = sctp_pick_conn_idx_on_state (sctp_conn->state);
+	  u8 idx = vnet_buffer (b0)->sctp.conn_idx;
 
 	  th0 = vlib_buffer_get_current (b0);
 
diff --git a/src/vnet/sctp/sctp_packet.h b/src/vnet/sctp/sctp_packet.h
index 3ca05b5a11e..9419c16241a 100644
--- a/src/vnet/sctp/sctp_packet.h
+++ b/src/vnet/sctp/sctp_packet.h
@@ -268,13 +268,13 @@ typedef struct
 #define CHUNK_FLAGS_MASK 0x00FF0000
 #define CHUNK_FLAGS_SHIFT 16
 
-#define CHUNK_UBIT_MASK 0x000F0000
+#define CHUNK_UBIT_MASK 0x00040000
 #define CHUNK_UBIT_SHIFT 18
 
-#define CHUNK_BBIT_MASK 0x000F0000
+#define CHUNK_BBIT_MASK 0x00020000
 #define CHUNK_BBIT_SHIFT 17
 
-#define CHUNK_EBIT_MASK 0x000F0000
+#define CHUNK_EBIT_MASK 0x00010000
 #define CHUNK_EBIT_SHIFT 16
 
 #define CHUNK_LENGTH_MASK 0x0000FFFF