Feat : LWIP integration part2

Change-Id: I2431581d611659beb9b9b4d6d95d6985e53a8061
Signed-off-by: sharath <sharathkumarboyanapally@gmail.com>

28 files changed, 13306 insertions, 0 deletions

diff --git a/stacks/lwip_stack/lwip_src/instance/spl_instance.c b/stacks/lwip_stack/lwip_src/instance/spl_instance.c
new file mode 100644
index 0000000..69b9d82
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/instance/spl_instance.c
@@ -0,0 +1,49 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include "nsfw_msg_api.h"
+#include "nsfw_init.h"
+#include "stackx/spl_sbr.h"
+#include "stackx/stackx_instance.h"
+
+stackx_instance *p_def_stack_instance = NULL;
+
+/**
+ * process message from other module, but the MT module message will be delayed
+ * to handle in the end of the loop to avoid to lose the message dequeued out.
+ *
+ * @param m the data_com_msg to handle
+ */
+int
+spl_process (data_com_msg * m)
+{
+  return call_msg_fun (m);
+}
+
+void
+add_disp_netif (struct netif *netif)
+{
+  struct disp_netif_list *item = malloc (sizeof (struct disp_netif_list));
+  if (!item)
+    {
+      NSPOL_LOGERR ("malloc failed");
+      return;
+    }
+
+  item->netif = netif;
+  item->next = p_def_stack_instance->netif_list;
+  p_def_stack_instance->netif_list = item;
+}
diff --git a/stacks/lwip_stack/lwip_src/ip_module/configuration_reader.c b/stacks/lwip_stack/lwip_src/ip_module/configuration_reader.c
new file mode 100644
index 0000000..2b71e07
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/ip_module/configuration_reader.c
@@ -0,0 +1,1045 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include <unistd.h>
+#include <sys/wait.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <netinet/in.h>
+#include <time.h>
+#include <pthread.h>
+#include <sys/types.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include "configuration_reader.h"
+#include "container_ip.h"
+#include "network.h"
+#include "nstack_log.h"
+#include "nstack_securec.h"
+#include "json.h"
+#include "spl_tcpip.h"
+
+#include <types.h>
+#include <nsfw_mgr_com_api.h>
+#include <nsfw_base_linux_api.h>
+#include "nsfw_maintain_api.h"
+
+NSTACK_STATIC struct config_data g_ip_module_buff;
+NSTACK_STATIC struct config_data *g_config_data;
+NSTACK_STATIC char ip_module_unix_socket[IP_MODULE_MAX_PATH_LEN + 1];
+NSTACK_STATIC char ip_module_unix_socket_dir_path[IP_MODULE_MAX_PATH_LEN + 1];
+//static unsigned long int g_thread_id = 0;
+
+#define MAX_CONNECTION_NUMBER 5
+#define TCP_OOS_LEN_MAX 250
+
+NSTACK_STATIC int read_configuration ();
+NSTACK_STATIC int unix_socket_listen (const char *servername);
+NSTACK_STATIC int process_query ();
+
+/*****************************************************************************
+*   Prototype    : is_digit_str
+*   Description  : check if a string contains only digit
+*   Input        : char *input
+*   Output       : 1 for yes, 0 for no
+*   Return Value : int
+*   Calls        :
+*   Called By    : get_main_pid
+*
+*****************************************************************************/
+NSTACK_STATIC int
+is_digit_str (char *input)
+{
+  if (NULL == input || '\0' == input[0])
+    {
+      return 0;
+    }
+
+  while (*input)
+    {
+      if (*input > '9' || *input < '0')
+        {
+          return 0;
+        }
+      input++;
+    }
+
+  return 1;
+}
+
+/*****************************************************************************
+*   Prototype    : process_query
+*   Description  : ./nStackCtrl -a query
+*   Input        : none
+*   Output       : none
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+NSTACK_STATIC int
+process_query ()
+{
+  int retval;
+
+  if (0 == g_config_data->param.type[0])
+    {
+      return process_post (NULL, IP_MODULE_ALL, IP_MODULE_OPERATE_QUERY);
+    }
+
+  /*name & p are freed inside process_post */
+  if (0 == strcmp (g_config_data->param.type, IP_MODULE_TYPE_PORT))
+    {
+      struct ip_action_param *p = malloc (sizeof (struct ip_action_param));
+      if (p == NULL)
+        {
+          NSOPR_LOGERR ("name allocation failed!");
+          NSOPR_SET_ERRINFO (NSCRTL_ERR, "mem alloc error!");
+          return -1;
+        }
+
+      retval =
+        MEMSET_S (p, sizeof (struct ip_action_param), 0,
+                  sizeof (struct ip_action_param));
+      if (EOK != retval)
+        {
+          NSOPR_LOGERR ("MEMSET_S failed]ret=%d", retval);
+          NSOPR_SET_ERRINFO (NSCRTL_ERR, "MEMSET_S error!");
+          free (p);
+          return -1;
+        }
+
+      retval =
+        STRCPY_S (p->container_id, sizeof (p->container_id),
+                  g_config_data->param.container_id);
+      if (EOK != retval)
+        {
+          NSOPR_LOGERR ("STRCPY_S failed]ret=%d", retval);
+          NSOPR_SET_ERRINFO (NSCRTL_ERR, "STRCPY_S error!");
+          free (p);
+          return -1;
+        }
+
+      retval =
+        STRCPY_S (p->port_name, sizeof (p->port_name),
+                  g_config_data->param.name);
+      if (EOK != retval)
+        {
+          NSOPR_LOGERR ("STRCPY_S failed]ret=%d", retval);
+          NSOPR_SET_ERRINFO (NSCRTL_ERR, "STRCPY_S error!");
+          free (p);
+          return -1;
+        }
+
+      return process_post ((void *) p, IP_MODULE_IP, IP_MODULE_OPERATE_QUERY);
+    }
+  else if (0 == strcmp (g_config_data->param.type, IP_MODULE_TYPE_NETWORK))
+    {
+      if (0 == g_config_data->param.name[0])
+        {
+          return process_post (NULL, IP_MODULE_NETWORK_ALL,
+                               IP_MODULE_OPERATE_QUERY);
+        }
+      else
+        {
+          char *name = malloc (sizeof (g_config_data->param.name));
+          if (NULL == name)
+            {
+              NSOPR_LOGERR ("name allocation failed!");
+              NSOPR_SET_ERRINFO (NSCRTL_ERR, "mem alloc error!");
+              return -1;
+            }
+
+          retval =
+            MEMSET_S (name, sizeof (g_config_data->param.name), 0,
+                      sizeof (g_config_data->param.name));
+          if (EOK != retval)
+            {
+              NSOPR_LOGERR ("MEMSET_S failed]ret=%d", retval);
+              NSOPR_SET_ERRINFO (NSCRTL_ERR, "MEMSET_S error!");
+              free (name);
+              return -1;
+            }
+
+          retval =
+            STRCPY_S (name, sizeof (g_config_data->param.name),
+                      g_config_data->param.name);
+          if (EOK != retval)
+            {
+              NSOPR_LOGERR ("STRCPY_S failed]ret=%d", retval);
+              NSOPR_SET_ERRINFO (NSCRTL_ERR, "STRCPY_S error!");
+              free (name);
+              return -1;
+            }
+
+          return process_post ((void *) name, IP_MODULE_NETWORK,
+                               IP_MODULE_OPERATE_QUERY);
+        }
+    }
+  else if (0 == strcmp (g_config_data->param.type, IP_MODULE_TYPE_IP))
+    {
+      if (0 == g_config_data->param.name[0])
+        {
+          return process_post (NULL, IP_MODULE_IP_ALL,
+                               IP_MODULE_OPERATE_QUERY);
+        }
+      else
+        {
+          NSOPR_SET_ERRINFO (NSCRTL_INPUT_ERR, "input error!");
+        }
+    }
+  else
+    {
+      NSOPR_SET_ERRINFO (NSCRTL_INPUT_ERR, "input error!");
+    }
+
+  return -1;
+}
+
+int
+read_ipmoduleoperatesetnet_configuration ()
+{
+  if (strcmp (g_config_data->param.type, IP_MODULE_TYPE_SETLOG) == 0)
+    {
+      if (NSCRTL_OK ==
+          setlog_level_value (g_config_data->param.name,
+                              g_config_data->param.value))
+        {
+          NSOPR_LOGDBG ("set log level ok!");
+        }
+      else
+        {
+          NSOPR_SET_ERRINFO (NSCRTL_INPUT_ERR, "input error!");
+        }
+    }
+  else if (strcmp (g_config_data->param.type, TCP_MODULE_TYPE_SET_OOS_LEN) ==
+           0)
+    {
+      if (is_digit_str (g_config_data->param.value) == 0)
+        {
+          NSOPR_SET_ERRINFO (NSCRTL_INPUT_ERR,
+                             "Invalid value:value must be digital and smaller than %u]value=\"%s\"",
+                             TCP_OOS_LEN_MAX, g_config_data->param.value);
+          return 0;
+        }
+
+      unsigned int value_len = strlen (g_config_data->param.value);
+      if ((value_len >= 2) && (g_config_data->param.value[0] == '0'))
+        {
+          NSOPR_SET_ERRINFO (NSCRTL_INPUT_ERR,
+                             "Invalid value:value cannot start with 0");
+          return 0;
+        }
+    }
+  else
+    {
+      NSOPR_SET_ERRINFO (NSCRTL_INPUT_ERR, "input error!");
+    }
+
+  return 0;
+}
+
+/*****************************************************************************
+*   Prototype    : read_version
+*   Description  : Query Version by nStackCtrl
+*   Input        : None
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+read_version ()
+{
+  int retVal;
+  json_object *version = json_object_new_object ();
+
+  if (NULL == version)
+    {
+      NSOPR_SET_ERRINFO (NSCRTL_ERR, "internal error for version=NULL!");
+      return NSCRTL_ERR;
+    }
+
+  json_object_object_add (version, "moudle",
+                          json_object_new_string (NSTACK_GETVER_MODULE));
+  json_object_object_add (version, "version",
+                          json_object_new_string (NSTACK_GETVER_VERSION));
+  json_object_object_add (version, "buildtime",
+                          json_object_new_string (NSTACK_GETVER_BUILDTIME));
+
+  json_object *version_array = json_object_new_array ();
+  if (NULL == version_array)
+    {
+      json_object_put (version);
+      NSOPR_SET_ERRINFO (NSCRTL_ERR,
+                         "internal error for version_array=NULL!");
+      return NSCRTL_ERR;
+    }
+
+  retVal = json_object_array_add (version_array, version);
+
+  if (0 != retVal)
+    {
+      json_object_put (version_array);
+      json_object_put (version);
+      NSOPR_SET_ERRINFO (NSCRTL_ERR,
+                         "internal error for json_object_array_add failed!");
+      return NSCRTL_ERR;
+    }
+
+  const char *str = json_object_to_json_string (version_array);
+
+  if (NULL == str)
+    {
+      json_object_put (version_array);
+      NSOPR_SET_ERRINFO (NSCRTL_ERR, "internal error for str=NULL!");
+      return NSCRTL_ERR;
+    }
+
+  size_t str_len = strlen (str);
+  if (str_len >= sizeof (get_config_data ()->json_buff))
+    {
+      json_object_put (version_array);
+      NSOPR_SET_ERRINFO (NSCRTL_ERR, "internal error!");
+      return NSCRTL_ERR;
+    }
+
+  retVal =
+    STRNCPY_S (get_config_data ()->json_buff,
+               sizeof (get_config_data ()->json_buff), str, str_len);
+  if (EOK != retVal)
+    {
+      json_object_put (version_array);
+      NSOPR_SET_ERRINFO (NSCRTL_ERR, "STRNCPY_S error!");
+      return NSCRTL_ERR;
+    }
+
+  json_object_put (version_array);
+  return NSCRTL_OK;
+}
+
+void
+reset_config_data (void)
+{
+  int retval = MEMSET_S (g_config_data, sizeof (struct config_data), 0,
+                         sizeof (struct config_data));
+  if (EOK != retval)
+    {
+      printf ("MEMSET_S failed]retval=%d.\n", retval);
+      exit (1);
+    }
+}
+
+int
+get_network_json_data ()
+{
+  STRCPY_S (g_config_data->param.type, sizeof (g_config_data->param.type),
+            "network");
+  g_config_data->param.action = IP_MODULE_OPERATE_ADD;
+
+  char *tmp_config_path;
+  tmp_config_path = realpath ("./network_data_tonStack.json", NULL);
+  if (!tmp_config_path)
+    {
+      exit (1);
+    }
+
+  int fp = open (tmp_config_path, O_RDONLY);
+  if (-1 == fp)
+    {
+      free (tmp_config_path);
+      NSTCP_LOGINF ("network file open failed.\n");
+      exit (1);
+    }
+  free (tmp_config_path);
+
+  int nread = read (fp, g_config_data->json_buff,
+                    sizeof (g_config_data->json_buff) - 1);
+  if (nread <= 0)
+    {
+      close (fp);
+      NSTCP_LOGINF ("read failed %d.\n", nread);
+      exit (1);
+    }
+
+  /* though MEMSET_S is done above, MEMSET_S can be removed */
+  g_config_data->json_buff[nread] = '\0';
+  close (fp);
+
+  struct network_configuration *network = NULL;
+  struct network_configuration *tmp = NULL;
+
+  /* input shouldnot be same with return */
+  network = parse_network_json (g_config_data->json_buff, NULL);
+  if (!network)
+    {
+      NSTCP_LOGINF ("Invalid network data!");
+      NSOPR_SET_ERRINFO (NSCRTL_INPUT_ERR, "Invalid network data!");
+      return -1;
+    }
+
+  /* run process_post for each network, not only the head node */
+  while (network)
+    {
+      tmp = network;
+      network = network->next;
+      int retval =
+        add_network_configuration ((struct network_configuration *) tmp);
+
+      /* When network exceeds max number, just log warning at operation.log */
+      if (retval == NSCRTL_NETWORK_COUNT_EXCEED)
+        {
+          NSTCP_LOGINF
+            ("Warning!! Network count exceed max allowed number]max=%d",
+             MAX_NETWORK_COUNT);
+        }
+      else
+        {
+
+          NSTCP_LOGINF ("add_network_configuration %d", retval);
+          NSOPR_SET_ERRINFO (retval, "add_network_configuration return %d",
+                             retval);
+        }
+
+      if (!retval)
+        {
+          /*init DPDK eth */
+          if ((retval = init_new_network_configuration ()) != ERR_OK)
+            {
+              NSTCP_LOGINF ("process_configuration failed! %d", retval);
+              free_network_configuration ((struct network_configuration *)
+                                          tmp, IP_MODULE_TRUE);
+              NSOPR_SET_ERRINFO (retval,
+                                 "init_new_network_configuration return %d",
+                                 retval);
+              return -1;
+            }
+        }
+    }
+  NSTCP_LOGINF ("Get_network_json_data done!");
+
+  return 0;
+}
+
+int
+get_ip_json_data ()
+{
+  NSTCP_LOGINF ("get_ip_json_data start!");
+
+  STRCPY_S (g_config_data->param.type, sizeof (g_config_data->param.type),
+            "ip");
+  g_config_data->param.action = IP_MODULE_OPERATE_ADD;
+
+  char *tmp_config_path;
+  tmp_config_path = realpath ("./ip_data.json", NULL);
+  if (!tmp_config_path)
+    {
+      exit (1);
+    }
+
+  int fp = open (tmp_config_path, O_RDONLY);
+  if (-1 == fp)
+    {
+      free (tmp_config_path);
+      NSTCP_LOGINF ("network file open failed\n");
+      exit (1);
+    }
+  free (tmp_config_path);
+
+  int nread = read (fp, g_config_data->json_buff,
+                    sizeof (g_config_data->json_buff) - 1);
+  if (nread <= 0)
+    {
+      close (fp);
+      NSTCP_LOGINF ("read failed %d.\n", nread);
+      exit (1);
+    }
+
+  /* though MEMSET_S is done above, MEMSET_S can be removed */
+  g_config_data->json_buff[nread] = '\0';
+  close (fp);
+
+  struct container_ip *container =
+    parse_container_ip_json (g_config_data->json_buff);
+  if (container)
+    {
+      int retval = add_container (container);
+
+      NSTCP_LOGINF ("add_container %d", retval);
+      NSOPR_SET_ERRINFO (retval, "add_container return %d", retval);
+    }
+  else
+    {
+      NSTCP_LOGINF ("Invalid IP config data!");
+      NSOPR_SET_ERRINFO (NSCRTL_INPUT_ERR, "Invalid IP config data!");
+      return -1;
+    }
+  NSTCP_LOGINF ("get_ip_json_data done!");
+
+  return 0;
+}
+
+int
+read_ipmoduleoperateadd_configuration ()
+{
+  struct network_configuration *tmp = NULL;
+  if (strcmp (g_config_data->param.type, IP_MODULE_TYPE_IP) == 0)
+    {
+      struct container_ip *container =
+        parse_container_ip_json (g_config_data->json_buff);
+      if (container)
+        {
+          return process_post ((void *) container, IP_MODULE_IP,
+                               IP_MODULE_OPERATE_ADD);
+        }
+      else
+        {
+          NSOPR_LOGERR ("Invalid IP config data!");
+          NSOPR_SET_ERRINFO (NSCRTL_INPUT_ERR, "Invalid IP config data!");
+          return -1;
+        }
+    }
+  else if (strcmp (g_config_data->param.type, IP_MODULE_TYPE_NETWORK) == 0)
+    {
+      struct network_configuration *network = NULL;
+
+      //Read network.json
+
+      /* input shouldnot be same with return */
+      network = parse_network_json (g_config_data->json_buff, NULL);
+      if (!network)
+        {
+          NSOPR_LOGERR ("Invalid network data!");
+          NSOPR_SET_ERRINFO (NSCRTL_INPUT_ERR, "Invalid network data!");
+          return -1;
+        }
+
+      /* run process_post for each network, not only the head node */
+      while (network)
+        {
+          tmp = network;
+          network = network->next;
+          int ret = process_post ((void *) tmp, IP_MODULE_NETWORK,
+                                  IP_MODULE_OPERATE_ADD);
+          if (ret == -1)
+            {
+              NSOPR_LOGERR ("process_configuration failed!");
+              return -1;
+            }
+        }
+      return 0;
+    }
+  else
+    {
+      NSOPR_SET_ERRINFO (NSCRTL_INPUT_ERR, "input error!");
+      return -1;
+    }
+}
+
+int
+read_ipmoduleoperatedel_configuration ()
+{
+  int retval;
+
+  if (strcmp (g_config_data->param.type, IP_MODULE_TYPE_IP) == 0)
+    {
+      struct ip_action_param *p = malloc (sizeof (struct ip_action_param));
+      if (NULL == p)
+        {
+          NSOPR_LOGERR ("ip_action_param allocation failed!");
+          NSOPR_SET_ERRINFO (NSCRTL_ERR, "mem alloc error!");
+          return -1;
+        }
+
+      retval =
+        MEMSET_S (p, sizeof (struct ip_action_param), 0,
+                  sizeof (struct ip_action_param));
+      if (EOK != retval)
+        {
+          NSOPR_LOGERR ("MEMSET_S failed]ret=%d", retval);
+          NSOPR_SET_ERRINFO (NSCRTL_ERR, "MEMSET_S error!");
+          free (p);
+          return -1;
+        }
+
+      retval =
+        STRCPY_S (p->container_id, sizeof (p->container_id),
+                  g_config_data->param.container_id);
+      if (EOK != retval)
+        {
+          NSOPR_LOGERR ("STRCPY_S failed]ret=%d", retval);
+          NSOPR_SET_ERRINFO (NSCRTL_ERR, "STRCPY_S error!");
+          free (p);
+          return -1;
+        }
+
+      retval =
+        STRCPY_S (p->port_name, sizeof (p->port_name),
+                  g_config_data->param.name);
+      if (EOK != retval)
+        {
+          NSOPR_LOGERR ("STRCPY_S failed]ret=%d", retval);
+          NSOPR_SET_ERRINFO (NSCRTL_ERR, "STRCPY_S error!");
+          free (p);
+          return -1;
+        }
+
+      return process_post ((void *) p, IP_MODULE_IP, IP_MODULE_OPERATE_DEL);
+    }
+  else if (strcmp (g_config_data->param.type, IP_MODULE_TYPE_NETWORK) == 0)
+    {
+      char *name = malloc (sizeof (g_config_data->param.name));
+      if (name == NULL)
+        {
+          NSOPR_LOGERR ("name allocation failed!");
+          NSOPR_SET_ERRINFO (NSCRTL_ERR, "mem alloc error!");
+          return -1;
+        }
+
+      retval =
+        MEMSET_S (name, sizeof (g_config_data->param.name), 0,
+                  sizeof (g_config_data->param.name));
+      if (EOK != retval)
+        {
+          NSOPR_LOGERR ("MEMSET_S failed]ret=%d", retval);
+          NSOPR_SET_ERRINFO (NSCRTL_ERR, "MEMSET_S error!");
+          free (name);
+          return -1;
+        }
+
+      retval =
+        STRCPY_S (name, sizeof (g_config_data->param.name),
+                  g_config_data->param.name);
+      if (EOK != retval)
+        {
+          NSOPR_LOGERR ("STRCPY_S failed]ret=%d", retval);
+          NSOPR_SET_ERRINFO (NSCRTL_ERR, "STRCPY_S error!");
+          free (name);
+          return -1;
+        }
+
+      return process_post ((void *) name, IP_MODULE_NETWORK,
+                           IP_MODULE_OPERATE_DEL);
+    }
+  else
+    {
+      NSOPR_SET_ERRINFO (NSCRTL_INPUT_ERR, "input error!");
+      return -1;
+    }
+}
+
+NSTACK_STATIC int
+read_configuration ()
+{
+  int retval = -1;
+  //u64 traceid = 0;
+
+  /* initialise default memory */
+  g_config_data->param.error = NSCRTL_OK;
+
+  /* Make sure error_desc is inited to null string */
+  g_config_data->param.error_desc[0] = '\0';
+
+  //traceid = g_config_data->param.traceid;
+
+  NSOPR_LOGINF
+    ("g_config_data]type=%s,name=%s,value=%s,container_id=%s,action=%d,Json_buf=%s, traceid=%llu",
+     g_config_data->param.type, g_config_data->param.name,
+     g_config_data->param.value, g_config_data->param.container_id,
+     g_config_data->param.action, g_config_data->json_buff,
+     g_config_data->param.traceid);
+
+  retval =
+    MEMSET_S (g_config_data->param.error_desc,
+              sizeof (g_config_data->param.error_desc), 0,
+              sizeof (g_config_data->param.error_desc));
+  if (0 != retval)
+    {
+      NSOPR_SET_ERRINFO (NSCRTL_ERR, "ERR:internal error, MEMSET_S failed]");
+      return -1;
+    }
+
+  switch (g_config_data->param.action)
+    {
+    case IP_MODULE_OPERATE_DEL:
+      {
+        retval = read_ipmoduleoperatedel_configuration ();
+        break;
+      }
+    case IP_MODULE_OPERATE_QUERY:
+      {
+        retval = process_query ();
+        break;
+      }
+    case IP_MODULE_OPERATE_ADD:
+      {
+        retval = read_ipmoduleoperateadd_configuration ();
+        break;
+      }
+    case IP_MODULE_OPERATE_SET:
+      retval = read_ipmoduleoperatesetnet_configuration ();
+      break;
+    case IP_MODULE_GET_VERSION:
+      {
+        retval = read_version ();
+        break;
+      }
+
+    default:
+      {
+        retval = -1;            //here, must set retval to -1
+        NSOPR_SET_ERRINFO (NSCRTL_INPUT_ERR, "input error!");
+        break;
+      }
+    }
+
+  return retval;
+}
+
+NSTACK_STATIC int
+unix_socket_listen (const char *servername)
+{
+  int fd, retval;
+  unsigned int len;
+  struct stat st;
+  struct sockaddr_un un;
+
+  if (stat (ip_module_unix_socket_dir_path, &st) == 0)
+    {
+      NSOPR_LOGDBG (" /directory is present");
+    }
+  else
+    {
+      NSOPR_LOGERR (" /var/run/nStack/ directory is not present ");
+      return (-1);
+    }
+
+  if ((fd = nsfw_base_socket (AF_UNIX, SOCK_STREAM, 0)) < 0)
+    {
+      return -1;
+    }
+
+  retval = unlink (servername); /* in case it already exists */
+  if (0 != retval)
+    {
+      NSOPR_LOGWAR ("unlink failed]retval=%d,errno=%d", retval, errno);
+    }
+
+  retval = MEMSET_S (&un, sizeof (un), 0, sizeof (un));
+  if (EOK != retval)
+    {
+      (void) nsfw_base_close (fd);
+      NSOPR_LOGERR ("MEMSET_S failed]ret=%d", retval);
+      return -1;
+    }
+
+  un.sun_family = AF_UNIX;
+  retval = STRCPY_S (un.sun_path, sizeof (un.sun_path), servername);
+  if (EOK != retval)
+    {
+      (void) nsfw_base_close (fd);
+      NSOPR_LOGERR ("STRCPY_S failed]ret=%d", retval);
+      return -1;
+    }
+
+  len =
+    (unsigned int) (offsetof (struct sockaddr_un, sun_path) +
+                    strlen (servername));
+
+  if (nsfw_base_bind (fd, (struct sockaddr *) &un, len) < 0)
+    {
+      (void) nsfw_base_close (fd);
+      return -1;
+    }
+  else
+    {
+      if (nsfw_base_listen (fd, MAX_CONNECTION_NUMBER) < 0)
+        {
+          (void) nsfw_base_close (fd);
+          return -1;
+        }
+      else
+        {
+          return fd;
+        }
+    }
+}
+
+/*****************************************************************************
+*   Prototype    : read_fn
+*   Description  : process new ip module msg
+*   Input        : i32 fd
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+void
+read_fn (i32 fd)
+{
+  ssize_t size;
+  ssize_t offset = 0;
+  size_t left = MAX_IP_MODULE_BUFF_SIZE;
+  while (left > 0)
+    {
+      size = nsfw_base_recv (fd, (char *) g_config_data + offset, left, 0);
+      if (size > 0)
+        {
+          offset += size;
+          left -= (size_t) size;
+        }
+      else
+        {
+          NSOPR_LOGERR ("Error when recieving]errno=%d,err_string=%s", errno,
+                        strerror (errno));
+          break;
+        }
+    }
+
+  if (left != 0)
+    {
+      (void) nsfw_base_close (fd);
+      return;
+    }
+
+  const char *old_hbt_cnt = "6";
+  const char *new_hbt_cnt = "60";
+  nsfw_set_soft_para (NSFW_PROC_MASTER, NSFW_HBT_COUNT_PARAM,
+                      (void *) new_hbt_cnt, sizeof (u16));
+  (void) read_configuration (); // if it returns -1, the err desc info will be wrote to g_config_data, so no need to check return value.
+  nsfw_set_soft_para (NSFW_PROC_MASTER, NSFW_HBT_COUNT_PARAM,
+                      (void *) old_hbt_cnt, sizeof (u16));
+
+  offset = 0;
+  left = MAX_IP_MODULE_BUFF_SIZE;
+  while (left > 0)
+    {
+      size =
+        nsfw_base_send (fd, (char *) g_config_data + offset, left,
+                        MSG_NOSIGNAL);
+
+      if (size > 0)
+        {
+          offset += size;
+          left -= (size_t) size;
+        }
+      else
+        {
+          NSOPR_LOGERR ("Error when Sending data]errno=%d", errno);
+          break;
+        }
+    }
+
+  (void) nsfw_base_close (fd);
+  return;
+}
+
+/*****************************************************************************
+*   Prototype    : ip_module_new_msg
+*   Description  : recv new config message
+*   Input        : i32 epfd
+*                  i32 fd
+*                  u32 events
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+ip_module_new_msg (i32 epfd, i32 fd, u32 events)
+{
+  if ((events & EPOLLERR) || (events & EPOLLHUP) || (!(events & EPOLLIN)))
+    {
+      nsfw_mgr_unreg_sock_fun (fd);
+      (void) nsfw_base_close (fd);
+      return TRUE;
+    }
+
+  nsfw_mgr_unreg_sock_fun (fd);
+  read_fn (fd);
+  return TRUE;
+}
+
+int
+init_ip_module_unix_socket_path ()
+{
+  const char *directory = "/var/run";
+  const char *home_dir = getenv ("HOME");
+
+  if (getuid () != 0 && home_dir != NULL)
+    directory = home_dir;
+
+  if (STRCPY_S
+      (ip_module_unix_socket_dir_path, IP_MODULE_MAX_PATH_LEN, directory) < 0)
+    {
+      NSOPR_LOGERR ("STRCPY_S fail]");
+      return -1;
+    }
+
+  if (STRCAT_S
+      (ip_module_unix_socket_dir_path, IP_MODULE_MAX_PATH_LEN,
+       "/ip_module") < 0)
+    {
+      NSOPR_LOGERR ("STRCAT_S fail]");
+      return -1;
+    }
+
+  if (STRCPY_S
+      (ip_module_unix_socket, IP_MODULE_MAX_PATH_LEN,
+       ip_module_unix_socket_dir_path) < 0)
+    {
+      NSOPR_LOGERR ("STRCPY_S fail]");
+      return -1;
+    }
+
+  if (STRCAT_S
+      (ip_module_unix_socket, IP_MODULE_MAX_PATH_LEN,
+       "/ip_module_unix_sock") < 0)
+    {
+      NSOPR_LOGERR ("STRCAT_S fail]");
+      return -1;
+    }
+
+  NSOPR_LOGINF ("ip_module_unix_socket=%s", ip_module_unix_socket);
+  NSOPR_LOGINF ("ip_module_unix_socket_dir_path=%s",
+                ip_module_unix_socket_dir_path);
+  return 0;
+}
+
+/*****************************************************************************
+*   Prototype    : ip_module_new_connection
+*   Description  : recv new connect for network config
+*   Input        : i32 epfd
+*                  i32 fd
+*                  u32 events
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+ip_module_new_connection (i32 epfd, i32 fd, u32 events)
+{
+  if ((events & EPOLLERR) || (events & EPOLLHUP) || (!(events & EPOLLIN)))
+    {
+      (void) nsfw_base_close (fd);
+      NSFW_LOGINF ("listen disconnect!]epfd=%d,listen=%d,event=0x%x", epfd,
+                   fd, events);
+      nsfw_mgr_unreg_sock_fun (fd);
+
+      if (init_ip_module_unix_socket_path () < 0)
+        {
+          NSFW_LOGERR ("Error when init path]epfd=%d,listen_fd=%d,event=0x%x",
+                       epfd, fd, events);
+          return FALSE;
+        }
+
+      i32 listen_fd = unix_socket_listen (ip_module_unix_socket);
+      if (listen_fd < 0)
+        {
+          NSFW_LOGERR ("get listen_fd faied!]epfd=%d,listen_fd=%d,event=0x%x",
+                       epfd, fd, events);
+          return FALSE;
+        }
+
+      if (FALSE ==
+          nsfw_mgr_reg_sock_fun (listen_fd, ip_module_new_connection))
+        {
+          (void) nsfw_base_close (listen_fd);
+          return FALSE;
+        }
+      return TRUE;
+    }
+
+  struct sockaddr in_addr;
+  socklen_t in_len;
+  int infd;
+  in_len = sizeof in_addr;
+
+  while (1)
+    {
+      infd = nsfw_base_accept (fd, &in_addr, &in_len);
+      if (infd == -1)
+        {
+          break;
+        }
+
+      if (FALSE == nsfw_mgr_reg_sock_fun (infd, ip_module_new_msg))
+        {
+          NSFW_LOGINF ("accept new fd but reg failed]new_mgr_fd=%d", infd);
+          return FALSE;
+        }
+      NSFW_LOGINF ("accept new fd]new_mgr_fd=%d", infd);
+    }
+
+  return TRUE;
+}
+
+int
+init_configuration_reader ()
+{
+  int error_number = 0;
+  INITPOL_LOGINF ("RTP", "init_configuration_reader", NULL_STRING,
+                  LOG_INVALID_VALUE, MODULE_INIT_START);
+  g_config_data = &g_ip_module_buff;
+
+  if (init_ip_module_unix_socket_path () < 0)
+    {
+      INITPOL_LOGERR ("RTP", "init_configuration_reader",
+                      "Error when init path", LOG_INVALID_VALUE,
+                      MODULE_INIT_FAIL);
+      return -1;
+    }
+
+  i32 listen_fd = unix_socket_listen (ip_module_unix_socket);
+  if (listen_fd < 0)
+    {
+      error_number = errno;
+      INITPOL_LOGERR ("RTP", "init_configuration_reader",
+                      "when listening ip_module_unix_socket", error_number,
+                      MODULE_INIT_FAIL);
+      return -1;
+    }
+
+  NSOPR_LOGINF ("start mgr_com module!]listern_fd=%d", listen_fd);
+
+  if (FALSE == nsfw_mgr_reg_sock_fun (listen_fd, ip_module_new_connection))
+    {
+      (void) nsfw_base_close (listen_fd);
+      NSOPR_LOGERR ("nsfw_mgr_reg_sock_fun failed]listen_fd=%d", listen_fd);
+      return -1;
+    }
+
+  INITPOL_LOGINF ("RTP", "init_configuration_reader", NULL_STRING,
+                  LOG_INVALID_VALUE, MODULE_INIT_SUCCESS);
+  return 0;
+}
+
+struct config_data *
+get_config_data ()
+{
+  return g_config_data;
+}
diff --git a/stacks/lwip_stack/lwip_src/ip_module/container_ip.c b/stacks/lwip_stack/lwip_src/ip_module/container_ip.c
new file mode 100644
index 0000000..3cbe9b9
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/ip_module/container_ip.c
@@ -0,0 +1,1168 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <string.h>
+#include "lwip/inet.h"
+#include "trp_rb_tree.h"
+#include "container_ip.h"
+#include "network.h"
+#include "netif.h"
+#include "nstack_log.h"
+#include "nstack_securec.h"
+#include "nstack_rd_mng.h"
+#include "config_common.h"
+#include "igmp.h"
+#include "spl_def.h"
+#include "stackx_ip_addr.h"
+#include "hal_api.h"
+#include "spl_hal.h"
+
+struct container_list g_container_list = { 0 };
+static trp_rb_root_t g_container_ip_root = { 0 };       //only handled in tcpip thread, no need protect it with a lock
+static trp_rb_root_t g_container_multicast_root = { 0 };        //only handled in tcpip thread, no need protect it with a lock
+
+static void free_container_port (struct container_port *port,
+                                 bool_t only_free);
+
+/*unsigned int value is typecasted into void * pointer and passed as argument to
+this function. so the value can never be > 0xFFFFFFFF. so can suppress the warning*/
+
+static int
+ip_compare (trp_key_t left, trp_key_t right)
+{
+  //return (int)((unsigned long)left - (unsigned long)right);
+
+  if (left > right)
+    {
+      return 1;
+    }
+  else if (left < right)
+    {
+      return -1;
+    }
+  else
+    {
+      return 0;
+    }
+}
+
+NSTACK_STATIC bool_t
+is_container_ok (struct container_ip * container)
+{
+  if (!container->ports_list)
+    {
+      return 0;
+    }
+
+  return 1;
+}
+
+NSTACK_STATIC void
+add_port (struct container_ip *container, struct container_port *port)
+{
+  if (port->ip_cidr_list)
+    {
+      port->next = container->ports_list;
+      container->ports_list = port;
+    }
+  else
+    {
+      free_container_port (port, IP_MODULE_TRUE);
+    }
+}
+
+static void
+add_ip_cidr (struct container_port *port,
+             struct container_port_ip_cidr *ip_cidr)
+{
+  if (!ip_cidr)
+    {
+      return;
+    }
+
+  ip_cidr->next = port->ip_cidr_list;
+  port->ip_cidr_list = ip_cidr;
+  return;
+}
+
+NSTACK_STATIC void
+add_multilcast_ip (struct container_port *port,
+                   struct container_multicast_id *muticastIP)
+{
+  if (!muticastIP)
+    {
+      return;
+    }
+
+  muticastIP->next = port->multicast_list;
+  port->multicast_list = muticastIP;
+  return;
+}
+
+NSTACK_STATIC void
+free_container_port_ip_cidr (struct container_port_ip_cidr *ip_cidr,
+                             bool_t only_free)
+{
+  output_api *api = get_output_api ();
+  struct container_port_ip_cidr *ip_cidr_tmp = NULL;
+
+  while (ip_cidr)
+    {
+      ip_cidr_tmp = ip_cidr;
+      ip_cidr = ip_cidr_tmp->next;
+      if (!only_free)
+        {
+          if (api->del_netif_ip)
+            {
+              struct network_configuration *network =
+                get_network_by_ip_with_tree (ip_cidr_tmp->ip);
+              if (network)
+                {
+                  if (network->phy_net->bond_name[0] != 0)
+                    {
+                      (void) api->del_netif_ip (network->phy_net->bond_name, ip_cidr_tmp->ip);  //fails only when netif_name not exist, no side effect so don't check return value.
+                    }
+                  else
+                    {
+                      (void) api->del_netif_ip (network->phy_net->
+                                                header->nic_name,
+                                                ip_cidr_tmp->ip);
+                    }
+                }
+              else
+                {
+                  NSOPR_LOGERR ("can't find network by]IP=%u",
+                                ip_cidr_tmp->ip);
+                }
+            }
+
+          trp_rb_erase ((void *) (u64_t) ip_cidr_tmp->ip,
+                        &g_container_ip_root, ip_compare);
+        }
+
+      free (ip_cidr_tmp);
+      ip_cidr_tmp = NULL;
+    }
+}
+
+/*note:::the ip must be local order*/
+void
+container_multicast_rd (unsigned int ip, int op)
+{
+  rd_ip_data rd_ip = { 0 };
+  int ret = 0;
+
+  rd_ip.addr = ip;
+  rd_ip.masklen = 32;
+  if (0 == op)
+    {
+      ret = nstack_rd_ip_node_insert ("nstack-dpdk", &rd_ip);
+    }
+  else
+    {
+      ret = nstack_rd_ip_node_delete (&rd_ip);
+    }
+
+  if (0 != ret)
+    {
+      NSOPR_LOGERR ("nstack rd multicast ip:0x%x %s fail", ip,
+                    (0 == op ? "insert" : "delete"));
+    }
+  else
+    {
+      NSOPR_LOGDBG ("nstack rd multicast ip:0x%x %s success", ip,
+                    (0 == op ? "insert" : "delete"));
+    }
+
+  return;
+}
+
+static void
+free_container_multicast (struct container_multicast_id *multicast,
+                          bool_t only_free)
+{
+  struct container_multicast_id *tmp = NULL;
+
+  while (multicast)
+    {
+      tmp = multicast;
+      multicast = multicast->next;
+      if (!only_free)
+        {
+          /*note:::multicast ip is network, need to change to local order, delete multicast ip from rd. */
+          container_multicast_rd (spl_ntohl (tmp->ip), 1);
+
+          trp_rb_erase ((void *) (u64_t) tmp->ip, &g_container_multicast_root,
+                        ip_compare);
+        }
+
+      free (tmp);
+      tmp = NULL;
+    }
+}
+
+static void
+free_container_port (struct container_port *port, bool_t only_free)
+{
+  struct container_port *port_tmp = NULL;
+  struct container_port *port_curr = port;
+
+  while (port_curr)
+    {
+      port_tmp = port_curr;
+      port_curr = port_tmp->next;
+
+      free_container_multicast (port_tmp->multicast_list, only_free);
+      free_container_port_ip_cidr (port_tmp->ip_cidr_list, only_free);
+
+      if (port_tmp->buffer)
+        {
+          free_port_buffer (port_tmp->buffer);
+          port_tmp->buffer = NULL;
+        }
+
+      free (port_tmp);
+      port_tmp = NULL;
+    }
+}
+
+void
+free_container (struct container_ip *container, bool_t only_free)
+{
+  struct container_ip *container_tmp = NULL;
+  struct container_ip *container_curr = container;
+
+  while (container_curr)
+    {
+      container_tmp = container_curr;
+      container_curr = container_tmp->next;
+      if (container_tmp->ports_list)
+        {
+          free_container_port (container_tmp->ports_list, only_free);
+        }
+
+      free (container_tmp);
+      container_tmp = NULL;
+    }
+}
+
+struct container_port *
+parse_port_obj (struct json_object *port_obj)
+{
+  int retval;
+  struct json_object *port_name_obj = NULL;
+  struct json_object *ip_cidr_list_obj = NULL;
+  struct json_object *mcIDObj = NULL;
+
+  if (!port_obj)
+    {
+      NSOPR_LOGERR ("port_obj is null");
+      return NULL;
+    }
+
+  struct container_port *port = malloc (sizeof (struct container_port));
+  if (!port)
+    {
+      NSOPR_LOGERR ("malloc failed");
+      return NULL;
+    }
+
+  retval =
+    MEMSET_S (port, sizeof (struct container_port), 0,
+              sizeof (struct container_port));
+  if (EOK != retval)
+    {
+      NSOPR_LOGERR ("MEMSET_S failed]ret=%d", retval);
+      free (port);
+      return NULL;
+    }
+
+  json_object_object_get_ex (port_obj, "port_name", &port_name_obj);
+  if (port_name_obj)
+    {
+      const char *port_name = json_object_get_string (port_name_obj);
+      if ((NULL == port_name)
+          || (strlen (port_name) >= IP_MODULE_MAX_NAME_LEN))
+        {
+          NSOPR_LOGERR ("port name is not ok");
+          goto RETURN_ERROR;
+        }
+
+      retval =
+        STRCPY_S (port->port_name, sizeof (port->port_name), port_name);
+
+      if (EOK != retval)
+        {
+          NSOPR_LOGERR ("STRCPY_S failed]ret=%d", retval);
+          goto RETURN_ERROR;
+        }
+    }
+
+  json_object_object_get_ex (port_obj, "ip_cidr", &ip_cidr_list_obj);
+  if (ip_cidr_list_obj)
+    {
+      int j;
+      int ip_cidr_num = json_object_array_length (ip_cidr_list_obj);
+      for (j = 0; j < ip_cidr_num; ++j)
+        {
+          struct json_object *ip_cidr_obj =
+            json_object_array_get_idx (ip_cidr_list_obj, j);
+          if (ip_cidr_obj)
+            {
+              char tmp[IP_MODULE_LENGTH_32] = { 0 };
+              struct container_port_ip_cidr *port_ip_cidr =
+                malloc (sizeof (struct container_port_ip_cidr));
+              if (NULL == port_ip_cidr)
+                {
+                  NSOPR_LOGERR ("malloc failed");
+                  goto RETURN_ERROR;
+                }
+
+              retval =
+                MEMSET_S (port_ip_cidr,
+                          sizeof (struct container_port_ip_cidr), 0,
+                          sizeof (struct container_port_ip_cidr));
+              if (EOK != retval)
+                {
+                  NSOPR_LOGERR ("MEMSET_S failed]ret=%d", retval);
+                  free (port_ip_cidr);
+                  port_ip_cidr = NULL;
+                  goto RETURN_ERROR;
+                }
+
+              const char *ip_cidr = json_object_get_string (ip_cidr_obj);
+              if ((NULL == ip_cidr) || (ip_cidr[0] == 0))
+                {
+                  NSOPR_LOGERR ("ip_cidr is not ok");
+                  free (port_ip_cidr);
+                  port_ip_cidr = NULL;
+                  goto RETURN_ERROR;
+                }
+
+              const char *sub = strstr (ip_cidr, "/");
+              if ((NULL == sub)
+                  || (sizeof (tmp) - 1 < (unsigned int) (sub - ip_cidr))
+                  || (strlen (sub) > sizeof (tmp) - 1))
+                {
+                  NSOPR_LOGERR
+                    ("Error : Ipaddress notation must be in ip cidr notation!");
+                  free (port_ip_cidr);
+                  port_ip_cidr = NULL;
+                  goto RETURN_ERROR;
+                }
+
+              retval =
+                STRNCPY_S (tmp, sizeof (tmp), ip_cidr,
+                           (size_t) (sub - ip_cidr));
+              if (EOK != retval)
+                {
+                  NSOPR_LOGERR ("STRNCPY_S failed]ret=%d", retval);
+                  free (port_ip_cidr);
+                  port_ip_cidr = NULL;
+                  goto RETURN_ERROR;
+                }
+
+              struct in_addr addr;
+              int iRet;
+              retval = MEMSET_S (&addr, sizeof (addr), 0, sizeof (addr));
+              if (EOK != retval)
+                {
+                  NSOPR_LOGERR ("MEMSET_S failed]ret=%d", retval);
+                  free (port_ip_cidr);
+                  port_ip_cidr = NULL;
+                  goto RETURN_ERROR;
+                }
+
+              iRet = spl_inet_aton (tmp, &addr);
+              if (0 == iRet)
+                {
+                  NSOPR_LOGERR ("spl_inet_aton failed");
+                  free (port_ip_cidr);
+                  port_ip_cidr = NULL;
+                  goto RETURN_ERROR;
+                }
+
+              port_ip_cidr->ip = addr.s_addr;
+              iRet = atoi (sub + 1);
+              if ((iRet <= 0) || (iRet > IP_MODULE_LENGTH_32))
+                {
+                  NSOPR_LOGERR ("IP mask length is not correct");
+                  free (port_ip_cidr);
+                  port_ip_cidr = NULL;
+                  goto RETURN_ERROR;
+                }
+
+              port_ip_cidr->mask_len = (unsigned int) iRet;
+              add_ip_cidr (port, port_ip_cidr);
+            }
+        }
+    }
+
+  json_object_object_get_ex (port_obj, "multicast_id", &mcIDObj);
+  if (mcIDObj)
+    {
+      int j;
+      int arrLen = json_object_array_length (mcIDObj);
+      if (0 == arrLen)
+        {
+          NSOPR_LOGERR ("arrLen is 0");
+          goto RETURN_ERROR;
+        }
+
+      for (j = 0; j < arrLen; ++j)
+        {
+          struct json_object *elemObj =
+            json_object_array_get_idx (mcIDObj, j);
+
+          if (elemObj)
+            {
+              struct json_object *tObj = NULL;
+              const char *tStr;
+              int ret;
+              struct in_addr addr;
+
+              struct container_multicast_id *mcID =
+                malloc (sizeof (struct container_multicast_id));
+              if (NULL == mcID)
+                {
+                  NSOPR_LOGERR ("Can't alloc container multicast id");
+                  goto RETURN_ERROR;
+                }
+
+              json_object_object_get_ex (elemObj, "group_ip", &tObj);
+              if (NULL == tObj)
+                {
+                  NSOPR_LOGERR ("No group_IP");
+                  free (mcID);
+                  mcID = NULL;
+                  goto RETURN_ERROR;
+                }
+
+              tStr = json_object_get_string (tObj);
+              if (NULL == tStr)
+                {
+                  NSOPR_LOGERR ("Get Multiple cast group IP Failed");
+                  free (mcID);
+                  mcID = NULL;
+                  goto RETURN_ERROR;
+                }
+
+              ret = spl_inet_aton (tStr, &addr);
+              if (0 == ret)
+                {
+                  NSOPR_LOGERR ("Parse group IP Failed");
+                  free (mcID);
+                  mcID = NULL;
+                  goto RETURN_ERROR;
+                }
+
+              mcID->ip = addr.s_addr;
+              add_multilcast_ip (port, mcID);
+            }
+        }
+    }
+
+  const char *port_json = json_object_get_string (port_obj);
+  if ((NULL == port_json) || (0 == strlen (port_json)))
+    {
+      NSOPR_LOGERR ("json_object_get_string failed");
+      goto RETURN_ERROR;
+    }
+
+  port->buffer = malloc_port_buffer ();
+  if (!port->buffer)
+    {
+      goto RETURN_ERROR;
+    }
+
+  retval =
+    STRCPY_S (get_port_json (port), IP_MODULE_PORT_JSON_LEN, port_json);
+  if (EOK != retval)
+    {
+      NSOPR_LOGERR ("STRCPY_S failed]ret=%d", retval);
+      goto RETURN_ERROR;
+    }
+
+  return port;
+RETURN_ERROR:
+  free_container_port (port, IP_MODULE_TRUE);
+  return NULL;
+}
+
+struct container_ip *
+parse_container_ip_json (char *param)
+{
+  int retval;
+  struct json_object *obj = json_tokener_parse (param);
+  struct json_object *container_id_obj = NULL;
+  struct json_object *ports_list_obj = NULL;
+
+  if (!obj)
+    {
+      return NULL;
+    }
+
+  struct container_ip *container = malloc (sizeof (struct container_ip));
+  if (container == NULL)
+    {
+      json_object_put (obj);
+      return NULL;
+    }
+
+  retval =
+    MEMSET_S (container, sizeof (struct container_ip), 0,
+              sizeof (struct container_ip));
+  if (EOK != retval)
+    {
+      NSOPR_LOGERR ("MEMSET_S failed]ret=%d", retval);
+      goto RETURN_ERROR;
+    }
+
+  json_object_object_get_ex (obj, "containerID", &container_id_obj);
+  if (container_id_obj)
+    {
+      const char *container_id = json_object_get_string (container_id_obj);
+      if ((container_id == NULL) || (container_id[0] == 0)
+          || (strlen (container_id) >= IP_MODULE_MAX_NAME_LEN))
+        {
+          goto RETURN_ERROR;
+        }
+
+      retval =
+        MEMSET_S (container->container_id, sizeof (container->container_id),
+                  0, sizeof (container->container_id));
+      if (EOK != retval)
+        {
+          NSOPR_LOGERR ("MEMSET_S failed]ret=%d", retval);
+          goto RETURN_ERROR;
+        }
+
+      retval =
+        STRCPY_S (container->container_id, sizeof (container->container_id),
+                  container_id);
+
+      if (EOK != retval)
+        {
+          NSOPR_LOGERR ("STRCPY_S failed]ret=%d", retval);
+          goto RETURN_ERROR;
+        }
+    }
+  else
+    {
+      /* this mandatory parameter */
+      goto RETURN_ERROR;
+    }
+
+  json_object_object_get_ex (obj, "ports_list", &ports_list_obj);
+  if (ports_list_obj)
+    {
+      int i;
+      int port_num = json_object_array_length (ports_list_obj);
+
+      if (port_num == 0)
+        {
+          /* this mandatory parameter */
+          goto RETURN_ERROR;
+        }
+
+      for (i = 0; i < port_num; i++)
+        {
+          struct json_object *port_obj =
+            json_object_array_get_idx (ports_list_obj, i);
+          struct container_port *port = parse_port_obj (port_obj);
+          if (!port)
+            {
+              goto RETURN_ERROR;
+            }
+
+          add_port (container, port);
+        }
+    }
+  else
+    {
+      /* mandatory parameter */
+      goto RETURN_ERROR;
+    }
+
+  /* Check if this function is required, or needs more check inside this function,
+     as some of them are alraedy validated
+   */
+  if (!is_container_ok (container))
+    {
+      goto RETURN_ERROR;
+    }
+
+  json_object_put (obj);
+  return container;
+
+RETURN_ERROR:
+  json_object_put (obj);
+  free_container (container, IP_MODULE_TRUE);
+  return NULL;
+}
+
+bool_t
+is_ip_match_netif (unsigned int ip, char *netif_name)
+{
+  if (!netif_name)
+    {
+      return 0;
+    }
+
+  if (trp_rb_search ((void *) (u64_t) ip, &g_container_ip_root, ip_compare))
+    {
+      struct network_configuration *network =
+        get_network_by_ip_with_tree (ip);
+      if (network && network->phy_net && network->phy_net->header)
+        {
+          if (0 ==
+              strncmp (netif_name, network->phy_net->header->nic_name,
+                       HAL_MAX_NIC_NAME_LEN))
+            {
+              return 1;
+            }
+        }
+    }
+
+  return 0;
+}
+
+inline bool_t
+is_ip_exist (unsigned int ip)
+{
+  if (trp_rb_search ((void *) (u64_t) ip, &g_container_ip_root, ip_compare))
+    {
+      return 1;
+    }
+
+  return 0;
+}
+
+int
+validate_addcontainerconfig (struct container_ip *container)
+{
+  struct container_port *port;
+  struct container_ip *old =
+    get_container_by_container_id (container->container_id);
+  struct container_port *tmp = container->ports_list;
+
+  if (old)
+    {
+      struct container_port *last = NULL;
+
+      while (tmp)
+        {
+          if (get_port (container->container_id, tmp->port_name))
+            {
+              NSOPR_LOGERR ("port=%s already exists!", tmp->port_name);
+              return NSCRTL_RD_EXIST;
+            }
+
+          last = tmp;
+          tmp = tmp->next;
+        }
+
+      (void) last;
+    }
+
+  /* check if port_name duplicates in one json configuration */
+  tmp = container->ports_list;
+  while (tmp)
+    {
+      if (get_port_from_container (tmp))
+        {
+          NSOPR_LOGERR ("port=%s duplicates!", tmp->port_name);
+          return NSCRTL_RD_EXIST;
+        }
+
+      tmp = tmp->next;
+    }
+
+  bool_t is_nstack_dpdk_port;
+  struct container_port **ref = &container->ports_list;
+  while ((port = *ref))
+    {
+      is_nstack_dpdk_port = 1;
+      struct container_port_ip_cidr *ip_cidr = port->ip_cidr_list;
+      while (ip_cidr)
+        {
+          struct network_configuration *network =
+            get_network_by_ip_with_tree (ip_cidr->ip);
+          if (network && (0 == strcmp (network->type_name, "nstack-dpdk")))
+            {
+              struct netif *pnetif;
+              if (get_netif_by_ip (ip_cidr->ip))
+                {
+                  NSOPR_LOGERR ("ip  exists]IP=0x%08x", ip_cidr->ip);
+                  return NSCRTL_RD_EXIST;
+                }
+
+              if (network->phy_net->bond_name[0] != 0)
+                {
+                  pnetif =
+                    find_netif_by_if_name (network->phy_net->bond_name);
+                }
+              else
+                {
+                  pnetif =
+                    find_netif_by_if_name (network->phy_net->
+                                           header->nic_name);
+                }
+
+              if (!pnetif)
+                {
+                  NSOPR_LOGERR ("can't find netif, network json:%s",
+                                get_network_json (network));
+                  return NSCRTL_ERR;
+                }
+
+              if (0 == port->port_name[0])
+                {
+                  NSOPR_LOGINF
+                    ("ip=0x%08x is in nstack dpdk network, but port_name is null, json:%s",
+                     ip_cidr->ip, get_port_json (port));
+                  is_nstack_dpdk_port = 0;
+                  break;
+                }
+            }
+          else
+            {
+              NSOPR_LOGINF ("port %s is not in nstack dpdk network, json:%s",
+                            port->port_name, get_port_json (port));
+              is_nstack_dpdk_port = 0;
+              break;
+            }
+
+          ip_cidr = ip_cidr->next;
+        }
+
+      /* only use nstack dpdk port */
+      if (is_nstack_dpdk_port)
+        {
+          ref = &port->next;
+        }
+      else
+        {
+          *ref = port->next;
+          port->next = NULL;
+          free_container_port (port, IP_MODULE_TRUE);
+        }
+    }
+
+  return (!container->ports_list) ? NSCRTL_FREE_ALL_PORT : NSCRTL_OK;
+}
+
+/* get the num of IPs in a container , which  in a certain subnet */
+extern struct network_list g_network_list;
+extern inline int is_in_subnet (unsigned int ip, struct ip_subnet *subnet);
+NSTACK_STATIC inline int get_network_ip_count
+  (struct container_ip *container, struct ip_subnet *subnet)
+{
+  int ip_count = 0;
+  struct container_port *port_list = NULL;
+  struct container_ip *ci = container;
+
+  while (ci)
+    {
+      port_list = ci->ports_list;
+      while (port_list)
+        {
+          struct container_port_ip_cidr *ip_list = port_list->ip_cidr_list;
+          while (ip_list)
+            {
+              if (!is_in_subnet (ip_list->ip, subnet))
+                {
+                  ip_count++;
+                }
+
+              ip_list = ip_list->next;
+            }
+
+          port_list = port_list->next;
+        }
+
+      ci = ci->next;
+    }
+
+  return ip_count;
+}
+
+int
+check_ip_count (struct container_ip *container)
+{
+  int cur_count = 0;
+  int new_count = 0;
+
+  if (NULL == container)
+    {
+      return 1;
+    }
+
+  struct network_configuration *network = g_network_list.header;
+  while (network)
+    {
+      cur_count =
+        get_network_ip_count (g_container_list.header, network->ip_subnet);
+      new_count = get_network_ip_count (container, network->ip_subnet);
+
+      if ((cur_count > MAX_NETWORK_IP_COUNT)
+          || (new_count > MAX_NETWORK_IP_COUNT)
+          || (cur_count + new_count > MAX_NETWORK_IP_COUNT))
+        {
+          NSOPR_LOGERR
+            ("reach ip addr max count]network=%s, max=%d, current=%d, new=%d.",
+             network->network_name, MAX_NETWORK_IP_COUNT, cur_count,
+             new_count);
+          return 0;
+        }
+
+      network = network->next;
+    }
+
+  return 1;
+}
+
+int
+match_groupaddr (struct container_multicast_id *multi_list,
+                 spl_ip_addr_t * groupaddr)
+{
+  struct container_multicast_id *group_info = multi_list;
+
+  while (group_info)
+    {
+      if (group_info->ip == groupaddr->addr)
+        {
+          return 1;
+        }
+
+      group_info = group_info->next;
+    }
+
+  return 0;
+}
+
+int
+add_container (struct container_ip *container)
+{
+  int retVal = 0;
+
+  /* need to check if any of the netif operation failed, then we should return fail */
+  retVal = validate_addcontainerconfig (container);
+  if (retVal != NSCRTL_OK)
+    {
+      free_container (container, IP_MODULE_TRUE);
+      return (NSCRTL_FREE_ALL_PORT == retVal) ? NSCRTL_OK : retVal;
+    }
+
+  /* control max network and ipaddress count */
+  if (!check_ip_count (container))
+    {
+      free_container (container, IP_MODULE_TRUE);
+      return NSCRTL_IP_COUNT_EXCEED;
+    }
+
+  struct container_port *last = NULL;
+  struct container_ip *old =
+    get_container_by_container_id (container->container_id);
+  if (old)
+    {
+      struct container_port *tmp = container->ports_list;
+      while (tmp)
+        {
+          /* here we don't need to check "if tmp->port_name == NULL", as validate_addcontainerconfig() has done this. */
+          if (get_port (container->container_id, tmp->port_name))
+            {
+              free_container (container, IP_MODULE_TRUE);
+              NSOPR_LOGERR ("port exist!");
+              return NSCRTL_RD_EXIST;
+            }
+
+          last = tmp;
+          tmp = tmp->next;
+        }
+    }
+  else
+    {
+      container->next = g_container_list.header;
+      g_container_list.header = container;
+    }
+
+  output_api *api = get_output_api ();
+  struct container_port *port = container->ports_list;
+  while (port)
+    {
+      struct container_port_ip_cidr *ip_cidr = port->ip_cidr_list;
+      while (ip_cidr)
+        {
+          if (api->add_netif_ip)
+            {
+              struct network_configuration *network =
+                get_network_by_ip_with_tree (ip_cidr->ip);
+              if (network)
+                {
+                  unsigned int mask = ~0;
+                  mask =
+                    (mask << (IP_MODULE_SUBNET_MASK_LEN - ip_cidr->mask_len));
+                  mask = spl_htonl (mask);
+                  if (network->phy_net->bond_name[0] != 0)
+                    {
+                      (void) api->add_netif_ip (network->phy_net->bond_name, ip_cidr->ip, mask);        //no need to check return value, validate_addcontainerconfig() has been checked parameters.
+                    }
+                  else
+                    {
+                      (void) api->add_netif_ip (network->phy_net->
+                                                header->nic_name, ip_cidr->ip,
+                                                mask);
+                    }
+                }
+              else
+                {
+                  NSOPR_LOGERR ("can't find network by]IP=%u,port_name=%s",
+                                ip_cidr->ip, port->port_name);
+                }
+            }
+
+          retVal =
+            trp_rb_insert ((void *) (u64_t) ip_cidr->ip, (void *) port,
+                           &g_container_ip_root, ip_compare);
+
+          if (0 != retVal)
+            {
+              NSOPR_LOGERR ("trp_rb_insert failed]ip_cidr->ip=%u",
+                            ip_cidr->ip);
+            }
+
+          ip_cidr = ip_cidr->next;
+        }
+      port = port->next;
+    }
+
+  if (old)
+    {
+      if (last)
+        {
+          last->next = old->ports_list;
+          old->ports_list = container->ports_list;
+        }
+
+      container->ports_list = NULL;
+      free_container (container, IP_MODULE_FALSE);
+    }
+
+  return NSCRTL_OK;
+}
+
+struct container_ip *
+get_container_by_container_id (char *container_id)
+{
+  if (NULL == container_id)
+    {
+      NSOPR_LOGERR ("Param input container ID is NULL");
+      return NULL;
+    }
+
+  struct container_ip *container = g_container_list.header;
+  while (container)
+    {
+      if (0 == strcmp (container->container_id, container_id))
+        {
+          return container;
+        }
+
+      container = container->next;
+    }
+
+  return NULL;
+}
+
+/*****************************************************************************
+*   Prototype    : getIpCfgAll
+*   Description  : Get All ip configurations
+*   Input        : char *
+*                     size_t
+*   Output       : char * patern:[***,***,***]
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+getIpCfgAll (char *jsonBuf, size_t size)
+{
+  int retval;
+
+  if (NULL == jsonBuf)
+    {
+      return NSCRTL_ERR;
+    }
+
+  if (size < 2)
+    {
+      NSOPR_LOGERR ("get all ip cfg error, buffer is not enough.");
+      return NSCRTL_STATUS_ERR;
+    }
+
+  char bfirstData = 1;
+  *jsonBuf = '[';
+  jsonBuf = jsonBuf + 1;
+
+  /*need another two char to keep [and ] */
+  size_t len = size - 2;
+  size_t strsize = 0;
+  struct container_port *port = NULL;
+  struct container_ip *container = g_container_list.header;
+  while (container)
+    {
+      port = container->ports_list;
+      while (port)
+        {
+          if (NULL == port->buffer)
+            {
+              port = port->next;
+              continue;
+            }
+
+          strsize = strlen (get_port_json (port)) + 1;
+
+          /*always reserve 1 char */
+          if ((strsize > 0) && (strsize < len))
+            {
+              if (bfirstData)
+                {
+                  bfirstData = 0;
+                }
+              else
+                {
+                  *jsonBuf = ',';
+                  jsonBuf = jsonBuf + 1;
+                  len = len - 1;
+                }
+
+              retval = STRCPY_S (jsonBuf, len, get_port_json (port));
+              if (EOK != retval)
+                {
+                  NSOPR_LOGERR ("STRCPY_S failed]ret=%d", retval);
+                  return NSCRTL_ERR;
+                }
+
+              len = len - strlen (get_port_json (port));
+              jsonBuf = jsonBuf + strlen (get_port_json (port));
+            }
+          else
+            {
+              NSOPR_LOGERR ("get all ip cfg error, buffer is not enough.");
+              return NSCRTL_STATUS_ERR;
+            }
+
+          port = port->next;
+        }
+
+      container = container->next;
+    }
+
+  *jsonBuf = ']';
+  return 0;
+}
+
+int
+del_port (char *container_id, char *port_name)
+{
+  struct container_port *port = NULL;
+  struct container_port **ref = NULL;
+  struct container_ip *container = NULL;
+  struct container_ip **container_ref = &g_container_list.header;
+
+  while ((container = *container_ref))
+    {
+      NSOPR_LOGDBG ("container->container_id=%s,container_id=%p",
+                    container->container_id, container_id);
+      if (strcmp (container->container_id, container_id) == 0)
+        {
+          ref = &container->ports_list;
+          while ((port = *ref))
+            {
+              if (strcmp (port_name, port->port_name) == 0)
+                {
+                  *ref = port->next;
+                  port->next = NULL;
+                  free_container_port (port, IP_MODULE_FALSE);
+                  return 0;
+                }
+
+              ref = &port->next;
+            }
+
+          break;
+        }
+
+      container_ref = &container->next;
+    }
+
+  return NSCRTL_RD_NOT_EXIST;
+}
+
+struct container_port *
+get_port (char *container_id, char *port_name)
+{
+  struct container_port *port = NULL;
+  struct container_ip *container = g_container_list.header;
+
+  while (container)
+    {
+      if (strcmp (container->container_id, container_id) == 0)
+        {
+          port = container->ports_list;
+          while (port)
+            {
+              if (strcmp (port_name, port->port_name) == 0)
+                {
+                  return port;
+                }
+
+              port = port->next;
+            }
+        }
+
+      container = container->next;
+    }
+
+  return NULL;
+}
+
+struct container_port *
+get_port_from_container (struct container_port *port)
+{
+  char *port_name = port->port_name;
+  struct container_port *tmp = port->next;
+
+  while (tmp)
+    {
+      if (strcmp (port_name, tmp->port_name) == 0)
+        {
+          return tmp;
+        }
+
+      tmp = tmp->next;
+    }
+
+  return NULL;
+}
diff --git a/stacks/lwip_stack/lwip_src/ip_module/ip_module_api.c b/stacks/lwip_stack/lwip_src/ip_module/ip_module_api.c
new file mode 100644
index 0000000..0ae61ed
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/ip_module/ip_module_api.c
@@ -0,0 +1,347 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include <stdlib.h>
+#include <string.h>
+#include "inet.h"
+#include "spl_ip_addr.h"
+#include "ip_module_api.h"
+#include "container_ip.h"
+#include "network.h"
+#include "config_common.h"
+#include "configuration_reader.h"
+#include "nstack_log.h"
+#include "nstack_securec.h"
+#include "spl_hal.h"
+#include "stackx_spl_share.h"
+#include "stackx/spl_api.h"
+#include "stackx_common.h"
+#include "spl_tcpip.h"
+
+static output_api g_output_api = { 0 };
+
+void
+regist_output_api (output_api * api)
+{
+  if (NULL == api)
+    {
+      NSOPR_LOGERR ("error!!!param api==NULL]");
+      return;
+    }
+
+  g_output_api = *api;
+}
+
+output_api *
+get_output_api ()
+{
+  return &g_output_api;
+}
+
+int
+process_post (void *arg, ip_module_type Type,
+              ip_module_operate_type operate_type)
+{
+  /* only when add network, in other words:
+     only when (IP_MODULE_NETWORK == Type) && (IP_MODULE_OPERATE_ADD == operate_type),
+     process_configuration() is called in read_fn thread itself.
+     other cases, will post_to tcpip_thread to handle them.
+
+     tips: when add network, it need to post msg(add netif) to tcpip thread and wait a sem,
+     but after the msg is handled by tcpip thread, the sem could be post.
+     so adding netword is handled in read_fn thread, can't be moved to tcpip thread.
+
+     But we should know, many global and static variables maybe not safe(netif,sc_dpdk etc.),
+     because they can be visited by multi-thread. */
+  if ((IP_MODULE_NETWORK == Type) && (IP_MODULE_OPERATE_ADD == operate_type))
+    {
+      return process_configuration (arg, Type, operate_type);
+    }
+
+  if (g_output_api.post_to)
+    {
+      int retval = g_output_api.post_to (arg, Type, operate_type);
+      if ((post_ip_module_msg == g_output_api.post_to) && (retval != ERR_OK)
+          && (arg != NULL))
+        {
+          if ((IP_MODULE_IP == Type)
+              && (IP_MODULE_OPERATE_ADD == operate_type))
+            {
+              free_container ((struct container_ip *) arg, IP_MODULE_TRUE);
+            }
+          else
+            {
+              free (arg);
+            }
+
+          arg = NULL;
+          NSOPR_LOGERR ("post failed]ret=%d", retval);
+          NSOPR_SET_ERRINFO (NSCRTL_INPUT_ERR, "process_post failed!");
+        }
+
+      return retval;
+    }
+  else
+    {
+      NSOPR_LOGERR ("g_output_api.post_to==NULL");
+      NSOPR_SET_ERRINFO (NSCRTL_ERR,
+                         "ERR:internal error, g_output_api.post_to==NULL]");
+      return -1;
+    }
+}
+
+/* arg can be NULL, no need check */
+int
+process_configuration (void *arg, ip_module_type Type,
+                       ip_module_operate_type operate_type)
+{
+  NSOPR_LOGINF ("precoess begin]arg=%p,action=%d,type=%d", arg, operate_type,
+                Type);
+  int retval = 0;
+
+  switch (Type)
+    {
+    case IP_MODULE_NETWORK:
+      switch (operate_type)
+        {
+        case IP_MODULE_OPERATE_ADD:
+          retval =
+            add_network_configuration ((struct network_configuration *) arg);
+
+          /* When network exceeds max number, just log warning at operation.log */
+          if (retval == NSCRTL_NETWORK_COUNT_EXCEED)
+            {
+              NSOPR_LOGWAR
+                ("Warning!! Network count exceed max allowed number]max=%d",
+                 MAX_NETWORK_COUNT);
+            }
+          else
+            {
+              NSOPR_SET_ERRINFO (retval,
+                                 "add_network_configuration return %d",
+                                 retval);
+            }
+
+          if (!retval)
+            {
+              /*init DPDK eth */
+              if ((retval = init_new_network_configuration ()) != ERR_OK)
+                {
+                  free_network_configuration ((struct network_configuration *)
+                                              arg, IP_MODULE_TRUE);
+                  NSOPR_SET_ERRINFO (retval,
+                                     "init_new_network_configuration return %d",
+                                     retval);
+                }
+            }
+          break;
+        case IP_MODULE_OPERATE_DEL:
+          {
+            retval = del_network_by_name ((char *) arg);
+            NSOPR_SET_ERRINFO (retval, "del_network_by_name return %d",
+                               retval);
+            free (arg);
+            arg = NULL;
+          }
+          break;
+        case IP_MODULE_OPERATE_QUERY:
+          {
+            struct network_configuration *network =
+              get_network_by_name ((char *) arg);
+            if (!network)
+              {
+                retval = NSCRTL_RD_NOT_EXIST;
+                NSOPR_SET_ERRINFO (retval, "get_network_by_name return %d",
+                                   retval);
+              }
+            else
+              {
+                if (strlen (get_network_json (network)) >
+                    sizeof (get_config_data ()->json_buff) - 1)
+                  {
+                    NSOPR_LOGERR
+                      ("length of network->network_json too big]len=%u",
+                       strlen (get_network_json (network)));
+                    NSOPR_SET_ERRINFO (NSCRTL_ERR,
+                                       "ERR:internal error, buf is not enough]");
+                    retval = NSCRTL_ERR;
+                  }
+
+                retval =
+                  STRCPY_S (get_config_data ()->json_buff,
+                            sizeof (get_config_data ()->json_buff),
+                            get_network_json (network));
+                if (EOK != retval)
+                  {
+                    NSOPR_LOGERR ("STRCPY_S failed]ret=%d", retval);
+                    NSOPR_SET_ERRINFO (NSCRTL_ERR,
+                                       "ERR:internal error, STRCPY_S failed]ret=%d",
+                                       retval);
+                    retval = NSCRTL_ERR;
+                  }
+
+              }
+
+            free (arg);
+            arg = NULL;
+          }
+          break;
+        default:
+          break;
+        }
+
+      break;
+    case IP_MODULE_IP:
+      switch (operate_type)
+        {
+        case IP_MODULE_OPERATE_ADD:
+          retval = add_container ((struct container_ip *) arg);
+          NSOPR_SET_ERRINFO (retval, "add_container return %d", retval);
+          break;
+        case IP_MODULE_OPERATE_DEL:
+          {
+            struct ip_action_param *p = (struct ip_action_param *) arg;
+            retval = del_port (p->container_id, p->port_name);
+            NSOPR_SET_ERRINFO (retval, "del_port return %d", retval);
+            free (arg);
+            arg = NULL;
+          }
+          break;
+        case IP_MODULE_OPERATE_QUERY:
+          {
+            struct ip_action_param *p = (struct ip_action_param *) arg;
+            struct container_port *port =
+              get_port (p->container_id, p->port_name);
+            if (!port)
+              {
+                retval = NSCRTL_RD_NOT_EXIST;
+                NSOPR_SET_ERRINFO (retval, "get_port return %d", retval);
+              }
+            else
+              {
+                if (strlen (get_port_json (port)) >
+                    sizeof (get_config_data ()->json_buff) - 1)
+                  {
+                    NSOPR_LOGERR
+                      ("length of network->network_json too big]len=%u",
+                       strlen (get_port_json (port)));
+                    retval = NSCRTL_ERR;
+                  }
+
+                retval =
+                  STRCPY_S (get_config_data ()->json_buff,
+                            sizeof (get_config_data ()->json_buff),
+                            get_port_json (port));
+                if (EOK != retval)
+                  {
+                    NSOPR_LOGERR ("STRCPY_S failed]ret=%d", retval);
+                    retval = NSCRTL_ERR;
+                  }
+
+              }
+
+            free (arg);
+            arg = NULL;
+          }
+          break;
+        default:
+          break;
+        }
+
+      break;
+    case IP_MODULE_NETWORK_ALL:
+      if (operate_type == IP_MODULE_OPERATE_QUERY)
+        {
+          retval =
+            get_network_all (get_config_data ()->json_buff,
+                             sizeof (get_config_data ()->json_buff));
+          NSOPR_SET_ERRINFO (retval, "get_network_all return %d", retval);
+        }
+
+      break;
+
+    case IP_MODULE_IP_ALL:
+      if (operate_type == IP_MODULE_OPERATE_QUERY)
+        {
+          retval =
+            getIpCfgAll (get_config_data ()->json_buff,
+                         sizeof (get_config_data ()->json_buff));
+          NSOPR_SET_ERRINFO (retval, "getIpCfgAll return %d", retval);
+        }
+
+      break;
+
+    case IP_MODULE_ALL:
+      if (operate_type == IP_MODULE_OPERATE_QUERY)
+        {
+          NSOPR_LOGERR
+            ("---------- IP_MODULE_ALL query is not implemented --------------");
+          NSOPR_SET_ERRINFO (NSCRTL_ERR,
+                             "ERR:This query interface is not implemented. ErrCode:%d",
+                             NSCRTL_ERR);
+        }
+
+      break;
+
+    default:
+      break;
+    }
+
+  NSOPR_LOGINF ("process finished]result=%d", retval);
+
+  return retval;
+}
+
+void
+ip_subnet_print (struct ip_subnet *subnet)
+{
+  spl_ip_addr_t ipAddr;
+
+  if (subnet == NULL)
+    {
+      return;
+    }
+
+  ipAddr.addr = spl_htonl (subnet->subnet);
+  NSPOL_LOGINF (IP_DEBUG, "]\t Subnet=%s/%u", spl_inet_ntoa (ipAddr),
+                subnet->mask_len);
+}
+
+port_buffer *
+malloc_port_buffer ()
+{
+  port_buffer *buffer = malloc (sizeof (port_buffer));
+  return buffer;
+}
+
+void
+free_port_buffer (port_buffer * buffer)
+{
+  free (buffer);
+}
+
+network_buffer *
+malloc_network_buffer ()
+{
+  network_buffer *buffer = malloc (sizeof (network_buffer));
+  return buffer;
+}
+
+void
+free_network_buffer (network_buffer * buffer)
+{
+  free (buffer);
+}
diff --git a/stacks/lwip_stack/lwip_src/ip_module/network.c b/stacks/lwip_stack/lwip_src/ip_module/network.c
new file mode 100644
index 0000000..83fbb7c
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/ip_module/network.c
@@ -0,0 +1,1127 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include <stdio.h>
+#include <sys/socket.h>
+#include <string.h>
+#include "json.h"
+#include "trp_rb_tree.h"
+#include "network.h"
+#include "nstack_log.h"
+#include "config_common.h"
+#include "stackx_spl_share.h"
+#include "stackx/spl_api.h"
+#include "sharedmemory.h"
+#include "nstack_securec.h"
+#include "nstack_rd_mng.h"
+#include "spl_hal.h"
+#include "inet.h"
+
+struct network_list g_network_list = { 0 };
+
+extern struct stackx_port_zone *p_stackx_port_zone;
+extern u32 spl_hal_is_nic_exist (const char *name);
+
+static bool_t
+is_phy_net_ok (struct phy_net *pst_phy_net)
+{
+  if (!pst_phy_net || !pst_phy_net->header)
+    {
+      NSOPR_LOGERR ("phy_net is not ok");
+      return 0;
+    }
+
+  return 1;
+}
+
+static bool_t
+is_network_configuration_ok (struct network_configuration *network)
+{
+  while (network)
+    {
+      if (!is_phy_net_ok (network->phy_net))
+        {
+          return 0;
+        }
+
+      network = network->next;
+    }
+
+  return 1;
+}
+
+static void
+add_ref_nic (struct phy_net *pst_phy_net, struct ref_nic *pst_ref_nic)
+{
+  pst_ref_nic->next = pst_phy_net->header;
+  pst_phy_net->header = pst_ref_nic;
+}
+
+static void
+free_ref_nic (struct ref_nic *pst_ref_nic)
+{
+  struct ref_nic *nic = pst_ref_nic;
+  struct ref_nic *tmp = NULL;
+
+  while (nic)
+    {
+      tmp = nic;
+      nic = tmp->next;
+      free (tmp);
+    }
+}
+
+static void
+free_phy_net (struct phy_net *pst_phy_net)
+{
+  if (pst_phy_net)
+    {
+      free_ref_nic (pst_phy_net->header);
+      free (pst_phy_net);
+    }
+}
+
+static void
+free_ip_subnet (struct ip_subnet *subnet, bool_t only_free)
+{
+  struct ip_subnet *tmp = NULL;
+
+  while (subnet)
+    {
+      tmp = subnet;
+      subnet = subnet->next;
+      free (tmp);
+    }
+}
+
+void
+free_network_configuration (struct network_configuration *network,
+                            bool_t only_free)
+{
+  if (network)
+    {
+      free_ip_subnet (network->ip_subnet, only_free);
+      free_phy_net (network->phy_net);
+
+      if (network->buffer)
+        {
+          free_network_buffer (network->buffer);
+          network->buffer = NULL;
+        }
+
+      free (network);
+    }
+}
+
+inline int
+is_in_subnet (unsigned int ip, struct ip_subnet *subnet)
+{
+  unsigned int mask = ~0;
+  unsigned int seg_ip, seg;
+
+  mask = mask << (IP_MODULE_SUBNET_MASK_LEN - subnet->mask_len);
+
+  seg_ip = ip & mask;
+  seg = subnet->subnet & mask;
+
+  return seg_ip != seg ? 1 : 0;
+}
+
+inline struct network_configuration *
+get_network_by_ip_with_tree (unsigned int ip)
+{
+  unsigned int h_ip = spl_ntohl (ip);
+  struct network_configuration *p = g_network_list.header;
+
+  while (p)
+    {
+      if (!is_in_subnet (h_ip, p->ip_subnet))
+        {
+          return p;
+        }
+
+      p = p->next;
+    }
+
+  return NULL;
+}
+
+struct network_configuration *
+get_network_by_name (char *name)
+{
+  struct network_configuration *network = g_network_list.header;
+
+  while (network)
+    {
+      if (strcasecmp (name, network->network_name) == 0)
+        {
+          return network;
+        }
+
+      network = network->next;
+    }
+
+  return NULL;
+}
+
+struct network_configuration *
+get_network_by_nic_name (char *name)
+{
+  if (NULL == name)
+    {
+      return NULL;
+    }
+
+  struct ref_nic *refnic = NULL;
+  struct phy_net *phynet = NULL;
+  struct network_configuration *network = g_network_list.header;
+  while (network)
+    {
+      phynet = network->phy_net;
+      if (phynet)
+        {
+          if ((phynet->bond_mode != -1)
+              && strcmp (name, phynet->bond_name) == 0)
+            {
+              return network;
+            }
+
+          refnic = phynet->header;
+          while (refnic)
+            {
+              if (strcmp (name, refnic->nic_name) == 0)
+                {
+                  return network;
+                }
+
+              refnic = refnic->next;
+            }
+        }
+
+      network = network->next;
+    }
+
+  return NULL;
+}
+
+struct network_configuration *
+get_network_by_name_from_json (struct network_configuration *network)
+{
+  char *name = network->network_name;
+  struct network_configuration *tmp = network->next;
+
+  while (tmp)
+    {
+      if (strcasecmp (name, tmp->network_name) == 0)
+        {
+          return tmp;
+        }
+
+      tmp = tmp->next;
+    }
+
+  return NULL;
+}
+
+NSTACK_STATIC inline int
+get_network_count ()
+{
+  int count = 0;
+  struct network_configuration *network = g_network_list.header;
+
+  while (network)
+    {
+      count++;
+      network = network->next;
+    }
+
+  return count;
+}
+
+int
+get_network_all (char *jsonBuf, size_t size)
+{
+  if (NULL == jsonBuf)
+    {
+      return NSCRTL_ERR;
+    }
+
+  if (size < 2)
+    {
+      NSOPR_LOGERR ("get all ip cfg error, buffer is not enough.");
+      return NSCRTL_STATUS_ERR;
+    }
+
+  int retVal;
+  char bfirstData = 1;
+  *jsonBuf = '[';
+  jsonBuf = jsonBuf + 1;
+
+  size_t len = size - 2;        // strlen("[]")
+  size_t strsize = 0;
+  struct network_configuration *network = g_network_list.header;
+  while (network)
+    {
+      if (NULL == network->buffer)
+        {
+          network = network->next;
+          continue;
+        }
+
+      if (bfirstData)
+        {
+          bfirstData = 0;
+        }
+      else
+        {
+          *jsonBuf = ',';
+          jsonBuf = jsonBuf + 1;
+          len = len - 1;
+        }
+
+      strsize = strlen (get_network_json (network)) + 1;
+      if ((strsize > 0) && (strsize < len))
+        {
+          retVal = STRCPY_S (jsonBuf, len, get_network_json (network));
+          if (EOK != retVal)
+            {
+              NSOPR_LOGERR ("STRCPY_S failed]ret=%d.", retVal);
+              return NSCRTL_STATUS_ERR;
+            }
+
+          len = len - strlen (get_network_json (network));
+          jsonBuf = jsonBuf + strlen (get_network_json (network));
+        }
+      else
+        {
+          NSOPR_LOGERR ("get all network cfg error, buffer is not enough.");
+          return NSCRTL_STATUS_ERR;
+        }
+
+      network = network->next;
+    }
+
+  *jsonBuf = ']';
+  return 0;
+}
+
+int
+nic_already_bind (const char *nic_name)
+{
+  struct ref_nic *pnic = NULL;
+  struct network_configuration *pnetwork = g_network_list.header;
+
+  while (pnetwork)
+    {
+      pnic = pnetwork->phy_net->header;
+      while (pnic)
+        {
+          if (0 == strcmp (pnic->nic_name, nic_name))
+            {
+              return 1;
+            }
+
+          pnic = pnic->next;
+        }
+
+      pnetwork = pnetwork->next;
+    }
+
+  return 0;
+}
+
+int
+nic_already_init (const char *nic_name)
+{
+  unsigned int i;
+
+  for (i = 0; i < p_stackx_port_zone->port_num; i++)
+    {
+      if (0 ==
+          strcmp (p_stackx_port_zone->stackx_one_port[i].linux_ip.if_name,
+                  nic_name))
+        {
+          return 1;
+        }
+    }
+
+  return 0;
+}
+
+extern struct stackx_port_info *head_used_port_list;
+int
+bonded_nic_already_bind (struct network_configuration *pnetwork,
+                         const char *nic_name)
+{
+  struct stackx_port_info *p_port_list = head_used_port_list;
+
+  while (p_port_list)
+    {
+      if (0 == strcmp (p_port_list->linux_ip.if_name, nic_name))
+        {
+          return 1;
+        }
+
+      p_port_list = p_port_list->next_use_port;
+    }
+
+  struct ref_nic *pnic = NULL;
+
+  while (pnetwork)
+    {
+      if (pnetwork->phy_net->bond_mode == -1)
+        {
+          pnic = pnetwork->phy_net->header;
+          if (0 == strcmp (pnic->nic_name, nic_name))
+            {
+              return 1;
+            }
+        }
+
+      pnetwork = pnetwork->next;
+    }
+
+  return 0;
+}
+
+extern struct bond_ports_info bond_ports_array;
+int
+nic_already_bond (struct network_configuration *pnetwork,
+                  const char *nic_name)
+{
+  u8_t i, j;
+  struct bond_set *s;
+
+  for (i = 0; i < bond_ports_array.cnt; i++)
+    {
+      s = &bond_ports_array.ports[i];
+      for (j = 0; j < s->slave_port_cnt; j++)
+        {
+          if (strcmp (s->slave_ports[j], nic_name) == 0)
+            {
+              return 1;
+            }
+        }
+    }
+
+  struct ref_nic *pnic = NULL;
+
+  while (pnetwork)
+    {
+      if (pnetwork->phy_net->bond_mode != -1)
+        {
+          pnic = pnetwork->phy_net->header;
+          while (pnic)
+            {
+              if (0 == strcmp (pnic->nic_name, nic_name))
+                {
+                  return 1;
+                }
+
+              pnic = pnic->next;
+            }
+        }
+
+      pnetwork = pnetwork->next;
+    }
+
+  return 0;
+}
+
+/*add network to rd*/
+void
+network_rd_proc (struct network_configuration *network, int op)
+{
+  struct ip_subnet *ptsubnet = NULL;
+  struct network_configuration *pn = NULL;
+  rd_ip_data rd_ip = { 0 };
+  int ret = 0;
+
+  pn = network;
+
+  while (pn)
+    {
+      if (0 == strcmp ("nstack-dpdk", pn->type_name))
+        {
+          ptsubnet = network->ip_subnet;
+          while (ptsubnet)
+            {
+              rd_ip.addr = ptsubnet->subnet;
+              rd_ip.masklen = ptsubnet->mask_len;
+              if (0 == op)
+                {
+                  ret = nstack_rd_ip_node_insert (network->type_name, &rd_ip);
+                }
+              else
+                {
+                  ret = nstack_rd_ip_node_delete (&rd_ip);
+                }
+
+              if (0 != ret)
+                {
+                  NSOPR_LOGERR ("nstack rd subnet:0x%x, masklen:0x%x %s fail",
+                                rd_ip.addr, rd_ip.masklen,
+                                (0 == op ? "insert" : "delete"));
+                }
+              else
+                {
+                  NSOPR_LOGDBG
+                    ("nstack rd subnet:0x%x, masklen:0x%x %s success",
+                     rd_ip.addr, rd_ip.masklen,
+                     (0 == op ? "insert" : "delete"));
+                }
+
+              ptsubnet = ptsubnet->next;
+            }
+        }
+
+      pn = pn->next;
+    }
+
+  return;
+}
+
+/* add network to list in descending sort */
+void
+add_network_to_list (struct network_configuration *network)
+{
+  struct network_configuration *curr = g_network_list.header;
+  struct network_configuration *prev = NULL;
+
+  network->next = NULL;
+
+  /*add network to rd */
+  network_rd_proc (network, 0);
+
+  while (curr)
+    {
+      if (network->ip_subnet->mask_len >= curr->ip_subnet->mask_len)
+        {
+          break;
+        }
+
+      prev = curr;
+      curr = curr->next;
+    }
+
+  if (NULL == prev)
+    {
+      network->next = curr;
+      g_network_list.header = network;
+    }
+  else
+    {
+      network->next = prev->next;
+      prev->next = network;
+    }
+}
+
+int
+add_network_configuration (struct network_configuration
+                           *pst_network_configuration)
+{
+  struct network_configuration *tmp = pst_network_configuration;
+  struct ref_nic *pheader;
+
+  if (tmp)
+    {
+      if (get_network_by_name (tmp->network_name)
+          || get_network_by_name_from_json (tmp))
+        {
+          NSOPR_LOGERR ("network exists or duplicates]network_name=%s",
+                        tmp->network_name);
+          free_network_configuration (pst_network_configuration,
+                                      IP_MODULE_TRUE);
+          return NSCRTL_RD_EXIST;
+        }
+
+      if (strcasecmp ("nstack-dpdk", tmp->type_name) != 0)
+        {
+          NSOPR_LOGWAR ("illegal network type]type_name=%s", tmp->type_name);
+        }
+
+      /* control max network and ipaddress count */
+      if (get_network_count () >= MAX_NETWORK_COUNT)
+        {
+          NSOPR_LOGERR ("network reach]max_count=%d", MAX_NETWORK_COUNT);
+          free_network_configuration (pst_network_configuration,
+                                      IP_MODULE_TRUE);
+          return NSCRTL_NETWORK_COUNT_EXCEED;
+        }
+
+      /* If nic is not existed or not initiated, return error */
+      pheader = tmp->phy_net->header;
+      while (pheader)
+        {
+          if (!spl_hal_is_nic_exist (pheader->nic_name)
+              && !nic_already_init (pheader->nic_name))
+            {
+              NSOPR_LOGERR ("Invalid configuration %s not exist Error! ",
+                            pheader->nic_name);
+              free_network_configuration (pst_network_configuration,
+                                          IP_MODULE_TRUE);
+              return NSCRTL_RD_NOT_EXIST;
+            }
+
+          pheader = pheader->next;
+        }
+
+      /* if a bonded nic has been inited in a non-bond network, return error */
+      if (tmp->phy_net->bond_mode != -1)
+        {
+          pheader = tmp->phy_net->header;
+          while (pheader)
+            {
+              if (bonded_nic_already_bind
+                  (pst_network_configuration, pheader->nic_name))
+                {
+                  NSOPR_LOGERR ("Invalid configuration %s already bind! ",
+                                pheader->nic_name);
+                  free_network_configuration (pst_network_configuration,
+                                              IP_MODULE_TRUE);
+                  return NSCRTL_INPUT_ERR;
+                }
+
+              pheader = pheader->next;
+            }
+        }
+
+      /* if a non-bond nic has been inited in a bonded network, return error */
+      if (tmp->phy_net->bond_mode == -1)
+        {
+          pheader = tmp->phy_net->header;
+          while (pheader)
+            {
+              if (nic_already_bond
+                  (pst_network_configuration, pheader->nic_name))
+                {
+                  NSOPR_LOGERR ("Invalid configuration %s already bind! ",
+                                pheader->nic_name);
+                  free_network_configuration (pst_network_configuration,
+                                              IP_MODULE_TRUE);
+                  return NSCRTL_INPUT_ERR;
+                }
+
+              pheader = pheader->next;
+            }
+        }
+    }
+  else
+    {
+      NSOPR_LOGERR ("Invalid network configuration!");
+      return NSCRTL_INPUT_ERR;
+    }
+
+  /*looping through each node has move to read_ipmoduleoperateadd_configuration */
+  ip_subnet_print (tmp->ip_subnet);
+  add_network_to_list (tmp);
+
+  return NSCRTL_OK;
+}
+
+struct network_configuration *
+parse_network_obj (struct json_object *network_obj)
+{
+  int retVal;
+  struct network_configuration *pst_network_configuration = NULL;
+  struct json_object *network_name_obj = NULL, *args_obj = NULL, *phy_obj =
+    NULL, *type_name_obj = NULL;
+  struct json_object *ref_nic_list_obj = NULL, *bond_mode_obj =
+    NULL, *bond_name_obj = NULL, *ipam_obj = NULL;
+  struct json_object *subnet_obj = NULL;
+
+  if (!network_obj)
+    {
+      NSOPR_LOGERR ("network_obj is null");
+      goto RETURN_ERROR;
+    }
+
+  pst_network_configuration = malloc (sizeof (struct network_configuration));
+  if (NULL == pst_network_configuration)
+    {
+      NSOPR_LOGERR ("malloc failed");
+      goto RETURN_ERROR;
+    }
+
+  retVal =
+    MEMSET_S (pst_network_configuration,
+              sizeof (struct network_configuration), 0,
+              sizeof (struct network_configuration));
+  if (EOK != retVal)
+    {
+      NSOPR_LOGERR ("MEMSET_S failed]ret=%d.", retVal);
+      goto RETURN_ERROR;
+    }
+
+  json_object_object_get_ex (network_obj, "name", &network_name_obj);
+  if (network_name_obj)
+    {
+      const char *network_name = json_object_get_string (network_name_obj);
+      if ((NULL == network_name) || (network_name[0] == 0)
+          || (strlen (network_name) >= IP_MODULE_MAX_NAME_LEN))
+        {
+          NSOPR_LOGERR ("network_name is not ok");
+          goto RETURN_ERROR;
+        }
+
+      retVal =
+        STRCPY_S (pst_network_configuration->network_name,
+                  sizeof (pst_network_configuration->network_name),
+                  network_name);
+
+      if (EOK != retVal)
+        {
+          NSOPR_LOGERR ("STRCPY_S failed]ret=%d.", retVal);
+          goto RETURN_ERROR;
+        }
+    }
+  else
+    {
+      NSOPR_LOGERR ("name is not ok");
+      goto RETURN_ERROR;
+    }
+
+  json_object_object_get_ex (network_obj, "type", &type_name_obj);
+  if (type_name_obj)
+    {
+      const char *type_name = json_object_get_string (type_name_obj);
+      if ((NULL == type_name) || (type_name[0] == 0)
+          || (strlen (type_name) >= IP_MODULE_MAX_NAME_LEN))
+        {
+          NSOPR_LOGERR ("type parse error");
+          goto RETURN_ERROR;
+        }
+
+      retVal =
+        STRCPY_S (pst_network_configuration->type_name,
+                  sizeof (pst_network_configuration->type_name), type_name);
+
+      if (EOK != retVal)
+        {
+          NSOPR_LOGERR ("STRCPY_S failed]ret=%d", retVal);
+          goto RETURN_ERROR;
+        }
+    }
+  else
+    {
+      NSOPR_LOGERR ("type is not ok");
+      goto RETURN_ERROR;
+    }
+
+  json_object_object_get_ex (network_obj, "args", &args_obj);
+  if (args_obj)
+    {
+      json_object_object_get_ex (args_obj, "phynet", &phy_obj);
+      if (phy_obj)
+        {
+          struct phy_net *pst_phy_net = malloc (sizeof (struct phy_net));
+          if (NULL == pst_phy_net)
+            {
+              NSOPR_LOGERR ("malloc failed");
+              goto RETURN_ERROR;
+            }
+
+          retVal =
+            MEMSET_S (pst_phy_net, sizeof (struct phy_net), 0,
+                      sizeof (struct phy_net));
+          if (EOK != retVal)
+            {
+              NSOPR_LOGERR ("MEMSET_S failed]ret=%d", retVal);
+              free_phy_net (pst_phy_net);
+              goto RETURN_ERROR;
+            }
+
+          json_object_object_get_ex (phy_obj, "ref_nic", &ref_nic_list_obj);
+          if (ref_nic_list_obj)
+            {
+              int j;
+              int ref_nic_num = json_object_array_length (ref_nic_list_obj);
+              if (0 == ref_nic_num)
+                {
+                  NSOPR_LOGERR ("ref_nic is empty");
+                  free_phy_net (pst_phy_net);
+                  goto RETURN_ERROR;
+                }
+
+              for (j = ref_nic_num - 1; j >= 0; j--)
+                {
+                  struct json_object *ref_nic_obj =
+                    json_object_array_get_idx (ref_nic_list_obj, j);
+                  if (ref_nic_obj)
+                    {
+                      const char *nic_name =
+                        json_object_get_string (ref_nic_obj);
+                      if ((NULL == nic_name) || (nic_name[0] == 0)
+                          || (strlen (nic_name) >= HAL_MAX_NIC_NAME_LEN))
+                        {
+                          NSOPR_LOGERR ("nic_name is not ok");
+                          free_phy_net (pst_phy_net);
+                          goto RETURN_ERROR;
+                        }
+
+                      struct ref_nic *pst_ref_nic =
+                        malloc (sizeof (struct ref_nic));
+                      if (NULL == pst_ref_nic)
+                        {
+                          NSOPR_LOGERR ("malloc failed");
+                          free_phy_net (pst_phy_net);
+                          goto RETURN_ERROR;
+                        }
+
+                      retVal =
+                        MEMSET_S (pst_ref_nic, sizeof (struct ref_nic), 0,
+                                  sizeof (struct ref_nic));
+                      if (EOK != retVal)
+                        {
+                          NSOPR_LOGERR ("MEMSET_S failed]ret=%d.", retVal);
+                          free (pst_ref_nic);
+                          pst_ref_nic = NULL;
+                          free_phy_net (pst_phy_net);
+                          goto RETURN_ERROR;
+                        }
+
+                      retVal =
+                        STRCPY_S (pst_ref_nic->nic_name,
+                                  sizeof (pst_ref_nic->nic_name), nic_name);
+
+                      if (EOK != retVal)
+                        {
+                          NSOPR_LOGERR ("STRCPY_S failed]ret=%d.", retVal);
+                          free (pst_ref_nic);
+                          free_phy_net (pst_phy_net);
+                          goto RETURN_ERROR;
+                        }
+
+                      add_ref_nic (pst_phy_net, pst_ref_nic);
+                    }
+                }
+            }
+          else
+            {
+              NSOPR_LOGERR ("ref_nic is not ok");
+              free_phy_net (pst_phy_net);
+              goto RETURN_ERROR;
+            }
+
+          json_object_object_get_ex (phy_obj, "bond_mode", &bond_mode_obj);
+          if (bond_mode_obj)
+            {
+              pst_phy_net->bond_mode = json_object_get_int (bond_mode_obj);
+              if (pst_phy_net->bond_mode != -1)
+                {
+                  json_object_object_get_ex (phy_obj, "bond_name",
+                                             &bond_name_obj);
+                  if (bond_name_obj)
+                    {
+                      const char *bond_name =
+                        json_object_get_string (bond_name_obj);
+                      if ((NULL == bond_name)
+                          || (strlen (bond_name) >= IP_MODULE_MAX_NAME_LEN))
+                        {
+                          NSOPR_LOGERR ("bond_name is not ok");
+                          free_phy_net (pst_phy_net);
+                          goto RETURN_ERROR;
+                        }
+
+                      retVal =
+                        MEMSET_S (pst_phy_net->bond_name,
+                                  sizeof (pst_phy_net->bond_name), 0,
+                                  sizeof (pst_phy_net->bond_name));
+
+                      if (EOK != retVal)
+                        {
+                          NSOPR_LOGERR ("MEMSET_S failed]ret=%d.", retVal);
+                          free_phy_net (pst_phy_net);
+                          goto RETURN_ERROR;
+                        }
+
+                      retVal =
+                        STRNCPY_S (pst_phy_net->bond_name,
+                                   sizeof (pst_phy_net->bond_name), bond_name,
+                                   strlen (bond_name));
+
+                      if (EOK != retVal)
+                        {
+                          NSOPR_LOGERR ("STRNCPY_S failed]retVal=%d", retVal);
+                          free_phy_net (pst_phy_net);
+                          goto RETURN_ERROR;
+                        }
+                    }
+                  else
+                    {
+                      NSOPR_LOGERR ("bond_name is not ok");
+                      free_phy_net (pst_phy_net);
+                      goto RETURN_ERROR;
+                    }
+                }
+            }
+          else
+            {
+              pst_phy_net->bond_mode = -1;
+            }
+
+          pst_network_configuration->phy_net = pst_phy_net;
+        }
+      else
+        {
+          NSOPR_LOGERR ("phy_net is not ok");
+          goto RETURN_ERROR;
+        }
+    }
+  else
+    {
+      NSOPR_LOGERR ("args is not ok");
+      goto RETURN_ERROR;
+    }
+
+  json_object_object_get_ex (network_obj, "ipam", &ipam_obj);
+  if (ipam_obj)
+    {
+      json_object_object_get_ex (ipam_obj, "subnet", &subnet_obj);
+      if (subnet_obj)
+        {
+          int iRet;
+          char tmp[IP_MODULE_LENGTH_32] = { 0 };
+          const char *subnet = json_object_get_string (subnet_obj);
+          struct in_addr addr;
+
+          retVal = MEMSET_S (&addr, sizeof (addr), 0, sizeof (addr));
+          if (EOK != retVal)
+            {
+              NSOPR_LOGERR ("MEMSET_S failed]ret=%d.", retVal);
+              goto RETURN_ERROR;
+            }
+
+          if ((NULL == subnet) || (subnet[0] == 0)
+              || (strlen (subnet) > IP_MODULE_LENGTH_32))
+            {
+              NSOPR_LOGERR ("subnet is not ok");
+              goto RETURN_ERROR;
+            }
+
+          const char *sub = strstr (subnet, "/");
+          if ((NULL == sub)
+              || (sizeof (tmp) - 1 <= (unsigned int) (sub - subnet))
+              || (strlen (sub) >= sizeof (tmp) - 1))
+            {
+              NSOPR_LOGERR ("subnet is not ok");
+              goto RETURN_ERROR;
+            }
+
+          iRet =
+            STRNCPY_S (tmp, sizeof (tmp), subnet, (size_t) (sub - subnet));
+          if (EOK != iRet)
+            {
+              NSOPR_LOGERR ("STRNCPY_S failed]ret=%d", iRet);
+              goto RETURN_ERROR;
+            }
+
+          iRet = spl_inet_aton (tmp, &addr);
+          if (0 == iRet)
+            {
+              NSOPR_LOGERR ("subnet is not ok");
+              goto RETURN_ERROR;
+            }
+
+          pst_network_configuration->ip_subnet =
+            (struct ip_subnet *) malloc (sizeof (struct ip_subnet));
+          if (!pst_network_configuration->ip_subnet)
+            {
+              NSOPR_LOGERR ("malloc failed");
+              goto RETURN_ERROR;
+            }
+
+          retVal =
+            MEMSET_S (pst_network_configuration->ip_subnet,
+                      sizeof (struct ip_subnet), 0,
+                      sizeof (struct ip_subnet));
+          if (EOK != retVal)
+            {
+              NSOPR_LOGERR ("MEMSET_S failed]ret=%d.", retVal);
+              goto RETURN_ERROR;
+            }
+
+          pst_network_configuration->ip_subnet->next = NULL;
+          pst_network_configuration->ip_subnet->subnet =
+            spl_ntohl (addr.s_addr);
+
+          iRet = atoi (sub + 1);
+          if ((iRet < 0) || (iRet > IP_MODULE_LENGTH_32))
+            {
+              NSOPR_LOGERR ("mask is not ok");
+              goto RETURN_ERROR;
+            }
+
+          pst_network_configuration->ip_subnet->mask_len =
+            (unsigned int) iRet;
+        }
+      else
+        {
+          NSOPR_LOGERR ("subnet is not ok");
+          goto RETURN_ERROR;
+        }
+    }
+  else
+    {
+      NSOPR_LOGERR ("ipam is not ok");
+      goto RETURN_ERROR;
+    }
+
+  const char *network_json = json_object_get_string (network_obj);
+  if ((NULL == network_json) || (network_json[0] == 0))
+    {
+      NSOPR_LOGERR ("json_object_get_string failed");
+      goto RETURN_ERROR;
+    }
+
+  pst_network_configuration->buffer = malloc_network_buffer ();
+  if (!pst_network_configuration->buffer)
+    {
+      NSOPR_LOGERR ("malloc_network_buffer failed");
+      goto RETURN_ERROR;
+    }
+
+  retVal =
+    STRCPY_S (get_network_json (pst_network_configuration),
+              IP_MODULE_NETWORK_JSON_LEN, network_json);
+  if (EOK != retVal)
+    {
+      NSOPR_LOGERR ("STRCPY_S failed]ret=%d.", retVal);
+      goto RETURN_ERROR;
+    }
+
+  return pst_network_configuration;
+
+RETURN_ERROR:
+  if (pst_network_configuration)
+    {
+      free_network_configuration (pst_network_configuration, IP_MODULE_TRUE);
+      pst_network_configuration = NULL;
+    }
+
+  return NULL;
+}
+
+struct network_configuration *
+parse_network_json (char *param, struct network_configuration *pnetwork_list)
+{
+  struct json_object *obj = json_tokener_parse (param);
+  struct network_configuration *pst_network_configuration = NULL;
+
+  if (!obj)
+    {
+      NSOPR_LOGERR ("parse error");
+      return NULL;
+    }
+
+  int network_num = json_object_array_length (obj);
+  if (0 == network_num)
+    {
+      json_object_put (obj);
+      return NULL;
+    }
+
+  int i;
+  for (i = 0; i < network_num; i++)
+    {
+      struct json_object *network_obj = json_object_array_get_idx (obj, i);
+      if (network_obj)
+        {
+          pst_network_configuration = parse_network_obj (network_obj);
+          if (!pst_network_configuration)
+            {
+              NSOPR_LOGERR ("parse_network_obj error");
+              goto RETURN_ERROR;
+            }
+
+          pst_network_configuration->next = pnetwork_list;
+          pnetwork_list = pst_network_configuration;
+          pst_network_configuration = NULL;
+        }
+      else
+        {
+          NSOPR_LOGERR ("network_obj is NULL");
+          goto RETURN_ERROR;
+        }
+    }
+
+  if (pnetwork_list)
+    {
+      if (!is_network_configuration_ok (pnetwork_list))
+        {
+          NSOPR_LOGERR ("network_configuration is not ok");
+          goto RETURN_ERROR;
+        }
+    }
+
+  json_object_put (obj);
+  return pnetwork_list;
+
+RETURN_ERROR:
+  free_network_configuration (pnetwork_list, IP_MODULE_TRUE);
+  json_object_put (obj);
+  return NULL;
+}
+
+int
+del_network_by_name (char *name)
+{
+  struct network_configuration *network = NULL;
+  struct network_configuration **ref = &g_network_list.header;
+
+  while ((network = *ref))
+    {
+      if (strcasecmp (name, network->network_name) == 0)
+        {
+          *ref = network->next;
+          network->next = NULL;
+
+          /*delete netework from rd */
+          network_rd_proc (network, 1);
+
+          free_network_configuration (network, IP_MODULE_FALSE);
+          return 0;
+        }
+
+      ref = &network->next;
+    }
+
+  return NSCRTL_RD_NOT_EXIST;
+}
+
+int
+is_in_same_network (unsigned int src_ip, unsigned int dst_ip)
+{
+  if (src_ip == dst_ip)
+    {
+      return 1;
+    }
+
+  struct network_configuration *src = get_network_by_ip_with_tree (src_ip);
+  struct network_configuration *dst = get_network_by_ip_with_tree (dst_ip);
+  if (src && (src == dst))
+    {
+      return 1;
+    }
+
+  return 0;
+}
+
+struct network_configuration *
+get_network_list ()
+{
+  return g_network_list.header;
+}
diff --git a/stacks/lwip_stack/lwip_src/ip_module/trp_rb_tree.c b/stacks/lwip_stack/lwip_src/ip_module/trp_rb_tree.c
new file mode 100644
index 0000000..1990f52
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/ip_module/trp_rb_tree.c
@@ -0,0 +1,563 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "trp_rb_tree.h"
+
+NSTACK_STATIC void
+__rb_rotate_left (struct trp_rb_node *X, struct trp_rb_root *root)
+{
+    /**************************
+     *  rotate Node X to left *
+     **************************/
+
+  struct trp_rb_node *Y = X->rb_right;
+
+  /* estblish X->Right link */
+  X->rb_right = Y->rb_left;
+  if (Y->rb_left != NULL)
+    Y->rb_left->rb_parent = X;
+
+  /* estblish Y->Parent link */
+  Y->rb_parent = X->rb_parent;
+  if (X->rb_parent)
+    {
+      if (X == X->rb_parent->rb_left)
+        X->rb_parent->rb_left = Y;
+      else
+        X->rb_parent->rb_right = Y;
+    }
+  else
+    {
+      root->rb_node = Y;
+    }
+
+  /* link X and Y */
+  Y->rb_left = X;
+  X->rb_parent = Y;
+
+  return;
+}
+
+NSTACK_STATIC void
+__rb_rotate_right (struct trp_rb_node *X, struct trp_rb_root *root)
+{
+    /****************************
+     *  rotate Node X to right  *
+     ****************************/
+
+  struct trp_rb_node *Y = X->rb_left;
+
+  /* estblish X->Left link */
+  X->rb_left = Y->rb_right;
+  if (Y->rb_right != NULL)
+    Y->rb_right->rb_parent = X;
+
+  /* estblish Y->Parent link */
+  Y->rb_parent = X->rb_parent;
+  if (X->rb_parent)
+    {
+      if (X == X->rb_parent->rb_right)
+        X->rb_parent->rb_right = Y;
+      else
+        X->rb_parent->rb_left = Y;
+    }
+  else
+    {
+      root->rb_node = Y;
+    }
+
+  /* link X and Y */
+  Y->rb_right = X;
+  X->rb_parent = Y;
+
+  return;
+}
+
+/* X, Y are for application */
+NSTACK_STATIC void
+__rb_erase_color (struct trp_rb_node *X, struct trp_rb_node *Parent,
+                  struct trp_rb_root *root)
+{
+    /*************************************
+     *  maintain red-black tree balance  *
+     *  after deleting node X            *
+     *************************************/
+
+  while (X != root->rb_node && (!X || X->color == RB_BLACK))
+    {
+
+      if (Parent == NULL)
+        {
+          break;
+        }
+
+      if (X == Parent->rb_left)
+        {
+          struct trp_rb_node *W = Parent->rb_right;
+          if (W->color == RB_RED)
+            {
+              W->color = RB_BLACK;
+              Parent->color = RB_RED;   /* Parent != NIL? */
+              __rb_rotate_left (Parent, root);
+              W = Parent->rb_right;
+            }
+
+          if ((!W->rb_left || W->rb_left->color == RB_BLACK)
+              && (!W->rb_right || W->rb_right->color == RB_BLACK))
+            {
+              W->color = RB_RED;
+              X = Parent;
+              Parent = X->rb_parent;
+            }
+          else
+            {
+              if (!W->rb_right || W->rb_right->color == RB_BLACK)
+                {
+                  if (W->rb_left != NULL)
+                    W->rb_left->color = RB_BLACK;
+                  W->color = RB_RED;
+                  __rb_rotate_right (W, root);
+                  W = Parent->rb_right;
+                }
+
+              W->color = Parent->color;
+              Parent->color = RB_BLACK;
+              if (W->rb_right->color != RB_BLACK)
+                {
+                  W->rb_right->color = RB_BLACK;
+                }
+              __rb_rotate_left (Parent, root);
+              X = root->rb_node;
+              break;
+            }
+        }
+      else
+        {
+
+          struct trp_rb_node *W = Parent->rb_left;
+          if (W->color == RB_RED)
+            {
+              W->color = RB_BLACK;
+              Parent->color = RB_RED;   /* Parent != NIL? */
+              __rb_rotate_right (Parent, root);
+              W = Parent->rb_left;
+            }
+
+          if ((!W->rb_left || (W->rb_left->color == RB_BLACK))
+              && (!W->rb_right || (W->rb_right->color == RB_BLACK)))
+            {
+              W->color = RB_RED;
+              X = Parent;
+              Parent = X->rb_parent;
+            }
+          else
+            {
+              if (!W->rb_left || (W->rb_left->color == RB_BLACK))
+                {
+                  if (W->rb_right != NULL)
+                    W->rb_right->color = RB_BLACK;
+                  W->color = RB_RED;
+                  __rb_rotate_left (W, root);
+                  W = Parent->rb_left;
+                }
+
+              W->color = Parent->color;
+              Parent->color = RB_BLACK;
+              if (W->rb_left->color != RB_BLACK)
+                {
+                  W->rb_left->color = RB_BLACK;
+                }
+              __rb_rotate_right (Parent, root);
+              X = root->rb_node;
+              break;
+            }
+        }
+    }
+
+  if (X)
+    {
+      X->color = RB_BLACK;
+    }
+
+  return;
+}
+
+static void
+rb_insert_color (struct trp_rb_node *X, struct trp_rb_root *root)
+{
+    /*************************************
+     *  maintain red-black tree balance  *
+     *  after inserting node X           *
+     *************************************/
+
+  /* check red-black properties */
+  while (X != root->rb_node && X->rb_parent->color == RB_RED)
+    {
+      /* we have a violation */
+      if (X->rb_parent == X->rb_parent->rb_parent->rb_left)
+        {
+          struct trp_rb_node *Y = X->rb_parent->rb_parent->rb_right;
+          if (Y && Y->color == RB_RED)
+            {
+
+              /* uncle is red */
+              X->rb_parent->color = RB_BLACK;
+              Y->color = RB_BLACK;
+              X->rb_parent->rb_parent->color = RB_RED;
+              X = X->rb_parent->rb_parent;
+            }
+          else
+            {
+
+              /* uncle is black */
+              if (X == X->rb_parent->rb_right)
+                {
+                  /* make X a left child */
+                  X = X->rb_parent;
+                  __rb_rotate_left (X, root);
+                }
+
+              /* recolor and rotate */
+              X->rb_parent->color = RB_BLACK;
+              X->rb_parent->rb_parent->color = RB_RED;
+              __rb_rotate_right (X->rb_parent->rb_parent, root);
+            }
+        }
+      else
+        {
+
+          /* miror image of above code */
+          struct trp_rb_node *Y = X->rb_parent->rb_parent->rb_left;
+          if (Y && (Y->color == RB_RED))
+            {
+
+              /* uncle is red */
+              X->rb_parent->color = RB_BLACK;
+              Y->color = RB_BLACK;
+              X->rb_parent->rb_parent->color = RB_RED;
+              X = X->rb_parent->rb_parent;
+            }
+          else
+            {
+
+              /* uncle is black */
+              if (X == X->rb_parent->rb_left)
+                {
+                  X = X->rb_parent;
+                  __rb_rotate_right (X, root);
+                }
+              X->rb_parent->color = RB_BLACK;
+              X->rb_parent->rb_parent->color = RB_RED;
+              __rb_rotate_left (X->rb_parent->rb_parent, root);
+            }
+        }
+    }
+
+  root->rb_node->color = RB_BLACK;
+
+  return;
+}
+
+static void
+rb_erase (struct trp_rb_node *node, struct trp_rb_root *root)
+{
+  struct trp_rb_node *child, *parent;
+  unsigned int color;
+
+  if (!node->rb_left)
+    {
+      child = node->rb_right;
+    }
+  else if (!node->rb_right)
+    {
+      child = node->rb_left;
+    }
+  else
+    {
+      struct trp_rb_node *old = node, *left;
+
+      node = node->rb_right;
+      while ((left = node->rb_left) != NULL)
+        {
+          node = left;
+        }
+
+      if (old->rb_parent)
+        {
+          if (old->rb_parent->rb_left == old)
+            {
+              old->rb_parent->rb_left = node;
+            }
+          else
+            {
+              old->rb_parent->rb_right = node;
+            }
+        }
+      else
+        {
+          root->rb_node = node;
+        }
+
+      child = node->rb_right;
+      parent = node->rb_parent;
+      color = node->color;
+
+      if (parent == old)
+        {
+          parent = node;
+        }
+      else
+        {
+          if (child)
+            {
+              child->rb_parent = parent;
+            }
+
+          parent->rb_left = child;
+
+          node->rb_right = old->rb_right;
+          old->rb_right->rb_parent = node;
+        }
+
+      node->color = old->color;
+      node->rb_parent = old->rb_parent;
+      node->rb_left = old->rb_left;
+      old->rb_left->rb_parent = node;
+
+      if (color == RB_BLACK)
+        {
+          __rb_erase_color (child, parent, root);
+        }
+
+      return;
+
+    }
+
+  parent = node->rb_parent;
+  color = node->color;
+
+  if (child)
+    {
+      child->rb_parent = parent;
+    }
+
+  if (parent)
+    {
+      if (parent->rb_left == node)
+        {
+          parent->rb_left = child;
+        }
+      else
+        {
+          parent->rb_right = child;
+        }
+    }
+  else
+    {
+      root->rb_node = child;
+    }
+
+  if (color == RB_BLACK)
+    {
+      __rb_erase_color (child, parent, root);
+    }
+
+  return;
+}
+
+NSTACK_STATIC trp_rb_node_t *
+rb_new_node (trp_key_t key, trp_data_t data,
+             trp_rb_node_t * parent /*, key_compare key_compare_fn */ )
+{
+  trp_rb_node_t *node = (trp_rb_node_t *) malloc (sizeof (trp_rb_node_t));
+  if (!node)
+    {
+      return NULL;
+    }
+  node->key = key;
+  node->data = data;
+  node->rb_parent = parent;
+  node->rb_left = node->rb_right = NULL;
+  node->color = RB_RED;
+  /*node->key_compare_fn = key_compare_fn; */
+  return node;
+}
+
+int
+trp_rb_insert (trp_key_t key, trp_data_t data, trp_rb_root_t * root,
+               key_compare key_compare_fn)
+{
+  trp_rb_node_t *node = root->rb_node, *parent = NULL;
+  int ret = 0;                  /* CID 24640 */
+  while (node)
+    {
+      parent = node;
+      ret = key_compare_fn (node->key, key);
+      if (0 < ret)
+        {
+          node = node->rb_left;
+        }
+      else if (0 > ret)
+        {
+          node = node->rb_right;
+        }
+      else
+        {
+          return -1;
+        }
+    }
+
+  node = rb_new_node (key, data, parent /*, key_compare_fn */ );
+  if (!node)
+    {
+      return -1;
+    }
+
+  if (parent)
+    {
+      if (ret > 0)
+        {
+          parent->rb_left = node;
+        }
+      else
+        {
+          parent->rb_right = node;
+        }
+    }
+  else
+    {
+      root->rb_node = node;
+    }
+
+  rb_insert_color (node, root);
+  return 0;
+}
+
+int
+trp_rb_insert_allow_same_key (trp_key_t key, trp_data_t data,
+                              trp_rb_root_t * root,
+                              key_compare key_compare_fn)
+{
+  trp_rb_node_t *node = root->rb_node, *parent = NULL;
+  int ret = 0;                  /*CID 24638 */
+  while (node)
+    {
+      parent = node;
+      ret = key_compare_fn (node->key, key);
+      if (0 < ret)
+        {
+          node = node->rb_left;
+        }
+      else
+        {
+          node = node->rb_right;
+        }
+    }
+
+  node = rb_new_node (key, data, parent /*, key_compare_fn */ );
+  if (!node)
+    {
+      return -1;
+    }
+
+  if (parent)
+    {
+      if (ret > 0)
+        {
+          parent->rb_left = node;
+        }
+      else
+        {
+          parent->rb_right = node;
+        }
+    }
+  else
+    {
+      root->rb_node = node;
+    }
+
+  rb_insert_color (node, root);
+  return 0;
+}
+
+NSTACK_STATIC trp_rb_node_t *
+trp_rb_search_inorder (trp_key_t key, trp_data_t data,
+                       trp_rb_node_t * node, int *count,
+                       key_compare key_compare_fn)
+{
+  if (!node)
+    {
+      return NULL;
+    }
+
+  int ret = key_compare_fn (node->key, key);;
+  if (0 == ret && data == node->data)
+    {
+      return node;
+    }
+
+  if ((NULL == count) || (0 >= --(*count)))
+    {
+      return NULL;
+    }
+
+  trp_rb_node_t *ret_node =
+    trp_rb_search_inorder (key, data, node->rb_left, count, key_compare_fn);
+  if (ret_node)
+    {
+      return ret_node;
+    }
+
+  ret_node =
+    trp_rb_search_inorder (key, data, node->rb_right, count, key_compare_fn);
+  return ret_node;
+}
+
+void
+trp_rb_erase_with_data (trp_key_t key, trp_data_t data, trp_rb_root_t * root,
+                        int count, key_compare key_compare_fn)
+{
+  trp_rb_node_t *node;
+  /* recursion operation need depth protect */
+  if (!
+      (node =
+       trp_rb_search_inorder (key, data, root->rb_node, &count,
+                              key_compare_fn)))
+    {
+      return;
+    }
+
+  rb_erase (node, root);
+  free (node);
+  node = NULL;
+}
+
+void
+trp_rb_erase (trp_key_t key, trp_rb_root_t * root, key_compare key_compare_fn)
+{
+  trp_rb_node_t *node;
+  if (!(node = trp_rb_search (key, root, key_compare_fn)))
+    {
+      return;
+    }
+
+  rb_erase (node, root);
+  free (node);
+  node = NULL;
+}
diff --git a/stacks/lwip_stack/lwip_src/netif/ethernetif.c b/stacks/lwip_stack/lwip_src/netif/ethernetif.c
new file mode 100644
index 0000000..6a3e810
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/netif/ethernetif.c
@@ -0,0 +1,167 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include "spl_opt.h"
+
+#include "spl_def.h"
+#include "mem.h"
+#include "stackx/spl_pbuf.h"
+//#include <stackx/stats.h>
+//#include "sockets.h"
+#include <netinet/in.h>
+
+#include "stackx_spl_share.h"
+#include "stackx/spl_api.h"
+#include "lwip/etharp.h"
+
+#include <sc_dpdk.h>
+#include "sc_dpdk.h"
+
+#include "cpuid.h"
+#include "nstack_log.h"
+#include "nstack_securec.h"
+#include "spl_hal.h"
+#include "hal_api.h"
+
+#include "sys.h"
+
+#define IFNAME0 'e'
+#define IFNAME1 'n'
+
+struct ethernetif
+{
+  struct eth_addr *ethaddr;
+
+};
+
+#if (DPDK_MODULE != 1)
+NSTACK_STATIC void
+low_level_init (struct netif *pnetif)
+{
+  struct ether_addr eth_addr;
+
+  struct netifExt *pnetifExt = NULL;
+  NSPOL_LOGINF (NETIF_DEBUG, "low_level_init \n");
+
+  pnetifExt = getNetifExt (pnetif->num);
+  if (NULL == pnetifExt)
+    return;
+
+  hal_get_macaddr (pnetifExt->hdl, &eth_addr);
+  NSPOL_LOGINF (SC_DPDK_INFO,
+                "low_level_init: Port %s, MAC : %02X:%02X:%02X:%02X:%02X:%02X",
+                pnetifExt->if_name, eth_addr.addr_bytes[0],
+                eth_addr.addr_bytes[1], eth_addr.addr_bytes[2],
+                eth_addr.addr_bytes[3], eth_addr.addr_bytes[4],
+                eth_addr.addr_bytes[5]);
+
+  pnetif->hwaddr_len = 6;
+  pnetif->hwaddr[0] = eth_addr.addr_bytes[0];   //0x00;
+  pnetif->hwaddr[1] = eth_addr.addr_bytes[1];   //0x1b;
+  pnetif->hwaddr[2] = eth_addr.addr_bytes[2];   //0x21;
+  pnetif->hwaddr[3] = eth_addr.addr_bytes[3];   //0x6b;
+  pnetif->hwaddr[4] = eth_addr.addr_bytes[4];   //0x24;
+  pnetif->hwaddr[5] = eth_addr.addr_bytes[5];   //0x40;
+  pnetif->mtu = SPL_FRAME_MTU;
+
+  /* don't set SPL_NETIF_FLAG_ETHARP if this device is not an ethernet one */
+  pnetif->flags =
+    NETIF_FLAG_BROADCAST | NETIF_FLAG_ETHARP | NETIF_FLAG_LINK_UP;
+  pnetif->flags |= NETIF_FLAG_IGMP;
+
+}
+#endif
+
+#if (DPDK_MODULE != 1)
+
+/**
+ * Should be called at the beginning of the program to set up the
+ * network interface. It calls the function low_level_init() to do the
+ * actual setup of the hardware.
+ *
+ * This function should be passed as a parameter to spl_netif_add().
+ *
+ * @param netif the lwip network interface structure for this ethernetif
+ * @return ERR_OK if the loopif is initialized
+ *         ERR_MEM if privatedata couldn't be allocated
+ *         any other err_t on error
+ */
+err_t
+ethernetif_init (struct netif * pnetif)
+{
+  struct ethernetif *eth_netif;
+
+  if (NULL == pnetif)
+    {
+      NSPOL_LOGERR ("netif=NULL");
+      return ERR_VAL;
+    }
+  NSPOL_LOGINF (NETIF_DEBUG, "ethernetif_init \n");
+
+  eth_netif = (struct ethernetif *) malloc (sizeof (struct ethernetif));
+  if (eth_netif == NULL)
+    {
+      NSPOL_LOGERR ("ethernetif_init: out of memory");
+      return ERR_MEM;
+    }
+
+  pnetif->state = eth_netif;
+  pnetif->name[0] = IFNAME0;
+  pnetif->name[1] = IFNAME1;
+
+  /* We directly use etharp_output() here to save a function call.
+   * You can instead declare your own function an call etharp_output()
+   * from it if you have to do some checks before sending (e.g. if link
+   * is available...) */
+  pnetif->output = etharp_output;
+  pnetif->linkoutput = spl_hal_output;
+
+  eth_netif->ethaddr = (struct eth_addr *) &(pnetif->hwaddr[0]);
+
+  // Add extra netif information here
+  if (0 != netifExt_add (pnetif))
+    {
+      return ERR_VAL;
+    }
+
+  /* initialize the hardware */
+  low_level_init (pnetif);
+  NSPOL_LOGINF (NETIF_DEBUG,
+                "ethernetif_init complete ifname [%c][%c][%d] \n",
+                pnetif->name[0], pnetif->name[1], pnetif->num);
+
+  return ERR_OK;
+}
+
+void
+ethernetif_packets_input (struct netif *pstnetif)
+{
+  struct spl_pbuf *p = NULL;
+  spl_hal_input (pstnetif, &p);
+
+  /* no packet could be read, silently ignore this */
+  if (p != NULL
+      && pstnetif->input (spl_convert_spl_pbuf_to_pbuf (p),
+                          pstnetif) != ERR_OK)
+    {
+      NSPOL_LOGERR ("netif->input failed]p=%p, netif=%p", p, pstnetif);
+    }
+
+  /* Free the spl pbuf */
+  spl_pbuf_free (p);
+}
+#endif
+//#endif /* 0 */
diff --git a/stacks/lwip_stack/lwip_src/netif/sc_dpdk.c b/stacks/lwip_stack/lwip_src/netif/sc_dpdk.c
new file mode 100644
index 0000000..95f3eec
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/netif/sc_dpdk.c
@@ -0,0 +1,574 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include "sc_dpdk.h"
+#include "common_mem_mbuf.h"
+#include "netif/common.h"
+#include "nstack_log.h"
+#include "nstack_securec.h"
+#include "nsfw_msg_api.h"
+#include "nsfw_maintain_api.h"
+#include "nsfw_recycle_api.h"
+#include "stackx_app_res.h"
+#include "stackx_pbuf.h"
+#ifdef SYS_MEM_RES_STAT
+#include "memp.h"
+#endif
+#include "spl_instance.h"
+#ifdef HAL_LIB
+#else
+#include "rte_memzone.h"
+#endif
+
+#define SPL_MEM_MODULE "spl_mem_module"
+
+#define TMR_TICK_LENGTH TCP_TMR_INTERVAL        /* define the tick length */
+
+u32_t uStackArgIndex = 0;
+int stackx_core_mask = 40;
+
+int g_nstack_bind_cpu = 0;
+int g_tcpip_thread_sleep_time = 0;
+
+extern int sbr_create_tx_pool ();
+extern int stackx_stat_zone_create ();
+
+#define GLOBAL_STACK_CORE_ARG "-c"
+#define GLOBAL_STACK_CORE_BINE "-bind_cpu"
+
+u32 g_type;
+struct memory_statics memory_used_size[80];
+
+void
+printmeminfo ()
+{
+  unsigned int i = 0;
+  long size = 0;
+
+  NSPOL_LOGDBG (SC_DPDK_INFO,
+                "*************************************************************");
+  for (i = 0; i < g_type; i++)
+    {
+      NSPOL_LOGDBG (SC_DPDK_INFO, "%s : %ld", memory_used_size[i].name,
+                    memory_used_size[i].size);
+      size += memory_used_size[i].size;
+    }
+
+  size += (g_type * sizeof (struct common_mem_memzone));
+  NSPOL_LOGDBG (SC_DPDK_INFO, "total size %ld", size);
+  NSPOL_LOGDBG (SC_DPDK_INFO,
+                "*************************************************************");
+}
+
+void
+print_call_stack ()
+{
+}
+
+/* Parse the argument given in the command line of the application */
+void
+smp_parse_stack_args (int argc, char **argv)
+{
+  int i = 0;
+
+  const unsigned int global_core_length = 2;    //GLOBAL_STACK_CORE_ARG "-c" string length is 2
+
+  for (i = uStackArgIndex + 1; i < argc; i++)
+    {
+      if ((i + 1) < argc)
+        {
+          if (strncmp (argv[i], "-sleep", 6) == 0)      //compare "-sleep" string, length is 6
+            {
+              g_tcpip_thread_sleep_time = atoi (argv[++i]);
+              NSPOL_LOGDBG (SC_DPDK_INFO, "g_tcpip_thread_sleep_time=%d",
+                            g_tcpip_thread_sleep_time);
+              continue;
+            }
+
+          if (strncmp (argv[i], GLOBAL_STACK_CORE_ARG, global_core_length) ==
+              0)
+            {
+              stackx_core_mask = atoi (argv[++i]);
+              if (stackx_core_mask < 1)
+                {
+                  NSPOL_LOGDBG (SC_DPDK_INFO,
+                                "Invalid Args:core_mask can't be less than 1,input value is:%s",
+                                argv[i]);
+                }
+
+              continue;
+            }
+
+          if (strncmp
+              (argv[i], GLOBAL_STACK_CORE_BINE,
+               sizeof (GLOBAL_STACK_CORE_BINE)) == 0)
+            {
+              if (argv[++i])
+                {
+                  g_nstack_bind_cpu = atoi (argv[i]);
+                }
+
+              if (g_nstack_bind_cpu < 0)
+                {
+                  g_nstack_bind_cpu = 0;
+                }
+
+              continue;
+            }
+        }
+      else
+        {
+          NSPOL_LOGDBG (SC_DPDK_INFO, "Invalid args:%s miss value ", argv[i]);  //now ,only support this format ,others maybe supported in future
+        }
+    }
+
+  return;
+}
+
+mpool_handle
+create_tx_mbuf_pool ()
+{
+  mpool_handle mbf_pool_handle = NULL;
+
+  nsfw_mem_mbfpool mbuf_pool;
+
+  mbuf_pool.stname.entype = NSFW_SHMEM;
+  int retval =
+    spl_snprintf (mbuf_pool.stname.aname, NSFW_MEM_NAME_LENGTH - 1, "%s",
+                  get_mempoll_tx_name ());
+  if (retval < 0)
+    {
+      NSPOL_LOGERR ("spl_snprintf failed");
+      return NULL;
+    }
+
+  mbuf_pool.usnum = TX_MBUF_POOL_SIZE - 1;
+  mbuf_pool.uscash_size = 0;
+  mbuf_pool.uspriv_size = 0;
+  mbuf_pool.usdata_room = TX_MBUF_MAX_LEN;
+  mbuf_pool.isocket_id = SOCKET_ID_ANY;
+  mbuf_pool.enmptype = NSFW_MRING_SPSC;
+  mbf_pool_handle = nsfw_mem_mbfmp_create (&mbuf_pool);
+  if (NULL == mbf_pool_handle)
+    {
+      NSPOL_LOGERR ("create_tx_mbuf_pool failed]name=%s, num=%u, room=%u",
+                    mbuf_pool.stname.aname, mbuf_pool.usnum,
+                    mbuf_pool.usdata_room);
+      return NULL;
+    }
+
+  NSPOL_LOGINF (SC_DPDK_INFO,
+                "tx_mempool_malloc=%p, num=%u, room=%u, total_mem=%d",
+                mbf_pool_handle, TX_MBUF_POOL_SIZE, mbuf_pool.usdata_room,
+                nsfw_mem_get_len (mbf_pool_handle, NSFW_MEM_MBUF));
+  DPDK_MEMORY_COUNT ((get_mempoll_tx_name ()),
+                     nsfw_mem_get_len (mbf_pool_handle, NSFW_MEM_MBUF));
+  MEM_STAT (SPL_MEM_MODULE, "spl_mbuf_pool", NSFW_SHMEM,
+            nsfw_mem_get_len (mbf_pool_handle, NSFW_MEM_MBUF));
+
+  return mbf_pool_handle;
+}
+
+mring_handle
+create_segment_pool ()
+{
+  nsfw_mem_sppool seg_pool;
+  seg_pool.stname.entype = NSFW_SHMEM;
+  int retval =
+    spl_snprintf (seg_pool.stname.aname, NSFW_MEM_NAME_LENGTH - 1, "%s",
+                  get_mempoll_seg_name ());
+  if (retval < 0)
+    {
+      NSPOL_LOGERR ("spl_snprintf failed");
+      return NULL;
+    }
+
+  seg_pool.usnum = 16;
+  seg_pool.useltsize = sizeof (struct common_pcb);
+  seg_pool.isocket_id = SOCKET_ID_ANY;
+  seg_pool.enmptype = NSFW_MRING_SPSC;
+
+  mring_handle seg_mp_handle = nsfw_mem_sp_create (&seg_pool);
+  if (NULL == seg_mp_handle)
+    {
+      NSPOL_LOGERR
+        ("create_segment_pool common failed]name=%s, num=%u, size=%u",
+         seg_pool.stname.aname, SPL_MEMP_NUM_TCP_SEG, seg_pool.useltsize);
+      return NULL;
+    }
+
+  NSPOL_LOGINF (SC_DPDK_INFO,
+                "common seg_mempool_malloc=%p, num=%u, size=%u, total_mem=%d",
+                seg_mp_handle, SPL_MEMP_NUM_TCP_SEG, seg_pool.useltsize,
+                nsfw_mem_get_len (seg_mp_handle, NSFW_MEM_SPOOL));
+  DPDK_MEMORY_COUNT ((get_mempoll_seg_name ()),
+                     nsfw_mem_get_len (seg_mp_handle, NSFW_MEM_SPOOL));
+  MEM_STAT (SPL_MEM_MODULE, "spl_seg_pool", NSFW_SHMEM,
+            nsfw_mem_get_len (seg_mp_handle, NSFW_MEM_SPOOL));
+  return seg_mp_handle;
+}
+
+mring_handle
+create_msg_pool ()
+{
+  nsfw_mem_sppool msg_pool;
+  msg_pool.stname.entype = NSFW_SHMEM;
+  int retval =
+    spl_snprintf (msg_pool.stname.aname, NSFW_MEM_NAME_LENGTH - 1, "%s",
+                  get_mempoll_msg_name ());
+  if (retval < 0)
+    {
+      NSPOL_LOGERR ("spl_snprintf fail");
+      return NULL;
+    }
+
+  msg_pool.usnum = TX_MSG_POOL_SIZE;
+  msg_pool.useltsize = sizeof (data_com_msg);
+  msg_pool.isocket_id = SOCKET_ID_ANY;
+  msg_pool.enmptype = NSFW_MRING_MPMC;
+  mring_handle msg_mp_handle = nsfw_mem_sp_create (&msg_pool);
+
+  if (NULL == msg_mp_handle)
+    {
+      NSPOL_LOGERR ("create_msg_pool failed]name=%s, num=%u, size=%u",
+                    msg_pool.stname.aname, TX_MSG_POOL_SIZE,
+                    msg_pool.useltsize);
+      return NULL;
+    }
+
+  NSPOL_LOGINF (SC_DPDK_INFO,
+                "msg_pool_malloc=%p, num=%u, size=%u, total_mem=%d",
+                msg_mp_handle, TX_MSG_POOL_SIZE, msg_pool.useltsize,
+                nsfw_mem_get_len (msg_mp_handle, NSFW_MEM_SPOOL));
+  DPDK_MEMORY_COUNT ((get_mempoll_msg_name ()),
+                     nsfw_mem_get_len (msg_mp_handle, NSFW_MEM_SPOOL));
+  MEM_STAT (SPL_MEM_MODULE, "spl_msg_pool", NSFW_SHMEM,
+            nsfw_mem_get_len (msg_mp_handle, NSFW_MEM_SPOOL));
+  return msg_mp_handle;
+}
+
+mring_handle
+create_primary_box ()
+{
+  nsfw_mem_mring mbox_pool;
+  mbox_pool.stname.entype = NSFW_SHMEM;
+  int retval =
+    spl_snprintf (mbox_pool.stname.aname, NSFW_MEM_NAME_LENGTH - 1, "%s",
+                  get_stackx_ring_name ());
+  if (retval < 0)
+    {
+      NSPOL_LOGERR ("spl_snprintf failed");
+      return NULL;
+    }
+
+  mbox_pool.usnum = MBOX_RING_SIZE - 1;
+  mbox_pool.isocket_id = SOCKET_ID_ANY;
+  mbox_pool.enmptype = NSFW_MRING_MPSC;
+  mring_handle mbox_handle = nsfw_mem_ring_create (&mbox_pool);
+  if (NULL == mbox_handle)
+    {
+      NSPOL_LOGERR ("create_primary_mbox failed]name=%s, num=%u",
+                    mbox_pool.stname.aname, mbox_pool.usnum + 1);
+      return NULL;
+    }
+
+  NSPOL_LOGINF (SC_DPDK_INFO, "primary_mbox_malloc=%p, num=%u, total_mem=%d",
+                mbox_handle, MBOX_RING_SIZE,
+                (nsfw_mem_get_len (mbox_handle, NSFW_MEM_RING)));
+  DPDK_MEMORY_COUNT ((get_stackx_ring_name ()),
+                     (nsfw_mem_get_len (mbox_handle, NSFW_MEM_RING)));
+  MEM_STAT (SPL_MEM_MODULE, "primary_mbox_ring", NSFW_SHMEM,
+            (nsfw_mem_get_len (mbox_handle, NSFW_MEM_RING)));
+  return mbox_handle;
+}
+
+mring_handle
+create_priority_box (u32 prio)
+{
+  nsfw_mem_mring mbox_pool;
+  mbox_pool.stname.entype = NSFW_SHMEM;
+  int retval =
+    spl_snprintf (mbox_pool.stname.aname, NSFW_MEM_NAME_LENGTH - 1, "%s",
+                  get_stackx_priority_ring_name (prio));
+  if (retval < 0)
+    {
+      NSPOL_LOGERR ("spl_snprintf failed");
+      return NULL;
+    }
+
+  mbox_pool.usnum = MBOX_RING_SIZE - 1;
+  mbox_pool.isocket_id = SOCKET_ID_ANY;
+  mbox_pool.enmptype = NSFW_MRING_MPSC;
+  mring_handle mbox_handle = nsfw_mem_ring_create (&mbox_pool);
+  if (NULL == mbox_handle)
+    {
+      NSPOL_LOGERR ("Create priority mbox fail]name=%s, num=%u",
+                    mbox_pool.stname.aname, mbox_pool.usnum + 1);
+      return NULL;
+    }
+
+  NSPOL_LOGINF (SC_DPDK_INFO, "prio=%u, mbox=%p, num=%u, total_mem=%d", prio,
+                mbox_handle, MBOX_RING_SIZE,
+                (nsfw_mem_get_len (mbox_handle, NSFW_MEM_RING)));
+  DPDK_MEMORY_COUNT ((get_stackx_priority_ring_name (prio)),
+                     (nsfw_mem_get_len (mbox_handle, NSFW_MEM_RING)));
+  MEM_STAT (SPL_MEM_MODULE, mbox_pool.stname.aname, NSFW_SHMEM,
+            (nsfw_mem_get_len (mbox_handle, NSFW_MEM_RING)));
+  return mbox_handle;
+
+}
+
+int
+init_instance ()
+{
+  int ret;
+  p_def_stack_instance =
+    (stackx_instance *) malloc (sizeof (stackx_instance));
+  if (NULL == p_def_stack_instance)
+    {
+      NSPOL_LOGERR ("malloc failed");
+      return -1;
+    }
+
+  ret = MEMSET_S (p_def_stack_instance, sizeof (stackx_instance), 0,
+                  sizeof (stackx_instance));
+  if (EOK != ret)
+    {
+      NSPOL_LOGERR ("MEMSET_S failed]ret=%d", ret);
+      return -1;
+    }
+
+  p_def_stack_instance->rss_queue_id = 0;
+  p_def_stack_instance->netif_list = NULL;
+  p_def_stack_instance->mp_tx = create_tx_mbuf_pool ();
+  if (!p_def_stack_instance->mp_tx)
+    {
+      return -1;
+    }
+
+  (void) spl_reg_res_tx_mgr (p_def_stack_instance->mp_tx);      // will only return 0, no need to check return value
+
+  /* Modified above code to hold common_pcb */
+  p_def_stack_instance->cpcb_seg = create_segment_pool ();
+  if (!p_def_stack_instance->cpcb_seg)
+    {
+      return -1;
+    }
+
+  p_def_stack_instance->lmsg_pool = create_msg_pool ();
+  if (!p_def_stack_instance->lmsg_pool)
+    {
+      return -1;
+    }
+
+  mring_handle mbox_array[SPL_MSG_BOX_NUM] = { NULL };
+  p_def_stack_instance->lstack.primary_mbox.llring = create_primary_box ();
+  if (!p_def_stack_instance->lstack.primary_mbox.llring)
+    {
+      return -1;
+    }
+  mbox_array[0] = p_def_stack_instance->lstack.primary_mbox.llring;
+
+  u32 m = 0;
+  while (m < MSG_PRIO_QUEUE_NUM)
+    {
+      p_def_stack_instance->lstack.priority_mbox[m].llring =
+        create_priority_box (m);
+      if (!p_def_stack_instance->lstack.priority_mbox[m].llring)
+        {
+          return -1;
+        }
+      mbox_array[m + 1] =
+        p_def_stack_instance->lstack.priority_mbox[m].llring;
+      m++;
+    }
+
+  (void) spl_add_mbox (mbox_array, SPL_MSG_BOX_NUM);
+
+  g_nsfw_rti_primary_stat = &p_def_stack_instance->lstat.primary_stat;  //save to g_nsfw_rti_primary_stat(this is a SH addr)
+  return 0;
+}
+
+void
+spl_free_msgs_in_box (mring_handle r)
+{
+  i32 count = 0, i = 0;
+
+  void **msgs = NULL;
+  data_com_msg *m = NULL;
+
+  while ((count = nsfw_mem_ring_dequeuev (r, msgs, 32)) > 0)
+    {
+      /* drop all of them */
+      if (msgs == NULL)
+        break;
+
+      for (i = 0; i < count; i++)
+        {
+          m = (data_com_msg *) msgs[i];
+          if (m->param.op_type == MSG_ASYN_POST)
+            ASYNC_MSG_FREE (m);
+          else
+            SYNC_MSG_ACK (m);
+        }
+    }
+}
+
+inline int
+spl_msg_malloc (data_com_msg ** p_msg_entry)
+{
+  mring_handle msg_pool = NULL;
+  int rslt;
+  stackx_instance *instance = p_def_stack_instance;
+  msg_pool = instance->lmsg_pool;
+  if (!msg_pool)
+    {
+      NSPOL_LOGERR ("msg_pool is NULL");
+      return -1;
+    }
+
+  rslt = nsfw_mem_ring_dequeue (msg_pool, (void **) p_msg_entry);
+  if ((rslt == 0) || (*p_msg_entry == NULL))
+    {
+      NSPOL_LOGERR ("failed to get msg from ring");
+      return -1;
+    }
+
+  res_alloc (&(*p_msg_entry)->param.res_chk);
+
+  (*p_msg_entry)->param.msg_from = msg_pool;
+  (*p_msg_entry)->param.err = ERR_OK;
+  return 0;
+}
+
+struct spl_pbuf *
+spl_mbuf_malloc (uint16_t len, spl_pbuf_type Type, u16_t * count)
+{
+  struct common_mem_mbuf *mbuf = NULL;
+  struct common_mem_mbuf *mbuf_first = NULL;
+  struct common_mem_mbuf *mbuf_tail = NULL;
+  struct spl_pbuf *buf = NULL;
+  struct spl_pbuf *first = NULL;
+  struct spl_pbuf *tail = NULL;
+
+  mpool_handle mp = NULL;
+
+  mp = p_def_stack_instance->mp_tx;
+  if (mp == NULL)
+    {
+      return NULL;              /*if mp is NULL when init app will Inform */
+    }
+
+  while (len > 0)
+    {
+      mbuf = (struct common_mem_mbuf *) nsfw_mem_mbf_alloc (mp, NSFW_SHMEM);
+      if (unlikely (mbuf == NULL))
+        {
+          if (mbuf_first != NULL)
+            {
+              if (res_free
+                  (&
+                   (((struct spl_pbuf *) ((char *) mbuf_first +
+                                          sizeof (struct
+                                                  common_mem_mbuf)))->res_chk)))
+                {
+                  NSPOL_LOGERR ("res_free failed");
+                }
+              spl_mbuf_free (mbuf_first);
+            }
+
+          return NULL;
+        }
+
+      uint16_t alloc = TX_MBUF_MAX_LEN;
+      if (len < TX_MBUF_MAX_LEN)
+        {
+          alloc = len;
+        }
+
+      (*count)++;
+      mbuf->data_len = alloc;
+      mbuf->next = NULL;
+      buf =
+        (struct spl_pbuf *) ((char *) mbuf + sizeof (struct common_mem_mbuf));
+      res_alloc (&buf->res_chk);
+
+      buf->next_a = 0;
+      buf->payload_a = ADDR_LTOSH_EXT (common_pktmbuf_mtod (mbuf, void *));
+      buf->tot_len = len;
+      buf->len = alloc;
+      buf->type = Type;
+      buf->flags = 0;
+
+      buf->freeNext = NULL;
+
+      buf->conn_a = 0;
+
+      if (first == NULL)
+        {
+          first = buf;
+          mbuf_first = mbuf;
+          tail = buf;
+          mbuf_tail = mbuf;
+          mbuf_first->nb_segs = 1;
+          mbuf_first->pkt_len = alloc;
+        }
+      else
+        {
+          /* Already there is a check for the return value of rtp_pktmbuf_alloc,
+             hence not an issue */
+
+          tail->next_a = ADDR_LTOSH_EXT (buf);
+          tail = buf;
+#ifdef HAL_LIB
+#else
+          mbuf_tail->next = mbuf;
+#endif
+          mbuf_tail = mbuf;
+
+          mbuf_first->pkt_len = (mbuf_first->pkt_len + mbuf->data_len);
+          mbuf_first->nb_segs++;
+
+        }
+
+      len -= alloc;
+    }
+
+  return first;
+}
+
+/*
+  * Ring distribution function: protocol stack once a packet processing, so there is no use of bulk package
+  * @param buf pbuf means * @param packet_inport the packet from which the port to enter, for the configuration table with the ip comparison
+  * @ Protocol stack add location: ip.c-> ip_input () -> (if (netif == NULL) branch)
+  * @ Return value: 0 for the send into * 0 for the transmission failed: send the original failure 1,
+  *   err = -20 did not match to the client, err = -1Ring full, overflow, will release the package
+*/
+
+inline void
+spl_mbuf_free (void *mbuf)
+{
+  (void) nsfw_mem_mbf_free ((mbuf_handle) mbuf, NSFW_SHMEM);
+}
+
+inline uint16_t
+spl_mbuf_refcnt_update (void *mbuf, int16_t value)
+{
+  common_mbuf_refcnt_set ((struct common_mem_mbuf *) mbuf,
+                          common_mbuf_refcnt_read ((struct common_mem_mbuf *)
+                                                   mbuf) + value);
+  return 1;
+}
diff --git a/stacks/lwip_stack/lwip_src/netif/spl_hal.c b/stacks/lwip_stack/lwip_src/netif/spl_hal.c
new file mode 100644
index 0000000..5f69a9d
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/netif/spl_hal.c
@@ -0,0 +1,1753 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include "sys_arch.h"
+#include "netif.h"
+#include "spl_sockets.h"
+//#include <netinet/in.h>
+
+#include "stackx_spl_share.h"
+#include "stackx_pbuf.h"
+#include "spl_api.h"
+#include "sharedmemory.h"
+//#include "nettool.h"
+#include "lwip/etharp.h"
+#include "ip_module_api.h"
+#include "sc_dpdk.h"
+#include "nstack_log.h"
+#include "common.h"
+#include "nstack_securec.h"
+//#include "ip.h"
+#include "configuration_reader.h"
+#include "spl_hal.h"
+#include "nsfw_maintain_api.h"
+#include "stackx_common.h"
+#include "spl_instance.h"
+#include <netinet/in.h>
+#include "prot/tcp.h"
+
+extern u32_t g_mbuf_size[MAX_THREAD_NUM];
+extern u32_t uStackArgIndex;
+extern void smp_parse_stack_args (int argc, char **argv);
+extern void spl_do_dump (struct spl_pbuf *p, u16 direction);
+
+#define SPL_HAL_SEND_TRY 100000
+
+#define SPL_HAL_MODULE "SPL_HAL_MODULE"
+extern u16_t g_offSetArry[SPL_PBUF_MAX_LAYER];
+
+/* structure to store the rx and tx packets. Put two per cache line as ports
+ * used in pairs */
+struct port_stats
+{
+  unsigned rx;
+  unsigned tx;
+  unsigned drop;
+  u64_t rx_size;
+  u64_t tx_size;
+  u64_t recv_last_cycles;
+  u64_t send_last_cycles;
+} __attribute__ ((aligned (COMMON_CACHE_LINE_SIZE / 2)));
+
+struct port_capa
+{
+  u32_t tx_ipv4_cksum_offload;
+  u32_t tx_udp_cksum_offload;
+  u32_t tx_tcp_cksum_offload;
+};
+
+struct rx_pkts
+{
+  u16_t num;
+  u16_t index;
+  struct common_mem_mbuf *pkts[PKT_BURST];
+};
+
+struct tx_pkts
+{
+  u16_t num;
+  struct common_mem_mbuf *pkts[PKT_BURST];
+};
+
+struct port_pkts
+{
+  struct rx_pkts rx;
+  struct tx_pkts tx;
+};
+
+struct psd_header
+{
+  u32_t src_addr;               /* IPaddress of source host. */
+  u32_t dst_addr;               /* IPaddress of destination host(s). */
+  u8_t zero;                    /* zero. */
+  u8_t proto;                   /* L4 protocol type. */
+  u16_t len;                    /* L4 length. */
+} __attribute__ ((__packed__));
+
+NSTACK_STATIC unsigned num_ports_NIC = 0;
+NSTACK_STATIC unsigned num_ports_NIC_start = 0;
+
+struct stackx_port_info *head_used_port_list;
+struct stackx_port_zone *p_stackx_port_zone = NULL;
+
+struct bond_ports_info bond_ports_array = {.cnt = 0 };
+
+static u8_t bond_ports_array_cnt_start = 0;
+
+static struct port_capa spl_hal_capa = { 0 };
+
+static struct port_pkts spl_hal_pkts[HAL_MAX_NIC_NUM];
+
+NSTACK_STATIC inline u16_t
+get_ipv4_16b_sum (u16_t * ptr16, u32_t nr)
+{
+  u32_t sum = 0;
+
+  while (nr > 1)
+    {
+      sum += *ptr16;
+      nr -= sizeof (u16_t);
+      ptr16++;
+
+      if (sum > UINT16_MAX)
+        {
+          sum -= UINT16_MAX;
+        }
+    }
+
+  /* If length is in odd bytes */
+  if (nr)
+    {
+      sum += *((u8_t *) ptr16);
+    }
+
+  sum = ((sum & 0xffff0000) >> 16) + (sum & 0xffff);
+  sum &= 0x0ffff;
+  return (u16_t) sum;
+}
+
+NSTACK_STATIC inline u16_t
+get_ipv4_bswap16 (u16_t x)
+{
+  return (u16_t) (((x & 0x00ffU) << 8) | ((x & 0xff00U) >> 8));
+}
+
+NSTACK_STATIC inline u16_t
+get_ipv4_psd_sum (struct ip_hdr * iphdr, u64_t ol_flags)
+{
+  struct psd_header psd_hdr;
+
+  psd_hdr.src_addr = iphdr->src.addr;
+  psd_hdr.dst_addr = iphdr->dest.addr;
+  psd_hdr.zero = 0;
+  psd_hdr.proto = iphdr->_proto;
+
+  if (ol_flags & PKT_TX_TCP_SEG)
+    {
+      psd_hdr.len = 0;
+    }
+  else
+    {
+      psd_hdr.len = get_ipv4_bswap16 ((get_ipv4_bswap16 (iphdr->_len)
+                                       - sizeof (struct ip_hdr)));
+    }
+
+  return get_ipv4_16b_sum ((u16_t *) & psd_hdr, sizeof (struct psd_header));
+}
+
+/* should be called after head_used_port_list is initialized */
+NSTACK_STATIC hal_hdl_t
+get_port_hdl_by_name (const char *name)
+{
+  unsigned int i = 0;
+  struct stackx_port_info *p = p_stackx_port_zone->stackx_one_port;
+
+  while (i < p_stackx_port_zone->port_num)
+    {
+      if (!strncasecmp (p->linux_ip.if_name, name, strlen (name)))
+        {
+          return p->linux_ip.hdl;
+        }
+
+      p = &p_stackx_port_zone->stackx_one_port[++i];
+    }
+
+  NSPOL_LOGERR ("failed to find port id]name=%s", name);
+  return hal_get_invalid_hdl ();
+}
+
+NSTACK_STATIC struct stackx_port_info *
+get_port_info_by_name (const char *name)
+{
+  struct stackx_port_info *p = p_stackx_port_zone->stackx_one_port;
+  unsigned int i = 0;
+
+  while (i < p_stackx_port_zone->port_num)
+    {
+      if (!strncasecmp (p->linux_ip.if_name, name, strlen (name)))
+        {
+          return p;
+        }
+
+      p = &p_stackx_port_zone->stackx_one_port[++i];
+    }
+
+  return NULL;
+}
+
+NSTACK_STATIC int
+del_port_in_port_list (const char *name)
+{
+  struct stackx_port_info *inf = head_used_port_list;
+  struct stackx_port_info *prev = NULL;
+
+  while (inf)
+    {
+      if (!strncasecmp (inf->linux_ip.if_name, name, strlen (name)))
+        {
+          if (prev != NULL)
+            {
+              prev->next_use_port = inf->next_use_port;
+            }
+          else
+            {
+              head_used_port_list = inf->next_use_port;
+            }
+
+          break;
+        }
+
+      prev = inf;
+      inf = inf->next_use_port;
+    }
+
+  return 0;
+}
+
+extern void create_netif (struct stackx_port_info *p_port_info);
+
+NSTACK_STATIC int
+add_port_in_port_list (struct stackx_port_info *p)
+{
+  char *name;
+  struct stackx_port_info *inf = head_used_port_list;
+  struct stackx_port_info *prev = NULL;
+
+  name = p->linux_ip.if_name;
+
+  while (inf)
+    {
+      if (!strncasecmp (inf->linux_ip.if_name, name, strlen (name)))
+        {
+          NSPOL_LOGERR ("ERROR: add an existing port!");
+          return -1;
+        }
+
+      prev = inf;
+      inf = inf->next_use_port;
+    }
+
+  if (prev == NULL)
+    {
+      head_used_port_list = p;
+    }
+  else
+    {
+      prev->next_use_port = p;
+    }
+
+  p->next_use_port = NULL;
+  create_netif (p);
+  return 0;
+}
+
+/* Queries the link status of a port and prints it to screen */
+NSTACK_STATIC void
+report_port_link_status (struct stackx_port_info *p)
+{
+  /* get link status */
+  u32 status;
+
+  status = hal_link_status (p->linux_ip.hdl);
+
+  if (status)
+    {
+      NSPOL_LOGINF (SC_DPDK_INFO, "Port=%s: Link Up", p->linux_ip.if_name);
+    }
+  else
+    {
+      NSPOL_LOGINF (SC_DPDK_INFO, "Port=%s: Link Down", p->linux_ip.if_name);
+    }
+}
+
+int
+spl_hal_ether_etoa (const unsigned char *e, int e_len, char *a, int a_len)
+{
+  char *c = a;
+  int i;
+  int retVal;
+
+  if (!e || !a || e_len < 0)
+    return -1;
+
+  if (e_len > NETIF_ETH_ADDR_LEN)
+    e_len = NETIF_ETH_ADDR_LEN;
+
+  if (a_len < e_len * 3)
+    return -1;
+
+  for (i = 0; i < e_len; i++)
+    {
+      if (i)
+        {
+          *c++ = ':';
+        }
+      retVal = SPRINTF_S (c, a_len - (c - a), "%02x", e[i] & 0xff);
+      if (-1 == retVal)
+        {
+          NSPOL_LOGERR ("SPRINTF_S failed]ret=%d.", retVal);
+          return -1;
+        }
+      c = c + retVal;
+    }
+
+  return 0;
+}
+
+NSTACK_STATIC inline void
+spl_hal_buf_convert (struct common_mem_mbuf *mbuf, struct spl_pbuf **buf)
+{
+  struct common_mem_mbuf *before = NULL;
+  struct spl_pbuf *last = NULL;
+  struct spl_pbuf *first = NULL;
+  struct spl_pbuf *tmp = NULL;
+
+  while (mbuf != NULL)
+    {
+      //dpdk 2.1
+      tmp =
+        (struct spl_pbuf *) ((char *) mbuf + sizeof (struct common_mem_mbuf));
+      res_alloc (&tmp->res_chk);
+      tmp->payload = common_pktmbuf_mtod (mbuf, void *);
+      tmp->tot_len = mbuf->pkt_len;
+      tmp->len = mbuf->data_len;
+      tmp->type = SPL_PBUF_HUGE;
+      tmp->proto_type = SPL_PBUF_PROTO_NONE;
+      tmp->next = NULL;
+      tmp->flags = 0;
+
+      if (first == NULL)
+        {
+          first = tmp;
+          last = first;
+        }
+      else
+        {
+          /* Always the "if(first == NULL)" code segment is executed and then
+             "else" segment code is executed, so the "last" variable is not
+             NULL always when "else" case is executed */
+          last->next = tmp;
+          last = tmp;
+        }
+
+      before = mbuf;
+      mbuf = mbuf->next;
+
+      before->next = NULL;
+    }
+
+  *buf = first;
+}
+
+NSTACK_STATIC int
+spl_hal_port_zone_init ()
+{
+  int retVal;
+  nsfw_mem_zone create_port_zone;
+  nsfw_mem_zone create_port_info;
+  struct stackx_port_info *mz_port_info;
+  INITPOL_LOGINF ("RTP", "spl_hal_port_zone_init", NULL_STRING,
+                  LOG_INVALID_VALUE, MODULE_INIT_START);
+
+  if ((CUR_CFG_HAL_PORT_NUM < 1)
+      || (SIZE_MAX / sizeof (struct stackx_port_info) < CUR_CFG_HAL_PORT_NUM))
+    {
+      NSPOL_LOGERR ("malloc parameter incorrect]max_linux_port=%u",
+                    CUR_CFG_HAL_PORT_NUM);
+      return -1;
+    }
+
+  if (spl_snprintf
+      (create_port_zone.stname.aname, NSFW_MEM_NAME_LENGTH - 1, "%s",
+       MP_STACKX_PORT_ZONE) < 0)
+    {
+      NSPOL_LOGERR ("spl_snprintf fail");
+
+      return -1;
+    }
+
+  create_port_zone.stname.entype = NSFW_SHMEM;
+  create_port_zone.isocket_id = SOCKET_ID_ANY;
+  create_port_zone.length = sizeof (struct stackx_port_zone);
+  create_port_zone.ireserv = 0;
+  p_stackx_port_zone =
+    (struct stackx_port_zone *) nsfw_mem_zone_create (&create_port_zone);
+
+  if (NULL == p_stackx_port_zone)
+    {
+      INITPOL_LOGERR ("RTP", "spl_hal_port_zone_init",
+                      "Cannot create memory zone for MP_STACKX_PORT_ZONE information",
+                      LOG_INVALID_VALUE, MODULE_INIT_FAIL);
+      common_exit (EXIT_FAILURE,
+                   "Cannot create memory zone for MP_STACKX_PORT_ZONE information");
+    }
+
+  retVal =
+    MEMSET_S (p_stackx_port_zone, sizeof (struct stackx_port_zone), 0,
+              sizeof (struct stackx_port_zone));
+
+  if (EOK != retVal)
+    {
+      INITPOL_LOGERR ("RTP", "spl_hal_port_zone_init", "MEMSET_S return fail",
+                      retVal, MODULE_INIT_FAIL);
+      nsfw_mem_zone_release (&create_port_zone.stname);
+      return -1;
+    }
+
+  if (spl_snprintf
+      (create_port_info.stname.aname, NSFW_MEM_NAME_LENGTH - 1, "%s",
+       MP_STACKX_PORT_INFO) < 0)
+    {
+      NSPOL_LOGERR ("VSNPRINTF_S fail");
+      return -1;
+    }
+
+  create_port_info.stname.entype = NSFW_SHMEM;
+  create_port_info.isocket_id = SOCKET_ID_ANY;
+  create_port_info.length =
+    CUR_CFG_HAL_PORT_NUM * sizeof (struct stackx_port_info);
+  create_port_info.ireserv = 0;
+  mz_port_info =
+    (struct stackx_port_info *) nsfw_mem_zone_create (&create_port_info);
+
+  if (NULL == mz_port_info)
+    {
+      INITPOL_LOGERR ("RTP", "spl_hal_port_zone_init",
+                      "Cannot create memory zone for MP_STACKX_PORT_INFO information",
+                      LOG_INVALID_VALUE, MODULE_INIT_FAIL);
+      common_exit (EXIT_FAILURE,
+                   "Cannot create memory zone for MP_STACKX_PORT_INFO information");
+    }
+
+  retVal =
+    MEMSET_S (mz_port_info, create_port_info.length, 0,
+              create_port_info.length);
+
+  if (EOK != retVal)
+    {
+      INITPOL_LOGERR ("RTP", "spl_hal_port_zone_init", "MEMSET_S return fail",
+                      retVal, MODULE_INIT_FAIL);
+      nsfw_mem_zone_release (&create_port_info.stname);
+      nsfw_mem_zone_release (&create_port_zone.stname);
+      return -1;
+    }
+
+  MEM_STAT (SPL_HAL_MODULE, create_port_zone.stname.aname, NSFW_SHMEM,
+            create_port_info.length);
+
+  p_stackx_port_zone->stackx_one_port = mz_port_info;
+
+  INITPOL_LOGINF ("RTP", "spl_hal_port_zone_init", NULL_STRING,
+                  LOG_INVALID_VALUE, MODULE_INIT_SUCCESS);
+
+  return 0;
+}
+
+int
+spl_hal_init (int argc, char *argv[])
+{
+  int retval = -1;
+  int idx_init;
+
+  NSPOL_LOGINF (SC_DPDK_INFO, "spl_hal_init start");
+
+  /* Get nstack args */
+  smp_parse_stack_args (argc, argv);
+
+  if (0 == uStackArgIndex)
+    {
+      NSPOL_LOGERR ("uStackArgIndex is 0, can lead to long loop]");
+      return retval;
+    }
+
+  /* Init DPDK */
+  argc = uStackArgIndex--;
+  INITPOL_LOGINF ("RTP", "hal_init_global", NULL_STRING, LOG_INVALID_VALUE,
+                  MODULE_INIT_START);
+
+  for (idx_init = 0; idx_init < argc; idx_init++)
+    {
+      NSPOL_LOGINF (SC_DPDK_INFO,
+                    "hal_init_global]idx_init=%d,argv[idx_init]=%s", idx_init,
+                    argv[idx_init]);
+    }
+
+  retval = hal_init_global (argc, argv);
+
+  if (0 != retval)
+    {
+      NSPOL_LOGERR ("call hal_init_global fail]retval = %d", retval);
+      return -1;
+    }
+
+  retval = hal_init_local ();
+
+  if (0 != retval)
+    {
+      NSPOL_LOGERR ("call hal_init_local fail]retval = %d", retval);
+      return -1;
+    }
+
+  retval = spl_hal_port_zone_init ();
+  if (0 != retval)
+    {
+      NSPOL_LOGERR ("call hal_init_local fail]retval = %d", retval);
+      return -1;
+    }
+
+  NSPOL_LOGDBG (SC_DPDK_INFO, "Finished Process Init");
+
+  return 0;
+
+}
+
+static inline int
+spl_hal_rx_mbuf_free (void *data, void *arg)
+{
+  struct spl_pbuf *tmp = NULL;
+  struct common_mem_mbuf *mbuf = (struct common_mem_mbuf *) data;
+  (void) arg;
+
+  tmp = (struct spl_pbuf *) ((char *) mbuf + sizeof (struct common_mem_mbuf));
+  if (tmp->res_chk.alloc_flag == TRUE)
+    return 1;
+
+  if (common_mbuf_refcnt_read (mbuf) == 0)
+    return 1;
+
+  NSPOL_LOGDBG (SC_DPDK_INFO, "rx_pool init in fault case: free mbuf=%p",
+                mbuf);
+  spl_mbuf_free (mbuf);
+  return 0;
+}
+
+struct common_mem_mempool *
+spl_hal_rx_pool_create (int nic_id, int queue_id, int start_type)
+{
+  int retval;
+  struct common_mem_mempool *mp;
+  nsfw_mem_mbfpool create_mbuf_pool;
+  nsfw_mem_name lookup_mbuf_pool;
+  struct common_mem_ring *ring;
+
+  if (start_type == 1)
+    {
+      create_mbuf_pool.stname.entype = NSFW_SHMEM;
+      create_mbuf_pool.uscash_size = 0;
+      create_mbuf_pool.uspriv_size = 0;
+      create_mbuf_pool.isocket_id = SOCKET_ID_ANY;
+      create_mbuf_pool.enmptype = NSFW_MRING_SPSC;
+
+      retval =
+        spl_snprintf (create_mbuf_pool.stname.aname, NSFW_MEM_NAME_LENGTH - 1,
+                      "%s", get_mempoll_rx_name (queue_id, nic_id));
+
+      if (-1 == retval)
+        {
+          NSPOL_LOGERR ("spl_snprintf fail");
+          return NULL;
+        }
+
+      create_mbuf_pool.usnum = RX_MBUF_POOL_SIZE - 1;
+      /*performance, rx buf cap is special, ((size - HEADROOM) >> 10) <<10, see ixgbe_dev_rx_init;
+         if want cap size == TX_MBUF_MAX_LEN, must let data_root=TX_MBUF_MAX_LEN+COMMON_PKTMBUF_HEADROOM and
+         TX_MBUF_MAX_LEN must N*1024;
+       */
+      create_mbuf_pool.usdata_room =
+        TX_MBUF_MAX_LEN + COMMON_PKTMBUF_HEADROOM;
+
+      NSPOL_LOGDBG (SC_DPDK_INFO, "hal_rx_pool.usnum=%u, usdata_room=%u",
+                    create_mbuf_pool.usnum, create_mbuf_pool.usdata_room);
+
+      mp =
+        (struct common_mem_mempool *)
+        nsfw_mem_mbfmp_create (&create_mbuf_pool);
+
+      if (mp == NULL)
+        {
+          NSPOL_LOGERR ("nsfw_mem_mbfmp_create fail");
+          return NULL;
+        }
+
+      MEM_STAT (SPL_HAL_MODULE, create_mbuf_pool.stname.aname, NSFW_SHMEM,
+                nsfw_mem_get_len (mp, NSFW_MEM_MBUF));
+      NSPOL_LOGDBG (SC_DPDK_INFO, "create:thread=%d,nic_id=%d,mp=%p,size=%d",
+                    queue_id, nic_id, mp, nsfw_mem_get_len (mp,
+                                                            NSFW_MEM_MBUF));
+
+      char rx_msg_arr_name[NSFW_MEM_NAME_LENGTH];
+      data_com_msg *rx_msg_array = NULL;
+      retval = spl_snprintf (rx_msg_arr_name, NSFW_MEM_NAME_LENGTH, "%s",
+                             get_mempoll_rxmsg_name (queue_id, nic_id));
+
+      if (-1 != retval)
+        {
+          rx_msg_array = (data_com_msg *) sbr_create_mzone (rx_msg_arr_name,
+                                                            (size_t)
+                                                            sizeof
+                                                            (data_com_msg) *
+                                                            RX_MBUF_POOL_SIZE);
+        }
+
+      if (!rx_msg_array)
+        {
+          NSSBR_LOGERR
+            ("Create rx_msg_array zone fail]name=%s, num=%u, size=%zu",
+             rx_msg_arr_name, RX_MBUF_POOL_SIZE,
+             (size_t) sizeof (data_com_msg) * RX_MBUF_POOL_SIZE);
+        }
+      else
+        {
+          /*bind msg to pbuf */
+          MEM_STAT (SPL_HAL_MODULE, rx_msg_arr_name, NSFW_SHMEM,
+                    (size_t) sizeof (data_com_msg) * RX_MBUF_POOL_SIZE);
+          NSSBR_LOGINF
+            ("Create rx_msg_array zone ok]name=%s, ptr=%p, num=%u, size=%zu",
+             rx_msg_arr_name, rx_msg_array, RX_MBUF_POOL_SIZE,
+             sizeof (data_com_msg) * RX_MBUF_POOL_SIZE);
+
+          struct common_mem_mbuf *mbuf = NULL;
+          struct spl_pbuf *buf = NULL;
+          u32 loop = 0;
+
+          for (; loop < RX_MBUF_POOL_SIZE; loop++)
+            {
+              mbuf = nsfw_mem_mbf_alloc (mp, NSFW_SHMEM);
+
+              if (!mbuf)
+                {
+                  /* alloc failed , still can work, no prebind success just not so faster */
+                  NSSBR_LOGERR
+                    ("nsfw_mem_mbf_alloc failed,this can not happen");
+                  break;
+                }
+
+              buf =
+                (struct spl_pbuf *) ((char *) mbuf +
+                                     sizeof (struct common_mem_mbuf));
+              sys_sem_init (&rx_msg_array[loop].param.op_completed);
+              rx_msg_array[loop].param.msg_from = NULL;
+              buf->msg = (void *) &rx_msg_array[loop];
+              (void) res_free (&buf->res_chk);  //no need to check return value, as it will do free operation depends on alloc_flag
+
+              if (nsfw_mem_mbf_free (mbuf, NSFW_SHMEM) < 0)
+                {
+                  /* free failed , still can work, no prebind work just not so faster */
+                  NSSBR_LOGERR
+                    ("nsfw_mem_mbf_free failed,this can not happen");
+                  break;
+                }
+            }
+
+        }
+
+    }
+  else
+    {
+      retval =
+        spl_snprintf (lookup_mbuf_pool.aname, NSFW_MEM_NAME_LENGTH - 1, "%s",
+                      get_mempoll_rx_name (queue_id, nic_id));
+
+      if (-1 == retval)
+        {
+          NSPOL_LOGERR ("spl_snprintf fail");
+          return NULL;
+        }
+
+      lookup_mbuf_pool.entype = NSFW_SHMEM;
+      lookup_mbuf_pool.enowner = NSFW_PROC_MAIN;
+      mp =
+        (struct common_mem_mempool *)
+        nsfw_mem_mbfmp_lookup (&lookup_mbuf_pool);
+
+      if (mp == NULL)
+        {
+          NSPOL_LOGERR ("nsfw_mem_mbfmp_lookup fail, name=%s, try to create",
+                        lookup_mbuf_pool.aname);
+          return spl_hal_rx_pool_create (nic_id, queue_id, 1);
+        }
+
+      NSPOL_LOGDBG (SC_DPDK_INFO, "lookup:thread=%d,nic_id=%d,mp=%p,size=%d",
+                    queue_id, nic_id, mp, nsfw_mem_get_len (mp,
+                                                            NSFW_MEM_MBUF));
+
+      /*We have to recycle RX mbufs hold by DPDK when fault recovering of upgrading nstack */
+      if (start_type == 3 || start_type == 2)
+        {
+          ring = (struct common_mem_ring *) (mp->pool_data);
+          NSPOL_LOGINF (SC_DPDK_INFO,
+                        "BEFORE clear rx_mpool]prod.head=%u, prod.tail=%u, "
+                        "cons.head=%u, cons.tail=%u", ring->prod.head,
+                        ring->prod.tail, ring->cons.head, ring->cons.tail);
+
+          if (nsfw_mem_mbuf_iterator (mp, 0, mp->size,
+                                      spl_hal_rx_mbuf_free, NULL) < 0)
+            {
+              NSPOL_LOGERR ("nsfw_mem_mbuf_iterator return fail");
+              return NULL;
+            }
+
+          NSPOL_LOGINF (SC_DPDK_INFO,
+                        "AFTER clear rx_mpool]prod.head=%u, prod.tail=%u, "
+                        "cons.head=%u, cons.tail=%u", ring->prod.head,
+                        ring->prod.tail, ring->cons.head, ring->cons.tail);
+        }
+
+    }
+
+  return mp;
+
+}
+
+int
+spl_hal_bond_config (struct network_configuration *network)
+{
+  struct phy_net *phynet = network->phy_net;
+  struct ref_nic *phead = phynet->header;
+  static u8_t bond_index = 0;   /* for auto-generating bond_name */
+  unsigned int check_bond = 0, check_name;
+  int retVal;
+  u8_t j, k, idx = 0;
+
+  /* get bond info from network configuration */
+  if (phynet->bond_mode != -1 && bond_ports_array.cnt < MAX_BOND_PORT_NUM)
+    {
+      struct ref_nic *phead_bond = phead;
+      char *name = phynet->bond_name;
+      struct bond_set *s = &bond_ports_array.ports[bond_ports_array.cnt];
+
+      while (phead_bond != NULL)
+        {
+          /* check slave name, repeated slave nic cannot be added to bond set. */
+          check_name = 0;
+
+          for (j = 0; j < idx; j++)
+            {
+              if (strcmp (s->slave_ports[j], phead_bond->nic_name) == 0)
+                {
+                  check_name = 1;
+                  break;
+                }
+            }
+
+          if (check_name)
+            {
+              break;
+            }
+
+          /* if this nic has been added to a bond_set, ignore it */
+          check_bond = 0;
+
+          for (k = 0; k < bond_ports_array.cnt && !check_bond; k++)
+            {
+              for (j = 0;
+                   j < bond_ports_array.ports[k].slave_port_cnt
+                   && !check_bond; j++)
+                {
+                  if (strcmp
+                      (bond_ports_array.ports[k].slave_ports[j],
+                       phead_bond->nic_name) == 0)
+                    {
+                      check_bond = 1;
+
+                      if (name[0] == 0)
+                        {
+                          retVal =
+                            STRNCPY_S (name, IP_MODULE_MAX_NAME_LEN,
+                                       bond_ports_array.
+                                       ports[k].bond_port_name,
+                                       strlen (bond_ports_array.
+                                               ports[k].bond_port_name));
+                          if (EOK != retVal)
+                            {
+                              NSPOL_LOGERR ("STRNCPY_S failed]ret=%d.",
+                                            retVal);
+                              return -1;
+                            }
+                        }
+
+                      if (strcmp
+                          (name,
+                           bond_ports_array.ports[k].bond_port_name) != 0)
+                        {
+                          NSOPR_SET_ERRINFO (-1, "%s init failed!\n", name);
+                          NSPOL_LOGERR
+                            ("%s init failed! %s in both %s and %s", name,
+                             phead_bond->nic_name, name,
+                             bond_ports_array.ports[k].bond_port_name);
+                          return -1;
+                        }
+                    }
+                }
+            }
+
+          if (check_bond == 1)
+            {
+              break;
+            }
+
+          /* copy slave ports name to bond array */
+          retVal =
+            STRNCPY_S (s->slave_ports[idx], HAL_MAX_NIC_NAME_LEN,
+                       phead_bond->nic_name, strlen (phead_bond->nic_name));
+
+          if (EOK != retVal)
+            {
+              NSPOL_LOGERR ("STRNCPY_S failed]ret=%d.", retVal);
+              return -1;
+            }
+
+          idx++;
+          phead_bond = phead_bond->next;
+
+          if (idx >= HAL_MAX_SLAVES_PER_BOND)
+            {
+              break;
+            }
+        }
+
+      if (check_bond == 0)
+        {
+          if (name[0] == 0)
+            {
+              /* if bond_name is a empty string, generate a new bond name */
+              retVal =
+                SPRINTF_S (name, HAL_MAX_NIC_NAME_LEN, "bond%u_auto",
+                           bond_index++);
+
+              if (-1 == retVal)
+                {
+                  NSPOL_LOGERR ("SPRINTF_S failed]ret=%d.", retVal);
+                  return -1;
+                }
+            }
+
+          /* copy bond_name to bond array */
+          retVal =
+            STRNCPY_S (s->bond_port_name, HAL_MAX_NIC_NAME_LEN, name,
+                       strlen (name));
+
+          if (EOK != retVal)
+            {
+              NSPOL_LOGERR ("STRNCPY_S failed]ret=%d.", retVal);
+              return -1;
+            }
+
+          s->slave_port_cnt = idx;
+          bond_ports_array.cnt++;
+          NSPOL_LOGINF (SC_DPDK_INFO,
+                        "bond_ports_array.cnt=%u,slave_port_cnt=%u",
+                        bond_ports_array.cnt, s->slave_port_cnt);
+        }
+    }
+
+  return 0;
+}
+
+int
+spl_hal_port_config (unsigned int *port_num)
+{
+  int retVal;
+  unsigned int check;
+  struct phy_net *phynet;
+
+  struct network_configuration *network = get_network_list ();
+
+  if (!network)
+    {
+      NSPOL_LOGERR ("fail to get_provider_node");
+      return -1;
+    }
+
+  unsigned int port_index = p_stackx_port_zone->port_num;
+
+  while (network && (phynet = network->phy_net))
+    {
+      struct ref_nic *phead = phynet->header;
+      NSPOL_LOGINF (SC_DPDK_INFO, "network=%p,network_name=%s", network,
+                    network->network_name);
+
+      if (spl_hal_bond_config (network) < 0)
+        {
+          NSPOL_LOGERR ("spl_hal_bond_config fail.");
+          return -1;
+        }
+
+      while (phead != NULL)
+        {
+          /* check if the NIC is inited */
+          for (check = 0; check < port_index; ++check)
+            {
+              if (strcmp
+                  (p_stackx_port_zone->stackx_one_port[check].
+                   linux_ip.if_name, phead->nic_name) == 0)
+                {
+                  break;
+                }
+            }
+
+          if (check != port_index)
+            {
+              phead = phead->next;
+              continue;
+            }
+
+          /* check if the number of VF exceeds MAX_VF_NUM */
+          if (port_index >= MAX_VF_NUM + p_stackx_port_zone->bonded_port_num)
+            {
+              NSOPR_SET_ERRINFO (-1, "Support Only %d VF. %s init failed!\n",
+                                 MAX_VF_NUM, phead->nic_name);
+              NSPOL_LOGERR ("Support Only %d VF. %s init failed!", MAX_VF_NUM,
+                            phead->nic_name);
+              NSOPR_SET_ERRINFO (-1, "Add network %s failed!\n",
+                                 network->network_name);
+              break;
+            }
+
+          if (strlen (phead->nic_name) >=
+              sizeof (p_stackx_port_zone->
+                      stackx_one_port[port_index].linux_ip.if_name) - 1
+              || strlen (phead->nic_name) <= 3)
+            {
+              NSPOL_LOGERR ("Invalid configuration");
+              return -1;
+            }
+
+          retVal =
+            STRCPY_S (p_stackx_port_zone->
+                      stackx_one_port[port_index].linux_ip.if_name,
+                      sizeof (p_stackx_port_zone->
+                              stackx_one_port[port_index].linux_ip.if_name),
+                      phead->nic_name);
+
+          if (EOK != retVal)
+            {
+              NSPOL_LOGERR ("STRCPY_S failed]ret=%d.", retVal);
+              return -1;
+            }
+
+          NSPOL_LOGINF (SC_DPDK_INFO, "if_name %s",
+                        p_stackx_port_zone->
+                        stackx_one_port[port_index].linux_ip.if_name);
+          retVal =
+            STRCPY_S (p_stackx_port_zone->
+                      stackx_one_port[port_index].linux_ip.ip_addr_linux,
+                      sizeof (p_stackx_port_zone->
+                              stackx_one_port[port_index].linux_ip.
+                              ip_addr_linux), "0.0.0.0");
+
+          if (EOK != retVal)
+            {
+              NSPOL_LOGERR ("STRCPY_S failed]ret=%d.", retVal);
+              return -1;
+            }
+
+          retVal =
+            STRCPY_S (p_stackx_port_zone->
+                      stackx_one_port[port_index].linux_ip.mask_linux,
+                      sizeof (p_stackx_port_zone->
+                              stackx_one_port[port_index].linux_ip.
+                              mask_linux), "0.0.0.0");
+
+          if (EOK != retVal)
+            {
+              NSPOL_LOGERR ("STRCPY_S failed]ret=%d.", retVal);
+              return -1;
+            }
+
+          retVal =
+            STRCPY_S (p_stackx_port_zone->
+                      stackx_one_port[port_index].linux_ip.bcast_linux,
+                      sizeof (p_stackx_port_zone->
+                              stackx_one_port[port_index].linux_ip.
+                              bcast_linux), "0.0.0.0");
+
+          if (EOK != retVal)
+            {
+              NSPOL_LOGERR ("STRCPY_S failed]ret=%d.", retVal);
+              return -1;
+            }
+
+          ++port_index;
+          NSPOL_LOGINF (SC_DPDK_INFO, "port_index=%u", port_index);
+
+          if (CUR_CFG_HAL_PORT_NUM <= port_index + bond_ports_array.cnt)
+            {
+              // TODO: Invalid configuration received,  return immediately
+              NSPOL_LOGERR
+                ("Insufficient nStack configuration when compared to configuration from network.json");
+              return -1;
+            }
+
+          /* [TA33636] [2017-04-11] Do not need provider.json */
+          if (phynet->bond_mode ==
+              -1 /*&& strncmp(network->network_name, "provider", 8) != 0 */ )
+            {
+              break;
+            }
+          else
+            {
+              phead = phead->next;
+            }
+        }
+
+      network = network->next;
+    }
+
+  *port_num = port_index;
+
+  return ERR_OK;
+}
+
+void
+spl_hal_capa_init ()
+{
+  u32_t ipv4_cksum_offload = 1;
+  u32_t udp_cksum_offload = 1;
+  u32_t tcp_cksum_offload = 1;
+  hal_netif_capa_t info = { 0 };
+  struct stackx_port_info *p_port_info = head_used_port_list;
+
+  while (p_port_info)
+    {
+      hal_get_capability (p_port_info->linux_ip.hdl, &info);
+
+      if ((info.tx_offload_capa & HAL_ETH_TX_OFFLOAD_IPV4_CKSUM) == 0)
+        {
+          ipv4_cksum_offload = 0;
+
+          NSPOL_LOGDBG (SC_DPDK_INFO, "Port %s TX_OFFLOAD_IPV4_CKSUM Disable",
+                        p_port_info->linux_ip.if_name);
+        }
+
+      if ((info.tx_offload_capa & HAL_ETH_TX_OFFLOAD_UDP_CKSUM) == 0)
+        {
+          udp_cksum_offload = 0;
+
+          NSPOL_LOGDBG (SC_DPDK_INFO, "Port %s TX_OFFLOAD_UDP_CKSUM Disable",
+                        p_port_info->linux_ip.if_name);
+        }
+
+      if ((info.tx_offload_capa & HAL_ETH_TX_OFFLOAD_TCP_CKSUM) == 0)
+        {
+          tcp_cksum_offload = 0;
+
+          NSPOL_LOGDBG (SC_DPDK_INFO, "Port %s TX_OFFLOAD_TCP_CKSUM Disable",
+                        p_port_info->linux_ip.if_name);
+        }
+
+      p_port_info = p_port_info->next_use_port;
+    }
+
+  spl_hal_capa.tx_ipv4_cksum_offload = ipv4_cksum_offload;
+  spl_hal_capa.tx_udp_cksum_offload = udp_cksum_offload;
+  spl_hal_capa.tx_tcp_cksum_offload = tcp_cksum_offload;
+
+  NSPOL_LOGINF (SC_DPDK_INFO,
+                "ipv4_cksum_offload(%u),udp_cksum_offload(%u),tcp_cksum_offload(%u)",
+                ipv4_cksum_offload, udp_cksum_offload, tcp_cksum_offload);
+
+}
+
+NSTACK_STATIC void
+spl_hal_bond_info_init (hal_hdl_t hdl, struct bond_set *s,
+                        struct stackx_port_info *p)
+{
+#define MAX_MAC_STR_LEN     20
+  char mac_string[MAX_MAC_STR_LEN];
+  int retVal;
+  struct ether_addr addr;
+
+  p->linux_ip.hdl = hdl;
+
+  struct stackx_port_info *slave_port;
+  slave_port = get_port_info_by_name (s->slave_ports[0]);
+
+  if (slave_port == NULL)
+    {
+      NSPOL_LOGERR ("get_port_info_by_name failed]bond_port_name=%s",
+                    s->bond_port_name);
+      return;
+    }
+
+  /* check the lenght of bond_port_name */
+  retVal =
+    STRCPY_S (p->linux_ip.if_name, sizeof (p->linux_ip.if_name),
+              s->bond_port_name);
+  if (EOK != retVal)
+    {
+      NSPOL_LOGERR ("STRCPY_S failed]ret=%d", retVal);
+      return;
+    }
+
+  hal_get_macaddr (hdl, &addr);
+  retVal =
+    spl_hal_ether_etoa (addr.addr_bytes, sizeof (addr.addr_bytes), mac_string,
+                        sizeof (mac_string));
+  if (retVal < 0)
+    {
+      NSPOL_LOGERR ("spl_hal_ether_etoa failed]ret=%d", retVal);
+      return;
+    }
+
+  retVal =
+    STRCPY_S (p->linux_ip.mac_addr, sizeof (p->linux_ip.mac_addr),
+              mac_string);
+  if (EOK != retVal)
+    {
+      NSPOL_LOGERR ("STRCPY_S failed]ret=%d", retVal);
+      return;
+    }
+
+  retVal =
+    STRCPY_S (p->linux_ip.ip_addr_linux, sizeof (p->linux_ip.ip_addr_linux),
+              slave_port->linux_ip.ip_addr_linux);
+  if (EOK != retVal)
+    {
+      NSPOL_LOGERR ("STRCPY_S failed]ret=%d", retVal);
+      return;
+    }
+
+  retVal =
+    STRCPY_S (p->linux_ip.mask_linux, sizeof (p->linux_ip.mask_linux),
+              slave_port->linux_ip.mask_linux);
+  if (EOK != retVal)
+    {
+      NSPOL_LOGERR ("STRCPY_S failed]ret=%d", retVal);
+      return;
+    }
+
+  retVal =
+    STRCPY_S (p->linux_ip.bcast_linux, sizeof (p->linux_ip.bcast_linux),
+              slave_port->linux_ip.bcast_linux);
+  if (EOK != retVal)
+    {
+      NSPOL_LOGERR ("STRCPY_S failed]ret=%d", retVal);
+      return;
+    }
+
+  NSPOL_LOGINF (SC_DPDK_INFO, "===== the bond port info ======");
+  NSPOL_LOGINF (SC_DPDK_INFO, "bond port name=%s", p->linux_ip.if_name);
+  NSPOL_LOGINF (SC_DPDK_INFO, "bond port mac=%s", p->linux_ip.mac_addr);
+  NSPOL_LOGINF (SC_DPDK_INFO, "bond port ip=%s", p->linux_ip.ip_addr_linux);
+  NSPOL_LOGINF (SC_DPDK_INFO, "bond port netmask=%s", p->linux_ip.mask_linux);
+  NSPOL_LOGINF (SC_DPDK_INFO, "bond port broad_cast addr=%s",
+                p->linux_ip.bcast_linux);
+
+}
+
+NSTACK_STATIC int
+spl_hal_bond_start (void)
+{
+  u8_t i, j = 0;
+  struct stackx_port_info *bond_port = NULL;
+  hal_hdl_t hdl;
+  hal_hdl_t slave_hdl[HAL_MAX_SLAVES_PER_BOND];
+
+  NSPOL_LOGINF (SC_DPDK_INFO, "bond_ports_array.cnt=%u",
+                bond_ports_array.cnt);
+
+  for (i = bond_ports_array_cnt_start; i < bond_ports_array.cnt; i++)
+    {
+      struct bond_set *s = &bond_ports_array.ports[i];
+      NSPOL_LOGINF (SC_DPDK_INFO, "i=%u,bond_port_name=%s", i,
+                    s->bond_port_name);
+
+      u8_t slave_num = 0;
+      for (j = 0; j < s->slave_port_cnt; j++)
+        {
+          NSPOL_LOGINF (SC_DPDK_INFO, "s->slave_ports[%u]=%s", j,
+                        s->slave_ports[j]);
+          hdl = get_port_hdl_by_name (s->slave_ports[j]);
+
+          if (!hal_is_valid (hdl))
+            {
+              continue;
+            }
+
+          slave_hdl[slave_num++] = hdl;
+
+          /* here we didn't release the port mem allocated in p_stackx_port_zone */
+          del_port_in_port_list (s->slave_ports[j]);
+        }
+
+      hdl = hal_bond (s->bond_port_name, slave_num, slave_hdl);
+
+      if (!hal_is_valid (hdl))
+        {
+          NSPOL_LOGERR ("hal_bond fail: bond_name =%s", s->bond_port_name);
+          return -1;
+        }
+
+      bond_port =
+        &p_stackx_port_zone->stackx_one_port[p_stackx_port_zone->port_num];
+      num_ports_NIC++;
+      p_stackx_port_zone->port_num++;
+      p_stackx_port_zone->bonded_port_num++;
+
+      spl_hal_bond_info_init (hdl, s, bond_port);
+      add_port_in_port_list (bond_port);
+
+    }
+
+  bond_ports_array_cnt_start = bond_ports_array.cnt;
+  return 0;
+}
+
+/*
+ * Initialises a given port using global settings and with the rx buffers
+ * coming from the mbuf_pool passed as parameter
+ */
+
+NSTACK_STATIC inline int
+spl_hal_port_start (uint16_t nic_id, struct stackx_port_info *p_port_info,
+                    u16_t num_queues)
+{
+  u16_t num_queues_request, q;
+  hal_hdl_t hdl;
+  struct common_mem_mempool *mp;
+  hal_netif_config_t conf;
+#define MAX_MAC_STR_LEN     20
+  char mac_string[MAX_MAC_STR_LEN];
+  int retVal;
+  struct ether_addr addr;
+
+  // change the queues number per configuration.
+  // even we only receive packets from one rx queue when dispatch mode is on, the tx queue
+  // shoule set to the queues number requested.
+  num_queues_request = num_queues;
+  if (num_queues_request > HAL_ETH_MAX_QUEUE_NUM)
+    {
+      NSPOL_LOGERR
+        ("no enougth  queue num for thread!]num_queues_request=%u,MAX_QUEUE_NUM=%u",
+         num_queues_request, HAL_ETH_MAX_QUEUE_NUM);
+      return -1;
+    }
+
+  NSPOL_LOGDBG (SC_DPDK_INFO, "# Initialising index=%s... ",
+                p_port_info->linux_ip.if_name);
+  /* used to have fflush,no use code ,remove it. */
+
+  conf.bit.hw_vlan_filter = 1;
+  conf.bit.hw_vlan_strip = 1;
+
+  conf.rx.queue_num = num_queues_request;
+  conf.tx.queue_num = num_queues_request;
+
+  for (q = 0; q < num_queues_request; q++)
+    {
+      mp =
+        (struct common_mem_mempool *) spl_hal_rx_pool_create (nic_id, q, 1);
+
+      if (mp == NULL)
+        {
+          NSPOL_LOGERR
+            ("spl_hal_rx_pool_create fail]mp=NULL,nic_id=%u,if_name=%s",
+             nic_id, p_port_info->linux_ip.if_name);
+          return -1;
+        }
+
+      (void) spl_reg_res_txrx_mgr ((mpool_handle *) mp);        // will only return 0, no need to check return value
+      conf.rx.ring_pool[q] = mp;
+      conf.rx.ring_size[q] = HAL_RX_RING_SIZE;
+      conf.tx.ring_size[q] = HAL_TX_RING_SIZE;
+    }
+
+  hdl = hal_create (p_port_info->linux_ip.if_name, &conf);
+
+  if (!hal_is_valid (hdl))
+    {
+      NSPOL_LOGERR ("hal_create fail]if_name =%s",
+                    p_port_info->linux_ip.if_name);
+      return -1;
+    }
+
+  p_port_info->linux_ip.hdl = hdl;
+
+  /* add mac address  */
+  hal_get_macaddr (hdl, &addr);
+  retVal =
+    spl_hal_ether_etoa (addr.addr_bytes, sizeof (addr.addr_bytes), mac_string,
+                        sizeof (mac_string));
+  if (retVal < 0)
+    {
+      NSPOL_LOGERR ("spl_hal_ether_etoa failed]ret=%d", retVal);
+      return -1;
+    }
+
+  retVal =
+    STRCPY_S (p_port_info->linux_ip.mac_addr,
+              sizeof (p_port_info->linux_ip.mac_addr), mac_string);
+  if (EOK != retVal)
+    {
+      NSPOL_LOGERR ("STRCPY_S failed]ret=%d", retVal);
+      return -1;
+    }
+
+  return 0;
+}
+
+NSTACK_STATIC int
+spl_hal_port_setup ()
+{
+  unsigned int i;
+  struct stackx_port_info *p_port_info = NULL;
+
+  INITPOL_LOGINF ("RTP", "spl_hal_port_setup", NULL_STRING, LOG_INVALID_VALUE,
+                  MODULE_INIT_START);
+
+  for (i = num_ports_NIC_start; i < num_ports_NIC; i++)
+    {
+      p_port_info = &(p_stackx_port_zone->stackx_one_port[i]);
+
+      if (spl_hal_port_start (i, p_port_info, (u16_t) 1) < 0)
+        {
+          NSPOL_LOGERR ("Error initialising]nic_id=%u", i);
+
+          INITPOL_LOGERR ("RTP", "spl_hal_port_setup", NULL_STRING,
+                          LOG_INVALID_VALUE, MODULE_INIT_FAIL);
+
+          return -1;
+        }
+      else
+        {
+          report_port_link_status (p_port_info);
+          add_port_in_port_list (p_port_info);
+        }
+    }
+
+  if (spl_hal_bond_start () < 0)
+    {
+      NSPOL_LOGERR ("bond port init failed!");
+
+      INITPOL_LOGERR ("RTP", "spl_hal_port_setup", NULL_STRING,
+                      LOG_INVALID_VALUE, MODULE_INIT_FAIL);
+
+      return -1;
+    }
+
+  spl_hal_capa_init ();
+
+  INITPOL_LOGINF ("RTP", "spl_hal_port_setup", NULL_STRING, LOG_INVALID_VALUE,
+                  MODULE_INIT_SUCCESS);
+
+  return 0;
+
+}
+
+int
+spl_hal_port_init ()
+{
+  int retval;
+  unsigned int i, port_num = 0;
+
+  int port_num_start = p_stackx_port_zone->port_num;
+  num_ports_NIC_start = num_ports_NIC;
+
+  //Read network info
+  INITPOL_LOGINF ("IP", "spl_hal_port_config", NULL_STRING, LOG_INVALID_VALUE,
+                  MODULE_INIT_START);
+  retval = spl_hal_port_config (&port_num);
+
+  if (retval != ERR_OK)
+    {
+      INITPOL_LOGERR ("IP", "spl_hal_port_config", NULL_STRING,
+                      LOG_INVALID_VALUE, MODULE_INIT_FAIL);
+      return -1;
+    }
+
+  p_stackx_port_zone->port_num = port_num;
+
+  NSPOL_LOGINF (SC_DPDK_INFO, "port_num=%u", port_num);
+  INITPOL_LOGINF ("IP", "spl_hal_port_config", NULL_STRING, LOG_INVALID_VALUE,
+                  MODULE_INIT_SUCCESS);
+
+  if (port_num_start == p_stackx_port_zone->port_num)
+    {
+      NSPOL_LOGERR ("No new NIC find.");
+      return 0;
+    }
+
+  //Get ports num
+  for (i = port_num_start; i < p_stackx_port_zone->port_num; i++)
+    {
+      if (p_stackx_port_zone->stackx_one_port[i].linux_ip.if_name[0] != 0)
+        {
+          /* right now hard coded, */
+          int eth_num =
+            atoi (p_stackx_port_zone->stackx_one_port[i].linux_ip.if_name +
+                  3);
+
+          num_ports_NIC++;
+
+          NSPOL_LOGDBG (SC_DPDK_INFO, "port_mask=%d ,eth_name=%s", eth_num,
+                        p_stackx_port_zone->stackx_one_port[i].
+                        linux_ip.if_name);
+        }
+    }
+
+  if (num_ports_NIC > HAL_MAX_NIC_NUM)
+    {
+      NSPOL_LOGERR ("just support one eth");
+      common_exit (EXIT_FAILURE, "just surport one eth");
+    }
+
+  if (num_ports_NIC == num_ports_NIC_start)
+    {
+      NSPOL_LOGERR ("No new NIC find.");
+      return 0;
+    }
+
+  retval = spl_hal_port_setup ();
+
+  if (retval == -1)
+    {
+      return -1;
+    }
+
+  NSPOL_LOGDBG (SC_DPDK_INFO, "Finished Process Init.");
+
+  return 1;
+}
+
+inline NSTACK_STATIC void
+spl_hal_send (struct netif *pnetif)
+{
+  u16_t i, sent = 0;
+  struct netifExt *pnetifExt = NULL;
+  u16_t netif_id = pnetif->num;
+  u16_t tx_num = spl_hal_pkts[netif_id].tx.num;
+  struct common_mem_mbuf **tx_ptks = spl_hal_pkts[netif_id].tx.pkts;
+
+  for (i = 0; i < tx_num; i++)
+    {
+      (void)
+        res_free (&
+                  (((struct spl_pbuf *) (((char *) tx_ptks[i]) +
+                                         sizeof (struct
+                                                 common_mem_mbuf)))->res_chk));
+    }
+
+  int _retry = 0;
+
+  pnetifExt = getNetifExt (pnetif->num);
+  if (NULL == pnetifExt)
+    return;
+
+  do
+    {
+      sent +=
+        hal_send_packet (pnetifExt->hdl, 0, &(tx_ptks[sent]), tx_num - sent);
+      _retry++;
+
+      if (_retry > SPL_HAL_SEND_TRY)
+        {
+          NSPOL_LOGERR ("send loop %d times but dpdk send data fail ",
+                        SPL_HAL_SEND_TRY);
+          break;
+        }
+    }
+  while (unlikely (sent != tx_num));
+
+  if (unlikely (sent != tx_num))
+    {
+      for (i = sent; i < tx_num; i++)
+        {
+          (void) nsfw_mem_mbf_free ((mbuf_handle) (tx_ptks[i]), NSFW_SHMEM);
+        }
+    }
+  for (i = 0; i < tx_num; i++)
+    {
+      /* set dpdk_send flag */
+      ((struct spl_pbuf *) (((char *) tx_ptks[i]) +
+                            sizeof (struct common_mem_mbuf)))->
+        res_chk.u8Reserve |= DPDK_SEND_FLAG;
+    }
+
+  spl_hal_pkts[netif_id].tx.num = 0;
+
+}
+
+inline u16_t
+spl_hal_recv (struct netif *pnetif, u8_t id)
+{
+  u16_t netif_id, rx_c = 0;
+  struct netifExt *pnetifExt = NULL;
+
+  netif_id = pnetif->num;
+
+  pnetifExt = getNetifExt (pnetif->num);
+  if (NULL == pnetifExt)
+    return 0;
+
+  rx_c =
+    hal_recv_packet (pnetifExt->hdl, 0, spl_hal_pkts[netif_id].rx.pkts,
+                     PKT_BURST);
+
+  if (rx_c <= 0)
+    {
+      return 0;
+    }
+
+  spl_hal_pkts[netif_id].rx.num = rx_c;
+  spl_hal_pkts[netif_id].rx.index = 0;
+
+  return rx_c;
+}
+
+/*needflush set 1 has pbuf release problem, ref maybe set 0 before release*/
+NSTACK_STATIC inline void
+spl_hal_set_cksum (struct spl_pbuf *buf, struct common_mem_mbuf *mbuf)
+{
+
+  //need to be careful, special when small packet oversize
+  if (buf->tot_len > mbuf->pkt_len)
+    {
+      NSPOL_LOGWAR (SC_DPDK_INFO,
+                    "small packet OVERSIZE]pbuf_len=%u,mbuf_len=%u", buf->len,
+                    mbuf->pkt_len);
+      mbuf->pkt_len = buf->len;
+    }
+
+  if (!spl_hal_tx_ip_cksum_enable () || !spl_hal_tx_tcp_cksum_enable ()
+      || !spl_hal_tx_udp_cksum_enable ())
+    {
+      struct tcp_hdr *t_hdr;
+      struct udp_hdr *u_hdr;
+      u16_t flag_offset;
+      u64_t ol_flags = (mbuf->ol_flags);        //& (~PKT_TX_L4_MASK));
+
+      struct eth_hdr *ethhdr = (struct eth_hdr *) ((char *) buf->payload);
+
+      if (ethhdr->type == 8)
+        {
+          struct ip_hdr *iphdr =
+            (struct ip_hdr *) ((char *) buf->payload +
+                               sizeof (struct eth_hdr));
+
+          if (!spl_hal_tx_ip_cksum_enable ())
+            {
+              ol_flags |= PKT_TX_IPV4 | PKT_TX_IP_CKSUM;
+              iphdr->_chksum = 0;
+            }
+
+          flag_offset = spl_ntohs (iphdr->_offset);
+
+          /*ip frag, only the first packet has udp or tcp head */
+          if (0 == (flag_offset & IP_OFFMASK))
+            {
+              switch (iphdr->_proto)
+                {
+                case IPPROTO_TCP:
+                  if (!spl_hal_tx_tcp_cksum_enable ())
+                    {
+                      t_hdr =
+                        (struct tcp_hdr *) ((char *) buf->payload +
+                                            sizeof (struct eth_hdr) +
+                                            sizeof (struct ip_hdr));
+                      t_hdr->chksum = get_ipv4_psd_sum (iphdr, ol_flags);
+                      ol_flags |= PKT_TX_TCP_CKSUM;
+                    }
+
+                  break;
+
+                case IPPROTO_UDP:
+                  {
+                    if ((mbuf->ol_flags & PKT_TX_UDP_CKSUM) ==
+                        PKT_TX_UDP_CKSUM)
+                      {
+                        u_hdr = (struct udp_hdr *) ((char *) buf->payload + sizeof (struct eth_hdr) + sizeof (struct ip_hdr));  //l2_len + l3_len);
+                        u_hdr->chksum =
+                          get_ipv4_psd_sum (iphdr, mbuf->ol_flags);
+                      }
+                  }
+
+                  break;
+
+                default:
+
+                  break;
+                }
+            }
+          mbuf->l2_len = sizeof (struct eth_hdr);       //l2_len;
+          mbuf->l3_len = sizeof (struct ip_hdr);
+          mbuf->ol_flags = ol_flags;
+        }
+    }
+}
+
+/*needflush set 1 has pbuf release problem, ref maybe set 0 before release*/
+err_t
+spl_hal_output (struct netif *pnetif, struct pbuf *buf)
+{
+  u16_t netif_id, idx;
+  struct common_mem_mbuf *mbuf;
+  struct spl_pbuf *spbuf = NULL;
+  //spl_pbuf_layer layer = SPL_PBUF_TRANSPORT;
+  //u16_t offset;
+
+  if (!p_def_stack_instance)
+    {
+      NSPOL_LOGERR ("p_def_stack_instance is NULL");
+      return -1;
+    }
+
+  u16_t proc_id = spl_get_lcore_id ();
+
+  NSPOL_LOGINF (SC_DPDK_INFO, "spl_hal_output. len %d totlen %d", buf->len,
+                buf->tot_len);
+  print_pbuf_payload_info (buf, true);
+
+  if (buf->tot_len > DEF_MBUF_DATA_SIZE)
+    {
+      NSPOL_LOGINF (TCP_DEBUG, "spl_pbuf_alloc_hugepage Failed!!!");
+      return ERR_MEM;
+
+    }
+  spbuf = spl_pbuf_alloc_hugepage (SPL_PBUF_RAW,
+                                   buf->tot_len,
+                                   SPL_PBUF_HUGE, proc_id, NULL);
+
+  if (!spbuf)
+    {
+      NSPOL_LOGINF (TCP_DEBUG, "spl_pbuf_alloc_hugepage Failed!!!");
+      return ERR_MEM;
+    }
+
+  if (ERR_OK != pbuf_to_splpbuf_copy (spbuf, buf))
+    {
+      NSPOL_LOGERR ("pbuf to splpbuf copy failed");
+      return -1;
+    }
+
+  mbuf =
+    (struct common_mem_mbuf *) ((char *) spbuf -
+                                sizeof (struct common_mem_mbuf));
+
+  if (spbuf->tot_len > mbuf->pkt_len)
+    {
+      NSPOL_LOGWAR (SC_DPDK_INFO,
+                    "small packet OVERSIZE]pbuf_len=%u,mbuf_len=%u",
+                    spbuf->len, mbuf->pkt_len);
+      mbuf->pkt_len = spbuf->len;
+    }
+
+  spl_hal_set_cksum (spbuf, mbuf);
+
+  netif_id = pnetif->num;
+  idx = spl_hal_pkts[netif_id].tx.num++;
+  spl_hal_pkts[netif_id].tx.pkts[idx] = mbuf;
+  spl_do_dump (spbuf, DUMP_SEND);
+  spl_hal_send (pnetif);
+
+  return 0;
+}
+
+void
+spl_hal_input (struct netif *pnetif, struct spl_pbuf **buf)
+{
+  u16_t netif_id;
+
+  struct common_mem_mbuf *mbuf;
+
+  netif_id = pnetif->num;
+
+  if (likely
+      (spl_hal_pkts[netif_id].rx.num > spl_hal_pkts[netif_id].rx.index))
+    {
+      mbuf = spl_hal_pkts[netif_id].rx.pkts[spl_hal_pkts[netif_id].rx.index];
+      spl_hal_pkts[netif_id].rx.index++;
+      spl_hal_buf_convert (mbuf, buf);
+      spl_do_dump (*buf, DUMP_RECV);
+    }
+  else
+    {
+      NSPOL_LOGERR
+        ("recv from spl_dev has a problem]pnetif=%p, num=%u, index=%u",
+         pnetif, spl_hal_pkts[netif_id].rx.num,
+         spl_hal_pkts[netif_id].rx.index);
+      *buf = NULL;
+    }
+  return;
+}
+
+int
+spl_hal_tx_ip_cksum_enable ()
+{
+  return !spl_hal_capa.tx_ipv4_cksum_offload;
+}
+
+int
+spl_hal_tx_udp_cksum_enable ()
+{
+  return !spl_hal_capa.tx_udp_cksum_offload;
+}
+
+int
+spl_hal_tx_tcp_cksum_enable ()
+{
+  return !spl_hal_capa.tx_tcp_cksum_offload;
+}
+
+u32
+spl_hal_is_nic_exist (const char *name)
+{
+  return hal_is_nic_exist (name);
+}
+
+int
+spl_hal_is_bond_netif (struct netif *pnetif)
+{
+  int i;
+  struct bond_set *s;
+  struct netifExt *pnetifExt = NULL;
+
+  pnetifExt = getNetifExt (pnetif->num);
+  if (NULL == pnetifExt)
+    return 0;
+
+  for (i = 0; i < bond_ports_array.cnt; i++)
+    {
+      s = &bond_ports_array.ports[i];
+      if (!strncmp
+          (pnetifExt->if_name, s->bond_port_name, HAL_MAX_NIC_NAME_LEN))
+        {
+          return 1;
+        }
+    }
+
+  return 0;
+}
diff --git a/stacks/lwip_stack/lwip_src/recycle/stackx_recycle.c b/stacks/lwip_stack/lwip_src/recycle/stackx_recycle.c
new file mode 100644
index 0000000..addc0e6
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/recycle/stackx_recycle.c
@@ -0,0 +1,676 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include "stackx_spl_share.h"
+#include "nsfw_recycle_api.h"
+#include "nstack_log.h"
+#include "nsfw_msg_api.h"
+#include "stackx_socket.h"
+#include "stackx_spl_msg.h"
+#include "stackx_app_res.h"
+#include "common.h"
+#include "sc_dpdk.h"
+#include "nsfw_mt_config.h"
+#include "spl_instance.h"
+
+#define SS_DELAY_CLOSE_SEC 5
+
+extern struct stackx_port_zone *p_stackx_port_zone;
+
+/*****************************************************************************
+*   Prototype    : sbr_recycle_rx_mbuf
+*   Description  : iterator and free rx mbufs with pid flags, when the app
+                   with certain pid is no longer exist
+*   Input        : void *data
+*                  void *arg
+*   Output       : None
+*   Return Value : static inline int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline int
+sbr_recycle_rx_mbuf (void *data, void *arg)
+{
+  u32 *recycle_flg;
+  pid_t *pid = (pid_t *) arg;
+  struct common_mem_mbuf *m_buf = (struct common_mem_mbuf *) data;
+#ifdef HAL_LIB
+#else
+  recycle_flg =
+    (u32 *) ((char *) (m_buf->buf_addr) + RTE_PKTMBUF_HEADROOM -
+             sizeof (u32));
+#endif
+  if (m_buf->refcnt > 0 && *recycle_flg == *pid)
+    {
+      NSSBR_LOGDBG ("free rx mbuf hold by app], mbuf=%p", m_buf);
+      *recycle_flg = MBUF_UNUSED;
+      spl_mbuf_free (m_buf);
+    }
+  return 0;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_recycle_rx_pool
+*   Description  : recycle rx mbufs hold by app when app crahes
+*   Input        : pid_t pid
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+NSTACK_STATIC int
+sbr_recycle_rx_pool (pid_t pid)
+{
+  static struct common_mem_mempool *rx_pool[MAX_VF_NUM * 2][MAX_THREAD_NUM] =
+    { {0} };
+  struct common_mem_mempool *mp;
+  nsfw_mem_name lookup_mbuf_pool;
+  u32 nic_id, queue_id = 0;
+  int retval;
+  struct common_mem_ring *ring;
+
+  for (nic_id = 0;
+       nic_id < p_stackx_port_zone->port_num && nic_id < MAX_VF_NUM * 2;
+       nic_id++)
+    {
+      mp = rx_pool[nic_id][queue_id];
+      if (mp == NULL)
+        {
+          retval =
+            spl_snprintf (lookup_mbuf_pool.aname, NSFW_MEM_NAME_LENGTH - 1,
+                          "%s", get_mempoll_rx_name (queue_id, nic_id));
+          if (-1 == retval)
+            {
+              NSPOL_LOGERR ("spl_snprintf fail");
+              break;
+            }
+
+          lookup_mbuf_pool.entype = NSFW_SHMEM;
+          lookup_mbuf_pool.enowner = NSFW_PROC_MAIN;
+          mp =
+            (struct common_mem_mempool *)
+            nsfw_mem_mbfmp_lookup (&lookup_mbuf_pool);
+          if (mp == NULL)
+            break;
+          rx_pool[nic_id][queue_id] = mp;
+        }
+      if (nsfw_mem_mbuf_iterator
+          (mp, 0, mp->size, sbr_recycle_rx_mbuf, (void *) &pid) < 0)
+        {
+          NSSBR_LOGERR ("nsfw_mem_mbuf_iterator return fail");
+          return -1;
+        }
+      ring = (struct common_mem_ring *) (mp->pool_data);
+      NSSBR_LOGINF
+        ("after recycling rx pbuf hold by app]ring=%p,prod.head=%u,prod.tail=%u,"
+         "cons.head=%u,cons.tail=%u.", ring, ring->prod.head, ring->prod.tail,
+         ring->cons.head, ring->cons.tail);
+    }
+
+  return 0;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_recycle_rx_pool
+*   Description  : recycle stackx tx mbufs hold by app when app crahes
+*   Input        : pid_t pid
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+NSTACK_STATIC int
+sbr_recycle_tx_pool (pid_t pid)
+{
+  struct common_mem_mempool *mp;
+  struct common_mem_ring *ring;
+
+  /* Try to free all the RX mbufs which are holded in stackx TX pool */
+  mp = (struct common_mem_mempool *) p_def_stack_instance->mp_tx;
+  if (mp == NULL)
+    return -1;
+
+  if (nsfw_mem_mbuf_iterator
+      (mp, 0, mp->size, sbr_recycle_rx_mbuf, (void *) &pid) < 0)
+    {
+      NSSBR_LOGERR ("nsfw_mem_mbuf_iterator return fail");
+      return -1;
+    }
+  ring = (struct common_mem_ring *) (mp->pool_data);
+  NSSBR_LOGINF
+    ("after recycling stackx tx pbuf hold by app]ring=%p,prod.head=%u,prod.tail=%u,"
+     "cons.head=%u,cons.tail=%u.", ring, ring->prod.head, ring->prod.tail,
+     ring->cons.head, ring->cons.tail);
+
+  return 0;
+}
+
+/*****************************************************************************
+*   Prototype    : ss_free_del_conn_msg
+*   Description  : free msg
+*   Input        : msg_delete_netconn *dmsg
+*   Output       : None
+*   Return Value : static inline void
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline void
+ss_free_del_conn_msg (msg_delete_netconn * dmsg)
+{
+  data_com_msg *msg = (data_com_msg *) ((char *) dmsg - MAX_MSG_PARAM_SIZE);
+
+  if (MSG_ASYN_POST == msg->param.op_type)      /* should check type for linger */
+    {
+      msg_free (msg);
+    }
+}
+
+extern int nsep_recycle_ep (u32 pid);
+
+/*****************************************************************************
+*   Prototype    : ss_recycle_done
+*   Description  : recycle done,need recycle ep and rx pool
+*   Input        : pid_t pid
+*   Output       : None
+*   Return Value : static inline void
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline void
+ss_recycle_done (pid_t pid)
+{
+  spl_free_tx_pool (pid);
+  spl_recycle_msg_pool (pid);
+  (void) sbr_recycle_rx_pool (pid);
+  (void) sbr_recycle_tx_pool (pid);
+  (void) nsep_recycle_ep (pid);
+  (void) nsfw_recycle_obj_end (pid);
+}
+
+/*****************************************************************************
+*   Prototype    : ss_notify_omc
+*   Description  : try to notify omc
+*   Input        : spl_netconn_t** conn_array
+*                  u32 conn_num
+*                  u8 notify_omc
+*                  pid_t pid
+*   Output       : None
+*   Return Value : static inline void
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline void
+ss_notify_omc (spl_netconn_t ** conn_array, u32 conn_num, u8 notify_omc,
+               pid_t pid)
+{
+  if (notify_omc)
+    {
+      u32 i;
+      for (i = 0; i < conn_num; ++i)
+        {
+          struct spl_netconn *conn = conn_array[i];
+          if (ss_is_pid_exist (conn, pid))
+            {
+              NSSBR_LOGINF ("there are still conn at work]pid=%d", pid);
+              break;
+            }
+
+          msg_delete_netconn *delay_msg = conn->recycle.delay_msg;
+          if (delay_msg && (delay_msg->pid == pid))
+            {
+              delay_msg->notify_omc = notify_omc;
+              NSSBR_LOGINF ("there are still conn at delay]pid=%d", pid);
+              break;
+            }
+        }
+
+      if (conn_num == i)
+        {
+          NSSBR_LOGINF ("recycle done,notify omc]pid=%d", pid);
+          ss_recycle_done (pid);
+        }
+    }
+}
+
+extern void do_pbuf_free (struct spl_pbuf *buf);
+
+static void
+ss_recycle_fd_share (sbr_fd_share * fd_share)
+{
+  /* to free pbufs which are attached to sbr_fd_share */
+  if (fd_share->recoder.head)
+    {
+      struct spl_pbuf *buf = fd_share->recoder.head;
+      fd_share->recoder.head = NULL;
+      fd_share->recoder.tail = NULL;
+      fd_share->recoder.totalLen = 0;
+      do_pbuf_free (buf);
+    }
+}
+
+extern void nsep_recycle_epfd (void *epinfo, u32 pid);
+extern void tcp_free_accept_ring (spl_netconn_t * conn);
+extern void free_conn_by_spl (spl_netconn_t * conn);
+extern void tcp_drop_conn (spl_netconn_t * conn);
+
+/*****************************************************************************
+*   Prototype    : ss_close_conn_now
+*   Description  : close netconn now
+*   Input        : spl_netconn_t *conn
+*                  msg_delete_netconn *dmsg
+*                  pid_t pid
+*                  ss_close_conn_fun close_conn
+*   Output       : None
+*   Return Value : static inline void
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline void
+ss_close_conn_now (spl_netconn_t * conn, msg_delete_netconn * dmsg, pid_t pid,
+                   ss_close_conn_fun close_conn)
+{
+  spl_netconn_t **conn_array = conn->recycle.group->conn_array;
+  u32 conn_num = conn->recycle.group->conn_num;
+  u8 notify_omc = dmsg->notify_omc;
+
+  close_conn (dmsg, 0);
+
+  nsep_recycle_epfd (conn->epInfo, pid);
+  conn->epInfo = NULL;          /*must be set to NULL */
+
+  u32 i;
+  if (conn->recycle.is_listen_conn)
+    {
+      /* drop the conn inside the accept ring */
+      tcp_free_accept_ring (conn);
+
+      /* app coredump and accept_from not changed, need recyle */
+      for (i = 0; i < conn_num; ++i)
+        {
+          struct spl_netconn *accept_conn = conn_array[i];
+          if ((accept_conn->recycle.accept_from == conn)
+              && ss_is_pid_array_empty (accept_conn))
+            {
+              NSSBR_LOGINF
+                ("recycle lost conn]listen_conn=%p,listen_private_data=%p,accept_conn=%p,accept_private_data=%p,pid=%d",
+                 conn, conn->private_data, accept_conn,
+                 accept_conn->private_data, pid);
+              data_com_msg *msg =
+                (data_com_msg *) ((char *) dmsg - MAX_MSG_PARAM_SIZE);
+              msg->param.receiver = ss_get_recv_obj (accept_conn);
+              dmsg->buf = NULL;
+              dmsg->time_started = sys_now ();
+              dmsg->shut = 0;
+              dmsg->conn = accept_conn;
+              close_conn (dmsg, 0);
+
+              nsep_recycle_epfd (accept_conn->epInfo, pid);
+              accept_conn->epInfo = NULL;       /*must be set to NULL */
+
+              /* lost conn need drop first, can't just free conn */
+              tcp_drop_conn (accept_conn);
+            }
+        }
+    }
+
+  if (SS_DELAY_STOPPED == conn->recycle.delay_flag)
+    {
+      sbr_fd_share *fd_share =
+        (sbr_fd_share *) ((char *) conn + SS_NETCONN_SIZE);
+      ss_recycle_fd_share (fd_share);
+      free_conn_by_spl (conn);
+    }
+
+  ss_free_del_conn_msg (dmsg);
+  ss_notify_omc (conn_array, conn_num, notify_omc, pid);
+}
+
+/*****************************************************************************
+*   Prototype    : ss_close_conn_delay
+*   Description  : delay to close conn
+*   Input        : spl_netconn_t *conn
+*                  pid_t pid
+*                  msg_delete_netconn *dmsg
+*                  ss_close_conn_fun close_conn
+*   Output       : None
+*   Return Value : static inline void
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline void
+ss_close_conn_delay (spl_netconn_t * conn, pid_t pid,
+                     msg_delete_netconn * dmsg, ss_close_conn_fun close_conn)
+{
+  NSSBR_LOGINF
+    ("ref > 0 and pid array is empty, start delay closing conn]conn=%p,pid=%d,private_data=%p",
+     conn, pid, conn->private_data);
+  close_conn (dmsg, SS_DELAY_CLOSE_SEC);
+}
+
+/*****************************************************************************
+*   Prototype    : ss_process_delay_up
+*   Description  : delay is up
+*   Input        : spl_netconn_t *conn
+*                  pid_t pid
+*                  msg_delete_netconn *dmsg
+*                  ss_close_conn_fun close_conn
+*   Output       : None
+*   Return Value : static inline void
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline void
+ss_process_delay_up (spl_netconn_t * conn, pid_t pid,
+                     msg_delete_netconn * dmsg, ss_close_conn_fun close_conn)
+{
+  spl_netconn_t **conn_array = conn->recycle.group->conn_array;
+  u32 conn_num = conn->recycle.group->conn_num;
+  u8 notify_omc = dmsg->notify_omc;
+
+  if (SS_DELAY_STARTED == conn->recycle.delay_flag)
+    {
+      if (ss_is_pid_array_empty (conn))
+        {
+          NSSBR_LOGINF
+            ("delay time is up,close conn now]conn=%p,pid=%d,private_data=%p",
+             conn, pid, conn->private_data);
+          conn->recycle.delay_flag = SS_DELAY_STOPPED;
+          conn->recycle.delay_msg = NULL;
+          ss_close_conn_now (conn, dmsg, pid, close_conn);
+          return;
+        }
+      else
+        {
+          NSSBR_LOGINF
+            ("stop delay closing conn,conn still working]conn=%p,pid=%d,private_data=%p",
+             conn, pid, conn->private_data);
+          conn->recycle.delay_flag = SS_DELAY_STOPPED;
+          conn->recycle.delay_msg = NULL;
+          ss_free_del_conn_msg (dmsg);
+          ss_notify_omc (conn_array, conn_num, notify_omc, pid);
+          return;
+        }
+    }
+  else if (SS_DELAY_AGAIN == conn->recycle.delay_flag)
+    {
+      if (ss_is_pid_array_empty (conn))
+        {
+          NSSBR_LOGINF
+            ("delay time is up,but need delay again]conn=%p,pid=%d,private_data=%p",
+             conn, pid, conn->private_data);
+          conn->recycle.delay_flag = SS_DELAY_STARTED;
+          ss_close_conn_delay (conn, pid, dmsg, close_conn);
+          return;
+        }
+      else
+        {
+          NSSBR_LOGINF
+            ("stop delay closing conn,conn still working]conn=%p,pid=%d,private_data=%p",
+             conn, pid, conn->private_data);
+          conn->recycle.delay_flag = SS_DELAY_STOPPED;
+          conn->recycle.delay_msg = NULL;
+          ss_free_del_conn_msg (dmsg);
+          ss_notify_omc (conn_array, conn_num, notify_omc, pid);
+          return;
+        }
+    }
+  else if (SS_DELAY_STOPPING == conn->recycle.delay_flag)
+    {
+      NSSBR_LOGINF
+        ("the conn has been closed,free conn]conn=%p,pid=%d,private_data=%p",
+         conn, pid, conn->private_data);
+      conn->recycle.delay_flag = SS_DELAY_STOPPED;
+      conn->recycle.delay_msg = NULL;
+      free_conn_by_spl (conn);
+      ss_free_del_conn_msg (dmsg);
+      ss_notify_omc (conn_array, conn_num, notify_omc, pid);
+      return;
+    }
+  else
+    {
+      NSSBR_LOGERR ("this can not happen]conn=%p,pid=%d,private_data=%p",
+                    conn, pid, conn->private_data);
+    }
+}
+
+/*****************************************************************************
+*   Prototype    : ss_recycle_conn
+*   Description  : recycle conn
+*   Input        : void *close_data
+*                  ss_close_conn_fun close_conn
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+ss_recycle_conn (void *close_data, ss_close_conn_fun close_conn)
+{
+  msg_delete_netconn *dmsg = (msg_delete_netconn *) close_data;
+  spl_netconn_t *conn = dmsg->conn;
+  pid_t pid = dmsg->pid;
+  u8 notify_omc = dmsg->notify_omc;
+  struct spl_netconn **conn_array = conn->recycle.group->conn_array;
+  u32 conn_num = conn->recycle.group->conn_num;
+
+  int ret = ss_del_pid (conn, pid);
+  if (0 == ret)
+    {
+      i32 ref = ss_dec_fork_ref (conn);
+      if (0 == ref)
+        {
+          if (conn->recycle.delay_flag != SS_DELAY_STOPPED)
+            {
+              conn->recycle.delay_flag = SS_DELAY_STOPPING;
+              NSSBR_LOGINF
+                ("stop delay closing conn,close conn now]conn=%p,pid=%d,private_data=%p",
+                 conn, pid, conn->private_data);
+            }
+          else
+            {
+              NSSBR_LOGINF ("close conn now]conn=%p,pid=%d,private_data=%p",
+                            conn, pid, conn->private_data);
+            }
+
+          ss_close_conn_now (conn, dmsg, pid, close_conn);
+          return 0;
+        }
+      else
+        {
+          if (ss_is_pid_array_empty (conn))
+            {
+              if (SS_DELAY_STOPPED == conn->recycle.delay_flag) /* only start one delay */
+                {
+                  conn->recycle.delay_flag = SS_DELAY_STARTED;
+                  conn->recycle.delay_msg = close_data;
+                  ss_close_conn_delay (conn, pid, dmsg, close_conn);
+                  return 0;
+                }
+              else if (SS_DELAY_STARTED == conn->recycle.delay_flag)
+                {
+                  conn->recycle.delay_flag = SS_DELAY_AGAIN;
+                  NSSBR_LOGINF
+                    ("ref > 0 and pid array is empty, delay again]conn=%p,pid=%d,private_data=%p",
+                     conn, pid, conn->private_data);
+                }
+            }
+        }
+    }
+  else
+    {
+      if (conn->recycle.delay_msg && (conn->recycle.delay_msg == close_data))   /* only the stater can process */
+        {
+          ss_process_delay_up (conn, pid, dmsg, close_conn);
+          return 0;
+        }
+    }
+
+  NSSBR_LOGINF ("go to notify omc]conn=%p,pid=%d,private_data=%p", conn, pid,
+                conn->private_data);
+  ss_free_del_conn_msg (dmsg);
+  ss_notify_omc (conn_array, conn_num, notify_omc, pid);
+  return 0;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_handle_recycle_conn
+*   Description  : post msg to spl
+*   Input        : spl_netconn_t* conn
+*                  pid_t pid
+*                  u8 notify_omc
+*   Output       : None
+*   Return Value : static inline int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline int
+sbr_handle_recycle_conn (spl_netconn_t * conn, pid_t pid, u8 notify_omc)
+{
+  data_com_msg *m = msg_malloc (ss_get_msg_pool (conn));
+
+  if (!m)
+    {
+      NSSBR_LOGERR ("malloc msg failed]conn=%p,pid=%d,private_data=%p", conn,
+                    pid, conn->private_data);
+      return -1;
+    }
+
+  NSSBR_LOGINF ("recycle conn]conn=%p,pid=%d,private_data=%p", conn, pid,
+                conn->private_data);
+
+  m->param.module_type = MSG_MODULE_SBR;
+  m->param.major_type = SPL_TCPIP_NEW_MSG_API;
+  m->param.minor_type = SPL_API_DO_DELCON;
+  m->param.err = 0;
+  m->param.op_type = MSG_ASYN_POST;
+  sys_sem_init (&m->param.op_completed);
+  m->param.receiver = ss_get_recv_obj (conn);
+  m->param.extend_member_bit = 0;
+
+  msg_delete_netconn *p = (msg_delete_netconn *) m->buffer;
+  p->extend_member_bit = 0;
+  p->time_started = sys_now ();
+  p->shut = 0;
+  p->pid = pid;
+  p->conn = conn;
+  p->notify_omc = notify_omc;
+  p->msg_box_ref = SPL_MSG_BOX_NUM;
+  p->buf = NULL;
+
+  /* to ensure that the last deal with SPL_API_DO_DELCON message */
+  int i;
+  for (i = 0; i < SPL_MSG_BOX_NUM; ++i)
+    {
+      if (msg_post_with_lock_rel
+          (m,
+           ss_get_instance_msg_box (ss_get_bind_thread_index (conn), i)) < 0)
+        {
+          msg_free (m);
+          NSSBR_LOGERR ("post msg failed]conn=%p,pid=%d,private_data=%p",
+                        conn, pid, conn->private_data);
+          return -1;
+        }
+    }
+
+  return 0;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_recycle_fd_share
+*   Description  : recycle sbr_fd_share
+*   Input        : sbr_fd_share* fd_share
+*                  pid_t pid
+*   Output       : None
+*   Return Value : static inline void
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline void
+sbr_recycle_fd_share (sbr_fd_share * fd_share, pid_t pid)
+{
+  if (fd_share->common_lock.locked == pid)
+    {
+      common_spinlock_unlock (&fd_share->common_lock);
+    }
+
+  if (fd_share->recv_lock.locked == pid)
+    {
+      common_spinlock_unlock (&fd_share->recv_lock);
+    }
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_recycle_conn
+*   Description  : recycle api,called by recycle module
+*   Input        : u32 exit_pid
+*                  void *pdata
+*                  u16 rec_type
+*   Output       : None
+*   Return Value : nsfw_rcc_stat
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+nsfw_rcc_stat
+sbr_recycle_conn (u32 exit_pid, void *pdata, u16 rec_type)
+{
+  NSSBR_LOGINF ("start recycle]pid=%d", exit_pid);
+
+  if (0 == exit_pid)
+    {
+      NSSBR_LOGERR ("pid is not ok]pid=%d", exit_pid);
+      return NSFW_RCC_CONTINUE;
+    }
+
+  spl_netconn_t **conn_array = spl_get_conn_array (exit_pid);
+  if (!conn_array)
+    {
+      NSSBR_LOGERR ("conn_array is NULL]pid=%d", exit_pid);
+      return NSFW_RCC_CONTINUE;
+    }
+
+  u32 num = spl_get_conn_num ();
+  spl_netconn_t *conn;
+  sbr_fd_share *fd_share;
+  u32 i;
+  for (i = 0; i < num; ++i)
+    {
+      conn = conn_array[i];
+      if (ss_is_pid_exist (conn, exit_pid))
+        {
+          fd_share = (sbr_fd_share *) ((char *) conn + SS_NETCONN_SIZE);
+          sbr_recycle_fd_share (fd_share, exit_pid);
+          sbr_handle_recycle_conn (conn, exit_pid, FALSE);
+        }
+    }
+
+  sbr_handle_recycle_conn (conn_array[0], exit_pid, TRUE);
+  return NSFW_RCC_SUSPEND;
+}
+
+REGIST_RECYCLE_OBJ_FUN (NSFW_REC_SBR_SOCKET, sbr_recycle_conn)
diff --git a/stacks/lwip_stack/lwip_src/socket/CMakeLists.txt b/stacks/lwip_stack/lwip_src/socket/CMakeLists.txt
new file mode 100644
index 0000000..e7915d2
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/socket/CMakeLists.txt
@@ -0,0 +1,76 @@
+#########################################################################
+#
+# Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#########################################################################
+
+SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -g -fPIE -pie -fPIC -m64 -mssse3 -std=gnu89")
+SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror -Wshadow -Wfloat-equal -Wformat=2")
+SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fstack-protector -fstack-protector-all")
+SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-z,relro,-z,now -Wl,--disable-new-dtags")
+SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-z,noexecstack -mcmodel=medium")
+
+SET(ADAPT_DIRECTORIES "${PROJECT_SOURCE_DIR}/src/adapt/")
+#SET(DMM_API "${PROJECT_SOURCE_DIR}/src/nSocket/include/")
+
+ADD_DEFINITIONS(-D_GNU_SOURCE -D_FORTIFY_SOURCE=2)
+ADD_DEFINITIONS(-DDPDK_MODULE=0)
+if(WITH_HAL_LIB)
+SET(RTP_CONFIG ${CMAKE_CURRENT_LIST_DIR}/../../../src/include/rtp_config.h)
+else()
+    SET(PAL_H_DIRECTORIES "/usr/include/dpdk/")
+    SET(RTP_CONFIG ${PROJECT_SOURCE_DIR}/../../src/framework/common/base/include/common/common_sys_config.h)
+    INCLUDE_DIRECTORIES(
+        ${PAL_H_DIRECTORIES}
+        ${ADAPT_DIRECTORIES}
+#        ${DMM_API}
+    )
+endif()
+SET(COMPLE_CONFIG ${CMAKE_CURRENT_LIST_DIR}/../../src/include/compile_config.h)
+#SET(MGR_COM ${PROJECT_SOURCE_DIR}/src/framework/ipc/mgr_com/mgr_com.h)
+SET(MGR_COM ${CMAKE_CURRENT_LIST_DIR}/../../src/include/mgr_com.h)
+ADD_DEFINITIONS(-include ${RTP_CONFIG})
+ADD_DEFINITIONS(-include ${COMPLE_CONFIG})
+ADD_DEFINITIONS(-include ${MGR_COM})
+if(WITH_SECUREC_LIB)
+LINK_LIBRARIES(pthread rt securec)
+else()
+LINK_LIBRARIES(pthread rt)
+endif()
+LINK_DIRECTORIES(${LIB_PATH_SHARED} ${LIB_PATH_STATIC})
+INCLUDE_DIRECTORIES(
+        ${CMAKE_CURRENT_LIST_DIR}/../../../../thirdparty/json/json-c-0.12.1/
+        ${CMAKE_CURRENT_LIST_DIR}/../../../../thirdparty/glog/glog-0.3.4/src/
+	${CMAKE_CURRENT_LIST_DIR}/../../src/include/
+#	${PROJECT_SOURCE_DIR}/src/framework/include/
+#	${PROJECT_SOURCE_DIR}/src/framework/common/include/
+#        ${ADAPT_DIRECTORIES}
+#        ${DMM_API}
+)
+
+FILE(GLOB COMMON ../common/*.c)
+FILE(GLOB SOCKET ./*.c)
+ADD_LIBRARY(socket STATIC ${COMMON} ${SOCKET})
+ADD_DEPENDENCIES(socket JSON GLOG DPDK)
+TARGET_INCLUDE_DIRECTORIES(
+	socket
+	PRIVATE
+	../common/
+	./
+	${CMAKE_CURRENT_LIST_DIR}/../../src/sbr/
+	${CMAKE_CURRENT_LIST_DIR}/../../src/include/
+#	${PROJECT_SOURCE_DIR}/src/framework/include/
+#	${PROJECT_SOURCE_DIR}/src/framework/common/include/
+#       ${ADAPT_DIRECTORIES}
+#       ${DMM_API}   
+)
diff --git a/stacks/lwip_stack/lwip_src/socket/stackx_cfg.h b/stacks/lwip_stack/lwip_src/socket/stackx_cfg.h
new file mode 100644
index 0000000..4f4749d
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/socket/stackx_cfg.h
@@ -0,0 +1,54 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#ifndef STACKX_CONTAINER_CFG_H
+#define STACKX_CONTAINER_CFG_H
+#include "types.h"
+
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+extern "C" {
+/* *INDENT-ON* */
+#endif
+
+#define SBR_MAX_CFG_PATH_LEN 256
+#define SBR_MAX_CONTAINER_IP_NUM 1024
+#define SBR_MAX_CFG_FILE_SIZE (1 * 1024 * 1024)
+
+typedef struct
+{
+  u32 ip;
+  u32 mask_len;
+} sbr_container_ip;
+
+typedef struct
+{
+  sbr_container_ip ip_array[SBR_MAX_CONTAINER_IP_NUM];
+  u32 ip_num;
+} sbr_container_ip_group;
+
+extern sbr_container_ip_group g_container_ip_group;
+
+int sbr_init_cfg ();
+int sbr_get_src_ip (u32 dst_ip, u32 * src_ip);
+
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+}
+/* *INDENT-ON* */
+#endif
+
+#endif
diff --git a/stacks/lwip_stack/lwip_src/socket/stackx_container_cfg.c b/stacks/lwip_stack/lwip_src/socket/stackx_container_cfg.c
new file mode 100644
index 0000000..b6580ed
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/socket/stackx_container_cfg.c
@@ -0,0 +1,332 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <arpa/inet.h>
+#include "stackx_cfg.h"
+#include "json.h"
+#include "nstack_log.h"
+#include "spl_def.h"
+#include "nstack_securec.h"
+#include "stackx_ip_addr.h"
+
+sbr_container_ip_group g_container_ip_group;
+
+/*****************************************************************************
+*   Prototype    : sbr_parse_container_ip_json
+*   Description  : parse port json
+*   Input        : char* param
+*   Output       : None
+*   Return Value : static void
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static void
+sbr_parse_container_ip_json (char *param)
+{
+  int retval;
+  struct json_object *obj = json_tokener_parse (param);
+  struct json_object *container_id_obj = NULL;
+  struct json_object *ports_list_obj = NULL;
+  struct json_object *ip_cidr_list_obj = NULL;
+
+  if (!obj)
+    {
+      NSSBR_LOGERR ("json_tokener_parse failed");
+      return;
+    }
+
+  json_object_object_get_ex (obj, "containerID", &container_id_obj);
+  if (!container_id_obj)
+    {
+      NSSBR_LOGERR ("can't get containerID");
+      goto RETURN_ERROR;
+    }
+
+  json_object_object_get_ex (obj, "ports_list", &ports_list_obj);
+  if (ports_list_obj)
+    {
+      int i;
+      int port_num = json_object_array_length (ports_list_obj);
+
+      if (0 == port_num)
+        {
+          NSSBR_LOGERR ("port num is 0");
+          goto RETURN_ERROR;
+        }
+
+      for (i = 0; i < port_num; i++)
+        {
+          struct json_object *port_obj =
+            json_object_array_get_idx (ports_list_obj, i);
+          json_object_object_get_ex (port_obj, "ip_cidr", &ip_cidr_list_obj);
+          if (ip_cidr_list_obj)
+            {
+              int j;
+              int ip_cidr_num = json_object_array_length (ip_cidr_list_obj);
+              for (j = 0; j < ip_cidr_num; ++j)
+                {
+                  struct json_object *ip_cidr_obj =
+                    json_object_array_get_idx (ip_cidr_list_obj, j);
+                  if (ip_cidr_obj)
+                    {
+                      char tmp[32] = { 0 };
+                      const char *ip_cidr =
+                        json_object_get_string (ip_cidr_obj);
+                      if ((NULL == ip_cidr) || (ip_cidr[0] == 0))
+                        {
+                          NSSBR_LOGERR ("ip is empty");
+                          goto RETURN_ERROR;
+                        }
+
+                      const char *sub = strstr (ip_cidr, "/");
+                      if ((NULL == sub)
+                          || (sizeof (tmp) - 1 <
+                              (unsigned int) (sub - ip_cidr))
+                          || (strlen (sub) > sizeof (tmp) - 1))
+                        {
+                          NSSBR_LOGERR ("ip format is not ok");
+                          goto RETURN_ERROR;
+                        }
+
+                      retval =
+                        STRNCPY_S (tmp, sizeof (tmp), ip_cidr,
+                                   (size_t) (sub - ip_cidr));
+                      if (EOK != retval)
+                        {
+                          NSSBR_LOGERR ("STRNCPY_S failed]ret=%d", retval);
+                          goto RETURN_ERROR;
+                        }
+
+                      struct in_addr addr;
+                      retval =
+                        MEMSET_S (&addr, sizeof (addr), 0, sizeof (addr));
+                      if (EOK != retval)
+                        {
+                          NSSBR_LOGERR ("MEMSET_S failed]ret=%d", retval);
+                          goto RETURN_ERROR;
+                        }
+
+                      retval = spl_inet_aton (tmp, &addr);
+                      if (0 == retval)
+                        {
+                          NSSBR_LOGERR ("spl_inet_aton failed]ret=%d",
+                                        retval);
+                          goto RETURN_ERROR;
+                        }
+
+                      g_container_ip_group.
+                        ip_array[g_container_ip_group.ip_num].ip =
+                        addr.s_addr;
+                      int mask_len = atoi (sub + 1);
+                      if ((mask_len <= 0) || (mask_len > 32))
+                        {
+                          NSSBR_LOGERR ("mask len is not ok");
+                          goto RETURN_ERROR;
+                        }
+
+                      g_container_ip_group.
+                        ip_array[g_container_ip_group.ip_num].mask_len =
+                        (u32) mask_len;
+                      g_container_ip_group.ip_num++;
+
+                      if (g_container_ip_group.ip_num >=
+                          SBR_MAX_CONTAINER_IP_NUM)
+                        {
+                          NSSBR_LOGWAR ("container ip num is full]ip_num=%u",
+                                        g_container_ip_group.ip_num);
+                          goto RETURN_OK;
+                        }
+                    }
+                }
+            }
+        }
+    }
+  else
+    {
+      NSSBR_LOGERR ("can't get ports_list");
+      goto RETURN_ERROR;
+    }
+
+RETURN_OK:
+  json_object_put (obj);
+  NSSBR_LOGINF ("container ip num is %u", g_container_ip_group.ip_num);
+  u32 idx;
+  for (idx = 0; idx < g_container_ip_group.ip_num; ++idx)
+    {
+      NSSBR_LOGDBG ("container ip=0x%08x",
+                    g_container_ip_group.ip_array[idx].ip);
+    }
+  return;
+
+RETURN_ERROR:
+  json_object_put (obj);
+  if (MEMSET_S
+      (&g_container_ip_group, sizeof (sbr_container_ip_group), 0,
+       sizeof (sbr_container_ip_group)) != EOK)
+    {
+      NSSBR_LOGERR ("MEMSET_S failed");
+    }
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_get_cfg_path
+*   Description  : get cfg path
+*   Input        : None
+*   Output       : None
+*   Return Value : NSTACK_STATIC const char*
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+NSTACK_STATIC const char *
+sbr_get_cfg_path ()
+{
+  static char cfg_file[SBR_MAX_CFG_PATH_LEN] = "/canal/output/portinfo.json";
+  return cfg_file;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_init_cfg
+*   Description  : init cfg from file
+*   Input        : None
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_init_cfg ()
+{
+  int ret;
+  off_t file_len = 0;
+  off_t buff_len = 0;
+  char *buff = NULL;
+  const char *cfg_file = sbr_get_cfg_path ();   /* no need check ret */
+
+  int fp = open (cfg_file, O_RDONLY);
+
+  if (fp < 0)
+    {
+      NSSBR_LOGWAR ("failed to open file]file name=%s", cfg_file);
+      goto RETURN_ERROR;
+    }
+
+  file_len = lseek (fp, 0, SEEK_END);
+  if (file_len <= 0)
+    {
+      NSSBR_LOGWAR ("failed to get file len]file name=%s", cfg_file);
+      goto RETURN_ERROR;
+    }
+
+  if (file_len > SBR_MAX_CFG_FILE_SIZE)
+    {
+      NSSBR_LOGWAR
+        ("file len is too big]file len=%d, max len=%d, file name=%s",
+         file_len, SBR_MAX_CFG_FILE_SIZE, cfg_file);
+      goto RETURN_ERROR;
+    }
+
+  ret = lseek (fp, 0, SEEK_SET);
+  if (ret < 0)
+    {
+      NSSBR_LOGWAR ("seek to start failed]file name=%s", cfg_file);
+      goto RETURN_ERROR;
+    }
+
+  buff_len = file_len + 1;
+  buff = (char *) malloc (buff_len);
+  if (!buff)
+    {
+      NSSBR_LOGWAR ("malloc buff failed]buff_len=%d", buff_len);
+      goto RETURN_ERROR;
+    }
+
+  ret = MEMSET_S (buff, buff_len, 0, buff_len);
+  if (EOK != ret)
+    {
+      NSSBR_LOGWAR ("MEMSET_S failed]ret=%d.", ret);
+      goto RETURN_ERROR;
+    }
+
+  ret = read (fp, buff, buff_len - 1);
+  if (ret <= 0)
+    {
+      NSSBR_LOGWAR ("read failed]ret=%d", ret);
+      goto RETURN_ERROR;
+    }
+
+  sbr_parse_container_ip_json (buff);
+  close (fp);
+  free (buff);
+  return 0;
+
+RETURN_ERROR:
+  if (fp >= 0)
+    {
+      close (fp);
+    }
+
+  if (buff)
+    {
+      free (buff);
+    }
+
+  return -1;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_get_src_ip
+*   Description  : get src ip from cfg
+*   Input        : u32 dst_ip
+*                  u32* src_ip
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_get_src_ip (u32 dst_ip, u32 * src_ip)
+{
+  if (!src_ip)
+    {
+      NSSBR_LOGERR ("src_ip is NULL");
+      return -1;
+    }
+
+  u32 i;
+  for (i = 0; i < g_container_ip_group.ip_num; ++i)
+    {
+      unsigned int mask = ~0;
+      mask = (mask << (32 - g_container_ip_group.ip_array[i].mask_len));
+      mask = htonl (mask);
+      if ((dst_ip & mask) == (g_container_ip_group.ip_array[i].ip & mask))
+        {
+          *src_ip = g_container_ip_group.ip_array[i].ip;
+          NSSBR_LOGDBG ("find src ip]container_ip=0x%08x,dest_ip=0x%08x",
+                        *src_ip, dst_ip);
+          return 0;
+        }
+    }
+
+  NSSBR_LOGDBG ("can't find src ip]dest_ip=0x%08x", dst_ip);
+  return -1;
+}
diff --git a/stacks/lwip_stack/lwip_src/socket/stackx_epoll_api.c b/stacks/lwip_stack/lwip_src/socket/stackx_epoll_api.c
new file mode 100644
index 0000000..0223ac9
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/socket/stackx_epoll_api.c
@@ -0,0 +1,150 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include "stackx_epoll_api.h"
+#include "stackx_spl_share.h"
+#include "common_pal_bitwide_adjust.h"
+#include "nstack_dmm_adpt.h"
+#include "nstack_dmm_api.h"
+
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+extern "C" {
+/* *INDENT-ON* */
+#endif /* _cplusplus */
+
+void
+epoll_triggle_event_from_api (sbr_socket_t * sock, int op)
+{
+  struct spl_netconn *conn = sbr_get_conn (sock);
+  void *epInfo = ADDR_SHTOL (conn->epInfo);
+  //NSPOL_LOGDBG(SOCKETS_DEBUG, "enter]fd=%d,op=%d", sock, op);
+  switch (op)
+    {
+    case EPOLL_API_OP_RECV:
+      break;
+    case EPOLL_API_OP_SEND:
+      if (conn->epoll_flag && epInfo)
+        {
+          nstack_event_callback (epInfo, EPOLLOUT);
+        }
+      break;
+    case EPOLL_API_OP_STACK_RECV:
+      if (conn->epoll_flag && epInfo)
+        {
+          nstack_event_callback (epInfo, EPOLLIN);
+        }
+      break;
+    default:
+      break;
+    }
+  return;
+}
+
+/*
+ *    This function will be registed to application
+ *    The context will be in the application
+ */
+unsigned int
+stackx_eventpoll_triggle (sbr_socket_t * sock, int triggle_ops,
+                          struct epoll_event *pevent, void *pdata)
+{
+  struct spl_netconn *conn = sbr_get_conn (sock);
+  unsigned int events = 0;
+  if (!conn)
+    {
+      NSPOL_LOGINF (SOCKETS_DEBUG, "get socket failed]fd=%d", sock->fd);
+      return -1;
+    }
+
+  NSPOL_LOGINF (SOCKETS_DEBUG,
+                "]fd=%d,triggle_ops=%d conn=%p event:%d pdata=%p", sock->fd,
+                triggle_ops, conn, pevent->events, pdata);
+  /*
+   *  sock->epoll_flag must be set before sock->rcvevent check.
+   *  Consider this Scenario : 1) network stack has got one packet, but event_callback not called yet
+   *                           2) Do epoll ctl add, then stackx triggle will check event, it will get 0
+   *                           3) network stack call event_callback , it will check epoll_flag
+   *  So, if epoll_flag not set before sock->rcvent check, neither of network stack and stackx triggle
+   *  will add this event to epoll. because : for network stack, event_callback check epoll_flag fail
+   *  for stackx triggle, event check fail.
+   */
+  if (nstack_ep_triggle_add == triggle_ops)
+    {
+      /*log info */
+      conn->epInfo = pdata;
+      __sync_fetch_and_add (&conn->epoll_flag, 1);
+    }
+
+  if ((pevent->events & EPOLLIN)
+      &&
+      (!((conn->rcvevent == 0) && (sbr_get_fd_share (sock)->lastdata == 0)
+         && (sbr_get_fd_share (sock)->lastoffset == 0))))
+    events |= EPOLLIN;
+  if ((pevent->events & EPOLLOUT) && conn->sendevent)
+    events |= EPOLLOUT;
+  if (conn->errevent)
+    events |= pevent->events & (conn->errevent);
+
+  switch (triggle_ops)
+    {
+    case nstack_ep_triggle_add:
+      break;
+    case nstack_ep_triggle_mod:
+      break;
+    case nstack_ep_triggle_del:
+      if (conn->epoll_flag > 0)
+        {
+          __sync_fetch_and_sub (&conn->epoll_flag, 1);
+        }
+      events = 0;
+      break;
+    default:
+      return -1;
+    }
+  return events;
+}
+
+/*
+ *    This function will be registed to application
+ *    The context will be in the application
+ *    RETURN VALUE : Event exists in current protocol
+ */
+unsigned int
+stackx_eventpoll_getEvt (sbr_socket_t * sock, unsigned int events)
+{
+  struct spl_netconn *conn = sbr_get_conn (sock);
+  unsigned int tevent = 0;
+  if ((events & EPOLLIN)
+      &&
+      (!((conn->rcvevent == 0) && (sbr_get_fd_share (sock)->lastdata == 0)
+         && (sbr_get_fd_share (sock)->lastoffset == 0))))
+    {
+      tevent |= EPOLLIN;
+    }
+
+  if ((events & EPOLLOUT) && conn->sendevent)
+    {
+      tevent |= EPOLLOUT;
+    }
+  return tevent;
+}
+
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+}
+/* *INDENT-ON* */
+#endif /* _cplusplus */
diff --git a/stacks/lwip_stack/lwip_src/socket/stackx_epoll_api.h b/stacks/lwip_stack/lwip_src/socket/stackx_epoll_api.h
new file mode 100644
index 0000000..59a1d21
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/socket/stackx_epoll_api.h
@@ -0,0 +1,48 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#ifndef STACKX_EPOLL_API_H
+#define STACKX_EPOLL_API_H
+#include "stackx_socket.h"
+
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+extern "C" {
+/* *INDENT-ON* */
+#endif
+
+typedef enum
+{
+  EPOLL_API_OP_RECV,
+  EPOLL_API_OP_SEND,
+  EPOLL_API_OP_STACK_RECV
+} EPOLL_TRIGGLE_EVENT_API_OPS_T;
+
+extern void epoll_triggle_event_from_api (sbr_socket_t * sock, int op);
+extern unsigned int stackx_eventpoll_getEvt (sbr_socket_t * sock,
+                                             unsigned int events);
+extern unsigned int stackx_eventpoll_triggle (sbr_socket_t * sock,
+                                              int triggle_ops,
+                                              struct epoll_event *pevent,
+                                              void *pdata);
+
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+}
+/* *INDENT-ON* */
+#endif
+
+#endif
diff --git a/stacks/lwip_stack/lwip_src/socket/stackx_event.c b/stacks/lwip_stack/lwip_src/socket/stackx_event.c
new file mode 100644
index 0000000..93e47a3
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/socket/stackx_event.c
@@ -0,0 +1,29 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include "stackx_spl_share.h"
+#include "common_pal_bitwide_adjust.h"
+#include "stackx_event.h"
+#include <netinet/in.h>
+
+#define FREE_FD_SET(readfd, writefd, exceptfd)  {\
+    if(readfd)\
+        free(readfd);\
+    if(writefd)\
+        free(writefd);\
+    if(exceptfd)\
+        free(exceptfd);\
+}
diff --git a/stacks/lwip_stack/lwip_src/socket/stackx_event.h b/stacks/lwip_stack/lwip_src/socket/stackx_event.h
new file mode 100644
index 0000000..f95b370
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/socket/stackx_event.h
@@ -0,0 +1,46 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#ifndef  __STACKX_EVENT_H__
+#define __STACKX_EVENT_H__
+
+#include <sys/types.h>
+#include <sys/time.h>
+#include <stdio.h>
+#include <unistd.h>
+#include "stackx_socket.h"
+#include "nstack_securec.h"
+#include "sbr_res_mgr.h"
+#include "sbr_index_ring.h"
+#define NSTACK_SETSIZE  SBR_MAX_FD_NUM
+
+typedef struct
+{
+  unsigned char fds_bits[(NSTACK_SETSIZE + 7) / 8];
+} __attribute__ ((packed)) nstack_fd_set;
+
+#define NSTACK_FD_SET(n, p)		((p)->fds_bits[(n)/8]|=1U<<((n)&0x07))
+#define NSTACK_FD_ISSET(n,p)	(((p)->fds_bits[(n)/8]&(1U<<((n)&0x07)))?1:0)
+#define NSTACK_FD_CLR(n,p)		((p)->fds_bits[(n)/8]&=~(1U<<((n)&0x07)))
+#define NSTACK_FD_ZERO(p)		(MEMSET_S((void *)(p), sizeof(*(p)),0,sizeof(*(p))))
+#define NSTACK_FD_OR(p1 ,p2)     {\
+    int i;\
+    for(i = 0; i < (NSTACK_SETSIZE+7)/8; i++){\
+        (p1)->fds_bits[i] |= (p2)->fds_bits[i];\
+    }\
+}
+
+#endif /* __STACKX_EVENT_H__ */
diff --git a/stacks/lwip_stack/lwip_src/socket/stackx_macro.h b/stacks/lwip_stack/lwip_src/socket/stackx_macro.h
new file mode 100644
index 0000000..96a0b91
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/socket/stackx_macro.h
@@ -0,0 +1,24 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#ifndef STACKX_MACRO_H
+#define STACKX_MACRO_H
+
+#ifndef SBR_USE_LOCK
+#define SBR_USE_LOCK
+#endif
+
+#endif
diff --git a/stacks/lwip_stack/lwip_src/socket/stackx_msg_handler.c b/stacks/lwip_stack/lwip_src/socket/stackx_msg_handler.c
new file mode 100644
index 0000000..6fcde44
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/socket/stackx_msg_handler.c
@@ -0,0 +1,775 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include "stackx_msg_handler.h"
+#include "stackx_spl_share.h"
+#include "stackx_spl_msg.h"
+#include "nsfw_msg_api.h"
+#include "common_pal_bitwide_adjust.h"
+#include "stackx_res_mgr.h"
+#include "stackx_prot_com.h"
+#include "nstack_securec.h"
+//#include "stackx_dfx_api.h"
+
+#define SBR_MSG_MALLOC(sk) msg_malloc(ss_get_msg_pool(sbr_get_conn(sk)))
+#define SBR_MSG_FREE(msg) msg_free(msg)
+
+#define SBR_MSG_POST(msg, ring) \
+    do { \
+        if (MSG_ASYN_POST == msg->param.op_type)\
+        {\
+            if (msg_post(msg, ring) < 0)\
+            {\
+                SBR_MSG_FREE(msg); \
+                NSSBR_LOGERR("msg_post failed]major=%u,minor=%u,type=%u", \
+                     msg->param.major_type, msg->param.minor_type, msg->param.op_type); \
+            }\
+        } \
+        else \
+        {\
+            if (msg_post(msg, ring) < 0)\
+            {\
+                 msg->param.err = ECONNABORTED; \
+                 NSSBR_LOGERR("msg_post_with_ref failed]major=%u,minor=%u,type=%u", \
+                     msg->param.major_type, msg->param.minor_type, msg->param.op_type); \
+            }\
+        } \
+    } while (0)
+
+#define SBR_MSG_POST_RET(msg, ring, ret) \
+    do { \
+        if (MSG_ASYN_POST == msg->param.op_type)\
+        {\
+            if ((ret = msg_post(msg, ring)) < 0)\
+            {\
+                SBR_MSG_FREE(msg); \
+                NSSBR_LOGERR("msg_post failed]major=%u,minor=%u,type=%u", \
+                     msg->param.major_type, msg->param.minor_type, msg->param.op_type); \
+            }\
+        } \
+        else \
+        {\
+            if ((ret = msg_post(msg, ring)) < 0)\
+            {\
+                 msg->param.err = ECONNABORTED; \
+                 NSSBR_LOGERR("msg_post_with_ref failed]major=%u,minor=%u,type=%u", \
+                     msg->param.major_type, msg->param.minor_type, msg->param.op_type); \
+            }\
+        } \
+    } while (0)
+
+NSTACK_STATIC inline void
+_sbr_construct_msg (data_com_msg * m, u16 major_type, u16 minor_type,
+                    u16 type, sbr_socket_t * sk)
+{
+  m->param.module_type = MSG_MODULE_SBR;
+  m->param.major_type = major_type;
+  m->param.minor_type = minor_type;
+  m->param.err = 0;
+  m->param.op_type = type;
+  sys_sem_init (&m->param.op_completed);
+  m->param.receiver = ss_get_recv_obj (sbr_get_conn (sk));
+  m->param.comm_receiver = ss_get_comm_private_data (sbr_get_conn (sk));
+  m->param.extend_member_bit = 0;
+}
+
+#define sbr_construct_msg(m, major_type, minor_type, type, sk) { \
+        _sbr_construct_msg(m, major_type, minor_type, type, sk); \
+        NSSBR_LOGINF("fd=%d,conn=%p,private_data=%p", sk->fd, sbr_get_conn(sk), ss_get_private_data(sbr_get_conn(sk))); \
+    }
+
+#define sbr_construct_msg_dbg(m, major_type, minor_type, type, sk) { \
+        _sbr_construct_msg(m, major_type, minor_type, type, sk); \
+        NSSBR_LOGDBG("fd=%d,conn=%p,private_data=%p", sk->fd, sbr_get_conn(sk), ss_get_private_data(sbr_get_conn(sk))); \
+    }
+
+/*****************************************************************************
+*   Prototype    : sbr_attach_msg
+*   Description  : use buf's msg first
+*   Input        : sbr_socket_t * sk
+*                  struct pbuf* buf
+*   Output       : None
+*   Return Value : static inline data_com_msg*
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline data_com_msg *
+sbr_attach_msg (sbr_socket_t * sk, struct spl_pbuf *buf)
+{
+  data_com_msg *m = NULL;
+  if (!sk)
+    {
+      return m;
+    }
+
+  if (buf && buf->msg)
+    {
+      m = (data_com_msg *) ADDR_SHTOL (buf->msg);
+    }
+  else
+    {
+      m = SBR_MSG_MALLOC (sk);
+    }
+
+  return m;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_handle_socket
+*   Description  : create socket
+*   Input        : sbr_socket_t * sk
+*                  netconn_type_t type
+*                  u8 proto
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_handle_socket (sbr_socket_t * sk, spl_netconn_type_t type, u8 proto)
+{
+  data_com_msg *m = SBR_MSG_MALLOC (sk);
+
+  if (!m)
+    {
+      NSSBR_LOGERR ("malloc msg failed]fd=%d", sk->fd);
+      sbr_set_sk_errno (sk, ENOMEM);
+      return -1;
+    }
+
+  sbr_construct_msg (m, SPL_TCPIP_NEW_MSG_API, SPL_API_DO_NEWCONN,
+                     MSG_SYN_POST, sk);
+  msg_new_netconn *p = (msg_new_netconn *) m->buffer;
+  p->conn = (spl_netconn_t *) ADDR_LTOSH (sbr_get_conn (sk));
+  p->type = type;
+  p->proto = proto;
+  p->socket = sk->fd;
+  p->extend_member_bit = 0;
+  SBR_MSG_POST (m, sbr_get_msg_box (sk));
+  int err = sbr_spl_err_to_errno (m->param.err);
+  SBR_MSG_FREE (m);
+  if (err != 0)
+    {
+      NSSBR_LOGERR ("handle socket failed]fd=%d,type=%d,proto=%u,err=%d",
+                    sk->fd, type, proto, err);
+      sbr_set_sk_errno (sk, err);
+      return -1;
+    }
+
+  return 0;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_handle_bind
+*   Description  : bind
+*   Input        : sbr_socket_t * sk
+*                  spl_ip_addr_t * addr
+*                  u16 port
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_handle_bind (sbr_socket_t * sk, spl_ip_addr_t * addr, u16 port)
+{
+  data_com_msg *m = SBR_MSG_MALLOC (sk);
+
+  if (!m)
+    {
+      NSSBR_LOGERR ("malloc msg failed]fd=%d", sk->fd);
+      sbr_set_sk_errno (sk, ENOMEM);
+      return -1;
+    }
+
+  sbr_construct_msg (m, SPL_TCPIP_NEW_MSG_API, SPL_API_DO_BIND, MSG_SYN_POST,
+                     sk);
+  msg_bind *p = (msg_bind *) m->buffer;
+  p->ipaddr = *addr;
+  p->port = port;
+  p->extend_member_bit = 0;
+  SBR_MSG_POST (m, sbr_get_msg_box (sk));
+  int err = sbr_spl_err_to_errno (m->param.err);
+  SBR_MSG_FREE (m);
+  if (err != 0)
+    {
+      NSSBR_LOGERR ("handle bind failed]fd=%d,err=%d", sk->fd, err);
+      sbr_set_sk_errno (sk, err);
+      return -1;
+    }
+
+  return 0;
+}
+
+int
+sbr_handle_listen (sbr_socket_t * sk, int backlog)
+{
+  data_com_msg *m = SBR_MSG_MALLOC (sk);
+
+  if (!m)
+    {
+      sbr_set_sk_errno (sk, ENOMEM);
+      return -1;
+    }
+
+  sbr_construct_msg (m, SPL_TCPIP_NEW_MSG_API, SPL_API_DO_LISTEN,
+                     MSG_SYN_POST, sk);
+  msg_listen *p = (msg_listen *) m->buffer;
+  p->conn_pool = sbr_get_conn_pool ();
+  p->backlog = backlog;
+  p->extend_member_bit = 0;
+  SBR_MSG_POST (m, sbr_get_msg_box (sk));
+  int err = sbr_spl_err_to_errno (m->param.err);
+  SBR_MSG_FREE (m);
+  if (err != 0)
+    {
+      NSSBR_LOGERR ("handle listen failed]fd=%d,err=%d", sk->fd, err);
+      sbr_set_sk_errno (sk, err);
+      return -1;
+    }
+
+  return 0;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_handle_connect
+*   Description  : connect
+*   Input        : sbr_socket_t * sk
+*                  spl_ip_addr_t * addr
+*                  u16 port
+*                  spl_ip_addr_t* local_ip
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_handle_connect (sbr_socket_t * sk, spl_ip_addr_t * addr, u16 port,
+                    spl_ip_addr_t * local_ip)
+{
+  data_com_msg *m = SBR_MSG_MALLOC (sk);
+
+  if (!m)
+    {
+      NSSBR_LOGERR ("malloc msg failed]fd=%d", sk->fd);
+      sbr_set_sk_errno (sk, ENOMEM);
+      return -1;
+    }
+
+  sbr_construct_msg (m, SPL_TCPIP_NEW_MSG_API, SPL_API_DO_CONNECT,
+                     MSG_SYN_POST, sk);
+  msg_connect *p = (msg_connect *) m->buffer;
+  p->local_ip = *local_ip;
+  p->ipaddr = *addr;
+  p->port = port;
+  p->extend_member_bit = 0;
+  SBR_MSG_POST (m, sbr_get_msg_box (sk));
+  int err = sbr_spl_err_to_errno (m->param.err);
+  SBR_MSG_FREE (m);
+  if (err != 0)
+    {
+      NSSBR_LOGERR ("handle connect failed]fd=%d,err=%d", sk->fd, err);
+      sbr_set_sk_errno (sk, err);
+      return -1;
+    }
+
+  return 0;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_handle_get_name
+*   Description  : get name
+*   Input        : sbr_socket_t * sk
+*                  struct sockaddr * name
+*                  socklen_t * namelen
+*                  u8 cmd
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_handle_get_name (sbr_socket_t * sk, struct sockaddr *name,
+                     socklen_t * namelen, u8 cmd)
+{
+  data_com_msg *m = SBR_MSG_MALLOC (sk);
+
+  if (!m)
+    {
+      NSSBR_LOGERR ("malloc msg failed]fd=%d", sk->fd);
+      sbr_set_sk_errno (sk, ENOMEM);
+      return -1;
+    }
+
+  sbr_construct_msg (m, SPL_TCPIP_NEW_MSG_API, SPL_API_DO_GETSOCK_NAME,
+                     MSG_SYN_POST, sk);
+  msg_getaddrname *p = (msg_getaddrname *) m->buffer;
+  p->cmd = cmd;
+  p->extend_member_bit = 0;
+  SBR_MSG_POST (m, sbr_get_msg_box (sk));
+  int err = sbr_spl_err_to_errno (m->param.err);
+  if (err != 0)
+    {
+      NSSBR_LOGERR ("handle get name failed]fd=%d,err=%d", sk->fd, err);
+      goto error;
+    }
+  else
+    {
+      if (*namelen > sizeof (p->sock_addr))
+        {
+          *namelen = sizeof (p->sock_addr);
+        }
+
+      int ret = MEMCPY_S (name, *namelen, &p->sock_addr, *namelen);
+      if (0 != ret)
+        {
+          NSSBR_LOGERR ("MEMCPY_S failed]fd=%d,ret=%d", sk->fd, ret);
+          err = EINVAL;
+          goto error;
+        }
+
+      *namelen = sizeof (p->sock_addr);
+    }
+
+  SBR_MSG_FREE (m);
+  return 0;
+
+error:
+  sbr_set_sk_errno (sk, err);
+  SBR_MSG_FREE (m);
+  return -1;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_handle_setsockopt
+*   Description  : msg box will changed in IP_TOS
+*   Input        : sbr_socket_t * sk
+*                  int level
+*                  int optname
+*                  const void *optval
+*                  socklen_t optlen
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_handle_setsockopt (sbr_socket_t * sk, int level, int optname,
+                       const void *optval, socklen_t optlen)
+{
+  data_com_msg *m = SBR_MSG_MALLOC (sk);
+
+  if (!m)
+    {
+      NSSBR_LOGERR ("malloc msg failed]fd=%d", sk->fd);
+      sbr_set_sk_errno (sk, ENOMEM);
+      return -1;
+    }
+
+  sbr_construct_msg (m, SPL_TCPIP_NEW_MSG_API, SPL_API_DO_SET_SOCK_OPT,
+                     MSG_SYN_POST, sk);
+  msg_setgetsockopt *p = (msg_setgetsockopt *) m->buffer;
+  p->extend_member_bit = 0;
+  p->level = level;
+  p->optname = optname;
+  p->msg_box = NULL;            /* init the value to avoid unexpected consequences */
+
+  if (optlen > sizeof (p->optval))
+    {
+      optlen = sizeof (p->optval);
+    }
+
+  p->optlen = optlen;
+  int err;
+  int ret = MEMCPY_S (&p->optval, sizeof (p->optval), optval, optlen);
+  if (0 != ret)
+    {
+      NSSBR_LOGERR ("MEMCPY_S failed]fd=%d,ret=%d", sk->fd, ret);
+      err = EINVAL;
+      goto error;
+    }
+
+  SBR_MSG_POST (m, sbr_get_msg_box (sk));
+  err = sbr_spl_err_to_errno (m->param.err);
+  if (err != 0)
+    {
+      NSSBR_LOGERR ("handle setsockopt failed]fd=%d,err=%d", sk->fd, err);
+      goto error;
+    }
+
+  if (IPPROTO_IP == level && IP_TOS == optname && p->msg_box)
+    {
+      ss_set_msg_box (sbr_get_conn (sk),
+                      (mring_handle) ADDR_SHTOL (p->msg_box));
+    }
+
+  SBR_MSG_FREE (m);
+  return 0;
+
+error:
+  sbr_set_sk_errno (sk, err);
+  SBR_MSG_FREE (m);
+  return -1;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_handle_getsockopt
+*   Description  : get sockopt
+*   Input        : sbr_socket_t * sk
+*                  int level
+*                  int optname
+*                  void *optval
+*                  socklen_t* optlen
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_handle_getsockopt (sbr_socket_t * sk, int level, int optname,
+                       void *optval, socklen_t * optlen)
+{
+  data_com_msg *m = SBR_MSG_MALLOC (sk);
+
+  if (!m)
+    {
+      NSSBR_LOGERR ("malloc msg failed]fd=%d", sk->fd);
+      sbr_set_sk_errno (sk, ENOMEM);
+      return -1;
+    }
+
+  sbr_construct_msg (m, SPL_TCPIP_NEW_MSG_API, SPL_API_DO_GET_SOCK_OPT,
+                     MSG_SYN_POST, sk);
+  msg_setgetsockopt *p = (msg_setgetsockopt *) m->buffer;
+  p->extend_member_bit = 0;
+  p->level = level;
+  p->optname = optname;
+
+  if (*optlen > sizeof (p->optval))
+    {
+      *optlen = sizeof (p->optval);
+    }
+
+  p->optlen = *optlen;
+  int err;
+  int ret = MEMCPY_S (&p->optval, sizeof (p->optval), optval, *optlen);
+  if (0 != ret)
+    {
+      NSSBR_LOGERR ("MEMCPY_S failed]fd=%d,ret=%d", sk->fd, ret);
+      err = EINVAL;
+      goto error;
+    }
+
+  SBR_MSG_POST (m, sbr_get_msg_box (sk));
+
+  err = sbr_spl_err_to_errno (m->param.err);
+  if (err != 0)
+    {
+      NSSBR_LOGERR ("handle getsockopt failed]fd=%d,err=%d", sk->fd, err);
+      goto error;
+    }
+
+  ret = MEMCPY_S (optval, *optlen, &p->optval.int_optval, *optlen);
+  if (0 != ret)
+    {
+      NSSBR_LOGERR ("MEMCPY_S failed]fd=%d,ret=%d", sk->fd, ret);
+      err = EINVAL;
+      goto error;
+    }
+
+  SBR_MSG_FREE (m);
+  return 0;
+
+error:
+  sbr_set_sk_errno (sk, err);
+  SBR_MSG_FREE (m);
+  return -1;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_handle_close
+*   Description  : need care msg_box_ref,make sure to finalize this message
+*   Input        : sbr_socket_t * sk
+*                  u8 how
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_handle_close (sbr_socket_t * sk, u8 how)
+{
+  data_com_msg *m = sbr_attach_msg (sk,
+                                    (struct spl_pbuf *)
+                                    ADDR_SHTOL (sbr_get_fd_share
+                                                (sk)->recoder.head));
+
+  if (!m)
+    {
+      NSSBR_LOGERR ("attach msg failed]fd=%d", sk->fd);
+      sbr_set_sk_errno (sk, ENOMEM);
+      return -1;
+    }
+
+  sbr_construct_msg (m, SPL_TCPIP_NEW_MSG_API, SPL_API_DO_DELCON,
+                     MSG_ASYN_POST, sk);
+  msg_delete_netconn *p = (msg_delete_netconn *) m->buffer;
+  p->extend_member_bit = 0;
+  p->buf = NULL;
+  p->time_started = sys_now ();
+  p->shut = how;
+  p->pid = get_sys_pid ();
+  p->conn = (spl_netconn_t *) ADDR_LTOSH (sbr_get_conn (sk));
+  p->notify_omc = FALSE;
+  p->msg_box_ref = SPL_MSG_BOX_NUM;
+
+  /* to ensure that the last deal with SPL_API_DO_DELCON message */
+  int i;
+  for (i = 0; i < SPL_MSG_BOX_NUM; ++i)
+    {
+      SBR_MSG_POST (m,
+                    ss_get_instance_msg_box (ss_get_bind_thread_index
+                                             (sbr_get_conn (sk)), i));
+    }
+
+  return 0;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_handle_udp_send
+*   Description  : udp send
+*   Input        : sbr_socket_t * sk
+*                  struct netbuf *buf
+*                  spl_ip_addr_t* local_ip
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_handle_udp_send (sbr_socket_t * sk, struct spl_netbuf *buf,
+                     spl_ip_addr_t * local_ip)
+{
+  data_com_msg *m =
+    sbr_attach_msg (sk, (struct spl_pbuf *) ADDR_SHTOL (buf->p));
+
+  if (!m)
+    {
+      NSSBR_LOGERR ("attach msg failed]fd=%d", sk->fd);
+      sbr_set_sk_io_errno (sk, ENOMEM);
+      return -1;
+    }
+
+  sbr_construct_msg_dbg (m, SPL_TCPIP_NEW_MSG_API, SPL_API_DO_SEND,
+                         MSG_ASYN_POST, sk);
+  msg_send_buf *p = (msg_send_buf *) m->buffer;
+  p->local_ip.addr = local_ip->addr;
+  int ret = MEMCPY_S (&p->addr, sizeof (spl_ip_addr_t), &buf->addr,
+                      sizeof (spl_ip_addr_t));
+  if (ret != 0)
+    {
+      NSSBR_LOGERR ("MEMCPY_S failed]fd=%d,ret=%d", sk->fd, ret);
+      sbr_set_sk_io_errno (sk, EINVAL);
+      SBR_MSG_FREE (m);
+      return -1;
+    }
+
+  p->p = buf->p;
+  p->port = buf->port;
+  p->extend_member_bit = 0;
+  SBR_MSG_POST_RET (m, sbr_get_msg_box (sk), ret);
+
+  if (0 == ret)
+    {
+      return 0;
+    }
+  else
+    {
+      NSSBR_LOGERR ("post msg failed]fd=%d", sk->fd);
+      sbr_set_sk_io_errno (sk, EAGAIN);
+      return -1;
+    }
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_handle_free_recv_buf
+*   Description  : free recv buf,can't free buf in app
+*   Input        : sbr_socket_t * sk
+*   Output       : None
+*   Return Value : void
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+void
+sbr_handle_free_recv_buf (sbr_socket_t * sk)
+{
+  data_com_msg *m = sbr_attach_msg (sk,
+                                    (struct spl_pbuf *)
+                                    ADDR_SHTOL (sbr_get_fd_share
+                                                (sk)->recoder.head));
+
+  if (!m)
+    {
+      NSSBR_LOGERR ("attach msg failed]fd=%d", sk->fd);
+      return;
+    }
+
+  sbr_construct_msg_dbg (m, SPL_TCPIP_NEW_MSG_API, SPL_API_DO_PBUF_FREE,
+                         MSG_ASYN_POST, sk);
+  msg_free_buf *p = (msg_free_buf *) m->buffer;
+  p->extend_member_bit = 0;
+  p->buf = sbr_get_fd_share (sk)->recoder.head;
+  sbr_get_fd_share (sk)->recoder.head = NULL;
+  sbr_get_fd_share (sk)->recoder.tail = NULL;
+  sbr_get_fd_share (sk)->recoder.totalLen = 0;
+  SBR_MSG_POST (m, sbr_get_msg_box (sk));
+}
+
+void
+sbr_handle_free_send_buf (sbr_socket_t * sk, struct spl_pbuf *buf)
+{
+  if (buf != NULL)
+    {
+      data_com_msg *m = sbr_attach_msg (sk, buf);
+
+      if (!m)
+        {
+          NSSBR_LOGERR ("attach msg failed]fd=%d", sk->fd);
+          return;
+        }
+
+      sbr_construct_msg_dbg (m, SPL_TCPIP_NEW_MSG_API, SPL_API_DO_PBUF_FREE,
+                             MSG_ASYN_POST, sk);
+      msg_free_buf *p = (msg_free_buf *) m->buffer;
+      p->extend_member_bit = 0;
+      p->buf = (struct spl_pbuf *) ADDR_LTOSH (buf);
+      SBR_MSG_POST (m, sbr_get_msg_box (sk));
+    }
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_handle_shutdown
+*   Description  : shut down
+*   Input        : sbr_socket_t * sk
+*                  u8 how
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_handle_shutdown (sbr_socket_t * sk, u8 how)
+{
+  int err;
+  data_com_msg *m = SBR_MSG_MALLOC (sk);
+
+  if (!m)
+    {
+      NSSBR_LOGERR ("malloc msg failed]fd=%d", sk->fd);
+      sbr_set_sk_errno (sk, ENOMEM);
+      return -1;
+    }
+
+  sbr_construct_msg (m, SPL_TCPIP_NEW_MSG_API, SPL_API_DO_CLOSE, MSG_SYN_POST,
+                     sk);
+  msg_close *p = (msg_close *) m->buffer;
+  p->extend_member_bit = 0;
+  p->time_started = sys_now ();
+  p->shut = how;
+  SBR_MSG_POST (m, sbr_get_msg_box (sk));
+  err = sbr_spl_err_to_errno (m->param.err);
+  SBR_MSG_FREE (m);
+  if (err != 0)
+    {
+      NSSBR_LOGERR ("handle getsockopt failed]fd=%d,err=%d", sk->fd, err);
+      sbr_set_sk_errno (sk, err);
+      return -1;
+    }
+
+  return 0;
+}
+
+void
+sbr_handle_tcp_recv (sbr_socket_t * sk, u32 len, struct spl_pbuf *buf)
+{
+  data_com_msg *m = sbr_attach_msg (sk, buf);
+
+  if (!m)
+    {
+      NSSBR_LOGERR ("attach msg failed]fd=%d", sk->fd);
+      return;
+    }
+
+  sbr_construct_msg_dbg (m, SPL_TCPIP_NEW_MSG_API, SPL_API_DO_RECV,
+                         MSG_ASYN_POST, sk);
+  msg_recv_buf *p = (msg_recv_buf *) m->buffer;
+  p->extend_member_bit = 0;
+  p->len = len;
+  p->p = (struct spl_pbuf *) ADDR_LTOSH (buf);
+  SBR_MSG_POST (m, sbr_get_msg_box (sk));
+}
+
+int
+sbr_handle_tcp_send (sbr_socket_t * sk, size_t size, struct spl_pbuf *buf,
+                     u8 api_flag)
+{
+  int ret;
+  data_com_msg *m = sbr_attach_msg (sk, buf);
+
+  if (!m)
+    {
+      NSSBR_LOGERR ("attach msg failed]fd=%d", sk->fd);
+      sbr_set_sk_io_errno (sk, ENOMEM);
+      return -1;
+    }
+
+  sbr_construct_msg_dbg (m, SPL_TCPIP_NEW_MSG_API, SPL_API_DO_WRITE,
+                         MSG_ASYN_POST, sk);
+  msg_write_buf *p = (msg_write_buf *) m->buffer;
+  p->extend_member_bit = 0;
+  p->len = size;
+  p->p = (struct spl_pbuf *) ADDR_LTOSH (buf);
+  p->apiflags = api_flag;
+  SBR_MSG_POST_RET (m, sbr_get_msg_box (sk), ret);
+
+  if (0 == ret)
+    {
+      return 0;
+    }
+  else
+    {
+      NSSBR_LOGERR ("post msg failed]fd=%d", sk->fd);
+      sbr_set_sk_io_errno (sk, EAGAIN);
+      return -1;
+    }
+}
+
+/* need delete sbr_handle_app_touch */
+void
+sbr_handle_app_touch (void)
+{
+}
diff --git a/stacks/lwip_stack/lwip_src/socket/stackx_msg_handler.h b/stacks/lwip_stack/lwip_src/socket/stackx_msg_handler.h
new file mode 100644
index 0000000..a52d026
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/socket/stackx_msg_handler.h
@@ -0,0 +1,78 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#ifndef STACKX_MSG_HANDLER_H
+#define STACKX_MSG_HANDLER_H
+#include "stackx_socket.h"
+#include "stackx_ip_addr.h"
+#include "stackx_netbuf.h"
+#include "stackx_spl_share.h"
+
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+extern "C" {
+/* *INDENT-ON* */
+#endif
+
+#define SBR_GET_SOCK_NAME 1
+#define SBR_GET_PEER_NAME 0
+
+int sbr_handle_socket (sbr_socket_t * sk, spl_netconn_type_t type, u8 proto);
+
+int sbr_handle_bind (sbr_socket_t * sk, spl_ip_addr_t * addr, u16 port);
+
+int sbr_handle_listen (sbr_socket_t * sk, int backlog);
+
+int sbr_handle_connect (sbr_socket_t * sk, spl_ip_addr_t * addr, u16 port,
+                        spl_ip_addr_t * local_ip);
+
+int sbr_handle_get_name (sbr_socket_t * sk, struct sockaddr *name,
+                         socklen_t * namelen, u8 cmd);
+
+int sbr_handle_setsockopt (sbr_socket_t * sk, int level, int optname,
+                           const void *optval, socklen_t optlen);
+
+int sbr_handle_getsockopt (sbr_socket_t * sk, int level, int optname,
+                           void *optval, socklen_t * optlen);
+
+int sbr_handle_close (sbr_socket_t * sk, u8 how);
+
+int sbr_handle_udp_send (sbr_socket_t * sk, struct spl_netbuf *buf,
+                         spl_ip_addr_t * local_ip);
+
+void sbr_handle_free_recv_buf (sbr_socket_t * sk);
+
+void sbr_handle_free_send_buf (sbr_socket_t * sk, struct spl_pbuf *buf);
+
+int sbr_handle_shutdown (sbr_socket_t * sk, u8 how);
+
+void sbr_handle_tcp_recv (sbr_socket_t * sk, u32 len, struct spl_pbuf *buf);
+
+int sbr_handle_tcp_send (sbr_socket_t * sk, size_t size, struct spl_pbuf *buf,
+                         u8 api_flag);
+
+int sbr_handle_custom_send (sbr_socket_t * sk, struct spl_pbuf *buf,
+                            spl_ip_addr_t * src, spl_ip_addr_t * dst, u8 tos);
+
+int sbr_handle_custom_close (sbr_socket_t * sk, spl_ip_addr_t * addr);
+
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+}
+/* *INDENT-ON* */
+#endif
+
+#endif
diff --git a/stacks/lwip_stack/lwip_src/socket/stackx_prot_com.c b/stacks/lwip_stack/lwip_src/socket/stackx_prot_com.c
new file mode 100644
index 0000000..6af2a77
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/socket/stackx_prot_com.c
@@ -0,0 +1,651 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include "stackx_msg_handler.h"
+#include "stackx_prot_com.h"
+#include "common_pal_bitwide_adjust.h"
+#include "stackx_err.h"
+#include "nstack_securec.h"
+#include "nsfw_rti.h"
+//#include "stackx_custom.h"
+
+#define FAST_SLEEP_TIME 10000
+#define FAST_RETRY_COUNT 100
+#define MAX_WAIT_TIMEOUT 0x7FFFFFFF
+
+/*****************************************************************************
+*   Prototype    : sbr_getsockopt_sol_socket
+*   Description  : get sol socket
+*   Input        : sbr_socket_t * sk
+*                  int optname
+*                  void * optval
+*                  socklen_t optlen
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_getsockopt_sol_socket (sbr_socket_t * sk, int optname, void *optval,
+                           socklen_t optlen)
+{
+  int err = 0;
+
+  switch (optname)
+    {
+    case SO_ERROR:
+      {
+        if (optlen < sizeof (int))
+          {
+            return EINVAL;
+          }
+
+        /* only overwrite ERR_OK or tempoary errors */
+        err = sbr_get_sk_errno (sk);
+
+        if ((0 == err) || (EINPROGRESS == err))
+          {
+            err =
+              sbr_spl_err_to_errno (ss_get_last_errno (sbr_get_conn (sk)));
+            sbr_set_sk_errno (sk, err);
+          }
+        else
+          {
+            sbr_set_sk_errno (sk, 0);
+          }
+
+        *(int *) optval = sbr_get_sk_errno (sk);
+
+        return 0;
+      }
+    case SO_BROADCAST:
+    case SO_KEEPALIVE:
+    case SO_RCVBUF:
+    case SO_SNDBUF:
+    case SO_REUSEADDR:
+      if (optlen < sizeof (int))
+        {
+          err = EINVAL;
+        }
+
+      break;
+    case SO_RCVTIMEO:
+    case SO_SNDTIMEO:
+      if (optlen < sizeof (struct timeval))
+        {
+          err = EINVAL;
+        }
+
+      break;
+    case SO_LINGER:
+      if (optlen < sizeof (struct linger))
+        {
+          err = EINVAL;
+        }
+
+      break;
+    default:
+      err = ENOPROTOOPT;
+      break;
+    }
+
+  return err;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_getsockopt_ipproto_ip
+*   Description  : get ipproto ip
+*   Input        : int optname
+*                  void * optval
+*                  socklen_t optlen
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_getsockopt_ipproto_ip (int optname, void *optval, socklen_t optlen)
+{
+  int err = 0;
+
+  switch (optname)
+    {
+    case IP_TOS:
+      if (optlen < sizeof (u8))
+        {
+          err = EINVAL;
+        }
+
+      break;
+    default:
+      err = ENOPROTOOPT;
+      break;
+    }
+
+  return err;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_pick_timeout
+*   Description  : pick time
+*   Input        : const void * optval
+*                  socklen_t optlen
+*                  i32* timeout
+*   Output       : None
+*   Return Value : static inline int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline int
+sbr_pick_timeout (const void *optval, socklen_t optlen, i32 * timeout)
+{
+  if (optlen < sizeof (struct timeval))
+    {
+      return EINVAL;
+    }
+
+  struct timeval *time_val = (struct timeval *) optval;
+  if ((time_val->tv_usec < 0) || (time_val->tv_usec > USEC_TO_SEC))
+    {
+      return EDOM;
+    }
+  else
+    {
+      if (time_val->tv_sec < 0)
+        {
+          *timeout = 0;
+        }
+      else
+        {
+          *timeout = MAX_WAIT_TIMEOUT;
+          if ((time_val->tv_sec != 0) || (time_val->tv_usec != 0))
+            {
+              if (time_val->tv_sec < ((MAX_WAIT_TIMEOUT / 1000) - 1))
+                {
+                  *timeout =
+                    time_val->tv_sec * 1000 + time_val->tv_usec / 1000;
+                }
+            }
+        }
+    }
+
+  return 0;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_setsockopt_sol_socket
+*   Description  : set sol socket
+*   Input        : sbr_socket_t * sk
+*                  int optname
+*                  const void * optval
+*                  socklen_t optlen
+*                  netconn_type_t type
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_setsockopt_sol_socket (sbr_socket_t * sk, int optname, const void *optval,
+                           socklen_t optlen, spl_netconn_type_t type)
+{
+  int err = 0;
+
+  switch (optname)
+    {
+    case SO_REUSEADDR:
+    case SO_BROADCAST:
+    case SO_KEEPALIVE:
+    case SO_RCVBUF:
+    case SO_SNDBUF:
+      if (optlen < sizeof (int))
+        {
+          err = EINVAL;
+        }
+
+      break;
+    case SO_RCVTIMEO:
+      err =
+        sbr_pick_timeout (optval, optlen,
+                          &sbr_get_fd_share (sk)->recv_timeout);
+      break;
+    case SO_SNDTIMEO:
+      err =
+        sbr_pick_timeout (optval, optlen,
+                          &sbr_get_fd_share (sk)->send_timeout);
+      break;
+    case SO_LINGER:
+      if (optlen < sizeof (struct linger))
+        {
+          err = EINVAL;
+        }
+
+      break;
+    default:
+      err = ENOPROTOOPT;
+      break;
+    }
+
+  return err;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_setsockopt_ipproto_ip
+*   Description  : set ipproto ip
+*   Input        : int optname
+*                  const void * optval
+*                  socklen_t optlen
+*                  netconn_type_t type
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_setsockopt_ipproto_ip (int optname, const void *optval, socklen_t optlen,
+                           spl_netconn_type_t type)
+{
+  int err = 0;
+
+  switch (optname)
+    {
+    case IP_TOS:
+      if (optlen < sizeof (u8))
+        {
+          err = EINVAL;
+        }
+
+      break;
+    case IP_MULTICAST_TTL:
+      if (optlen < sizeof (u8))
+        {
+          err = EINVAL;
+          break;
+        }
+
+      if (type != SPL_NETCONN_UDP)
+        {
+          err = EAFNOSUPPORT;
+          break;
+        }
+
+      break;
+    case IP_MULTICAST_IF:
+      if (optlen < sizeof (struct in_addr))
+        {
+          err = EINVAL;
+          break;
+        }
+
+      if (type != SPL_NETCONN_UDP)
+        {
+          err = EAFNOSUPPORT;
+          break;
+        }
+
+      break;
+    case IP_MULTICAST_LOOP:
+      if (optlen < sizeof (u8))
+        {
+          err = EINVAL;
+          break;
+        }
+
+      if (type != SPL_NETCONN_UDP)
+        {
+          err = EAFNOSUPPORT;
+          break;
+        }
+
+      break;
+    default:
+      err = ENOPROTOOPT;
+      break;
+    }
+
+  return err;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_dequeue_buf
+*   Description  : dequeue buf
+*   Input        : sbr_socket_t * sk
+*                  void **buf
+*                  i32 timeout
+*                  u8 use_l4_ring
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_dequeue_buf (sbr_socket_t * sk, void **buf, i32 timeout)
+{
+  mring_handle ring = ss_get_recv_ring (sbr_get_conn (sk));
+
+  struct timespec start, end;
+  long timediff;
+  long timediff_sec;
+  long timeout_sec = (long) (timeout / 1000);
+  unsigned int retry_count = 0;
+
+  if (timeout > 0)
+    {
+      if (unlikely (0 != clock_gettime (CLOCK_MONOTONIC, &start)))
+        {
+          NSSBR_LOGERR ("Failed to get time, errno = %d", errno);
+        }
+    }
+
+  if (!ss_recv_ring_valid (sbr_get_conn (sk)))
+    {
+      NSSBR_LOGDBG ("ring is invalid]fd=%d", sk->fd);
+      sbr_set_sk_io_errno (sk, ENOTCONN);
+      return -1;
+    }
+
+  int dequeue_ret = 0;
+  pid_t pid = get_sys_pid ();
+
+  while (1)
+    {
+      if (ss_is_shut_rd (sbr_get_conn (sk)))
+        {
+          NSSBR_LOGDBG ("is shut rd]fd=%d", sk->fd);
+          sbr_set_sk_io_errno (sk, EINVAL);
+          return -1;
+        }
+
+      dequeue_ret = nsfw_mem_ring_dequeue (ring, buf);
+      if (1 == dequeue_ret)
+        {
+          pbuf_set_recycle_flg ((struct spl_pbuf *) *buf, pid); /*release buf hold by app on abnormal exit */
+          return 0;
+        }
+      else if (0 == dequeue_ret)
+        {
+          /*If the peer reset connect, try to receive data only once */
+          if (ss_can_not_recv (sbr_get_conn (sk)))
+            {
+              NS_LOG_CTRL (LOG_CTRL_RECV_QUEUE_FULL, LOGSBR, "NSSBR",
+                           NSLOG_WAR, "try to fetch one more time]fd=%d",
+                           sk->fd);
+                /**
+                 * l4_ring will not be processed here as can_not_recv flag is
+                 * set by TCP only.
+                 */
+              if (1 == nsfw_mem_ring_dequeue (ring, buf))
+                {
+                  pbuf_set_recycle_flg ((struct spl_pbuf *) *buf, pid);
+                  return 0;
+                }
+
+              sbr_set_sk_io_errno (sk, ENOTCONN);
+              return -1;
+            }
+
+          int err = ss_get_last_errno (sbr_get_conn (sk));
+          if (SPL_ERR_IS_FATAL (err) || err == ERR_TIMEOUT)     /* have to handle ERR_TIMEOUT here, when TCP keepalive timeout. */
+            {
+              NS_LOG_CTRL (LOG_CTRL_RECV_QUEUE_FULL, LOGSBR, "NSSBR",
+                           NSLOG_ERR, "connection fatal error!err=%d", err);
+
+              /* l4_ring need to be handled in the future */
+              if (1 == nsfw_mem_ring_dequeue (ring, buf))
+                {
+                  pbuf_set_recycle_flg ((struct spl_pbuf *) *buf, pid);
+                  return 0;
+                }
+
+              sbr_set_sk_io_errno (sk, sbr_spl_err_to_errno (err));
+              return -1;
+            }
+
+          if (0 > timeout)
+            {
+              sbr_set_sk_io_errno (sk, EWOULDBLOCK);
+              return -1;
+            }
+
+          if (retry_count < FAST_RETRY_COUNT)
+            {
+              sys_sleep_ns (0, FAST_SLEEP_TIME);
+              retry_count++;
+            }
+          else
+            {
+              sys_sleep_ns (0, sbr_get_fd_share (sk)->block_polling_time);
+            }
+
+          if (timeout > 0)
+            {
+              if (unlikely (0 != clock_gettime (CLOCK_MONOTONIC, &end)))
+                {
+                  NSSBR_LOGERR ("Failed to get time, errno = %d", errno);
+                }
+              timediff_sec = end.tv_sec - start.tv_sec;
+              if (timediff_sec >= timeout_sec)
+                {
+                  timediff = end.tv_nsec > start.tv_nsec ?
+                    (timediff_sec * 1000) + (end.tv_nsec -
+                                             start.tv_nsec) /
+                    USEC_TO_SEC : (timediff_sec * 1000) -
+                    ((start.tv_nsec - end.tv_nsec) / USEC_TO_SEC);
+                  if (timediff > timeout)
+                    {
+                      NSSBR_LOGDBG ("recv timeout]fd=%d", sk->fd);
+                      sbr_set_sk_io_errno (sk, EWOULDBLOCK);
+                      return -1;
+                    }
+                }
+            }
+        }
+      else
+        {
+          NSSBR_LOGERR ("dequeue failed]fd=%d", sk->fd);
+          sbr_set_sk_io_errno (sk, EINVAL);
+          return -1;
+        }
+    }
+}
+
+int
+sbr_com_peak (sbr_socket_t * sk)
+{
+  NSSBR_LOGERR ("not implement]fd=%d", sk->fd);
+  return -1;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_com_try_lock_recv
+*   Description  : try lock recv
+*   Input        : sbr_socket_t * sk
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_com_try_lock_recv (sbr_socket_t * sk)
+{
+#ifdef SBR_USE_LOCK
+  return common_spinlock_try_lock_with_pid (&sbr_get_fd_share (sk)->recv_lock,
+                                            get_sys_pid ());
+#else
+  return 1;
+#endif
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_com_lock_common
+*   Description  : lock common
+*   Input        : sbr_socket_t * sk
+*   Output       : None
+*   Return Value : void
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+void
+sbr_com_lock_common (sbr_socket_t * sk)
+{
+#ifdef SBR_USE_LOCK
+  while (!common_spinlock_try_lock_with_pid
+         (&sbr_get_fd_share (sk)->common_lock, get_sys_pid ()))
+    {
+      sys_sleep_ns (0, 0);
+    }
+#endif
+
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_com_unlock_common
+*   Description  : unlock common
+*   Input        : sbr_socket_t * sk
+*   Output       : None
+*   Return Value : void
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+void
+sbr_com_unlock_common (sbr_socket_t * sk)
+{
+#ifdef SBR_USE_LOCK
+  common_spinlock_unlock (&sbr_get_fd_share (sk)->common_lock);
+#endif
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_com_free_recv_buf
+*   Description  : free recv buf,can't free buf in app
+*   Input        : sbr_socket_t * sk
+*                  struct spl_pbuf *p
+*   Output       : None
+*   Return Value : void
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+void
+sbr_com_free_recv_buf (sbr_socket_t * sk, struct spl_pbuf *p)
+{
+  struct spl_pbuf *p_orig = p;
+  if (p)
+    {
+      p->freeNext = NULL;
+      p = (struct spl_pbuf *) ADDR_LTOSH (p);
+
+      if (sbr_get_fd_share (sk)->recoder.totalLen > 0)
+        {
+          ((struct spl_pbuf *)
+           ADDR_SHTOL (sbr_get_fd_share (sk)->recoder.tail))->freeNext = p;
+          sbr_get_fd_share (sk)->recoder.tail = p;
+        }
+      else
+        {
+          sbr_get_fd_share (sk)->recoder.head = p;
+          sbr_get_fd_share (sk)->recoder.tail = p;
+        }
+
+      sbr_get_fd_share (sk)->recoder.totalLen++;
+    }
+
+  /* send MSG only if it's a big packet or number of packets larger than 32 */
+  if ((p_orig && p_orig->tot_len > MAX_RECV_FREE_LEN) ||
+      (sbr_get_fd_share (sk)->recoder.totalLen >= MAX_RECV_FREE_BUF))
+    {
+      sbr_handle_free_recv_buf (sk);
+    }
+
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_get_sockaddr_and_len
+*   Description  : get addr and len
+*   Input        : u16 port
+*                  spl_ip_addr_t * ipaddr
+*                  struct sockaddr * addr
+*                  socklen_t * addrlen
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_get_sockaddr_and_len (u16 port, spl_ip_addr_t * ipaddr,
+                          struct sockaddr *addr, socklen_t * addrlen)
+{
+  int ret;
+  struct sockaddr_in sin;
+
+  ret = MEMSET_S (&sin, sizeof (sin), 0, sizeof (sin));
+  if (0 != ret)
+    {
+      NSSBR_LOGERR ("MEMSET_S failed]ret=%d.", ret);
+      return -1;
+    }
+
+  sin.sin_family = AF_INET;
+  sin.sin_port = htons (port);
+  inet_addr_from_ipaddr (&sin.sin_addr, ipaddr);
+  if (*addrlen > sizeof (struct sockaddr))
+    {
+      *addrlen = sizeof (struct sockaddr);
+    }
+
+  if (*addrlen > 0)
+    {
+      ret = MEMCPY_S (addr, sizeof (struct sockaddr), &sin, *addrlen);
+      if (0 != ret)
+        {
+          NSSBR_LOGERR ("MEMCPY_S failed]ret=%d", ret);
+
+          return -1;
+        }
+    }
+
+  return 0;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_com_set_app_info
+*   Description  : set app info to netconn
+*   Input        : sbr_socket_t * sk
+*                  void* appinfo
+*   Output       : None
+*   Return Value : void
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+void
+sbr_com_set_app_info (sbr_socket_t * sk, void *appinfo)
+{
+  return;
+}
diff --git a/stacks/lwip_stack/lwip_src/socket/stackx_prot_com.h b/stacks/lwip_stack/lwip_src/socket/stackx_prot_com.h
new file mode 100644
index 0000000..054393b
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/socket/stackx_prot_com.h
@@ -0,0 +1,159 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#ifndef STACKX_PROT_COM_H
+#define STACKX_PROT_COM_H
+#include "stackx_socket.h"
+#include "stackx_ip_addr.h"
+#include "stackx_res_mgr.h"
+#include "stackx_pbuf.h"
+#include "stackx_macro.h"
+
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+extern "C" {
+/* *INDENT-ON* */
+#endif
+
+#define MAX_RECV_FREE_BUF 32
+#define MAX_RECV_FREE_LEN (SPL_FRAME_MTU - SPL_IP_HLEN)
+#define USEC_TO_SEC 1000000
+
+#ifndef NSTACK_SOCKOPT_CHECK
+#define NSTACK_SOCKOPT_CHECK
+/* setsockopt level type*/
+enum
+{
+  NSTACK_SOCKOPT = 0xff02
+};
+/*setsockopt optname type*/
+enum
+{
+  NSTACK_SEM_SLEEP = 0X001
+};
+#endif
+
+#define sbr_malloc_tx_pbuf(len, offset) sbr_malloc_pbuf(sbr_get_tx_pool(), len, TX_MBUF_MAX_LEN, offset)
+
+static inline int
+sbr_spl_err_to_errno (i32 err)
+{
+  static int table[] = {
+    0,                          /* ERR_OK             0      No error, everything OK.    */
+    ENOMEM,                     /* ERR_MEM            -1      Out of memory error.       */
+    ENOBUFS,                    /* ERR_BUF            -2      Buffer error.              */
+    ETIMEDOUT,                  /* ERR_TIMEOUT        -3      Timeout                    */
+    EHOSTUNREACH,               /* ERR_RTE            -4      Routing problem.           */
+    EINPROGRESS,                /* ERR_INPROGRESS     -5      Operation in progress      */
+    EINVAL,                     /* ERR_VAL            -6      Illegal value.             */
+    EWOULDBLOCK,                /* ERR_WOULDBLOCK     -7      Operation would block.     */
+    EADDRINUSE,                 /* ERR_USE            -8      Address in use.            */
+    EISCONN,                    /* ERR_ISCONN         -9      Already connected.         */
+    ECONNABORTED,               /* ERR_ABRT           -10     Connection aborted.        */
+    ECONNRESET,                 /* ERR_RST            -11     Connection reset.          */
+    ENOTCONN,                   /* ERR_CLSD           -12     Connection closed.         */
+    ENOTCONN,                   /* ERR_CONN           -13     Not connected.             */
+    EIO,                        /* ERR_ARG            -14     Illegal argument.          */
+    -1,                         /* ERR_IF             -15     Low-level netif error      */
+    EALREADY,                   /*ERR_ALREADY         -16     previous connect attemt has not yet completed */
+    EPROTOTYPE,                 /*ERR_PROTOTYPE       -17     prototype error or some other generic error.
+                                   the operation is not allowed on current socket              */
+    EINVAL,                     /* ERR_CALLBACK       -18     callback error             */
+    EADDRNOTAVAIL,              /* ERR_CANTASSIGNADDR -19 Cannot assign requested address */
+    EIO,                        /* ERR_CONTAINER_ID  -20 Illegal container id           */
+    ENOTSOCK,                   /*ERR_NOTSOCK         -21 not a socket                   */
+    -1,                         /* ERR_CLOSE_WAIT     -22 closed in established state */
+    EPROTONOSUPPORT,            /* ERR_EPROTONOSUPPORT -23 Protocol not supported */
+    ECONNABORTED                /* ERR_FAULTRECOVERY   -24 SPL just recovered from a fatal fault */
+  };
+
+  if (((-(err)) >= 0)
+      && (-(err)) < (i32) (sizeof (table) / sizeof (table[0])))
+    {
+      return table[-(err)];
+    }
+  else
+    {
+      return EIO;
+    }
+}
+
+int sbr_getsockopt_sol_socket (sbr_socket_t * sk, int optname, void *optval,
+                               socklen_t optlen);
+int sbr_getsockopt_ipproto_ip (int optname, void *optval, socklen_t optlen);
+int sbr_setsockopt_sol_socket (sbr_socket_t * sk, int optname,
+                               const void *optval, socklen_t optlen,
+                               spl_netconn_type_t type);
+int sbr_setsockopt_ipproto_ip (int optname, const void *optval,
+                               socklen_t optlen, spl_netconn_type_t type);
+int sbr_dequeue_buf (sbr_socket_t * sk, void **buf, i32 timeout);
+int sbr_com_peak (sbr_socket_t * sk);
+int sbr_com_try_lock_recv (sbr_socket_t * sk);
+/*****************************************************************************
+*   Prototype    : sbr_com_lock_recv
+*   Description  : lock recv
+*   Input        : sbr_socket_t * sk
+*   Output       : None
+*   Return Value : void
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline void
+sbr_com_lock_recv (sbr_socket_t * sk)
+{
+#ifdef SBR_USE_LOCK
+  while (!common_spinlock_try_lock_with_pid
+         (&sbr_get_fd_share (sk)->recv_lock, get_sys_pid ()))
+    {
+      sys_sleep_ns (0, 0);
+    }
+#endif
+
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_com_unlock_recv
+*   Description  : unlock recv
+*   Input        : sbr_socket_t * sk
+*   Output       : None
+*   Return Value : void
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline void
+sbr_com_unlock_recv (sbr_socket_t * sk)
+{
+#ifdef SBR_USE_LOCK
+  common_spinlock_unlock (&sbr_get_fd_share (sk)->recv_lock);
+#endif
+}
+
+void sbr_com_lock_common (sbr_socket_t * sk);
+void sbr_com_unlock_common (sbr_socket_t * sk);
+void sbr_com_free_recv_buf (sbr_socket_t * sk, struct spl_pbuf *p);
+int sbr_get_sockaddr_and_len (u16 port, spl_ip_addr_t * ip_addr,
+                              struct sockaddr *addr, socklen_t * addrlen);
+void sbr_com_set_app_info (sbr_socket_t * sk, void *appinfo);
+
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+}
+/* *INDENT-ON* */
+#endif
+
+#endif
diff --git a/stacks/lwip_stack/lwip_src/socket/stackx_protocol_api.c b/stacks/lwip_stack/lwip_src/socket/stackx_protocol_api.c
new file mode 100644
index 0000000..94ef483
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/socket/stackx_protocol_api.c
@@ -0,0 +1,94 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include "sbr_protocol_api.h"
+#include "stackx_epoll_api.h"
+#include "stackx_socket.h"
+#include "stackx_res_mgr.h"
+#include "stackx_common_opt.h"
+#include "common_mem_api.h"
+#include "stackx_event.h"
+
+extern sbr_fdopt tcp_fdopt;
+extern sbr_fdopt udp_fdopt;
+extern sbr_fdopt raw_fdopt;
+extern sbr_fdopt custom_fdopt;
+
+/*****************************************************************************
+*   Prototype    : sbr_init_protocol
+*   Description  : init protocol
+*   Input        : None
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_init_protocol ()
+{
+  return sbr_init_stackx ();
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_get_fdopt
+*   Description  : get fdopt by domain type protocol
+*   Input        : int domain
+*                  int type
+*                  int protocol
+*   Output       : None
+*   Return Value : sbr_fdopt*
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+sbr_fdopt *
+sbr_get_fdopt (int domain, int type, int protocol)
+{
+  if (domain != AF_INET)
+    {
+      NSSBR_LOGERR ("domain is not AF_INET]domain=%d", domain);
+      sbr_set_errno (EAFNOSUPPORT);
+      return NULL;
+    }
+
+  sbr_fdopt *fdopt = NULL;
+
+  switch (type & (~(O_NONBLOCK | SOCK_CLOEXEC)))
+    {
+    case SOCK_DGRAM:
+      fdopt = &udp_fdopt;
+      break;
+    case SOCK_STREAM:
+      fdopt = &tcp_fdopt;
+      break;
+
+    default:
+      NSSBR_LOGDBG ("type is unknown]type=%d", type);
+      sbr_set_errno (ESOCKTNOSUPPORT);
+      return NULL;
+    }
+
+  return fdopt;
+}
+
+/* app send its version info to nStackMain */
+extern void sbr_handle_app_touch (void);
+void
+sbr_app_touch_in (void)
+{
+  sbr_handle_app_touch ();
+}
diff --git a/stacks/lwip_stack/lwip_src/socket/stackx_res_mgr.c b/stacks/lwip_stack/lwip_src/socket/stackx_res_mgr.c
new file mode 100644
index 0000000..331eee1
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/socket/stackx_res_mgr.c
@@ -0,0 +1,246 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include "stackx_res_mgr.h"
+#include "stackx_common.h"
+#include "nstack_securec.h"
+#include "nsfw_msg.h"
+#include "stackx_common.h"
+#include "nsfw_mgr_com_api.h"
+#include "stackx_cfg.h"
+#include "nsfw_maintain_api.h"
+//#include "stackx_dfx_api.h"
+#include "stackx_app_res.h"
+
+sbr_share_group g_share_group = { 0 };
+
+#define SLOW_SLEEP_TIME 500000
+
+NSTACK_STATIC inline void
+sbr_reset_fd_share (sbr_fd_share * fd_share)
+{
+  common_mem_spinlock_init (&fd_share->recv_lock);
+  common_mem_spinlock_init (&fd_share->common_lock);
+  fd_share->err = 0;
+  fd_share->lastoffset = 0;
+  fd_share->lastdata = NULL;
+  fd_share->recoder.head = NULL;
+  fd_share->recoder.tail = NULL;
+  fd_share->recoder.totalLen = 0;
+  fd_share->recv_timeout = 0;
+  fd_share->send_timeout = 0;
+  fd_share->rcvlowat = 1;
+  fd_share->block_polling_time = SLOW_SLEEP_TIME;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_init_tx_pool
+*   Description  : get tx buf pool
+*   Input        : None
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+NSTACK_STATIC int
+sbr_init_tx_pool ()
+{
+  mpool_handle pool[1];
+  pool[0] = NULL;
+
+  (void) sbr_malloc_tx_pool (get_sys_pid (), pool, 1);
+  if (pool[0])
+    {
+      g_share_group.tx_pool = pool[0];
+      return 0;
+    }
+
+  return -1;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_init_app_res
+*   Description  : get msg, conn pool
+*   Input        : None
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+NSTACK_STATIC int
+sbr_init_app_res ()
+{
+  g_share_group.conn_pool = sbr_get_instance_conn_pool (0);
+  if (!g_share_group.conn_pool)
+    {
+      return -1;
+    }
+
+  return 0;
+}
+
+/*=========== get share config for app =============*/
+NSTACK_STATIC inline int
+get_share_config ()
+{
+  static nsfw_mem_name g_cfg_mem_info =
+    { NSFW_SHMEM, NSFW_PROC_MAIN, NSTACK_SHARE_CONFIG };
+
+  mzone_handle base_cfg_mem = nsfw_mem_zone_lookup (&g_cfg_mem_info);
+  if (NULL == base_cfg_mem)
+    {
+      NSSOC_LOGERR ("get config share mem failed.");
+      return -1;
+    }
+
+  if (get_share_cfg_from_mem (base_cfg_mem) < 0)
+    {
+      NSSOC_LOGERR ("get share config failed.");
+      return -1;
+    }
+
+  NSSOC_LOGDBG ("get share config success.");
+  return 0;
+}
+
+int
+nstack_set_share_config ()
+{
+  return 0;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_init_stackx
+*   Description  : init stackx res
+*   Input        : None
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_init_stackx ()
+{
+  sbr_init_cfg ();
+  if (get_share_config () < 0)
+    {
+      NSSBR_LOGERR ("get_share_config  failed");
+      return -1;
+    }
+
+  if (sbr_attach_group_array () != 0)
+    {
+      NSSBR_LOGERR ("sbr_attach_group_array failed");
+      return -1;
+    }
+
+  NSSBR_LOGDBG ("sbr_attach_group_array ok");
+
+  if (sbr_init_tx_pool () != 0)
+    {
+      NSSBR_LOGERR ("init tx pool failed");
+      return -1;
+    }
+
+  NSSBR_LOGDBG ("init tx pool ok");
+
+  if (sbr_init_app_res () != 0)
+    {
+      NSSBR_LOGERR ("sbr_init_app_res failed");
+      return -1;
+    }
+
+  NSSBR_LOGDBG ("sbr_init_app_res ok");
+  NSSBR_LOGDBG ("sbr_init_stackx ok");
+  return 0;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_malloc_conn_for_sk
+*   Description  : malloc netconn for sk,need add pid
+*   Input        : sbr_socket_t* sk
+*                  netconn_type_t type
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_malloc_conn_for_sk (sbr_socket_t * sk, spl_netconn_type_t type)
+{
+  spl_netconn_t *conn = ss_malloc_conn_app (g_share_group.conn_pool, type);
+
+  if (!conn)
+    {
+      NSSBR_LOGERR ("malloc conn failed]fd=%d", sk->fd);
+      sbr_set_errno (ENOBUFS);
+      return -1;
+    }
+
+  NSSBR_LOGINF ("malloc conn ok]fd=%d,conn=%p", sk->fd, conn);
+
+  u16 thread_index = 0;
+  ss_set_bind_thread_index (conn, thread_index);
+  ss_set_msg_box (conn, ss_get_instance_msg_box (thread_index, 0));
+
+  sbr_fd_share *fd_share = (sbr_fd_share *) ((char *) conn + SS_NETCONN_SIZE);
+  sbr_reset_fd_share (fd_share);
+
+  sk->stack_obj = (void *) conn;
+  sk->sk_obj = (void *) fd_share;
+  return 0;
+}
+
+int
+sbr_init_conn_for_accept (sbr_socket_t * sk, spl_netconn_t * conn)
+{
+  if (!conn)
+    {
+      sbr_set_sk_errno (sk, ENOBUFS);
+      return -1;
+    }
+
+  NSSBR_LOGINF ("accept conn ok]fd=%d,conn=%p,private_data=%p", sk->fd, conn,
+                conn->private_data);
+
+  if (ss_add_pid (conn, get_sys_pid ()) < 0)
+    {
+      NSSBR_LOGERR ("ss_add_pid failed]fd=%d", sk->fd);
+    }
+
+  ss_set_accept_from (conn, NULL);      /* need clear flag */
+
+  sbr_fd_share *fd_share = (sbr_fd_share *) ((char *) conn + SS_NETCONN_SIZE);
+  sbr_reset_fd_share (fd_share);
+
+  sk->stack_obj = (void *) conn;
+  sk->sk_obj = (void *) fd_share;
+
+  return 0;
+}
+
+void
+sbr_free_conn_from_sk (sbr_socket_t * sk)
+{
+  ss_free_conn (sbr_get_conn (sk));
+  sk->stack_obj = NULL;
+  sk->sk_obj = NULL;
+  NSSBR_LOGDBG ("free conn ok]fd=%d", sk->fd);
+}
diff --git a/stacks/lwip_stack/lwip_src/socket/stackx_res_mgr.h b/stacks/lwip_stack/lwip_src/socket/stackx_res_mgr.h
new file mode 100644
index 0000000..e139644
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/socket/stackx_res_mgr.h
@@ -0,0 +1,69 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#ifndef STACKX_RES_MGR_H
+#define STACKX_RES_MGR_H
+#include "stackx_socket.h"
+#include "stackx_ip_tos.h"
+#include "stackx_tx_box.h"
+#include "stackx_app_res.h"
+
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+extern "C" {
+/* *INDENT-ON* */
+#endif
+
+typedef struct
+{
+  mring_handle conn_pool;
+  mpool_handle tx_pool;
+} sbr_share_group;
+
+extern sbr_share_group g_share_group;
+
+static inline mpool_handle
+sbr_get_tx_pool ()
+{
+  return g_share_group.tx_pool;
+}
+
+static inline mring_handle
+sbr_get_conn_pool ()
+{
+  return (mring_handle) ADDR_LTOSH (g_share_group.conn_pool);
+}
+
+static inline mring_handle
+sbr_get_spl_msg_box (sbr_socket_t * sk, u8 tos)
+{
+  return
+    ss_get_instance_msg_box (ss_get_bind_thread_index (sbr_get_conn (sk)),
+                             stackx_get_prio (tos));
+}
+
+int sbr_init_stackx ();
+int sbr_malloc_conn_for_sk (sbr_socket_t * sk, spl_netconn_type_t type);
+int sbr_init_conn_for_accept (sbr_socket_t * sk, spl_netconn_t * conn);
+void sbr_free_conn_from_sk (sbr_socket_t * sk);
+
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+}
+/* *INDENT-ON* */
+#endif
+
+#endif
diff --git a/stacks/lwip_stack/lwip_src/socket/stackx_socket.h b/stacks/lwip_stack/lwip_src/socket/stackx_socket.h
new file mode 100644
index 0000000..e344535
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/socket/stackx_socket.h
@@ -0,0 +1,141 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#ifndef STACKX_SOCKET_H
+#define STACKX_SOCKET_H
+#include "sbr_protocol_api.h"
+#include "stackx_spl_share.h"
+#include "nstack_log.h"
+#include "stackx_pbuf.h"
+#include "common_mem_spinlock.h"
+
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+extern "C" {
+/* *INDENT-ON* */
+#endif
+
+typedef struct
+{
+  PRIMARY_ADDR struct spl_pbuf *head;
+  PRIMARY_ADDR struct spl_pbuf *tail;
+  int totalLen;
+} sbr_recvbuf_recoder;
+
+/* need fork and recycle */
+typedef struct
+{
+  common_mem_spinlock_t recv_lock;
+  common_mem_spinlock_t common_lock;
+  PRIMARY_ADDR void *lastdata;
+  u32 lastoffset;
+  sbr_recvbuf_recoder recoder;
+  i32 recv_timeout;
+  i32 send_timeout;
+  i32 rcvlowat;
+  int err;
+  u64 block_polling_time;
+  i64 extend_member_bit;
+} sbr_fd_share;
+
+/* check sbr_fd_share size */
+SIZE_OF_TYPE_NOT_LARGER_THAN (sbr_fd_share, SBR_FD_SIZE);
+
+#define sbr_get_fd_share(sk) ((sbr_fd_share*)sk->sk_obj)
+
+#define sbr_get_conn(sk) ((spl_netconn_t*)sk->stack_obj)
+
+#define sbr_get_msg_box(sk) ss_get_msg_box(sbr_get_conn(sk))
+
+/*****************************************************************************
+*   Prototype    : sbr_set_sk_errno
+*   Description  : set errno for sk
+*   Input        : sbr_socket_t * sk
+*                  int err
+*   Output       : None
+*   Return Value : static inline void
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline void
+sbr_set_sk_errno (sbr_socket_t * sk, int err)
+{
+  sbr_get_fd_share (sk)->err = err;
+  if (err != 0)
+    {
+      if (sbr_get_conn (sk))
+        {
+          NSSBR_LOGERR ("fd=%d,errno=%d,conn=%p,private_data=%p", sk->fd, err,
+                        sbr_get_conn (sk),
+                        ss_get_private_data (sbr_get_conn (sk)));
+        }
+
+      sbr_set_errno (err);
+    }
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_set_sk_io_errno
+*   Description  : set errno for sk in send/recv func, in case of too many logs
+*   Input        : sbr_socket_t * sk
+*                  int err
+*   Output       : None
+*   Return Value : static inline void
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline void
+sbr_set_sk_io_errno (sbr_socket_t * sk, int err)
+{
+  sbr_get_fd_share (sk)->err = err;
+  if (err != 0)
+    {
+      if (sbr_get_conn (sk))
+        {
+          NSSBR_LOGDBG ("fd=%d,errno=%d,conn=%p,private_data=%p", sk->fd, err,
+                        sbr_get_conn (sk),
+                        ss_get_private_data (sbr_get_conn (sk)));
+        }
+
+      sbr_set_errno (err);
+    }
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_get_sk_errno
+*   Description  : get sk's errno
+*   Input        : sbr_socket_t * sk
+*   Output       : None
+*   Return Value : static inline int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline int
+sbr_get_sk_errno (sbr_socket_t * sk)
+{
+  return sbr_get_fd_share (sk)->err;
+}
+
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+}
+/* *INDENT-ON* */
+#endif
+
+#endif
diff --git a/stacks/lwip_stack/lwip_src/socket/stackx_tcp.c b/stacks/lwip_stack/lwip_src/socket/stackx_tcp.c
new file mode 100644
index 0000000..c64dc7b
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/socket/stackx_tcp.c
@@ -0,0 +1,1696 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include "stackx_prot_com.h"
+#include "stackx_msg_handler.h"
+#include "stackx_pbuf.h"
+#include "stackx_epoll_api.h"
+#include "nstack_securec.h"
+#include "common_pal_bitwide_adjust.h"
+#include "common_mem_mbuf.h"
+#include "stackx_spl_share.h"
+#include "stackx_err.h"
+#include "stackx_cfg.h"
+#include "spl_opt.h"
+//#include "stackx_tcp_sctl.h"
+#include "stackx_tcp_opt.h"
+//#include "stackx_dfx_api.h"
+#include "stackx_spl_msg.h"
+#ifdef HAL_LIB
+#else
+#include "rte_memcpy.h"
+#endif
+
+static void
+sbr_tcp_init_conn (spl_netconn_t * conn)
+{
+  conn->tcp_sndbuf = CONN_TCP_MEM_DEF_LINE;
+  conn->conn_pool = sbr_get_conn_pool ();
+}
+
+/* need close after accept failed */
+static void
+sbr_tcp_accept_failed (sbr_socket_t * sk)
+{
+  (void) sbr_handle_close (sk, 0);
+  sk->stack_obj = NULL;
+  sk->sk_obj = NULL;
+}
+
+NSTACK_STATIC int
+sbr_tcp_socket (sbr_socket_t * sk, int domain, int type, int protocol)
+{
+  int err = 0;
+
+  if (sbr_malloc_conn_for_sk (sk, SPL_NETCONN_TCP) != 0)
+    {
+      return -1;
+    }
+
+  sbr_tcp_init_conn (sbr_get_conn (sk));
+
+  err = sbr_handle_socket (sk, SPL_NETCONN_TCP, 0);
+  if (0 == err)
+    {
+      /* Prevent automatic window updates, we do this on our own! */
+      ss_set_noautorecved_flag (sbr_get_conn (sk), 1);
+
+      ss_set_nonblock_flag (sbr_get_conn (sk), (type & O_NONBLOCK));
+
+      ss_add_recv_event (sbr_get_conn (sk));
+
+      ss_set_send_event (sbr_get_conn (sk), 1);
+
+      NSSBR_LOGINF ("add write and read events]fd=%d", sk->fd);
+
+      return 0;
+    }
+  else
+    {
+      sbr_free_conn_from_sk (sk);
+      return err;
+    }
+}
+
+NSTACK_STATIC int
+sbr_tcp_bind (sbr_socket_t * sk, const struct sockaddr *name,
+              socklen_t namelen)
+{
+  const struct sockaddr_in *addr_in = (const struct sockaddr_in *) name;
+  NSSBR_LOGINF ("fd=%d,addr=%s,port=%d,conn=%p,private_data=%p", sk->fd,
+                spl_inet_ntoa (addr_in->sin_addr), ntohs (addr_in->sin_port),
+                sbr_get_conn (sk), ss_get_private_data (sbr_get_conn (sk)));
+  spl_ip_addr_t local_addr;
+  inet_addr_to_ipaddr (&local_addr, &addr_in->sin_addr);
+  u16 local_port = addr_in->sin_port;
+  return sbr_handle_bind (sk, &local_addr, ntohs (local_port));
+}
+
+NSTACK_STATIC int
+sbr_tcp_listen (sbr_socket_t * sk, int backlog)
+{
+  ss_set_is_listen_conn (sbr_get_conn (sk), 1);
+  return sbr_handle_listen (sk, backlog);
+}
+
+static inline int
+sbr_tcp_recv_is_timeout (sbr_socket_t * sk, struct timespec *starttm)
+{
+  struct timespec currtm;
+  i64 timediff_ms, timediff_sec;
+  i64 timeout_thr_ms, timeout_thr_sec;
+
+  timeout_thr_ms = sbr_get_fd_share (sk)->recv_timeout;
+  if (0 == timeout_thr_ms)
+    {
+      return ERR_OK;
+    }
+
+  timeout_thr_sec = (i64) (timeout_thr_ms / 1000);
+
+  /* Handle system time change side-effects */
+  if (unlikely (0 != clock_gettime (CLOCK_MONOTONIC, &currtm)))
+    {
+      NSSBR_LOGERR ("Failed to get time, errno = %d", errno);
+    }
+
+  timediff_sec = currtm.tv_sec - starttm->tv_sec;
+  if (timediff_sec >= timeout_thr_sec)
+    {
+      timediff_ms = currtm.tv_nsec > starttm->tv_nsec ?
+        (timediff_sec * 1000) + (currtm.tv_nsec -
+                                 starttm->tv_nsec) /
+        USEC_TO_SEC : (timediff_sec * 1000) -
+        ((starttm->tv_nsec - currtm.tv_nsec) / USEC_TO_SEC);
+
+      /*NOTE: if user configured the timeout as say 0.5 ms, then timediff value
+         will be negetive if still 0.5 ms is not elapsed. this is intended and we should
+         not typecast to any unsigned type during this below if check */
+      if (timediff_ms > timeout_thr_ms)
+        {
+          sbr_set_sk_errno (sk, ETIMEDOUT);
+          return ETIMEDOUT;
+        }
+    }
+
+  return ERR_OK;
+}
+
+static inline int
+sbr_tcp_wait_new_conn (sbr_socket_t * sk, void **new_buf)
+{
+  int ret = 0;
+  int elem_num;
+  u32 timeout_thr_sec;
+  struct timespec starttm = { 0, 0 };
+  unsigned int retry_count = 0;
+  mring_handle ring = NULL;
+
+#define FAST_SLEEP_TIME 10000
+#define SLOW_SLEEP_TIME 500000
+#define FAST_RETRY_COUNT 100
+
+  ring = ss_get_recv_ring (sbr_get_conn (sk));  //clear codeDEX warning , CID:24284
+  timeout_thr_sec = sbr_get_fd_share (sk)->recv_timeout / 1000;
+  if (0 != timeout_thr_sec)
+    {
+      if (unlikely (0 != clock_gettime (CLOCK_MONOTONIC, &starttm)))
+        {
+          NSSBR_LOGERR ("Failed to get time, errno = %d", errno);
+        }
+    }
+
+  while (1)
+    {
+      if (ss_is_shut_rd (sbr_get_conn (sk)))
+        {
+          sbr_set_sk_errno (sk, EINVAL);
+          ret = EINVAL;
+          break;
+        }
+
+      elem_num = nsfw_mem_ring_dequeue (ring, new_buf);
+      if (1 == elem_num)
+        {
+          break;
+        }
+      else if (0 == elem_num)
+        {
+          ret = sbr_tcp_recv_is_timeout (sk, &starttm);
+          if (0 != ret)
+            {
+              break;
+            }
+
+          /* reduce CPU usage in blocking mode- Begin */
+          if (retry_count < FAST_RETRY_COUNT)
+            {
+              sys_sleep_ns (0, FAST_SLEEP_TIME);
+              retry_count++;
+            }
+          else
+            {
+              sys_sleep_ns (0, sbr_get_fd_share (sk)->block_polling_time);
+            }
+
+          continue;
+        }
+      else
+        {
+          sbr_set_sk_errno (sk, EINVAL);
+          ret = EINVAL;
+          break;
+        }
+    }
+
+  return ret;
+}
+
+NSTACK_STATIC inline int
+sbr_tcp_get_sockaddr (sbr_socket_t * sk, struct sockaddr *addr,
+                      socklen_t * addrlen)
+{
+  int ret;
+  spl_netconn_t *conn = sbr_get_conn (sk);
+
+  ret = (addr
+         && addrlen) ? sbr_get_sockaddr_and_len (ss_get_remote_port (conn),
+                                                 ss_get_remote_ip (conn),
+                                                 addr, addrlen) : 0;
+  if (0 != ret)
+    {
+      sbr_set_sk_io_errno (sk, EINVAL);
+      NSSBR_LOGERR ("sbr_tcp_get_sockaddr]fd=%d", sk->fd);
+      return -1;
+    }
+
+  return 0;
+}
+
+static int
+sbr_tcp_accept_socket (sbr_socket_t * sk, sbr_socket_t * new_sk,
+                       struct sockaddr *addr, socklen_t * addrlen)
+{
+  int err;
+  spl_netconn_t *newconn = NULL;
+
+  err = sbr_tcp_wait_new_conn (sk, (void **) &newconn);
+  if (ERR_OK != err)
+    {
+      return err;
+    }
+
+  err = sbr_init_conn_for_accept (new_sk, newconn);
+  if (ERR_OK != err)
+    {
+      /*When conn is null, return err; then do not need to free conn */
+      return err;
+    }
+
+  err = sbr_tcp_get_sockaddr (new_sk, addr, addrlen);
+  if (ERR_OK != err)
+    {
+      NSSBR_LOGERR ("sbr_get_sockaddr_and_socklen]ret=%d.", err);
+      sbr_tcp_accept_failed (new_sk);
+      return -1;
+    }
+
+  NSSBR_LOGINF
+    ("return]listen fd=%d,listen conn=%p,listen private_data=%p,accept fd=%d,accept conn=%p,accept private_data=%p",
+     sk->fd, sbr_get_conn (sk), ss_get_private_data (sbr_get_conn (sk)),
+     new_sk->fd, sbr_get_conn (new_sk),
+     ss_get_private_data (sbr_get_conn (new_sk)));
+
+  //ip_addr_info_print(SOCKETS_DEBUG, &sbr_get_conn(new_sk)->share_remote_ip);
+
+  /* test_epollCtl_003_001: Accept a conn. Add epoll_ctl with IN event and
+     send a msg from peer. The event will be given once epoll_wait is called.
+     Now, modify to IN|OUT. Calling epoll_event should return immediately since
+     the socket is writable. Currently, returns 0 events.
+     This issue is fixed here
+   */
+
+  /* Prevent automatic window updates, we do this on our own! */
+  ss_set_noautorecved_flag (sbr_get_conn (new_sk), 1);
+
+  ss_set_send_event (sbr_get_conn (new_sk), 1);
+
+  /* don't set conn->last_err: it's only ERR_OK, anyway */
+  return ERR_OK;
+}
+
+NSTACK_STATIC int
+sbr_tcp_accept (sbr_socket_t * sk, sbr_socket_t * new_sk,
+                struct sockaddr *addr, socklen_t * addrlen)
+{
+  int err;
+
+  /* If conn is not in listen state then return failure with error code: EINVAL(22) */
+  if (!ss_is_listen_state (sbr_get_conn (sk)))
+    {
+      NSSBR_LOGERR ("fd is not listening for connections]fd=%d,err=%d",
+                    sk->fd, EINVAL);
+      sbr_set_sk_errno (sk, EINVAL);
+
+      return -1;
+    }
+
+  if (ss_is_nonblock_flag (sbr_get_conn (sk))
+      && (0 >= ss_get_recv_event (sbr_get_conn (sk))))
+    {
+      NSSBR_LOGERR ("fd is nonblocking and rcvevent<=0]fd=%d,err=%d", sk->fd,
+                    EWOULDBLOCK);
+      sbr_set_sk_errno (sk, EWOULDBLOCK);
+
+      return -1;
+    }
+
+  err = ss_get_last_errno (sbr_get_conn (sk));
+  if (SPL_ERR_IS_FATAL (err))
+    {
+      /* don't recv on fatal errors: this might block the application task
+         waiting on acceptmbox forever! */
+      sbr_set_sk_errno (sk, sbr_spl_err_to_errno (err));
+
+      return -1;
+    }
+
+  /* wait for a new connection */
+  err = sbr_tcp_accept_socket (sk, new_sk, addr, addrlen);
+  if (ERR_OK != err)
+    {
+      NSSBR_LOGERR ("sbr_tcp_accept_socket failed]fd=%d,err=%d", sk->fd, err);
+
+      return -1;
+    }
+
+  /* Prevent automatic window updates, we do this on our own! */
+  ss_sub_recv_event (sbr_get_conn (sk));
+
+  sbr_set_sk_errno (sk, ERR_OK);
+
+  /* test_epollCtl_003_001: Accept a conn. Add epoll_ctl with IN event and
+     send a msg from peer. The event will be given once epoll_wait is called.
+     Now, modify to IN|OUT. Calling epoll_event should return immediately since
+     the socket is writable. Currently, returns 0 events.
+     This issue is fixed here
+   */
+
+  ss_set_send_event (sbr_get_conn (new_sk), 1);
+
+  return new_sk->fd;
+}
+
+NSTACK_STATIC int
+sbr_tcp_accept4 (sbr_socket_t * sk, sbr_socket_t * new_sk,
+                 struct sockaddr *addr, socklen_t * addrlen, int flags)
+{
+  int fd = sbr_tcp_accept (sk, new_sk, addr, addrlen);
+
+  if (0 > fd)
+    {
+      return fd;
+    }
+
+  if (flags & O_NONBLOCK)
+    {
+      int opts = new_sk->fdopt->fcntl (new_sk, F_GETFL, 0);
+      if (opts < 0)
+        {
+          NSSBR_LOGERR ("sbr_tcp_fcntl(sock,GETFL)");
+          sbr_tcp_accept_failed (new_sk);
+          return -1;
+        }
+
+      opts = opts | O_NONBLOCK;
+
+      if (new_sk->fdopt->fcntl (new_sk, F_SETFL, opts) < 0)
+
+        {
+          NSSBR_LOGERR ("sbr_tcp_fcntl(sock,F_SETFL,opts)");
+          sbr_tcp_accept_failed (new_sk);
+          return -1;
+        }
+    }
+
+  return new_sk->fd;
+}
+
+NSTACK_STATIC int
+sbr_tcp_connect (sbr_socket_t * sk, const struct sockaddr *name,
+                 socklen_t namelen)
+{
+  const struct sockaddr_in *addr_in = (const struct sockaddr_in *) name;
+
+  NSSBR_LOGINF ("fd=%d,addr=%s,port=%d,conn=%p,private_data=%p", sk->fd,
+                spl_inet_ntoa (addr_in->sin_addr), ntohs (addr_in->sin_port),
+                sbr_get_conn (sk), ss_get_private_data (sbr_get_conn (sk)));
+  spl_ip_addr_t remote_addr;
+
+  inet_addr_to_ipaddr (&remote_addr, &addr_in->sin_addr);
+  u16 remote_port = addr_in->sin_port;
+
+  spl_ip_addr_t local_addr;
+  if (IPADDR_ANY == ss_get_local_ip (sbr_get_conn (sk))->addr)
+    {
+      if (sbr_get_src_ip (remote_addr.addr, &local_addr.addr) != 0)
+        {
+          sbr_set_sk_errno (sk, EHOSTUNREACH);
+          NSSBR_LOGERR ("get src ip failed]fd=%d", sk->fd);
+          return -1;
+        }
+    }
+
+  if (sbr_handle_connect (sk, &remote_addr, ntohs (remote_port), &local_addr)
+      != 0)
+    {
+      NSSBR_LOGERR ("fail]fd=%d", sk->fd);
+      return -1;
+    }
+
+  if (ss_is_shut_rd (sbr_get_conn (sk)))
+    {
+      NSSBR_LOGERR ("shut_rd]fd=%d", sk->fd);
+      sbr_set_sk_errno (sk, ECONNRESET);
+      return -1;
+    }
+
+  NSSBR_LOGINF ("succeeded]fd=%d", sk->fd);
+  sbr_set_sk_errno (sk, ERR_OK);
+
+  return 0;
+}
+
+static u8 netconn_shutdown_opt[] = {
+  SPL_NETCONN_SHUT_RD,
+  SPL_NETCONN_SHUT_WR,
+  SPL_NETCONN_SHUT_RDWR
+};
+
+NSTACK_STATIC int
+sbr_tcp_shutdown (sbr_socket_t * sk, int how)
+{
+  err_t err;
+
+  if (ss_is_listen_state (sbr_get_conn (sk)))
+    {
+      return 0;
+    }
+
+  ss_set_shut_status (sbr_get_conn (sk), how);
+
+  err = sbr_handle_shutdown (sk, netconn_shutdown_opt[how]);
+
+  return (err == ERR_OK ? 0 : -1);
+}
+
+NSTACK_STATIC int
+sbr_tcp_getsockname (sbr_socket_t * sk, struct sockaddr *name,
+                     socklen_t * namelen)
+{
+
+  NSSBR_LOGINF ("sockname]fd=%d,tcp_state=%d", sk->fd,
+                ss_get_tcp_state (sbr_get_conn (sk)));
+
+  return sbr_handle_get_name (sk, name, namelen, SBR_GET_SOCK_NAME);
+}
+
+NSTACK_STATIC int
+sbr_tcp_getpeername (sbr_socket_t * sk, struct sockaddr *name,
+                     socklen_t * namelen)
+{
+
+  if (SPL_CLOSED == ss_get_tcp_state (sbr_get_conn (sk)))
+    {
+      NSSBR_LOGERR ("connection not exist]fd=%d", sk->fd);
+      sbr_set_sk_errno (sk, ENOTCONN);
+      return -1;
+    }
+
+  NSSBR_LOGINF ("peername]fd=%d,tcp_state=%d", sk->fd,
+                ss_get_tcp_state (sbr_get_conn (sk)));
+
+  return sbr_handle_get_name (sk, name, namelen, SBR_GET_PEER_NAME);
+}
+
+static int
+sbr_getsockopt_ipproto_tcp (int optname, void *optval, socklen_t optlen)
+{
+  int err = ERR_OK;
+
+  switch (optname)
+    {
+    case SPL_TCP_NODELAY:
+      break;
+    case SPL_TCP_KEEPIDLE:
+    case SPL_TCP_KEEPINTVL:
+    case SPL_TCP_KEEPCNT:
+      break;
+    default:
+      err = ENOPROTOOPT;
+      break;
+    }
+
+  return err;
+}
+
+NSTACK_STATIC int inline
+sbr_tcp_set_sockopt_err (sbr_socket_t * sk, int err)
+{
+  if (ENOPROTOOPT == err)
+    {
+      return 0;
+    }
+  else
+    {
+      sbr_set_sk_errno (sk, err);
+      return -1;
+    }
+}
+
+NSTACK_STATIC int inline
+sbr_tcp_get_sockopt_err (sbr_socket_t * sk, int err)
+{
+  sbr_set_sk_errno (sk, err);
+  return -1;
+}
+
+NSTACK_STATIC int
+sbr_tcp_getsockopt (sbr_socket_t * sk, int level, int optname, void *optval,
+                    socklen_t * optlen)
+{
+  int err = 0;
+
+  switch (level)
+    {
+    case SOL_SOCKET:
+      err = sbr_getsockopt_sol_socket (sk, optname, optval, *optlen);
+      break;
+
+    case IPPROTO_IP:
+      err = sbr_getsockopt_ipproto_ip (optname, optval, *optlen);
+      break;
+
+    case IPPROTO_TCP:
+      err = sbr_getsockopt_ipproto_tcp (optname, optval, *optlen);
+      break;
+
+    case NSTACK_SOCKOPT:
+      if ((optname == NSTACK_SEM_SLEEP) || (*optlen < sizeof (u32_t)))
+        {
+          *(u32_t *) optval =
+            sbr_get_fd_share (sk)->block_polling_time / 1000;
+          NSSOC_LOGINF
+            ("tcp get recv sleep time success, usleep time = %d,fd = %d",
+             sbr_get_fd_share (sk)->block_polling_time, sk->fd);
+          return ERR_OK;
+        }
+      else
+        {
+          NSSOC_LOGINF ("get recv sleep time failed, fd = %d", sk->fd);
+          sbr_set_sk_io_errno (sk, EINVAL);
+          return -1;
+        }
+    default:
+      err = ENOPROTOOPT;
+      break;
+    }
+
+  if (0 != err)
+    {
+      NSSBR_LOGERR ("fail]fd=%d,level=%d,optname=%d,err=%d", sk->fd, level,
+                    optname, err);
+      /* for option not support ,getsockopt() should return fail */
+      return sbr_tcp_get_sockopt_err (sk, err);
+    }
+
+  return sbr_handle_getsockopt (sk, level, optname, optval, optlen);
+}
+
+int
+sbr_setsockopt_ipproto_tcp (int optname, socklen_t optlen)
+{
+  int err = 0;
+
+  if (optlen < sizeof (int))
+    {
+      return EINVAL;
+    }
+
+  switch (optname)
+    {
+    case SPL_TCP_KEEPIDLE:
+    case SPL_TCP_KEEPINTVL:
+    case SPL_TCP_KEEPCNT:
+      break;
+    default:
+      err = ENOPROTOOPT;
+      break;
+    }
+
+  return err;
+}
+
+NSTACK_STATIC int
+sbr_tcp_setsockopt (sbr_socket_t * sk, int level, int optname,
+                    const void *optval, socklen_t optlen)
+{
+  int err = 0;
+
+  switch (level)
+    {
+    case SOL_SOCKET:
+      err =
+        sbr_setsockopt_sol_socket (sk, optname, optval, optlen,
+                                   SPL_NETCONN_TCP);
+      break;
+    case IPPROTO_IP:
+      err =
+        sbr_setsockopt_ipproto_ip (optname, optval, optlen, SPL_NETCONN_TCP);
+      break;
+    case IPPROTO_TCP:
+      err = sbr_setsockopt_ipproto_tcp (optname, optlen);
+      break;
+    case NSTACK_SOCKOPT:
+      {
+        u32_t sleep_time = *(u32_t *) optval;
+        /*sleep time should less than 1s */
+        if ((optname == NSTACK_SEM_SLEEP) && (optlen >= sizeof (u32_t))
+            && (sleep_time < 1000000))
+          {
+            sbr_get_fd_share (sk)->block_polling_time = sleep_time * 1000;
+            NSSOC_LOGINF
+              ("tcp set recv sleep time success, usleep time = %d,fd = %d",
+               sbr_get_fd_share (sk)->block_polling_time, sk->fd);
+            return ERR_OK;
+          }
+        else
+          {
+            NSSOC_LOGINF ("set recv sleep time failed, fd = %d", sk->fd);
+            sbr_set_sk_io_errno (sk, EINVAL);
+            return -1;
+          }
+      }
+
+    default:
+      err = ENOPROTOOPT;
+      break;
+    }
+
+  if (0 != err)
+    {
+      NSSBR_LOGERR ("fail]fd=%d,level=%d,optname=%d,err=%d", sk->fd, level,
+                    optname, err);
+
+      return sbr_tcp_set_sockopt_err (sk, err);
+    }
+
+  return sbr_handle_setsockopt (sk, level, optname, optval, optlen);
+}
+
+static inline u16
+sbr_tcp_mbuf_count (struct spl_pbuf *p)
+{
+  u16 count = 0;
+  struct spl_pbuf *buf = p;
+  struct common_mem_mbuf *mbuf;
+
+  while (buf)
+    {
+      mbuf =
+        (struct common_mem_mbuf *) ((char *) buf -
+                                    sizeof (struct common_mem_mbuf));
+      while (mbuf)
+        {
+          count++;
+#ifdef HAL_LIB
+#else
+          mbuf = mbuf->next;
+#endif
+        }
+
+      buf = (struct spl_pbuf *) ADDR_SHTOL (buf->next_a);
+    }
+
+  return count;
+}
+
+static inline void
+sbr_tcp_free_recvbuf (sbr_socket_t * sk, struct spl_pbuf *p)
+{
+  int len;
+
+  if (ss_is_noautorecved_flag (sbr_get_conn (sk)))
+    {
+      len = sbr_tcp_mbuf_count (p);
+      sbr_handle_tcp_recv (sk, len, p);
+    }
+}
+
+static inline void
+sbr_tcp_recv_no_peak (sbr_socket_t * sk, struct spl_pbuf *buf, u32 buflen,
+                      u32 copylen)
+{
+  if ((buflen - copylen) > 0)
+    {
+      sbr_get_fd_share (sk)->lastdata = (void *) ADDR_LTOSH (buf);
+      sbr_get_fd_share (sk)->lastoffset += copylen;
+    }
+  else
+    {
+      sbr_get_fd_share (sk)->lastdata = 0;
+      sbr_get_fd_share (sk)->lastoffset = 0;
+      sbr_tcp_free_recvbuf (sk, buf);
+    }
+}
+
+static inline int
+sbr_tcp_recv_from_ring (sbr_socket_t * sk, struct spl_pbuf **buf, i32 timeout)
+{
+  int err;
+  spl_netconn_t *conn = sbr_get_conn (sk);
+
+  err = ss_get_last_errno (conn);
+  if (SPL_ERR_IS_FATAL (err))
+    {
+      /* don't recv on fatal errors: this might block the application task
+         waiting on recvmbox forever! */
+
+      /* @todo: this does not allow us to fetch data that has been put into recvmbox
+         before the fatal error occurred - is that a problem? */
+      NSSBR_LOGDBG ("last err when recv:]=%d", err);
+      if (ERR_CLSD != err)
+        {
+          sbr_set_sk_io_errno (sk, sbr_spl_err_to_errno (err));
+          return -1;
+        }
+    }
+
+  *buf = NULL;
+  if (0 != sbr_dequeue_buf (sk, (void **) buf, timeout))
+    {
+      return -1;
+    }
+
+  ss_sub_recv_avail (sbr_get_conn (sk), (*buf)->tot_len);
+
+  ss_sub_recv_event (sbr_get_conn (sk));
+
+  return 0;
+}
+
+static inline int
+sbr_tcp_recv_state_check (sbr_socket_t * sk)
+{
+
+  spl_tcp_state_t state = ss_get_tcp_state (sbr_get_conn (sk));
+
+  NSSBR_LOGDBG ("tcp state when recv:]=%d", state);
+
+  //close_wait state also can recive data
+  //no connect cannot recive data
+  if (SPL_ESTABLISHED > state)
+    {
+      if (SPL_SHUT_WR != ss_get_shut_status (sbr_get_conn (sk)))
+        {
+          /* after all data retrnasmission, connection is active */
+          /* patch solution as last_err is not maintained properly */
+          if ((SPL_CLOSED == state)
+              && (ERR_TIMEOUT == ss_get_last_errno (sbr_get_conn (sk))))
+            {
+              sbr_set_sk_io_errno (sk, ETIMEDOUT);
+            }
+          else if ((SPL_CLOSED == state)
+                   && (ERR_RST == ss_get_last_errno (sbr_get_conn (sk))))
+            {
+              sbr_set_sk_io_errno (sk, ECONNRESET);
+            }
+          else
+            {
+              sbr_set_sk_io_errno (sk, ENOTCONN);
+            }
+
+          return -1;
+        }
+    }
+
+  return 0;
+}
+
+NSTACK_STATIC int
+sbr_tcp_recvfrom (sbr_socket_t * sk, void *mem, size_t len, int flags,
+                  struct sockaddr *from, socklen_t * fromlen)
+{
+  struct spl_pbuf *p;
+  u32 buflen;
+  u32 copylen;
+  u32 off = 0;
+  u8 done = 0;
+  int para_len = len;
+  int retval = 0;
+
+  sbr_com_lock_recv (sk);
+  NSSOC_LOGINF ("recv start, fd = %d last data %p flags to be set %d", sk->fd,
+                ADDR_SHTOL (sbr_get_fd_share (sk)->lastdata), MSG_DONTWAIT);
+
+  if (0 != sbr_tcp_recv_state_check (sk))
+    {
+      retval = -1;
+      goto sbr_tcp_recvfrom_exit;
+    }
+
+  do
+    {
+      if (sbr_get_fd_share (sk)->lastdata)
+        {
+          p = ADDR_SHTOL (sbr_get_fd_share (sk)->lastdata);
+        }
+      else
+        {
+          if ((flags & MSG_DONTWAIT)
+              || ss_is_nonblock_flag (sbr_get_conn (sk)))
+            {
+              NSSOC_LOGINF ("call ss_get_recv_event");
+              if (0 >= ss_get_recv_event (sbr_get_conn (sk)))
+                {
+                  NSSOC_LOGINF ("no recv event]fd=%d", sk->fd);
+                  sbr_set_sk_io_errno (sk, EWOULDBLOCK);
+                  retval = -1;
+                  goto sbr_tcp_recvfrom_exit;
+                }
+            }
+
+          if (0 !=
+              sbr_tcp_recv_from_ring (sk, &p,
+                                      sbr_get_fd_share (sk)->recv_timeout))
+            {
+              /* already received data, return that */
+              if (off > 0)
+                {
+                  sbr_set_sk_io_errno (sk, ERR_OK);
+                  retval = off;
+                  goto sbr_tcp_recvfrom_exit;
+                }
+
+              /* we tell the user the connection is closed by returning zero */
+              if (sbr_get_sk_errno (sk) == ENOTCONN)
+                {
+                  retval = 0;
+                  goto sbr_tcp_recvfrom_exit;
+                }
+
+              retval = -1;
+              goto sbr_tcp_recvfrom_exit;
+            }
+
+          sbr_get_fd_share (sk)->lastdata = (void *) ADDR_LTOSH (p);
+        }
+
+      buflen = p->tot_len - sbr_get_fd_share (sk)->lastoffset;
+      copylen = len > buflen ? buflen : len;
+
+      if ((copylen > 0)
+          && 0 == spl_pbuf_copy_partial (p, (u8 *) mem + off, copylen,
+                                         sbr_get_fd_share (sk)->lastoffset))
+        {
+          NSSBR_LOGERR ("copy failed]fd=%d", sk->fd);
+          sbr_set_sk_io_errno (sk, EFAULT);
+          retval = -1;
+          goto sbr_tcp_recvfrom_exit;
+        }
+
+      off += copylen;
+
+      len -= copylen;
+
+      if ((len == 0) || (ss_get_recv_event (sbr_get_conn (sk)) <= 0)
+          || ((flags & MSG_PEEK) != 0))
+        {
+          if ((off >= sbr_get_fd_share (sk)->rcvlowat)
+              || (para_len <= sbr_get_fd_share (sk)->rcvlowat))
+            {
+              done = 1;
+            }
+        }
+
+      if (done)
+        {
+          if (sbr_tcp_get_sockaddr (sk, from, fromlen) != 0)
+            {
+              retval = -1;
+              goto sbr_tcp_recvfrom_exit;
+            }
+        }
+
+      /* If this is a TCP socket, check if there is data left in the buffer,
+         If so, it should be saved in the sock structure for next  time around. */
+      if (!(flags & MSG_PEEK))
+        {
+          sbr_tcp_recv_no_peak (sk, p, buflen, copylen);
+        }
+    }
+  while (!done);
+
+  retval = off;
+
+  NSSOC_LOGINF ("recv done, fd = %d last data %p", sk->fd);
+sbr_tcp_recvfrom_exit:
+
+  NSSOC_LOGINF ("recv exit, fd = %d last data %p", sk->fd);
+  sbr_com_unlock_recv (sk);
+  return retval;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_tcp_recvdata
+*   Description  : recvdata
+*   Input        : sbr_socket_t* sk
+*                  const struct iovec* iov
+*                  int iovcnt
+*                  int flags
+*   Output       : None
+*   Return Value : static inline int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline int
+sbr_tcp_recvdata (sbr_socket_t * sk, const struct iovec *iov, int iovcnt)
+{
+  int max = SBR_MAX_INTEGER;
+  int len = 0;
+  int ret = 0;
+  int i = 0;
+
+  do
+    {
+      len += ret;
+
+      if (!iov[i].iov_base || (0 == iov[i].iov_len))
+        {
+          ret = 0;
+          continue;
+        }
+
+      ret = sbr_tcp_recvfrom (sk, (char *) iov[i].iov_base, iov[i].iov_len, 0,
+                              NULL, NULL);
+    }
+  while ((ret == (int) iov[i].iov_len) && (iovcnt > (++i))
+         && (max - len - ret > (int) iov[i].iov_len));
+
+  if (len == 0)
+    {
+      return ret;
+    }
+  else
+    {
+      return (ret == -1 ? len : len + ret);
+    }
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_tcp_readv
+*   Description  : readv
+*   Input        : sbr_socket_t* sk
+*                  const struct iovec* iov
+*                  int iovcnt
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static int
+sbr_tcp_readv (sbr_socket_t * sk, const struct iovec *iov, int iovcnt)
+{
+  return sbr_tcp_recvdata (sk, iov, iovcnt);
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_tcp_recvmsg
+*   Description  : recvmsg,unsupport flags
+*   Input        : sbr_socket_t* sk
+*                  struct msghdr* msg
+*                  int flags
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static int
+sbr_tcp_recvmsg (sbr_socket_t * sk, struct msghdr *msg, int flags)
+{
+  if (sbr_tcp_get_sockaddr
+      (sk, (struct sockaddr *) msg->msg_name, &msg->msg_namelen) != 0)
+    {
+      return -1;
+    }
+
+  return sbr_tcp_recvdata (sk, msg->msg_iov, msg->msg_iovlen);
+}
+
+static int
+sbr_tcp_send_is_timeout (sbr_socket_t * sk, struct timespec *starttm)
+{
+  struct timespec currtm;
+  i64 timediff_ms, timediff_sec;
+  i64 timeout_thr_ms, timeout_thr_sec;
+
+  timeout_thr_ms = sbr_get_fd_share (sk)->send_timeout;
+  if (0 == timeout_thr_ms)
+    {
+      return 0;
+    }
+
+  /* it is possible that starttm don't be inited,
+     if send_timeout is change when this write function is running */
+  timeout_thr_sec = (timeout_thr_ms + 240) >> 10;
+
+  /* Handle system time change side-effects */
+  if (unlikely (0 != clock_gettime (CLOCK_MONOTONIC, &currtm)))
+    {
+      NSSBR_LOGERR ("Failed to get time, errno = %d", errno);
+    }
+
+  timediff_sec = currtm.tv_sec - starttm->tv_sec;
+  if (timediff_sec >= timeout_thr_sec)
+    {
+      timediff_ms = currtm.tv_nsec > starttm->tv_nsec ?
+        (timediff_sec * 1000) + (currtm.tv_nsec -
+                                 starttm->tv_nsec) /
+        USEC_TO_SEC : (timediff_sec * 1000) -
+        ((starttm->tv_nsec - currtm.tv_nsec) / USEC_TO_SEC);
+
+      /*NOTE: if user configured the timeout as say 0.5 ms, then timediff value
+         will be negetive if still 0.5 ms is not elapsed. this is intended and we should
+         not typecast to any unsigned type during this below if check */
+      if (timediff_ms > timeout_thr_ms)
+        {
+          return 1;
+        }
+    }
+
+  return 0;
+}
+
+static inline int
+sbr_tcp_write_is_wait (sbr_socket_t * sk, struct timespec *starttm,
+                       int noneblockFlg)
+{
+  if (noneblockFlg || sbr_tcp_send_is_timeout (sk, starttm))
+    {
+      return 0;
+    }
+  else
+    {
+      return 1;
+    }
+}
+
+static struct spl_pbuf *
+sbr_tcp_write_alloc_buf (sbr_socket_t * sk, size_t seglen, u8 api_flag,
+                         struct timespec *starttm, u8 * errno_flag)
+{
+  spl_netconn_t *conn = sbr_get_conn (sk);
+
+  struct spl_pbuf *curr_buf = NULL;
+  size_t head_len = SPL_TCP_HLEN + SPL_PBUF_IP_HLEN + SPL_PBUF_LINK_HLEN;
+  int noneblockFlg = (api_flag & SPL_NETCONN_DONTBLOCK)
+    || ss_is_nonblock_flag (sbr_get_conn (sk));
+
+  do
+    {
+      /* When packages are lost more than TCP_MAXRTX times,
+       * conn will be closed and pcb will be removed. */
+      if (ss_get_tcp_state (conn) == SPL_CLOSED)
+        {
+          NSSBR_LOGERR ("pcb SPL_CLOSED]conn=%p", conn);
+          sbr_set_sk_io_errno (sk, ECONNABORTED);
+          /* Must set errno_flag when set errno, to avoid errnno overwritten by sbr_tcp_write */
+          *errno_flag = 1;
+          return NULL;
+        }
+
+      curr_buf = sbr_malloc_tx_pbuf (seglen + head_len, head_len);
+      if (NULL == curr_buf)
+        {
+          if (!sbr_tcp_write_is_wait (sk, starttm, noneblockFlg))
+            {
+              return NULL;
+            }
+
+          int err = ss_get_last_errno (sbr_get_conn (sk));
+          if (SPL_ERR_IS_FATAL (err))
+            {
+              NSSBR_LOGERR ("connection fatal error!err=%d", err);
+              sbr_set_sk_io_errno (sk, sbr_spl_err_to_errno (err));
+              *errno_flag = 1;
+              return NULL;
+            }
+
+          sched_yield ();
+        }
+    }
+  while (curr_buf == NULL);
+
+  return curr_buf;
+}
+
+static inline void
+sbr_tcp_write_rel_buf (sbr_socket_t * sk, struct spl_pbuf *buf,
+                       u32 thread_index)
+{
+  if (buf != NULL)
+    {
+      sbr_handle_free_send_buf (sk, buf);
+    }
+}
+
+static inline int
+sbr_tcp_write_fill_buf (const void *data, size_t * pos,
+                        struct spl_pbuf *seg_buf, size_t seglen,
+                        size_t optlen)
+{
+  size_t start = *pos;
+  size_t copy = seglen;
+  struct spl_pbuf *first = seg_buf;
+
+  while ((0 < copy) && (NULL != first))
+    {
+      char *dst_ptr = PTR_SHTOL (char *, first->payload_a) + optlen;
+
+      if (NULL ==
+          common_memcpy (dst_ptr, (u8_t *) data + start, first->len - optlen))
+        {
+          NSSBR_LOGERR ("common_memcpy error]dst=%p,src=%p,len=%u",
+                        dst_ptr, (u8_t *) data + start, first->len);
+          return -1;
+        }
+
+      start += (first->len - optlen);
+      copy -= (first->len - optlen);
+      first = ADDR_SHTOL (first->next_a);
+    }
+
+  (*pos) = start;
+
+  return 0;
+}
+
+static inline int
+sbr_tcp_writev_fill_buf (const struct iovec *iov, size_t * iov_pos,
+                         int *iov_var, size_t * pos, struct spl_pbuf *seg_buf,
+                         size_t seglen, size_t optlen)
+{
+  size_t valid_copy_len;
+  size_t iov_data_left;
+
+  size_t copy = seglen;
+  size_t start = *pos;
+  size_t current_iov_pos = *iov_pos;
+  int current_iov_var = *iov_var;
+
+  u32 pbuf_offset = optlen;
+  u32 pbuf_data_len;
+  struct spl_pbuf *first = seg_buf;
+
+  while ((0 < copy) && (NULL != first))
+    {
+      iov_data_left = iov[current_iov_var].iov_len - current_iov_pos;
+      if (seglen == copy)
+        {
+          pbuf_offset = optlen;
+        }
+
+      pbuf_data_len = first->len - pbuf_offset;
+      valid_copy_len =
+        (iov_data_left > pbuf_data_len ? pbuf_data_len : iov_data_left);
+      if (NULL ==
+          common_memcpy ((char *) ADDR_SHTOL (first->payload_a) + pbuf_offset,
+                         (u8_t *) iov[current_iov_var].iov_base +
+                         current_iov_pos, valid_copy_len))
+        {
+          NSSBR_LOGERR
+            ("common_memcpy error]current_iov_var=%d, dst=%p,src=%p,len=%zu",
+             current_iov_var,
+             (char *) ADDR_SHTOL (first->payload_a) + pbuf_offset,
+             (u8_t *) iov[current_iov_var].iov_base + current_iov_pos,
+             valid_copy_len);
+          return -1;
+        }
+
+      start += valid_copy_len;
+      copy -= valid_copy_len;
+
+      if (iov_data_left == pbuf_data_len)
+        {
+          first = PTR_SHTOL (struct spl_pbuf *, first->next_a);
+          pbuf_offset = optlen; //+= valid_copy_len;
+          current_iov_var++;
+          current_iov_pos = 0;
+        }
+      else if (iov_data_left > pbuf_data_len)
+        {
+          first = PTR_SHTOL (struct spl_pbuf *, first->next_a);
+          pbuf_offset = optlen; //+= valid_copy_len;
+          current_iov_pos += valid_copy_len;
+        }
+      else
+        {
+          pbuf_offset += valid_copy_len;
+
+          current_iov_var++;
+          current_iov_pos = 0;
+        }
+    }
+
+  *iov_pos = current_iov_pos;
+  *iov_var = current_iov_var;
+  *pos = start;
+
+  return 0;
+}
+
+static inline void
+sbr_tcp_write_add_buf_to_list (struct spl_pbuf **p_head,
+                               struct spl_pbuf **p_tail,
+                               struct spl_pbuf *seg_buf, size_t seglen,
+                               size_t optlen)
+{
+  seg_buf->len = seglen + optlen;
+  seg_buf->tot_len = seglen + optlen;
+  seg_buf->next_a = 0;
+
+  /*put seg_buf after p_head */
+  if (NULL == (*p_head))
+    {
+      (*p_head) = seg_buf;
+      (*p_tail) = seg_buf;
+    }
+  else
+    {
+      (*p_tail)->next_a = ADDR_LTOSH (seg_buf);
+      (*p_tail) = seg_buf;
+    }
+}
+
+NSTACK_STATIC int
+sbr_tcp_write (sbr_socket_t * sk, const void *data, size_t size, u8 api_flag,
+               size_t * written)
+{
+  err_t err = -1;
+  size_t pos = 0, left, seglen;
+  u32 pbuf_seg_cnt = 0;
+  u32 thread_index = 0;
+  struct spl_pbuf *seg_buf = NULL;
+  struct spl_pbuf *p_head = NULL;
+  struct spl_pbuf *p_tail = p_head;
+  struct spl_netconn *conn = sbr_get_conn (sk);
+  u32 mss = ss_get_mss (sbr_get_conn (sk));
+
+  if (0 == size)
+    {
+      NSSBR_LOGERR ("fd=%d,size=%u", sk->fd, (u32) size);
+      return 0;
+    }
+
+  struct timespec ts;
+  if (unlikely (0 != clock_gettime (CLOCK_MONOTONIC, &ts)))
+    {
+      NSSBR_LOGERR ("Failed to get time, errno = %d", errno);
+    }
+
+  while (pos < size)
+    {
+      left = size - pos;
+      seglen = left > mss ? mss : left;
+      u8 errno_set = 0;
+      seg_buf =
+        sbr_tcp_write_alloc_buf (sk, seglen, api_flag, &ts, &errno_set);
+      if (NULL == seg_buf)
+        {
+          NSSBR_LOGINF ("sbr_tcp_write_alloc_buf failed......");
+          if (NULL != p_head)
+            {
+              err = sbr_handle_tcp_send (sk, size, p_head, api_flag);
+              if (ERR_OK != err)
+                {
+                  NSSBR_LOGERR ("sbr_handle_tcp_send error]err(%d)", err);
+                  goto err_ref_buf;
+                }
+            }
+
+          if (0 == pos)
+            {
+              /* If errno is already set in sbr_tcp_write_alloc_buf, do not overwrite here */
+              if (!errno_set)
+                {
+                  sbr_set_sk_io_errno (sk, EWOULDBLOCK);
+                }
+              return -1;
+            }
+
+          NSSBR_LOGDBG ("sent size %zu", pos);
+          *written = pos;
+
+          return ERR_OK;
+        }
+
+      if (0 != sbr_tcp_write_fill_buf (data, &pos, seg_buf, seglen, 0))
+        {
+          sbr_set_sk_io_errno (sk, EFAULT);
+          NSSBR_LOGERR ("sbr_tcp_write_fill_buf error]");
+          goto err_ref_buf;
+        }
+
+      sbr_tcp_write_add_buf_to_list (&p_head, &p_tail, seg_buf, seglen, 0);
+
+      ++pbuf_seg_cnt;
+      if (p_head
+          && ((SPL_TCP_SEND_MAX_SEG_PER_MSG <= pbuf_seg_cnt)
+              || (pos >= size)))
+        {
+          pbuf_seg_cnt = 0;
+          err = sbr_handle_tcp_send (sk, size, p_head, api_flag);
+          if (ERR_OK != err)
+            {
+              NSSBR_LOGERR ("sbr_handle_tcp_send error]err(%d)", err);
+              goto err_ref_buf;
+            }
+
+          p_head = NULL;
+        }
+    }
+
+  *written = size;
+
+  (void) conn;
+  return ERR_OK;
+
+err_ref_buf:
+  sbr_tcp_write_rel_buf (sk, p_head, thread_index);
+  (void) conn;
+  return -1;
+}
+
+NSTACK_STATIC int
+sbr_tcp_writev (sbr_socket_t * sk, const struct iovec *iov, int iovcnt)
+{
+  err_t err = -1;
+  int idx = 0;
+  size_t pos = 0, left, seglen, optlen = 0;
+  u32 pbuf_seg_cnt = 0;
+  u32 thread_index = 0;
+  size_t size = 0;
+  size_t iov_pos = 0;
+  int iov_var = 0;
+  struct spl_pbuf *seg_buf = NULL;
+  struct spl_pbuf *p_head = NULL;
+  struct spl_pbuf *p_tail = p_head;
+  struct spl_netconn *conn = sbr_get_conn (sk);
+  u32 mss = ss_get_mss (sbr_get_conn (sk));
+
+  if (mss <= optlen)
+    {
+      NSSBR_LOGERR ("mss invalid]mss=%u,optlen=%zu,fd=%d", mss, optlen,
+                    sk->fd);
+      sbr_set_sk_io_errno (sk, EINVAL);
+      return -1;
+    }
+
+  /* mss dose't include the tcp options length */
+  mss -= optlen;
+
+  while (idx < iovcnt)
+    {
+      if (SBR_MAX_INTEGER - iov[idx].iov_len <= size)
+        {
+          size = SBR_MAX_INTEGER;
+          break;
+        }
+
+      size += iov[idx].iov_len;
+      idx++;
+    }
+
+  struct timespec starttm;
+  if (unlikely (0 != clock_gettime (CLOCK_MONOTONIC, &starttm)))
+    {
+      NSSBR_LOGERR ("Failed to get time, errno = %d", errno);
+    }
+
+  while (pos < size)
+    {
+      left = size - pos;
+
+      seglen = left > mss ? mss : left;
+      u8 errno_set = 0;
+      seg_buf =
+        sbr_tcp_write_alloc_buf (sk, seglen + optlen, SPL_NETCONN_COPY,
+                                 &starttm, &errno_set);
+      if (NULL == seg_buf)
+        {
+          if (NULL != p_head)
+            {
+              err = sbr_handle_tcp_send (sk, size, p_head, SPL_NETCONN_COPY);
+              /*If errno is already set in sbr_tcp_write_alloc_buf, do not overwrite here */
+              if (err != ERR_OK)
+                {
+                  NSSBR_LOGERR ("sbr_handle_tcp_send error]err(%d)", err);
+                  goto err_ref_buf;
+                }
+            }
+
+          /* [Start]
+             1)Set SO_SNDTIMEO to 10
+             2)Send a msg of larger buff size.and let the timeout happen for send (dont receive at peer side.)
+             3)iRet will be 0 and  errno received will be 11 (EAGAIN).
+
+             Above issue is fixed.
+           */
+          if (0 == pos)
+            {
+              if (!errno_set)
+                {
+                  sbr_set_sk_io_errno (sk, EWOULDBLOCK);
+                }
+              return -1;
+            }
+          /* [End] */
+
+          NSSBR_LOGDBG ("sent size %zu", pos);
+
+          return pos;
+        }
+
+      if (0 !=
+          sbr_tcp_writev_fill_buf (iov, &iov_pos, &iov_var, &pos, seg_buf,
+                                   seglen, optlen))
+        {
+          sbr_set_sk_io_errno (sk, EFAULT);
+          NSSBR_LOGERR ("sbr_tcp_writev_fill_buf error]");
+          goto err_ref_buf;
+        }
+
+      sbr_tcp_write_add_buf_to_list (&p_head, &p_tail, seg_buf, seglen,
+                                     optlen);
+
+      /* @todo: for non-blocking write, check if 'size' would ever fit into
+         snd_queue or snd_buf */
+      ++pbuf_seg_cnt;
+      if (p_head
+          && ((SPL_TCP_SEND_MAX_SEG_PER_MSG <= pbuf_seg_cnt)
+              || (pos >= size)))
+        {
+          pbuf_seg_cnt = 0;
+          err = sbr_handle_tcp_send (sk, size, p_head, SPL_NETCONN_COPY);
+          if (ERR_OK != err)
+            {
+              NSSBR_LOGERR ("sbr_handle_tcp_send error]err(%d)", err);
+              goto err_ref_buf;
+            }
+
+          p_head = NULL;
+        }
+    }
+  (void) conn;
+  return size;
+
+err_ref_buf:
+  sbr_tcp_write_rel_buf (sk, p_head, thread_index);
+  (void) conn;
+  return -1;
+}
+
+NSTACK_STATIC int
+sbr_tcp_sendto (sbr_socket_t * sk, const void *data, size_t size, int flags,
+                const struct sockaddr *to, socklen_t tolen)
+{
+  return sk->fdopt->send (sk, data, size, flags);
+}
+
+static inline int
+sbr_tcp_send_state_check (sbr_socket_t * sk)
+{
+  if ((SPL_SHUT_WR == ss_get_shut_status (sbr_get_conn (sk)))
+      || (SPL_SHUT_RDWR == ss_get_shut_status (sbr_get_conn (sk))))
+    {
+      sbr_set_sk_io_errno (sk, EPIPE);
+      return -1;
+    }
+
+  spl_tcp_state_t state = ss_get_tcp_state (sbr_get_conn (sk));
+  if ((SPL_ESTABLISHED != state) && (SPL_CLOSE_WAIT != state))
+    {
+      /* after all data retrnasmission, connection is active */
+      /* patch solution as last_err is not maintained properly */
+      if ((SPL_CLOSED == state)
+          && (ERR_TIMEOUT == ss_get_last_errno (sbr_get_conn (sk))))
+        {
+          sbr_set_sk_io_errno (sk, ETIMEDOUT);
+        }
+      else if ((SPL_CLOSED == state)
+               && (ERR_RST == ss_get_last_errno (sbr_get_conn (sk))))
+        {
+          sbr_set_sk_io_errno (sk, ECONNRESET);
+        }
+      else
+        {
+          sbr_set_sk_io_errno (sk, EPIPE);
+        }
+
+      return -1;
+    }
+
+  return 0;
+}
+
+NSTACK_STATIC int
+sbr_tcp_send (sbr_socket_t * sk, const void *data, size_t size, int flags)
+{
+  int err;
+  size_t written = 0;
+  u8 write_flags;
+
+  if (0 != sbr_tcp_send_state_check (sk))
+    {
+      NSSBR_LOGDBG ("tcp state not correct]fd=%d, err=%d", sk->fd,
+                    sbr_get_sk_errno (sk));
+      return -1;
+    }
+
+  write_flags = SPL_NETCONN_COPY |
+    ((flags & MSG_MORE) ? SPL_NETCONN_MORE : 0) |
+    ((flags & MSG_DONTWAIT) ? SPL_NETCONN_DONTBLOCK : 0);
+
+  NSSBR_LOGINF ("Sbr tcp write start");
+  err = sbr_tcp_write (sk, data, size, write_flags, &written);
+  NSSBR_LOGINF ("Sbr tcp write end written %d", written);
+
+  return (err == ERR_OK ? written : -1);
+}
+
+NSTACK_STATIC int
+sbr_tcp_sendmsg (sbr_socket_t * sk, const struct msghdr *pmsg, int flags)
+{
+  if (0 != sbr_tcp_send_state_check (sk))
+    {
+      NSSBR_LOGDBG ("tcp state not correct]fd=%d, err=%d", sk->fd,
+                    sbr_get_sk_errno (sk));
+      return -1;
+    }
+
+  return sbr_tcp_writev (sk, pmsg->msg_iov, pmsg->msg_iovlen);
+}
+
+NSTACK_STATIC int
+sbr_tcp_fcntl (sbr_socket_t * sk, int cmd, long arg)
+{
+  int ret = 0;
+
+  switch (cmd)
+    {
+    case F_GETFL:
+      ret = ss_get_nonblock_flag (sbr_get_conn (sk));
+      NSSBR_LOGDBG ("F_GETFL]fd=%d,ret=%d", sk->fd, ret);
+      break;
+
+    case F_SETFL:
+      if (arg & O_NONBLOCK)
+        {
+          NSSBR_LOGDBG ("F_SETFL set O_NONBLOCK val]fd=%d,arg=%ld", sk->fd,
+                        arg);
+          ss_set_nonblock_flag (sbr_get_conn (sk), (arg & O_NONBLOCK));
+        }
+      else
+        {
+          NSSBR_LOGDBG ("F_SETFL clean O_NONBLOCK val]fd=%d,arg=%ld", sk->fd,
+                        arg);
+          ss_set_nonblock_flag (sbr_get_conn (sk), 0);
+        }
+
+      break;
+
+    default:
+      NSSBR_LOGERR ("cmd is not support]fd=%d,cmd=%d", sk->fd, cmd);
+      ret = -1;
+      sbr_set_sk_errno (sk, EINVAL);
+      break;
+    }
+
+  return ret;
+}
+
+NSTACK_STATIC int
+sbr_tcp_ioctl (sbr_socket_t * sk, unsigned long cmd, void *arg)
+{
+  int ret = 0;
+  int recv_avail;
+
+  switch (cmd)
+    {
+    case FIONREAD:
+      {
+        if (ss_is_listen_state (sbr_get_conn (sk)))
+          {
+            ret = -1;
+            sbr_set_sk_errno (sk, EINVAL);
+            break;
+          }
+
+        recv_avail = ss_get_recv_avail (sbr_get_conn (sk));
+        *((u32 *) arg) = recv_avail >= 0 ? recv_avail : 0;
+        if (sbr_get_fd_share (sk)->lastdata)
+          {
+            struct spl_pbuf *buf =
+              ADDR_SHTOL (sbr_get_fd_share (sk)->lastdata);
+            *((u32 *) arg) +=
+              (buf->tot_len - sbr_get_fd_share (sk)->lastoffset);
+          }
+      }
+
+      break;
+
+    case FIONBIO:
+      {
+        u8 val = 0;
+
+        if (arg && *(u32 *) arg)
+          {
+            val = 1;
+          }
+
+        ss_set_nonblock_flag (sbr_get_conn (sk), val);
+        NSSBR_LOGDBG ("FIONBIO]fd=%d,val=%u", sk->fd, val);
+      }
+
+      break;
+
+    default:
+      {
+        NSSBR_LOGERR ("cmd is not support]fd=%d,cmd=%lu", sk->fd, cmd);
+        ret = -1;
+        sbr_set_sk_errno (sk, ENOTTY);
+      }
+      break;
+    }
+
+  return ret;
+}
+
+NSTACK_STATIC int
+sbr_tcp_close (sbr_socket_t * sk)
+{
+  if (sbr_get_fd_share (sk)->lastdata)
+    {
+      struct spl_netbuf *buf = ADDR_SHTOL (sbr_get_fd_share (sk)->lastdata);
+      struct spl_pbuf *p = (struct spl_pbuf *) ADDR_SHTOL (buf->p);
+      sbr_tcp_free_recvbuf (sk, p);
+    }
+
+  return sbr_handle_close (sk, 0);
+}
+
+sbr_fdopt tcp_fdopt = {
+  .socket = sbr_tcp_socket,
+  .bind = sbr_tcp_bind,
+  .listen = sbr_tcp_listen,
+  .accept = sbr_tcp_accept,
+  .accept4 = sbr_tcp_accept4,
+  .connect = sbr_tcp_connect,
+  .shutdown = sbr_tcp_shutdown,
+  .getsockname = sbr_tcp_getsockname,
+  .getpeername = sbr_tcp_getpeername,
+  .getsockopt = sbr_tcp_getsockopt,
+  .setsockopt = sbr_tcp_setsockopt,
+  .recvfrom = sbr_tcp_recvfrom,
+  .readv = sbr_tcp_readv,
+  .recvmsg = sbr_tcp_recvmsg,
+  .send = sbr_tcp_send,
+  .sendto = sbr_tcp_sendto,
+  .sendmsg = sbr_tcp_sendmsg,
+  .writev = sbr_tcp_writev,
+  .fcntl = sbr_tcp_fcntl,
+  .ioctl = sbr_tcp_ioctl,
+  .close = sbr_tcp_close,
+  .peak = sbr_com_peak,
+  .lock_common = sbr_com_lock_common,
+  .unlock_common = sbr_com_unlock_common,
+  .ep_getevt = stackx_eventpoll_getEvt,
+  .ep_ctl = stackx_eventpoll_triggle,
+  .set_close_stat = NULL,
+};
diff --git a/stacks/lwip_stack/lwip_src/socket/stackx_udp.c b/stacks/lwip_stack/lwip_src/socket/stackx_udp.c
new file mode 100644
index 0000000..cf08731
--- /dev/null
+++ b/stacks/lwip_stack/lwip_src/socket/stackx_udp.c
@@ -0,0 +1,1169 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include "stackx_prot_com.h"
+#include "stackx_msg_handler.h"
+#include "stackx_pbuf.h"
+#include "stackx_epoll_api.h"
+#include "stackx_err.h"
+#include "nstack_securec.h"
+#include "common_pal_bitwide_adjust.h"
+#include "stackx_cfg.h"
+#include <netinet/in.h>
+#ifdef HAL_LIB
+#else
+#include "rte_memcpy.h"
+#endif
+
+#define SPL_PBUF_UDP_LEN (SPL_FRAME_MTU + SPL_PBUF_LINK_HLEN)
+#define L2_L3_ROOM_LEN (SPL_PBUF_LINK_HLEN + SPL_PBUF_IP_HLEN)
+#define L4_ROOM_LEN SPL_PBUF_UDP_HLEN
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_socket
+*   Description  : create socket
+*   Input        : sbr_socket_t * sk
+*                  int domain
+*                  int type
+*                  int protocol
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static int
+sbr_udp_socket (sbr_socket_t * sk, int domain, int type, int protocol)
+{
+  if (sbr_malloc_conn_for_sk (sk, SPL_NETCONN_UDP) != 0)
+    {
+      return -1;
+    }
+
+  int ret = sbr_handle_socket (sk, SPL_NETCONN_UDP, 0);
+  if (ret != 0)
+    {
+      sbr_free_conn_from_sk (sk);
+      return ret;
+    }
+
+  ss_set_nonblock_flag (sbr_get_conn (sk), (type & O_NONBLOCK));
+  ss_set_send_event (sbr_get_conn (sk), 1);
+  return ret;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_bind
+*   Description  : udp bind
+*   Input        : sbr_socket_t * sk
+*                  const struct sockaddr * name
+*                  socklen_t namelen
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_udp_bind (sbr_socket_t * sk, const struct sockaddr *name,
+              socklen_t namelen)
+{
+  const struct sockaddr_in *addr_in = (const struct sockaddr_in *) name;
+
+  NSSBR_LOGINF ("fd=%d,addr=%s,port=%d,conn=%p,private_data=%p", sk->fd,
+                spl_inet_ntoa (addr_in->sin_addr), ntohs (addr_in->sin_port),
+                sbr_get_conn (sk), ss_get_private_data (sbr_get_conn (sk)));
+  spl_ip_addr_t local_addr;
+  inet_addr_to_ipaddr (&local_addr, &addr_in->sin_addr);
+  u16 local_port = addr_in->sin_port;
+  return sbr_handle_bind (sk, &local_addr, ntohs (local_port));
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_listen
+*   Description  : unsupport
+*   Input        : sbr_socket_t * sk
+*                  int backlog
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_udp_listen (sbr_socket_t * sk, int backlog)
+{
+  NSSBR_LOGERR ("type is not TCP]fd=%d", sk->fd);
+  sbr_set_sk_errno (sk, EOPNOTSUPP);
+  return -1;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_accept
+*   Description  : unsupport
+*   Input        : sbr_socket_t * sk
+*                  sbr_socket_t * new_sk
+*                  struct sockaddr * addr
+*                  socklen_t * addrlen
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_udp_accept (sbr_socket_t * sk, sbr_socket_t * new_sk,
+                struct sockaddr *addr, socklen_t * addrlen)
+{
+  NSSBR_LOGERR ("type is not TCP]fd=%d", sk->fd);
+  sbr_set_sk_errno (sk, EOPNOTSUPP);
+  return -1;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_accept4
+*   Description  : unsupport
+*   Input        : sbr_socket_t * sk
+*                  sbr_socket_t * new_sk
+*                  struct sockaddr * addr
+*                  socklen_t * addrlen
+*                  int flags
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_udp_accept4 (sbr_socket_t * sk, sbr_socket_t * new_sk,
+                 struct sockaddr *addr, socklen_t * addrlen, int flags)
+{
+  NSSBR_LOGERR ("type is not TCP]fd=%d", sk->fd);
+  sbr_set_sk_errno (sk, EOPNOTSUPP);
+  return -1;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_connect
+*   Description  : udp connect
+*   Input        : sbr_socket_t * sk
+*                  const struct sockaddr * name
+*                  socklen_t namelen
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_udp_connect (sbr_socket_t * sk, const struct sockaddr *name,
+                 socklen_t namelen)
+{
+  const struct sockaddr_in *addr_in = (const struct sockaddr_in *) name;
+
+  NSSBR_LOGINF ("fd=%d,addr=%s,port=%d,conn=%p,private_data=%p", sk->fd,
+                spl_inet_ntoa (addr_in->sin_addr), ntohs (addr_in->sin_port),
+                sbr_get_conn (sk), ss_get_private_data (sbr_get_conn (sk)));
+  spl_ip_addr_t remote_addr;
+
+  inet_addr_to_ipaddr (&remote_addr, &addr_in->sin_addr);
+  u16 remote_port = addr_in->sin_port;
+
+  spl_ip_addr_t local_addr;
+  if (IPADDR_ANY == ss_get_local_ip (sbr_get_conn (sk))->addr)
+    {
+      if (sbr_get_src_ip (remote_addr.addr, &local_addr.addr) != 0)
+        {
+          sbr_set_sk_errno (sk, EHOSTUNREACH);
+          NSSBR_LOGERR ("get src ip failed]fd=%d", sk->fd);
+          return -1;
+        }
+    }
+
+  return sbr_handle_connect (sk, &remote_addr, ntohs (remote_port),
+                             &local_addr);
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_shutdown
+*   Description  : udp shutdown
+*   Input        : sbr_socket_t * sk
+*                  int how
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static int
+sbr_udp_shutdown (sbr_socket_t * sk, int how)
+{
+  ss_set_shut_status (sbr_get_conn (sk), how);
+  NSSBR_LOGERR ("type is not TCP]fd=%d", sk->fd);
+  sbr_set_sk_errno (sk, EOPNOTSUPP);
+  return -1;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_getsockname
+*   Description  : get sock name
+*   Input        : sbr_socket_t * sk
+*                  struct sockaddr * name
+*                  socklen_t * namelen
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_udp_getsockname (sbr_socket_t * sk, struct sockaddr *name,
+                     socklen_t * namelen)
+{
+  return sbr_handle_get_name (sk, name, namelen, SBR_GET_SOCK_NAME);
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_getpeername
+*   Description  : get peer name
+*   Input        : sbr_socket_t * sk
+*                  struct sockaddr * name
+*                  socklen_t * namelen
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_udp_getpeername (sbr_socket_t * sk, struct sockaddr *name,
+                     socklen_t * namelen)
+{
+  return sbr_handle_get_name (sk, name, namelen, SBR_GET_PEER_NAME);
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_getsockopt
+*   Description  : get sockopt
+*   Input        : sbr_socket_t * sk
+*                  int level
+*                  int optname
+*                  void * optval
+*                  socklen_t * optlen
+*   Output       : None
+*   Return Value : int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+int
+sbr_udp_getsockopt (sbr_socket_t * sk, int level, int optname, void *optval,
+                    socklen_t * optlen)
+{
+  int err = 0;
+
+  switch (level)
+    {
+    case SOL_SOCKET:
+      err = sbr_getsockopt_sol_socket (sk, optname, optval, *optlen);
+      break;
+    case IPPROTO_IP:
+      err = sbr_getsockopt_ipproto_ip (optname, optval, *optlen);
+      break;
+
+    case NSTACK_SOCKOPT:
+      if ((optname == NSTACK_SEM_SLEEP) || (*optlen < sizeof (u32_t)))
+        {
+          *(u32_t *) optval =
+            sbr_get_fd_share (sk)->block_polling_time / 1000;
+          NSSOC_LOGINF
+            ("udp get recv sleep time success]usleep time=%d,fd=%d",
+             sbr_get_fd_share (sk)->block_polling_time, sk->fd);
+          return ERR_OK;
+        }
+      else
+        {
+          NSSOC_LOGINF ("get recv sleep time failed]fd=%d", sk->fd);
+          sbr_set_sk_io_errno (sk, EINVAL);
+          return -1;
+        }
+
+    default:
+      err = ENOPROTOOPT;
+      break;
+    }
+
+  if (err != 0)
+    {
+      NSSBR_LOGERR ("fail]fd=%d,level=%d,optname=%d,err=%d", sk->fd, level,
+                    optname, err);
+      /* for option not support ,getsockopt() should return fail */
+      sbr_set_sk_errno (sk, err);
+      return -1;
+    }
+
+  return sbr_handle_getsockopt (sk, level, optname, optval, optlen);
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_setsockopt
+*   Description  : set sockopt
+*   Input        : sbr_socket_t * sk
+*                  int level
+*                  int optname
+*                  const void * optval
+*                  socklen_t optlen
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static int
+sbr_udp_setsockopt (sbr_socket_t * sk, int level, int optname,
+                    const void *optval, socklen_t optlen)
+{
+  NSSBR_LOGDBG ("udp setsockopt]fd=%d,level=%d,optname=%d", sk->fd, level,
+                optname);
+  int err = 0;
+  switch (level)
+    {
+    case SOL_SOCKET:
+      err =
+        sbr_setsockopt_sol_socket (sk, optname, optval, optlen,
+                                   SPL_NETCONN_UDP);
+      break;
+    case IPPROTO_IP:
+      err =
+        sbr_setsockopt_ipproto_ip (optname, optval, optlen, SPL_NETCONN_UDP);
+      break;
+    case NSTACK_SOCKOPT:
+      {
+        u32_t sleep_time = *(u32_t *) optval;
+        /*sleep time should less than 1s */
+        if ((optname == NSTACK_SEM_SLEEP) && (optlen >= sizeof (u32_t))
+            && (sleep_time < 1000000))
+          {
+            sbr_get_fd_share (sk)->block_polling_time = sleep_time * 1000;
+            NSSOC_LOGINF
+              ("udp set recv sleep time success]usleep time=%d,fd=%d",
+               sbr_get_fd_share (sk)->block_polling_time, sk->fd);
+            return ERR_OK;
+          }
+        else
+          {
+            NSSOC_LOGINF ("set recv sleep time failed]fd=%d", sk->fd);
+            sbr_set_sk_io_errno (sk, EINVAL);
+            return -1;
+          }
+      }
+
+    default:
+      err = ENOPROTOOPT;
+      break;
+    }
+
+  if (err != 0)
+    {
+      NSSBR_LOGERR ("fail]fd=%d,level=%d,optname=%d,err=%d", sk->fd, level,
+                    optname, err);
+
+      if (ENOPROTOOPT == err)
+        {
+          return 0;
+        }
+      else
+        {
+          sbr_set_sk_errno (sk, err);
+          return -1;
+        }
+    }
+
+  return sbr_handle_setsockopt (sk, level, optname, optval, optlen);
+}
+
+static inline int
+sbr_udp_get_from_addr (sbr_socket_t * sk, struct sockaddr *from,
+                       socklen_t * fromlen, struct spl_netbuf *buf)
+{
+  int ret;
+  u16 port = netbuf_fromport (buf);
+  spl_ip_addr_t *addr = netbuf_fromaddr (buf);
+
+  ret = (from
+         && fromlen) ? sbr_get_sockaddr_and_len (port, addr, from,
+                                                 fromlen) : 0;
+  if (0 != ret)
+    {
+      sbr_set_sk_io_errno (sk, EINVAL);
+      NSSBR_LOGERR ("sbr_udp_get_from_addr]fd=%d", sk->fd);
+      return -1;
+    }
+
+  return 0;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_recv_from_ring
+*   Description  : recv buf from ring
+*   Input        : sbr_socket_t * sk
+*                  struct spl_netbuf ** buf
+*                  i32 timeout
+*   Output       : None
+*   Return Value : static inline int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline int
+sbr_udp_recv_from_ring (sbr_socket_t * sk, struct spl_netbuf **buf,
+                        i32 timeout)
+{
+  void *p = NULL;
+  spl_netconn_t *conn = sbr_get_conn (sk);
+
+  if (sbr_dequeue_buf (sk, &p, timeout) != 0)
+    {
+      return -1;
+    }
+
+  *buf = (struct spl_netbuf *) ((char *) p + sizeof (struct spl_pbuf));
+  ss_sub_recv_event (conn);
+  return 0;
+}
+
+/*****************************************************************************
+*   Prototype    : _sbr_udp_recvfrom
+*   Description  : base recvfrom,without lock
+*   Input        : sbr_socket_t * sk
+*                  void * mem
+*                  size_t len
+*                  int flags
+*                  struct sockaddr * from
+*                  socklen_t * fromlen
+*   Output       : None
+*   Return Value : static inline int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline int
+_sbr_udp_recvfrom (sbr_socket_t * sk, void *mem, size_t len, int flags,
+                   struct sockaddr *from, socklen_t * fromlen)
+{
+  struct spl_netbuf *buf = NULL;
+  struct spl_pbuf *p;
+  u32 buflen;
+  u32 copylen;
+  u32 off = 0;
+
+  if (sbr_get_fd_share (sk)->lastdata)
+    {
+      buf = ADDR_SHTOL (sbr_get_fd_share (sk)->lastdata);
+    }
+  else
+    {
+      if ((flags & MSG_DONTWAIT) || ss_is_nonblock_flag (sbr_get_conn (sk)))
+        {
+          /*
+           * return with last err when
+           * some fatal err occurs, for example, spl just recovered from a fault.
+           */
+          int err = ss_get_last_errno (sbr_get_conn (sk));
+          if (SPL_ERR_IS_FATAL (err))
+            {
+              NSSBR_LOGDBG ("connection fatal error]sk->fd=%d,errno=%d",
+                            sk->fd, err);
+              sbr_set_sk_io_errno (sk, sbr_spl_err_to_errno (err));
+              return -1;
+            }
+
+          if (ss_get_recv_event (sbr_get_conn (sk)) <= 0)
+            {
+              NSSBR_LOGDBG ("no recv event]fd=%d", sk->fd);
+              sbr_set_sk_io_errno (sk, EWOULDBLOCK);
+              return -1;
+            }
+        }
+
+      if (sbr_udp_recv_from_ring
+          (sk, &buf, sbr_get_fd_share (sk)->recv_timeout) != 0)
+        {
+          return -1;
+        }
+
+      sbr_get_fd_share (sk)->lastdata = (void *) ADDR_LTOSH (buf);
+    }
+
+  p = (struct spl_pbuf *) ADDR_SHTOL (buf->p);
+  buflen = p->tot_len;
+
+  if (mem)
+    {
+      copylen = len > buflen ? buflen : len;
+
+      if ((copylen > 0) && 0 == spl_pbuf_copy_partial (p, mem, copylen, 0))
+        {
+          NSSBR_LOGERR ("copy failed]fd=%d", sk->fd);
+          sbr_set_sk_io_errno (sk, EFAULT);
+          return -1;
+        }
+
+      off += copylen;
+    }
+
+  if (sbr_udp_get_from_addr (sk, from, fromlen, buf) != 0)
+    {
+      return -1;
+    }
+
+  if (!(flags & MSG_PEEK))
+    {
+      sbr_get_fd_share (sk)->lastdata = NULL;
+      sbr_com_free_recv_buf (sk, (struct spl_pbuf *) ADDR_SHTOL (buf->p));
+    }
+
+  return off;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_recvfrom
+*   Description  : recv from
+*   Input        : sbr_socket_t * sk
+*                  void * mem
+*                  size_t len
+*                  int flags
+*                  struct sockaddr * from
+*                  socklen_t * fromlen
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static int
+sbr_udp_recvfrom (sbr_socket_t * sk, void *mem, size_t len, int flags,
+                  struct sockaddr *from, socklen_t * fromlen)
+{
+  sbr_com_lock_recv (sk);
+  int ret = _sbr_udp_recvfrom (sk, mem, len, flags, from, fromlen);
+  sbr_com_unlock_recv (sk);
+  return ret;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_recvdata
+*   Description  : recv data
+*   Input        : sbr_socket_t * sk
+*                  const struct iovec* iov
+*                  int iovcnt
+*                  struct spl_netbuf *buf
+*   Output       : None
+*   Return Value : static inline
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline int
+sbr_udp_recvdata (sbr_socket_t * sk, const struct iovec *iov, int iovcnt,
+                  int flags, struct sockaddr *from, socklen_t * fromlen)
+{
+  sbr_com_lock_recv (sk);
+  if (-1 == _sbr_udp_recvfrom (sk, NULL, 0, MSG_PEEK, from, fromlen))
+    {
+      sbr_com_unlock_recv (sk);
+      return -1;
+    }
+
+  struct spl_netbuf *buf = ADDR_SHTOL (sbr_get_fd_share (sk)->lastdata);
+  struct spl_pbuf *p = (struct spl_pbuf *) ADDR_SHTOL (buf->p);
+  u32 buflen = p->tot_len;
+  u32 copylen = 0;
+  u32 offset = 0;
+
+  int i;
+
+  for (i = 0; i < iovcnt; ++i)
+    {
+      if (!iov[i].iov_base || (0 == iov[i].iov_len))
+        {
+          continue;
+        }
+
+      copylen = buflen > iov[i].iov_len ? iov[i].iov_len : buflen;
+      if ((copylen > 0)
+          && 0 == spl_pbuf_copy_partial (p, iov[i].iov_base, copylen, offset))
+        {
+          NSSBR_LOGERR ("copy failed]fd=%d", sk->fd);
+          goto done;
+        }
+
+      offset += copylen;
+      buflen -= copylen;
+
+      if (0 == buflen)
+        {
+          goto done;
+        }
+    }
+
+done:
+  if (!(flags & MSG_PEEK))
+    {
+      sbr_get_fd_share (sk)->lastdata = NULL;
+      sbr_com_free_recv_buf (sk, (struct spl_pbuf *) ADDR_SHTOL (buf->p));
+    }
+
+  sbr_com_unlock_recv (sk);
+  return offset;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_readv
+*   Description  : readv
+*   Input        : sbr_socket_t* sk
+*                  const struct iovec* iov
+*                  int iovcnt
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static int
+sbr_udp_readv (sbr_socket_t * sk, const struct iovec *iov, int iovcnt)
+{
+  return sbr_udp_recvdata (sk, iov, iovcnt, 0, NULL, NULL);
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_recvmsg
+*   Description  : recv msg
+*   Input        : sbr_socket_t* sk
+*                  struct msghdr* msg
+*                  int flags
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static int
+sbr_udp_recvmsg (sbr_socket_t * sk, struct msghdr *msg, int flags)
+{
+  return sbr_udp_recvdata (sk, msg->msg_iov, msg->msg_iovlen, flags,
+                           (struct sockaddr *) msg->msg_name,
+                           &msg->msg_namelen);
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_copy_iov
+*   Description  : copy iov
+*   Input        : sbr_socket_t * sk
+*                  const struct iovec * iov
+*                  int iovcnt
+*                  struct spl_pbuf* buf
+*   Output       : None
+*   Return Value : static inline int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline int
+sbr_copy_iov (sbr_socket_t * sk, const struct iovec *iov, int iovcnt,
+              struct spl_pbuf *buf)
+{
+  u32 buf_left = buf->len;
+  i8 *buf_data = (i8 *) ADDR_SHTOL (buf->payload);
+  u32 iov_left;
+  i8 *iov_data;
+  u32 copy_len;
+
+  int i;
+
+  for (i = 0; i < iovcnt; ++i)
+    {
+      if (!iov[i].iov_base || (0 == iov[i].iov_len))
+        {
+          continue;
+        }
+
+      iov_left = (u32) iov[i].iov_len;  /* to u32 is ok,len is checked in sbr_udp_senddata */
+      iov_data = (i8 *) iov[i].iov_base;
+      while (iov_left)
+        {
+          copy_len = buf_left > iov_left ? iov_left : buf_left;
+
+          if (NULL == common_memcpy (buf_data, iov_data, copy_len))
+            {
+              NSSBR_LOGERR ("common_memcpy error]fd=%d", sk->fd);
+              sbr_set_sk_errno (sk, EFAULT);
+              return -1;
+            }
+
+          buf_data += copy_len;
+          buf_left -= copy_len;
+          iov_data += copy_len;
+          iov_left -= copy_len;
+          if (0 == buf_left)
+            {
+              buf = (struct spl_pbuf *) ADDR_SHTOL (buf->next);
+              if (buf)
+                {
+                  buf_left = buf->len;
+                  buf_data = (i8 *) ADDR_SHTOL (buf->payload);
+                }
+              else
+                {
+                  return 0;
+                }
+            }
+        }
+    }
+
+  return 0;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_senddata
+*   Description  : send data
+*   Input        : sbr_socket_t * sk
+*                  const struct iovec * iov
+*                  int iovcnt
+*                  int flags
+*                  const struct sockaddr * to
+*                  socklen_t tolen
+*   Output       : None
+*   Return Value : static inline int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static inline int
+sbr_udp_senddata (sbr_socket_t * sk, const struct iovec *iov, int iovcnt,
+                  int flags, const struct sockaddr *to, socklen_t tolen)
+{
+  size_t size = 0;
+  int iov_idx;
+
+  for (iov_idx = 0; iov_idx < iovcnt; ++iov_idx)
+    {
+      if ((SPL_MAX_UDP_MSG_LEN - size) < iov[iov_idx].iov_len)
+        {
+          NSSBR_LOGERR
+            ("size > SPL_MAX_UDP_MSG_LEN]fd=%d,SPL_MAX_UDP_MSG_LEN=%u",
+             sk->fd, SPL_MAX_UDP_MSG_LEN);
+          sbr_set_sk_io_errno (sk, EMSGSIZE);
+          return -1;
+        }
+
+      size += iov[iov_idx].iov_len;
+    }
+
+  if (to == NULL)
+    {
+      /* if not bind , then dest address should not be NULL */
+      if (IPADDR_ANY == ss_get_remote_ip (sbr_get_conn (sk))->addr)
+        {
+          NSSBR_LOGERR ("dest address is null]fd=%d", sk->fd);
+          sbr_set_sk_io_errno (sk, EDESTADDRREQ);
+          return -1;
+        }
+    }
+  else if (to->sa_family != AF_INET)
+    {
+      NSSBR_LOGERR ("invalid address family]fd=%d,family=%d", sk->fd,
+                    to->sa_family);
+      sbr_set_sk_io_errno (sk, EAFNOSUPPORT);
+      return -1;
+    }
+  else if (tolen != sizeof (struct sockaddr_in))
+    {
+      NSSBR_LOGERR ("invalid address len]fd=%d,tolen=%u", sk->fd, tolen);
+      sbr_set_sk_io_errno (sk, EINVAL);
+      return -1;
+    }
+
+  struct spl_netbuf buf;
+  const struct sockaddr_in *to_in = (const struct sockaddr_in *) to;
+  buf.p = NULL;
+  if (to_in)
+    {
+      NSSBR_LOGDBG ("fd=%d,addr=%s,port=%d,conn=%p,private_data=%p", sk->fd,
+                    spl_inet_ntoa (to_in->sin_addr), ntohs (to_in->sin_port),
+                    sbr_get_conn (sk),
+                    ss_get_private_data (sbr_get_conn (sk)));
+      inet_addr_to_ipaddr (&buf.addr, &to_in->sin_addr);
+      netbuf_fromport (&buf) = ntohs (to_in->sin_port);
+    }
+  else
+    {
+      spl_ip_addr_set_any (&buf.addr);
+      netbuf_fromport (&buf) = 0;
+    }
+
+  spl_ip_addr_t local_ip;
+  if (IPADDR_ANY == ss_get_local_ip (sbr_get_conn (sk))->addr)
+    {
+      if (sbr_get_src_ip (buf.addr.addr, &local_ip.addr) != 0)
+        {
+          sbr_set_sk_io_errno (sk, EHOSTUNREACH);
+          NSSBR_LOGERR ("get src ip failed]fd=%d", sk->fd);
+          return -1;
+        }
+    }
+
+  int err = ss_get_last_errno (sbr_get_conn (sk));
+  if (SPL_ERR_IS_FATAL (err))
+    {
+      NS_LOG_CTRL (LOG_CTRL_SEND, LOGSBR, "NSSBR", NSLOG_ERR,
+                   "connection fatal error!err=%d", err);
+      sbr_set_sk_errno (sk, sbr_spl_err_to_errno (err));
+      return -1;
+    }
+
+  u16 remain_len = size;        //+ head_room_len;
+  struct spl_pbuf *p = NULL;
+  PRIMARY_ADDR struct spl_pbuf *header = NULL;
+  struct spl_pbuf **tail = &header;
+  u16 head_len = L2_L3_ROOM_LEN + L4_ROOM_LEN;
+  u16 copy_len;
+  u16 alloc_len;
+
+  do
+    {
+      copy_len =
+        remain_len >
+        (SPL_PBUF_UDP_LEN - head_len) ? (SPL_PBUF_UDP_LEN -
+                                         head_len) : remain_len;
+      alloc_len = head_len + copy_len;
+      p = sbr_malloc_tx_pbuf (alloc_len, head_len);
+      if (unlikely (!p))
+        {
+          NSSBR_LOGDBG ("malloc pbuf failed]fd=%d", sk->fd);
+          sbr_set_sk_io_errno (sk, ENOMEM);
+          sbr_handle_free_send_buf (sk,
+                                    (struct spl_pbuf *) ADDR_SHTOL (header));
+          // ss_set_send_event(sbr_get_conn(sk), 0);
+          return -1;
+        }
+
+      struct spl_pbuf *tmp = (struct spl_pbuf *) ADDR_SHTOL (header);
+      while (tmp)
+        {
+          tmp->tot_len += p->len;
+          tmp = (struct spl_pbuf *) ADDR_SHTOL (tmp->next);
+        }
+
+      *tail = (struct spl_pbuf *) ADDR_LTOSH (p);       /* header will changed to share */
+      tail = &p->next;
+
+      remain_len -= copy_len;
+      head_len = L2_L3_ROOM_LEN;
+    }
+  while (remain_len);
+
+  /*udp support len=0 */
+  if (size != 0)
+    {
+      if (sbr_copy_iov
+          (sk, iov, iovcnt, (struct spl_pbuf *) ADDR_SHTOL (header)) != 0)
+        {
+          sbr_handle_free_send_buf (sk,
+                                    (struct spl_pbuf *) ADDR_SHTOL (header));
+          return -1;
+        }
+    }
+
+  buf.p = header;
+  err = sbr_handle_udp_send (sk, &buf, &local_ip);
+  if (0 == err)
+    {
+      epoll_triggle_event_from_api (sk, EPOLL_API_OP_SEND);
+      //ss_set_send_event(sbr_get_conn(sk), 1);
+      return size;
+    }
+  else
+    {
+      sbr_handle_free_send_buf (sk, (struct spl_pbuf *) ADDR_SHTOL (buf.p));
+      return -1;
+    }
+
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_sendto
+*   Description  : sendto
+*   Input        : sbr_socket_t * sk
+*                  const void * data
+*                  size_t size
+*                  int flags
+*                  const struct sockaddr * to
+*                  socklen_t tolen
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static int
+sbr_udp_sendto (sbr_socket_t * sk, const void *data, size_t size, int flags,
+                const struct sockaddr *to, socklen_t tolen)
+{
+  struct iovec iov;
+
+  iov.iov_base = (void *) data;
+  iov.iov_len = size;
+  return sbr_udp_senddata (sk, &iov, 1, flags, to, tolen);
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_send
+*   Description  : send
+*   Input        : sbr_socket_t * sk
+*                  const void * data
+*                  size_t size
+*                  int flags
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static int
+sbr_udp_send (sbr_socket_t * sk, const void *data, size_t size, int flags)
+{
+  return sk->fdopt->sendto (sk, data, size, flags, NULL, 0);
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_sendmsg
+*   Description  : send msg
+*   Input        : sbr_socket_t * sk
+*                  const struct msghdr * pmsg
+*                  int flags
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static int
+sbr_udp_sendmsg (sbr_socket_t * sk, const struct msghdr *pmsg, int flags)
+{
+  return sbr_udp_senddata (sk, pmsg->msg_iov, pmsg->msg_iovlen, flags,
+                           (struct sockaddr *) pmsg->msg_name,
+                           pmsg->msg_namelen);
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_writev
+*   Description  : writev
+*   Input        : sbr_socket_t * sk
+*                  const struct iovec * iov
+*                  int iovcnt
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static int
+sbr_udp_writev (sbr_socket_t * sk, const struct iovec *iov, int iovcnt)
+{
+  return sbr_udp_senddata (sk, iov, iovcnt, 0, NULL, 0);
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_fcntl
+*   Description  : fcntl
+*   Input        : sbr_socket_t * sk
+*                  int cmd
+*                  long arg
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static int
+sbr_udp_fcntl (sbr_socket_t * sk, int cmd, long arg)
+{
+  int ret = 0;
+
+  switch (cmd)
+    {
+    case F_GETFL:
+      ret = ss_get_nonblock_flag (sbr_get_conn (sk));
+      NSSBR_LOGDBG ("F_GETFL]fd=%d,ret=%d", sk->fd, ret);
+      break;
+    case F_SETFL:
+      if (arg & O_NONBLOCK)
+        {
+          NSSBR_LOGDBG ("F_SETFL set O_NONBLOCK val]fd=%d,arg=%ld", sk->fd,
+                        arg);
+          ss_set_nonblock_flag (sbr_get_conn (sk), (arg & O_NONBLOCK));
+        }
+      else
+        {
+          NSSBR_LOGDBG ("F_SETFL clean O_NONBLOCK val]fd=%d,arg=%ld", sk->fd,
+                        arg);
+          ss_set_nonblock_flag (sbr_get_conn (sk), 0);
+        }
+
+      break;
+    default:
+      NSSBR_LOGERR ("cmd is not support]fd=%d,cmd=%d", sk->fd, cmd);
+      ret = -1;
+      sbr_set_sk_errno (sk, EINVAL);
+
+      break;
+    }
+
+  return ret;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_ioctl
+*   Description  : ioctl
+*   Input        : sbr_socket_t * sk
+*                  unsigned long cmd
+*                  void * arg
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static int
+sbr_udp_ioctl (sbr_socket_t * sk, unsigned long cmd, void *arg)
+{
+  int ret = 0;
+
+  switch (cmd)
+    {
+    case FIONREAD:
+      {
+        if (!sbr_com_try_lock_recv (sk))
+          {
+            return 0;
+          }
+
+        struct spl_pbuf *p = NULL;
+        struct spl_netbuf *buf = NULL;
+        if (!sbr_get_fd_share (sk)->lastdata)
+          {
+            ret = sbr_udp_recv_from_ring (sk, &buf, -1);
+            if (ret != 0)
+              {
+                sbr_com_unlock_recv (sk);
+                return EWOULDBLOCK == errno ? 0 : -1;
+              }
+
+            sbr_get_fd_share (sk)->lastdata = (void *) ADDR_LTOSH (buf);
+            p = (struct spl_pbuf *) ADDR_SHTOL (buf->p);
+          }
+        else
+          {
+            buf =
+              (struct spl_netbuf *)
+              ADDR_SHTOL (sbr_get_fd_share (sk)->lastdata);
+            p = (struct spl_pbuf *) ADDR_SHTOL (buf->p);
+          }
+
+        *((u32 *) arg) = p->tot_len;
+        sbr_com_unlock_recv (sk);
+      }
+      break;
+    case FIONBIO:
+      {
+        u8 val = 0;
+
+        if (arg && *(u32 *) arg)
+          {
+            val = 1;
+          }
+
+        ss_set_nonblock_flag (sbr_get_conn (sk), val);
+        NSSBR_LOGDBG ("FIONBIO]fd=%d,val=%u", sk->fd, val);
+      }
+      break;
+    default:
+      {
+        NSSBR_LOGERR ("cmd is not support]fd=%d,cmd=%lu", sk->fd, cmd);
+        ret = -1;
+        sbr_set_sk_errno (sk, ENOTTY);
+      }
+
+      break;
+    }
+
+  return ret;
+}
+
+/*****************************************************************************
+*   Prototype    : sbr_udp_close
+*   Description  : close
+*   Input        : sbr_socket_t * sk
+*   Output       : None
+*   Return Value : static int
+*   Calls        :
+*   Called By    :
+*
+*****************************************************************************/
+static int
+sbr_udp_close (sbr_socket_t * sk)
+{
+  if (sbr_get_fd_share (sk)->lastdata)
+    {
+      struct spl_netbuf *buf = ADDR_SHTOL (sbr_get_fd_share (sk)->lastdata);
+      struct spl_pbuf *p = (struct spl_pbuf *) ADDR_SHTOL (buf->p);
+      sbr_com_free_recv_buf (sk, p);
+    }
+
+  /* if failed,free it in recycle */
+  return sbr_handle_close (sk, 0);
+}
+
+sbr_fdopt udp_fdopt = {
+  .socket = sbr_udp_socket,
+  .bind = sbr_udp_bind,
+  .listen = sbr_udp_listen,
+  .accept = sbr_udp_accept,
+  .accept4 = sbr_udp_accept4,
+  .connect = sbr_udp_connect,
+  .shutdown = sbr_udp_shutdown,
+  .getsockname = sbr_udp_getsockname,
+  .getpeername = sbr_udp_getpeername,
+  .getsockopt = sbr_udp_getsockopt,
+  .setsockopt = sbr_udp_setsockopt,
+  .recvfrom = sbr_udp_recvfrom,
+  .readv = sbr_udp_readv,
+  .recvmsg = sbr_udp_recvmsg,
+  .send = sbr_udp_send,
+  .sendto = sbr_udp_sendto,
+  .sendmsg = sbr_udp_sendmsg,
+  .writev = sbr_udp_writev,
+  .fcntl = sbr_udp_fcntl,
+  .ioctl = sbr_udp_ioctl,
+  .close = sbr_udp_close,
+  .peak = sbr_com_peak,
+  .lock_common = sbr_com_lock_common,
+  .unlock_common = sbr_com_unlock_common,
+  .ep_getevt = stackx_eventpoll_getEvt,
+  .ep_ctl = stackx_eventpoll_triggle,
+  .set_close_stat = NULL,
+};