From 71b40478b341e21199bcdc1943a373270ee50d0e Mon Sep 17 00:00:00 2001
From: sharath <sharathkumarboyanapally@gmail.com>
Date: Mon, 21 Jan 2019 18:28:22 +0530
Subject: Feat: F-stack Integration

Change-Id: I0eec7617ff6782192bcbf6e4c41630f585bed54c
Signed-off-by: sharath <sharathkumarboyanapally@gmail.com>
---
 CMakeLists.txt                            |   1 +
 doc/DMM_DeveloperManual.md                | 132 ++++++++++++
 resources/F-Stack-RTC-epoll.png           | Bin 0 -> 98976 bytes
 scripts/build.sh                          |   2 +
 scripts/build_fstack.sh                   |  19 ++
 src/nSocket/include/dmm_pub_api.h         |   6 +
 src/nSocket/include/nstack_dmm_api.h      |   2 +
 src/nSocket/nstack/nstack_socket.c        | 124 ++++++++++--
 src/nSocket/nstack_rd/nstack_rd_init.c    |   1 +
 src/nSocket/nstack_rd/nstack_rd_ip.c      |   6 +-
 stacks/fstack/CMakeLists.txt              |  59 ++++++
 stacks/fstack/app/Makefile                |   9 +
 stacks/fstack/app/dmm_start.sh            |  65 ++++++
 stacks/fstack/app/fstack_client.c         |  49 +++++
 stacks/fstack/app/fstack_server.c         |  78 ++++++++
 stacks/fstack/app/multi_sock_tcp_epoll.c  | 322 ++++++++++++++++++++++++++++++
 stacks/fstack/app/single_sock_tcp_epoll.c | 217 ++++++++++++++++++++
 stacks/fstack/config/config.ini           | 124 ++++++++++++
 stacks/fstack/config/module_config.json   |  28 +++
 stacks/fstack/config/rd_config.json       |  10 +
 stacks/fstack/fstack.patch                |  12 ++
 stacks/fstack/run_fstack.txt              |  47 +++++
 stacks/fstack/src/fstack_adpt.c           | 241 ++++++++++++++++++++++
 23 files changed, 1534 insertions(+), 20 deletions(-)
 create mode 100644 resources/F-Stack-RTC-epoll.png
 create mode 100755 scripts/build_fstack.sh
 create mode 100644 src/nSocket/include/dmm_pub_api.h
 create mode 100644 stacks/fstack/CMakeLists.txt
 create mode 100644 stacks/fstack/app/Makefile
 create mode 100755 stacks/fstack/app/dmm_start.sh
 create mode 100644 stacks/fstack/app/fstack_client.c
 create mode 100644 stacks/fstack/app/fstack_server.c
 create mode 100644 stacks/fstack/app/multi_sock_tcp_epoll.c
 create mode 100644 stacks/fstack/app/single_sock_tcp_epoll.c
 create mode 100644 stacks/fstack/config/config.ini
 create mode 100644 stacks/fstack/config/module_config.json
 create mode 100644 stacks/fstack/config/rd_config.json
 create mode 100644 stacks/fstack/fstack.patch
 create mode 100644 stacks/fstack/run_fstack.txt
 create mode 100644 stacks/fstack/src/fstack_adpt.c

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 43b228d..98f6d94 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -214,4 +214,5 @@ ADD_SUBDIRECTORY(src)
 ADD_SUBDIRECTORY(app_example)
 
 ADD_SUBDIRECTORY(stacks/rsocket)
+ADD_SUBDIRECTORY(stacks/fstack)
 ADD_SUBDIRECTORY(thirdparty/apps)
diff --git a/doc/DMM_DeveloperManual.md b/doc/DMM_DeveloperManual.md
index 51e4be5..2646e60 100644
--- a/doc/DMM_DeveloperManual.md
+++ b/doc/DMM_DeveloperManual.md
@@ -29,6 +29,11 @@
 [**7 How to use the incidental apps to test the protocol stack**](#7-how-to-use-the-incidental-apps-to-test-the-protocol-stack)<br>
 [**7.1 App introduction**](#7.1-app-introduction)<br>
 [**7.2 Test implementation**](#7.2-Test-implementation)<br>
+[**8. DMM-RTC mode**](#8.-dmm-rtc-mode)<br>
+[**8.1 RTC Stack Init and Run**](#8.1-rtc-stack-init-and-run)<br>
+[**8.2 RTC epoll**](#8.2-rtc-epoll)<br>
+[**8.3 RTC Limitations**](#8.3-rtc-limitations)<br>
+[**8.3.1 F-Stack RTC Limitations**](#8.3.1-f-stack-rtc-limitations)<br>
 
 **1. Introduction**<br>
 ============================
@@ -1116,4 +1121,131 @@ The following table is an introduction to some parameters in the code.
   -x                        |            flag print
 
 
+**8. DMM-RTC mode**<br>
+============================
+DMM-RTC mode is one of the basic model types of DMM, where nSocket and stack belong to the same process.
+In this mode we need to handle the stack initialization at the time of DMM framework initialization.
+Spceial care need to be taken for epoll_wait.
+Need to mention the depoly type as 1 in module_config.json which tells DMM that nSocket and stack belong to the same process
+
+**8.1 RTC Stack Init and Run**<br>
+-------------------------
+RTC stack init need to be handled at the time of framework initialization.
+Showing F-Stack stack registration function where dmm_fstack_init handles the F-Stack initialization
+~~~C
+
+int
+fstack_stack_register (nstack_proc_cb * proc_fun, nstack_event_cb * event_ops)
+{
+#define NSTACK_MK_DECL(ret, fn, args) proc_fun->socket_ops.pf##fn = (void*)ff_##fn;
+#include "declare_syscalls.h"
+#undef NSTACK_MK_DECL
+  g_dmm_fstack.p_epoll_ctl = dlsym (event_ops->handle, "ff_epoll_ctl");
+  g_dmm_fstack.p_epoll_create = dlsym (event_ops->handle, "ff_epoll_create");
+  g_dmm_fstack.p_epoll_wait = dlsym (event_ops->handle, "ff_epoll_wait");
+  g_dmm_fstack.p_close = dlsym (event_ops->handle, "ff_close");
+  g_dmm_fstack.regVal = *event_ops;
+
+  proc_fun->socket_ops.pfselect = NULL;
+
+  proc_fun->extern_ops.module_init = dmm_fstack_init;
+  proc_fun->extern_ops.ep_ctl = fstack_ep_ctl_ops;
+  proc_fun->extern_ops.ep_getevt = NULL;
+  proc_fun->extern_ops.run = fr_run;
+  proc_fun->extern_ops.ep_prewait = dmm_fstack_epoll;
+  return 0;
+}
+
+~~~
+
+If a stack need to launch a function on all cores it need to call nstack_run with a loop function implemented by the user as its argument
+Consider the below F-Stack application which explains the usage of nstack_run
+~~~C
+
+int
+main (int argc, char *argv[])
+{
+
+  int opt = 1;
+  int addrlen = sizeof (address);
+
+  if ((server_fd = socket (AF_INET, SOCK_STREAM, 0)) == 0)
+    {
+      printf ("socket failed\n");
+      exit (EXIT_FAILURE);
+    }
+  if (bind (server_fd, (struct sockaddr *) &address, sizeof (address)) < 0)
+    {
+      printf ("bind failed");
+      exit (EXIT_FAILURE);
+    }
+  printf ("bind success\n");
+  if (listen (server_fd, 3) < 0)
+    {
+      printf ("listen");
+      exit (EXIT_FAILURE);
+    }
+  printf ("listen success\n");
+  nstack_run ((void *) loop);
+  return 0;
+}
+
+~~~
+~~~C
+int
+loop (void *arg)
+{
+  int new_socket, valread;
+  int addrlen = sizeof (address);
+  char buffer[1024] = { 0 };
+  char hello[1024] = { 0 };
+  sprintf (hello, "Hello from server\t PROC_ID : %d", getpid ());
+  printf ("before accept\n");
+  if ((new_socket = accept (server_fd, (struct sockaddr *) &address,
+                            (socklen_t *) & addrlen)) < 0)
+    {
+      printf ("accept failed");
+      return -1;
+    }
+
+  printf ("accept success\n");
+  valread = read (new_socket, buffer, 1024);
+  printf ("%s\n", buffer);
+  send (new_socket, hello, strlen (hello), 0);
+  printf ("Hello message sent\n");
+  return 0;
+}
+
+~~~
+
+**8.2 RTC epoll**
+-------------------------
+RTC stack module init creates an epoll fd by calling stack's epoll_create function and make the epoll_fd global.
+Whenever application calls epoll_ctl, stack's epoll_ctl will be called with the global epoll_fd.
+An RTC Stack application must call nstack_epoll_prewait before calling epoll_wait.
+nstack_epoll_prewait updates ep_item if any event occurs.
+epoll_wait checks ep_item whether any event has occured and updates the same to the application.
+
+Below figure depicts the DMM F-Stack-RTC  epoll architecture.
+
+
+![F-Stack-RTC epoll](../resources/F-Stack-RTC-epoll.png)
+-------------------------
+Figure: F-Stack-RTC-epoll
+
+During the F-Stack module init a global epoll fd will be created by calling ff_epoll_create.
+epoll_create allocates fd and nsep_epollInfo_t.
+epoll_ctl calls ff_epoll_ctl with the global epoll fd created during the module init.
+Non-blocking nstack_epoll_prewait will be called in a loop from application before epoll_wait to check any events have occured on global epoll fd, 
+in case of any events nstack_event_callback function will be called which updates the epitem.
+epoll_wait checks epitem whether some event has occured or not and returns updtae to the application.
+
+**8.3 RTC Limitations**
+-------------------------
+**8.3.1 F-Stack RTC Limitations**
+-------------------------------------------
+No need to call ff_init as regular F-Stack applications call.
+Need to call nstack_run insted of ff_run.
+Must call nstack_epoll_prewait before calling epoll_wait.
+In case of multi-stack and multi-socket application with F-Stack as one of the stacks, should not give timeout value for epoll_wait.
 
diff --git a/resources/F-Stack-RTC-epoll.png b/resources/F-Stack-RTC-epoll.png
new file mode 100644
index 0000000..1f34120
Binary files /dev/null and b/resources/F-Stack-RTC-epoll.png differ
diff --git a/scripts/build.sh b/scripts/build.sh
index eaa0f0e..11e89e4 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -61,6 +61,8 @@ if [ "${BUILD_ALL}" == "YES" ]; then
     bash -x $DMM_DIR/stacks/lwip_stack/vagrant/build.sh "from-base-build" || exit 1
     #============build rsocket============================
     bash -x $DMM_DIR/scripts/build_rsocket.sh || exit 1
+    #============build F-Stack============================
+    bash -x $DMM_DIR/scripts/build_fstack.sh || exit 1
     #=======other new stacks build can be called from here
     echo "SUCCESSFULLY built all the stacks"
 fi
diff --git a/scripts/build_fstack.sh b/scripts/build_fstack.sh
new file mode 100755
index 0000000..3f8e351
--- /dev/null
+++ b/scripts/build_fstack.sh
@@ -0,0 +1,19 @@
+#!/bin/bash -x
+
+#build F-stack dependencies
+sudo apt-get -y install libnuma-dev
+sudo apt-get -y install libssl-dev
+
+#build F-stack
+DMM_DIR=`dirname $(readlink -f $0)`/../
+BUILD_DIR=${DMM_DIR}/build
+
+cd ${BUILD_DIR}
+sudo make dmm_fstack
+if [ $? -eq 0 ]; then
+    echo "fstack build has SUCCESS"
+else
+    echo "fstack build has FAILED"
+    exit 1
+fi
+echo "fstack build finished"
diff --git a/src/nSocket/include/dmm_pub_api.h b/src/nSocket/include/dmm_pub_api.h
new file mode 100644
index 0000000..cfc20af
--- /dev/null
+++ b/src/nSocket/include/dmm_pub_api.h
@@ -0,0 +1,6 @@
+
+#ifndef __PUBLIC_DMM_API
+#define __PUBLIC_DMM_API
+int nstack_run (void *loop);
+int nstack_epoll_prewait ();
+#endif
diff --git a/src/nSocket/include/nstack_dmm_api.h b/src/nSocket/include/nstack_dmm_api.h
index ab06650..7c93f86 100644
--- a/src/nSocket/include/nstack_dmm_api.h
+++ b/src/nSocket/include/nstack_dmm_api.h
@@ -68,6 +68,8 @@ typedef struct __nstack_extern_ops
   int (*stack_fd_check) (int s, int flag);      /*check whether fd belong to stack, if belong, return 1, else return 0 */
   int (*stack_alloc_fd) ();     /*alloc a fd id for epoll */
   int (*peak) (int s);          /*used for stack-x , isource maybe no need */
+  int (*run) (void *fun);       /*used for stack-x , isource maybe no need */
+  void (*ep_prewait) ();        /*used for stack-x , isource maybe no need */
 } nstack_extern_ops;
 
 /*
diff --git a/src/nSocket/nstack/nstack_socket.c b/src/nSocket/nstack/nstack_socket.c
index 1b7a416..130d08f 100644
--- a/src/nSocket/nstack/nstack_socket.c
+++ b/src/nSocket/nstack/nstack_socket.c
@@ -144,12 +144,10 @@ int
 nstack_socket (int domain, int itype, int protocol)
 {
   int ret = -1;                 //tmp ret of a Single proctol mode.
-  int rdret = -1;               //the final Return judge vale.
   int modInx;
   nstack_socket_ops *ops;
   int ret_fd = -1;
   int protoFD[NSTACK_MAX_MODULE_NUM];
-  nstack_rd_key rdkey = { 0 };
   int selectmod = -1;
 
   /*check whether module init finish or not */
@@ -187,21 +185,6 @@ nstack_socket (int domain, int itype, int protocol)
   nstack_fd_local_lock_info_t *lock_info = get_fd_local_lock_info (ret_fd);
   LOCK_FOR_EP (lock_info);
 
-  /*check wether select stack create success, if no return fail */
-  rdkey.type = RD_DATA_TYPE_PROTO;
-  rdkey.proto_type = itype;
-  rdret = nstack_rd_get_stackid (&rdkey, &selectmod);
-  if ((0 != rdret) || (selectmod < 0) || (selectmod >= nstack_get_modNum ()))
-    {
-      NSSOC_LOGERR ("protocol:%d select stack fail", protocol);
-      selectmod = -1;
-    }
-  else
-    {
-      NSSOC_LOGINF ("Create socket of]select modName=%s",
-                    nstack_get_module_name_by_idx (selectmod));
-    }
-
   /*create socket by calling select module or all module */
   nstack_each_modOps (modInx, ops)
   {
@@ -392,6 +375,26 @@ nstack_bind (int fd, const struct sockaddr *addr, socklen_t addrlen)
                     nstack_get_module_name_by_idx (selectmod));
     }
 
+  if (selectmod == -1)
+    {
+      rdkey.type = RD_DATA_TYPE_PROTO;
+      rdkey.proto_type = fdInf->type;
+      retval = nstack_rd_get_stackid (&rdkey, &selectmod);
+      if ((0 != retval) || (selectmod < 0)
+          || (selectmod >= nstack_get_modNum ()))
+        {
+          NSSOC_LOGWAR
+            ("fd Can't select any module for]fd=%d,IP==%s using proto route",
+             fd, inet_ntoa ((iaddr->sin_addr)));
+          selectmod = -1;
+        }
+      else
+        {
+          NSSOC_LOGINF ("Bind socket of]select modName=%s",
+                        nstack_get_module_name_by_idx (selectmod));
+        }
+    }
+
   retval = -1;
   nstack_each_modInx (modIdx)
   {
@@ -905,6 +908,25 @@ nstack_connect (int fd, const struct sockaddr *addr, socklen_t addrlen)
       rdkey.type = RD_DATA_TYPE_IP;
       rdkey.ip_addr = iaddr->sin_addr.s_addr;
       retval = nstack_rd_get_stackid (&rdkey, &selectmod);
+      if ((0 != retval) || (selectmod == -1))
+        {
+          rdkey.type = RD_DATA_TYPE_PROTO;
+          rdkey.proto_type = fdInf->type;
+          retval = nstack_rd_get_stackid (&rdkey, &selectmod);
+          if ((0 != retval) || (selectmod < 0)
+              || (selectmod >= nstack_get_modNum ()))
+            {
+              NSSOC_LOGINF ("fd=%d addr=%s Select module=%s",
+                            fd, inet_ntoa (iaddr->sin_addr),
+                            nstack_get_module_name_by_idx (selectmod));
+              selectmod = -1;
+            }
+          else
+            {
+              NSSOC_LOGINF ("Connect socket of]select modName=%s",
+                            nstack_get_module_name_by_idx (selectmod));
+            }
+        }
       if (ns_success == retval && selectmod >= 0)
         {
           NSSOC_LOGINF ("fd=%d addr=%s Select module=%s",
@@ -1390,6 +1412,25 @@ nstack_sendto (int fd, const void *buf, size_t len, int flags,
   rdkey.type = RD_DATA_TYPE_IP;
   rdkey.ip_addr = iaddr->sin_addr.s_addr;
   retval = nstack_rd_get_stackid (&rdkey, &selectmod);
+  if ((0 != retval) || (selectmod == -1))
+    {
+      rdkey.type = RD_DATA_TYPE_PROTO;
+      rdkey.proto_type = fdInf->type;
+      retval = nstack_rd_get_stackid (&rdkey, &selectmod);
+      if ((0 != retval) || (selectmod < 0)
+          || (selectmod >= nstack_get_modNum ()))
+        {
+          NSSOC_LOGINF ("fd=%d addr=%s Select module=%s",
+                        fd, inet_ntoa (iaddr->sin_addr),
+                        nstack_get_module_name_by_idx (selectmod));
+          selectmod = -1;
+        }
+      else
+        {
+          NSSOC_LOGINF ("sendto socket of]select modName=%s",
+                        nstack_get_module_name_by_idx (selectmod));
+        }
+    }
   if ((ns_success == retval) && (selectmod >= 0))
     {
       NSSOC_LOGINF ("fd=%d,addr=%s,select_module=%s",
@@ -1477,6 +1518,25 @@ nstack_sendmsg (int fd, const struct msghdr * msg, int flags)
       rdkey.type = RD_DATA_TYPE_IP;
       rdkey.ip_addr = iaddr->sin_addr.s_addr;
       retval = nstack_rd_get_stackid (&rdkey, &selectmod);
+      if ((0 != retval) || (selectmod == -1))
+        {
+          rdkey.type = RD_DATA_TYPE_PROTO;
+          rdkey.proto_type = fdInf->type;
+          retval = nstack_rd_get_stackid (&rdkey, &selectmod);
+          if ((0 != retval) || (selectmod < 0)
+              || (selectmod >= nstack_get_modNum ()))
+            {
+              NSSOC_LOGINF ("fd=%d addr=%s Select module=%s",
+                            fd, inet_ntoa (iaddr->sin_addr),
+                            nstack_get_module_name_by_idx (selectmod));
+              selectmod = -1;
+            }
+          else
+            {
+              NSSOC_LOGINF ("Connect socket of]select modName=%s",
+                            nstack_get_module_name_by_idx (selectmod));
+            }
+        }
       if (ns_success == retval)
         {
           NSSOC_LOGINF ("fd=%d,addr=%s,select_module=%s",
@@ -2812,3 +2872,33 @@ nstack_fork (void)
   dmm_write_unlock (get_fork_lock ());
   return pid;
 }
+
+int
+nstack_run (void *loop)
+{
+  int i;
+
+  nstack_each_modInx (i)
+  {
+    if (nstack_extern_deal (i).run)
+      {
+        nstack_extern_deal (i).run (loop);
+      }
+  }
+  return 0;
+}
+
+int
+nstack_epoll_prewait ()
+{
+  int i;
+
+  nstack_each_modInx (i)
+  {
+    if (nstack_extern_deal (i).ep_prewait)
+      {
+        nstack_extern_deal (i).ep_prewait ();
+      }
+  }
+  return 0;
+}
diff --git a/src/nSocket/nstack_rd/nstack_rd_init.c b/src/nSocket/nstack_rd/nstack_rd_init.c
index b3d4158..363b4b6 100644
--- a/src/nSocket/nstack_rd/nstack_rd_init.c
+++ b/src/nSocket/nstack_rd/nstack_rd_init.c
@@ -42,6 +42,7 @@ rd_stack_plane_map g_nstack_plane_info[] = {
    {"rsocket", "nstack-rsocket", -1},
    {{RD_STACKX_NAME}, {RD_STACKX_PLANENAME}, -1},
    {"vpp_hoststack", "nstack-vpp", -1},
+   {"fstack", "nstack-fstack", -1}
 };
 /* *INDENT-ON* */
 
diff --git a/src/nSocket/nstack_rd/nstack_rd_ip.c b/src/nSocket/nstack_rd/nstack_rd_ip.c
index 5297d4a..f0563a5 100644
--- a/src/nSocket/nstack_rd/nstack_rd_ip.c
+++ b/src/nSocket/nstack_rd/nstack_rd_ip.c
@@ -66,12 +66,12 @@ nstack_rd_ip_item_insert (nstack_rd_list * hlist, void *rditem)
   nstack_rd_node *tempdata = NULL;
   struct hlist_node *tempnode = NULL;
   struct hlist_node *tem = NULL;
-  unsigned int ip_addr = 0;
-  unsigned int ip_masklen = 0;
-  unsigned int ip_maskv = MASK_V (ip_addr, ip_masklen);
   unsigned int tempip_addr = 0;
   unsigned int tempip_masklen = 0;
   rd_data_item *pitem = (rd_data_item *) rditem;
+  unsigned int ip_addr = pitem->ipdata.addr;
+  unsigned int ip_masklen = pitem->ipdata.masklen;
+  unsigned int ip_maskv = MASK_V (ip_addr, ip_masklen);
 
   ip_masklen = pitem->ipdata.masklen;
   NSSOC_LOGDBG ("stackid:%d, ipaddr:%u.%u.%u.%u masklen:0x%x was inserted",
diff --git a/stacks/fstack/CMakeLists.txt b/stacks/fstack/CMakeLists.txt
new file mode 100644
index 0000000..a058749
--- /dev/null
+++ b/stacks/fstack/CMakeLists.txt
@@ -0,0 +1,59 @@
+#########################################################################
+#
+# Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#########################################################################
+
+SET(fstack_dir f-stack-1.12)
+SET(dpdk_target x86_64-native-linuxapp-gcc)
+SET(dmm_inc_dir ${DMM_REL_INC_DIR})
+
+
+SET(fstack_url https://github.com/F-Stack/f-stack/archive/v1.12.tar.gz)
+
+INCLUDE(ExternalProject)
+ExternalProject_Add(
+    fstack
+    URL                 ${fstack_url}
+    TLS_VERIFY          false
+    SOURCE_DIR          ${CMAKE_CURRENT_LIST_DIR}/${fstack_dir}
+    DOWNLOAD_DIR        ${CMAKE_CURRENT_LIST_DIR}
+    BUILD_IN_SOURCE     1
+    PATCH_COMMAND       patch -p1 -i ../fstack.patch
+    CONFIGURE_COMMAND   make -C dpdk config T=${dpdk_target}
+                            build O=${dpdk_target} EXTRA_CFLAGS=-fPIC
+    BUILD_COMMAND       make -C lib "CONF_CFLAGS=-fPIC -g"
+    INSTALL_COMMAND     ""
+)
+set_target_properties(fstack PROPERTIES EXCLUDE_FROM_ALL TRUE)
+
+#####################################################################
+
+SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -g -fPIC -m64 -pthread")
+
+LINK_DIRECTORIES(${fstack_dir}/lib ${fstack_dir}/dpdk/${dpdk_target}/lib)
+ADD_LIBRARY(dmm_fstack  SHARED src/fstack_adpt.c)
+
+ADD_DEFINITIONS(-D_GNU_SOURCE)
+
+INCLUDE_DIRECTORIES(${DMM_REL_INC_DIR})
+INCLUDE_DIRECTORIES(${fstack_dir}/lib)
+
+TARGET_LINK_LIBRARIES(dmm_fstack
+    -Wl,--whole-archive     -lfstack -ldpdk
+    -Wl,--no-whole-archive  -lrt -lm -ldl -lcrypto -pthread -lnuma
+)
+
+ADD_DEPENDENCIES(dmm_fstack fstack post-compile)
+
+set_target_properties(dmm_fstack PROPERTIES EXCLUDE_FROM_ALL TRUE)
diff --git a/stacks/fstack/app/Makefile b/stacks/fstack/app/Makefile
new file mode 100644
index 0000000..42f5274
--- /dev/null
+++ b/stacks/fstack/app/Makefile
@@ -0,0 +1,9 @@
+LIBS+= -L../../../release/lib64 -Wl,--whole-archive,-lnStackAPI,--no-whole-archive
+TARGET="fstack"
+all:
+	cc -O -gdwarf-2 -Wl,--rpath,../../../release/lib64 -I../../../src/nSocket/include  -o ${TARGET}_multi_sock_epoll multi_sock_tcp_epoll.c ${LIBS}
+	cc -O -gdwarf-2 -Wl,--rpath,../../../release/lib64 -I../../../src/nSocket/include  -o ${TARGET}_server_no_epoll fstack_server.c ${LIBS}
+	cc -O -gdwarf-2 -Wl,--rpath,../../../release/lib64 -I../../../src/nSocket/include  -o ${TARGET}_single_sock_server single_sock_tcp_epoll.c ${LIBS}
+.PHONY: clean
+clean:
+	rm -f *.o ${TARGET}_multi_sock_epoll ${TARGET}_single_sock_server ${TARGET}_server_no_epoll
diff --git a/stacks/fstack/app/dmm_start.sh b/stacks/fstack/app/dmm_start.sh
new file mode 100755
index 0000000..1fe99ee
--- /dev/null
+++ b/stacks/fstack/app/dmm_start.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+function usage() {
+    echo "F-Stack app start tool"
+    echo "Options:"
+    echo " -c [conf]                Path of config file"
+    echo " -b [N]                   Path of binary"
+    echo " -o [N]                   Other ARGs for app"
+    echo " -h                       show this help"
+    exit
+}
+
+conf=config.ini
+bin=fstack_multi_proc
+
+while getopts "c:b:o:h" args
+do
+    case $args in
+         c)
+            conf=$OPTARG
+            ;;
+         b)
+            bin=$OPTARG
+            ;;
+         o)
+            others=$OPTARG
+            ;;
+         h)
+            usage
+            exit 0
+            ;;
+    esac
+done
+
+if ! type "bc" > /dev/null 2>&1; then
+    echo "please install bc"
+    exit
+fi
+
+allcmask0x=`cat ${conf}|grep lcore_mask|awk -F '=' '{print $2}'`
+((allcmask=16#$allcmask0x))
+
+num_procs=0
+PROCESSOR=$(grep 'processor' /proc/cpuinfo |sort |uniq |wc -l)
+for((i=0;i<${PROCESSOR};++i))
+do
+    mask=`echo "2^$i"|bc`
+    ((result=${allcmask} & ${mask}))
+    if [ ${result} != 0 ]
+    then
+        ((num_procs++));
+    fi
+done
+
+for((proc_id=0; proc_id<${num_procs}; ++proc_id))
+do
+    if ((proc_id == 0))
+    then
+        echo "number of processes ${num_procs}"
+	sudo LD_LIBRARY_PATH=../../../release/lib64/ LD_PRELOAD=../../../release/lib64/libnStackAPI.so F_PROC_ID=${proc_id} F_PROC_TYPE=primary FF_PATH=../f-stack-1.12/ FF_DPDK=../f-stack-1.12/dpdk/x86_64-native-linuxapp-gcc ./${bin} &
+        sleep 5
+    else
+	sudo LD_LIBRARY_PATH=../../../release/lib64/ LD_PRELOAD=../../../release/lib64/libnStackAPI.so F_PROC_ID=${proc_id} F_PROC_TYPE=secondary FF_PATH=../f-stack-1.12/ FF_DPDK=../f-stack-1.12/dpdk/x86_64-native-linuxapp-gcc ./${bin} & 
+    fi
+done
diff --git a/stacks/fstack/app/fstack_client.c b/stacks/fstack/app/fstack_client.c
new file mode 100644
index 0000000..a4b378c
--- /dev/null
+++ b/stacks/fstack/app/fstack_client.c
@@ -0,0 +1,49 @@
+// Client side C/C++ program to demonstrate Socket programming
+#include <stdio.h>
+#include <sys/socket.h>
+#include <stdlib.h>
+#include <netinet/in.h>
+#include <string.h>
+#include <arpa/inet.h>
+#include <unistd.h>
+#define PORT 8080
+
+int
+main (int argc, char const *argv[])
+{
+  struct sockaddr_in address;
+  int sock = 0, valread;
+  struct sockaddr_in serv_addr;
+  char *hello = "Hello from client";
+  char buffer[1024] = { 0 };
+  if ((sock = socket (AF_INET, SOCK_STREAM, 0)) < 0)
+    {
+      printf ("\n Socket creation error \n");
+      return -1;
+    }
+
+  memset (&serv_addr, '0', sizeof (serv_addr));
+
+  serv_addr.sin_family = AF_INET;
+  serv_addr.sin_port = htons (PORT);
+
+  // Convert IPv4 and IPv6 addresses from text to binary form
+  //if(inet_pton(AF_INET, "192.28.128.8", &serv_addr.sin_addr)<=0)
+  if (inet_pton (AF_INET, "172.28.128.13", &serv_addr.sin_addr) <= 0)
+    {
+      printf ("\nInvalid address/ Address not supported \n");
+      return -1;
+    }
+
+  if (connect (sock, (struct sockaddr *) &serv_addr, sizeof (serv_addr)) < 0)
+    {
+      printf ("\nConnection Failed \n");
+      return -1;
+    }
+  send (sock, hello, strlen (hello), 0);
+  printf ("Hello message sent\n");
+  valread = read (sock, buffer, 1024);
+  printf ("%s\n", buffer);
+  close (sock);
+  return 0;
+}
diff --git a/stacks/fstack/app/fstack_server.c b/stacks/fstack/app/fstack_server.c
new file mode 100644
index 0000000..241284b
--- /dev/null
+++ b/stacks/fstack/app/fstack_server.c
@@ -0,0 +1,78 @@
+#define _GNU_SOURCE
+#include <unistd.h>
+#include <stdio.h>
+#include <sys/socket.h>
+#include <stdlib.h>
+#include <netinet/in.h>
+#include <string.h>
+#include <arpa/inet.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include "dmm_pub_api.h"
+#define PORT 8080
+
+int server_fd;
+struct sockaddr_in address;
+int
+loop (void *arg)
+{
+  int new_socket, valread;
+  int addrlen = sizeof (address);
+  char buffer[1024] = { 0 };
+  char hello[1024] = { 0 };
+  sprintf (hello, "Hello from server\t PROC_ID : %d", getpid ());
+  printf ("before accept\n");
+  if ((new_socket = accept (server_fd, (struct sockaddr *) &address,
+                            (socklen_t *) & addrlen)) < 0)
+    {
+      printf ("accept failed");
+      return -1;
+    }
+
+  printf ("accept success\n");
+  valread = read (new_socket, buffer, 1024);
+  printf ("%s\n", buffer);
+  send (new_socket, hello, strlen (hello), 0);
+  printf ("Hello message sent\n");
+  return 0;
+}
+
+int
+main (int argc, char *argv[])
+{
+
+  int opt = 1;
+  int addrlen = sizeof (address);
+#if 0
+  if (-1 == nstack_conf (argc, argv))
+    {
+      printf ("nstack_conf failed\n");
+      return -1;
+    }
+#endif
+  if ((server_fd = socket (AF_INET, SOCK_STREAM, 0)) == 0)
+    {
+      printf ("socket failed\n");
+      exit (EXIT_FAILURE);
+    }
+  address.sin_family = AF_INET;
+  address.sin_addr.s_addr = inet_addr ("172.28.128.13");
+  address.sin_port = htons (PORT);
+
+  // Forcefully attaching socket to the port 8080
+  if (bind (server_fd, (struct sockaddr *) &address, sizeof (address)) < 0)
+    {
+      printf ("bind failed");
+      exit (EXIT_FAILURE);
+    }
+  printf ("bind success\n");
+  if (listen (server_fd, 3) < 0)
+    {
+      printf ("listen");
+      exit (EXIT_FAILURE);
+    }
+  printf ("listen success\n");
+  nstack_run ((void *) loop);
+  return 0;
+}
diff --git a/stacks/fstack/app/multi_sock_tcp_epoll.c b/stacks/fstack/app/multi_sock_tcp_epoll.c
new file mode 100644
index 0000000..56e6374
--- /dev/null
+++ b/stacks/fstack/app/multi_sock_tcp_epoll.c
@@ -0,0 +1,322 @@
+#define _GNU_SOURCE
+#include <unistd.h>
+#include <stdio.h>
+#include <sys/ioctl.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+#include <errno.h>
+#include <assert.h>
+#include <sys/epoll.h>
+#include "dmm_pub_api.h"
+#include <pthread.h>
+#include <fcntl.h>
+
+#define MAX_EVENTS 512
+
+int gepfd;
+int fstack_listen_fd, kernel_listen_fd, i, kernel_accept_fd, fstack_accept_fd,
+  readlen;
+char buffer[1024] = { 0 };
+
+char fstack_html[] =
+  "HTTP/1.1 200 OK\r\n"
+  "Server: F-Stack\r\n"
+  "Date: Sat, 25 Feb 2017 09:26:33 GMT\r\n"
+  "Content-Type: text/html\r\n"
+  "Content-Length: 439\r\n"
+  "Last-Modified: Tue, 21 Feb 2017 09:44:03 GMT\r\n"
+  "Connection: keep-alive\r\n"
+  "Accept-Ranges: bytes\r\n"
+  "\r\n"
+  "<!DOCTYPE html>\r\n"
+  "<html>\r\n"
+  "<head>\r\n"
+  "<title>Welcome to F-Stack!</title>\r\n"
+  "<style>\r\n"
+  "    body {  \r\n"
+  "        width: 35em;\r\n"
+  "        margin: 0 auto; \r\n"
+  "        font-family: Tahoma, Verdana, Arial, sans-serif;\r\n"
+  "    }\r\n"
+  "</style>\r\n"
+  "</head>\r\n"
+  "<body>\r\n"
+  "<h1>Welcome to F-Stack!</h1>\r\n"
+  "\r\n"
+  "<p>For online documentation and support please refer to\r\n"
+  "<a href=\"http://F-Stack.org/\">F-Stack.org</a>.<br/>\r\n"
+  "\r\n"
+  "<p><em>Thank you for using F-Stack.</em></p>\r\n" "</body>\r\n" "</html>";
+char kernel_html[] =
+  "HTTP/1.1 200 OK\r\n"
+  "Server: Kernel-Stack\r\n"
+  "Date: Sat, 25 Feb 2017 09:26:33 GMT\r\n"
+  "Content-Type: text/html\r\n"
+  "Content-Length: 439\r\n"
+  "Last-Modified: Tue, 21 Feb 2017 09:44:03 GMT\r\n"
+  "Connection: keep-alive\r\n"
+  "Accept-Ranges: bytes\r\n"
+  "\r\n"
+  "<!DOCTYPE html>\r\n"
+  "<html>\r\n"
+  "<head>\r\n"
+  "<title>You are using Kernel stack!</title>\r\n"
+  "<style>\r\n"
+  "    body {  \r\n"
+  "        width: 35em;\r\n"
+  "        margin: 0 auto; \r\n"
+  "        font-family: Tahoma, Verdana, Arial, sans-serif;\r\n"
+  "    }\r\n"
+  "</style>\r\n"
+  "</head>\r\n"
+  "<body>\r\n"
+  "<h1>You are using Kernel stack!</h1>\r\n"
+  "\r\n"
+  "<p>For online documentation and support please refer to\r\n"
+  "<a href=\"https://www.kernel.org/doc//\">kernel.org</a>.<br/>\r\n"
+  "\r\n"
+  "<p><em>Thank you for using kernel.</em></p>\r\n" "</body>\r\n" "</html>";
+struct epoll_event ev;
+struct epoll_event events[MAX_EVENTS];
+
+static struct sockaddr_in g_k_ser;
+static struct sockaddr_in g_f_ser;
+
+static void
+setArgsDefault ()
+{
+
+  memset (&g_k_ser, 0, sizeof (g_k_ser));
+  g_k_ser.sin_family = AF_INET;
+  g_k_ser.sin_addr.s_addr = inet_addr ("127.0.0.1");
+  g_k_ser.sin_port = htons (80);
+  bzero (&(g_k_ser.sin_zero), 8);
+
+  memset (&g_f_ser, 0, sizeof (g_f_ser));
+  g_f_ser.sin_family = AF_INET;
+  g_f_ser.sin_addr.s_addr = inet_addr ("0.0.0.0");
+  g_f_ser.sin_port = htons (80);
+  bzero (&(g_f_ser.sin_zero), 8);
+}
+
+static int
+process_arg (int argc, char *argv[])
+{
+  int opt = 0;
+  const char *optstring = "f:k:";
+
+  if (argc < 2)
+    {
+      printf
+        ("Param info :-f fstack server address; -k kernel server address; \n");
+      return 0;
+    }
+  while ((opt = getopt (argc, argv, optstring)) != -1)
+    {
+      switch (opt)
+        {
+        case 'f':
+          g_f_ser.sin_addr.s_addr = inet_addr (optarg);
+          break;
+        case 'k':
+          g_k_ser.sin_addr.s_addr = inet_addr (optarg);
+          break;
+        }
+    }
+  return 1;
+}
+
+void
+clear_fd (int clearFd)
+{
+  epoll_ctl (gepfd, EPOLL_CTL_DEL, clearFd, NULL);
+  close (clearFd);
+  if (clearFd == kernel_accept_fd)
+    {
+      kernel_accept_fd = 0;
+    }
+  else if (clearFd == fstack_accept_fd)
+    {
+      fstack_accept_fd = 0;
+    }
+
+  printf ("%d fd has been deleted from the epoll list\n\n\n", clearFd);
+}
+
+int
+loop (void *arg)
+{
+  /* Wait for events to happen */
+  (void) nstack_epoll_prewait ();
+  int nevents = epoll_wait (gepfd, events, MAX_EVENTS, 0);
+
+  for (i = 0; i < nevents; ++i)
+    {
+      /* Handle new connect */
+      if ((events[i].data.fd == fstack_listen_fd)
+          || (events[i].data.fd == kernel_listen_fd))
+        {
+          while (1)
+            {
+              int acceptFd = accept (events[i].data.fd, NULL, NULL);
+              if (acceptFd < 0)
+                {
+                  break;
+                }
+
+              if (events[i].data.fd == kernel_listen_fd)
+                {
+                  kernel_accept_fd = acceptFd;
+                  printf ("kernel listen fd = %d accept fd = %d\n",
+                          kernel_listen_fd, kernel_accept_fd);
+                }
+              else
+                {
+                  fstack_accept_fd = acceptFd;
+                  printf ("f-stack listen fd = %d accept fd = %d\n",
+                          kernel_listen_fd, fstack_accept_fd);
+                }
+
+              ev.data.fd = acceptFd;
+              ev.events = EPOLLIN;
+
+              /* Add to event list */
+              if (epoll_ctl (gepfd, EPOLL_CTL_ADD, acceptFd, &ev) != 0)
+                {
+                  printf ("epoll_ctl failed:%d, %s\n", errno,
+                          strerror (errno));
+                  break;
+                }
+            }
+        }
+      else
+        {
+          if (events[i].events & EPOLLERR)
+            {
+              /* Simply close socket */
+              clear_fd (events[i].data.fd);
+            }
+          else if (events[i].events & EPOLLIN)
+            {
+              readlen = recv (events[i].data.fd, buffer, sizeof (buffer), 0);
+              printf ("read event on fd = %d readlen %d\n", events[i].data.fd,
+                      readlen);
+              if (readlen > 0)
+                {
+                  printf ("received below buffer from client\n %s\n", buffer);
+                  if (events[i].data.fd == kernel_accept_fd)
+                    {
+                      if (strlen (kernel_html) !=
+                          send (events[i].data.fd, kernel_html,
+                                strlen (kernel_html), 0))
+                        {
+                          printf ("kernel send failed\n");
+                        }
+                      printf ("kernel send fd = %d\n", events[i].data.fd);
+                    }
+                  else
+                    {
+                      if (sizeof (fstack_html) !=
+                          send (events[i].data.fd, fstack_html,
+                                sizeof (fstack_html), 0))
+                        {
+                          printf ("fstack send failed\n");
+                        }
+                      printf ("fstack send fd = %d\n", events[i].data.fd);
+                    }
+                }
+              else
+                {
+                  clear_fd (events[i].data.fd);
+                }
+            }
+          else
+            {
+              printf ("unknown event: %8.8X\n", events[i].events);
+            }
+        }
+    }
+}
+
+void
+main (int argc, char *argv[])
+{
+  int on = 1, ret = 0;
+
+  setArgsDefault ();
+  ret = process_arg (argc, argv);
+  if (ret != 1)
+    {
+      printf ("The param error\n");
+      return;
+    }
+
+  // Creating socket file descriptor
+  if ((kernel_listen_fd = socket (AF_INET, SOCK_STREAM, 0)) == 0)
+    {
+      printf ("kernel socket failed\n");
+      exit (EXIT_FAILURE);
+    }
+
+  printf ("kernel socket success\n");
+  ioctl (kernel_listen_fd, FIONBIO, &on);
+
+  // Forcefully attaching socket to the port 8080
+  if (bind (kernel_listen_fd, (struct sockaddr *) &g_k_ser,
+            sizeof (g_k_ser)) < 0)
+    {
+      printf ("kernel bind failed\n");
+      exit (EXIT_FAILURE);
+    }
+
+  printf ("kernel bind success\n");
+  if (listen (kernel_listen_fd, 3) < 0)
+    {
+      printf ("kernel listen failed \n");
+      exit (EXIT_FAILURE);
+    }
+  printf ("kernel listen success \n");
+
+  assert ((gepfd = epoll_create (100)) > 0);
+  ev.data.fd = kernel_listen_fd;
+  ev.events = EPOLLIN;
+
+  epoll_ctl (gepfd, EPOLL_CTL_ADD, kernel_listen_fd, &ev);
+
+  /* fstack */
+  fstack_listen_fd = socket (AF_INET, SOCK_STREAM, 0);
+  printf ("sockfd:%d\n", fstack_listen_fd);
+  if (fstack_listen_fd < 0)
+    {
+      printf ("socket failed\n");
+      exit (1);
+    }
+
+  ioctl (fstack_listen_fd, FIONBIO, &on);
+
+  ret =
+    bind (fstack_listen_fd, (struct sockaddr *) &g_f_ser, sizeof (g_f_ser));
+  if (ret < 0)
+    {
+      printf ("bind failed\n");
+      exit (1);
+    }
+
+  ret = listen (fstack_listen_fd, MAX_EVENTS);
+  if (ret < 0)
+    {
+      printf ("listen failed\n");
+      exit (1);
+    }
+
+  ev.data.fd = fstack_listen_fd;
+  ev.events = EPOLLIN;
+  epoll_ctl (gepfd, EPOLL_CTL_ADD, fstack_listen_fd, &ev);
+  nstack_run ((void *) loop);
+  return;
+}
diff --git a/stacks/fstack/app/single_sock_tcp_epoll.c b/stacks/fstack/app/single_sock_tcp_epoll.c
new file mode 100644
index 0000000..2facd67
--- /dev/null
+++ b/stacks/fstack/app/single_sock_tcp_epoll.c
@@ -0,0 +1,217 @@
+#define _GNU_SOURCE
+#include <unistd.h>
+#include <stdio.h>
+#include <sys/ioctl.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+#include <errno.h>
+#include <assert.h>
+#include <sys/epoll.h>
+#include "dmm_pub_api.h"
+#include <pthread.h>
+#include <fcntl.h>
+
+#define MAX_EVENTS 512
+
+int gepfd;
+int fstack_listen_fd, i, readlen;
+char buffer[1024] = { 0 };
+
+char fstack_html[] =
+  "HTTP/1.1 200 OK\r\n"
+  "Server: F-Stack\r\n"
+  "Date: Sat, 25 Feb 2017 09:26:33 GMT\r\n"
+  "Content-Type: text/html\r\n"
+  "Content-Length: 439\r\n"
+  "Last-Modified: Tue, 21 Feb 2017 09:44:03 GMT\r\n"
+  "Connection: keep-alive\r\n"
+  "Accept-Ranges: bytes\r\n"
+  "\r\n"
+  "<!DOCTYPE html>\r\n"
+  "<html>\r\n"
+  "<head>\r\n"
+  "<title>Welcome to F-Stack!</title>\r\n"
+  "<style>\r\n"
+  "    body {  \r\n"
+  "        width: 35em;\r\n"
+  "        margin: 0 auto; \r\n"
+  "        font-family: Tahoma, Verdana, Arial, sans-serif;\r\n"
+  "    }\r\n"
+  "</style>\r\n"
+  "</head>\r\n"
+  "<body>\r\n"
+  "<h1>Welcome to F-Stack!</h1>\r\n"
+  "\r\n"
+  "<p>For online documentation and support please refer to\r\n"
+  "<a href=\"http://F-Stack.org/\">F-Stack.org</a>.<br/>\r\n"
+  "\r\n"
+  "<p><em>Thank you for using F-Stack.</em></p>\r\n" "</body>\r\n" "</html>";
+struct epoll_event ev;
+struct epoll_event events[MAX_EVENTS];
+
+static struct sockaddr_in g_f_ser;
+
+static void
+setArgsDefault ()
+{
+  memset (&g_f_ser, 0, sizeof (g_f_ser));
+  g_f_ser.sin_family = AF_INET;
+  g_f_ser.sin_addr.s_addr = inet_addr ("0.0.0.0");
+  g_f_ser.sin_port = htons (80);
+  bzero (&(g_f_ser.sin_zero), 8);
+}
+
+static int
+process_arg (int argc, char *argv[])
+{
+  int opt = 0;
+  const char *optstring = "f:";
+
+  if (argc < 1)
+    {
+      printf ("Param info :-f fstack server address;\n");
+      return 0;
+    }
+  while ((opt = getopt (argc, argv, optstring)) != -1)
+    {
+      switch (opt)
+        {
+        case 'f':
+          g_f_ser.sin_addr.s_addr = inet_addr (optarg);
+          break;
+        }
+    }
+  return 1;
+}
+
+void
+clear_fd (int clearFd)
+{
+  epoll_ctl (gepfd, EPOLL_CTL_DEL, clearFd, NULL);
+  close (clearFd);
+  printf ("%d fd has been deleted from the epoll list\n\n\n", clearFd);
+}
+
+int
+loop (void *arg)
+{
+  /* Wait for events to happen */
+  (void) nstack_epoll_prewait ();
+  int nevents = epoll_wait (gepfd, events, MAX_EVENTS, 0);
+
+  for (i = 0; i < nevents; ++i)
+    {
+      /* Handle new connect */
+      if (events[i].data.fd == fstack_listen_fd)
+        {
+          while (1)
+            {
+              int fstack_accept_fd = accept (events[i].data.fd, NULL, NULL);
+              if (fstack_accept_fd < 0)
+                {
+                  break;
+                }
+
+              printf ("f-stack listen fd = %d f-stack accept fd = %d\n",
+                      fstack_listen_fd, fstack_accept_fd);
+
+              ev.data.fd = fstack_accept_fd;
+              ev.events = EPOLLIN;
+
+              /* Add to event list */
+              if (epoll_ctl (gepfd, EPOLL_CTL_ADD, fstack_accept_fd, &ev) !=
+                  0)
+                {
+                  printf ("epoll_ctl failed:%d, %s\n", errno,
+                          strerror (errno));
+                  break;
+                }
+            }
+        }
+      else
+        {
+          if (events[i].events & EPOLLERR)
+            {
+              /* Simply close socket */
+              clear_fd (events[i].data.fd);
+            }
+          else if (events[i].events & EPOLLIN)
+            {
+              readlen = recv (events[i].data.fd, buffer, sizeof (buffer), 0);
+              printf ("read event on fd = %d readlen %d\n", events[i].data.fd,
+                      readlen);
+              if (readlen > 0)
+                {
+                  printf ("received below buffer from client\n %s\n", buffer);
+                  if (sizeof (fstack_html) !=
+                      send (events[i].data.fd, fstack_html,
+                            sizeof (fstack_html), 0))
+                    {
+                      printf ("fstack send failed\n");
+                    }
+                  printf ("fstack send fd = %d\n", events[i].data.fd);
+                }
+              else
+                {
+                  clear_fd (events[i].data.fd);
+                }
+            }
+          else
+            {
+              printf ("unknown event: %8.8X\n", events[i].events);
+            }
+        }
+    }
+}
+
+void
+main (int argc, char *argv[])
+{
+  int on = 1, ret = 0;
+
+  setArgsDefault ();
+  ret = process_arg (argc, argv);
+  if (ret != 1)
+    {
+      printf ("The param error\n");
+      return;
+    }
+
+  /* fstack */
+  fstack_listen_fd = socket (AF_INET, SOCK_STREAM, 0);
+  printf ("sockfd:%d\n", fstack_listen_fd);
+  if (fstack_listen_fd < 0)
+    {
+      printf ("socket failed\n");
+      exit (1);
+    }
+
+  ioctl (fstack_listen_fd, FIONBIO, &on);
+
+  ret =
+    bind (fstack_listen_fd, (struct sockaddr *) &g_f_ser, sizeof (g_f_ser));
+  if (ret < 0)
+    {
+      printf ("bind failed\n");
+      exit (1);
+    }
+
+  ret = listen (fstack_listen_fd, MAX_EVENTS);
+  if (ret < 0)
+    {
+      printf ("listen failed\n");
+      exit (1);
+    }
+
+  assert ((gepfd = epoll_create (100)) > 0);
+  ev.data.fd = fstack_listen_fd;
+  ev.events = EPOLLIN;
+  epoll_ctl (gepfd, EPOLL_CTL_ADD, fstack_listen_fd, &ev);
+  nstack_run ((void *) loop);
+  return;
+}
diff --git a/stacks/fstack/config/config.ini b/stacks/fstack/config/config.ini
new file mode 100644
index 0000000..19a7e75
--- /dev/null
+++ b/stacks/fstack/config/config.ini
@@ -0,0 +1,124 @@
+[dpdk]
+# Hexadecimal bitmask of cores to run on.
+lcore_mask=1
+
+# Number of memory channels.
+channel=4
+
+# Specify base virtual address to map.
+#base_virtaddr=0x7f0000000000
+
+# Promiscuous mode of nic, defualt: enabled.
+promiscuous=1
+numa_on=1
+
+# TCP segment offload, default: disabled.
+tso=0
+
+# HW vlan strip, default: enabled.
+vlan_strip=1
+
+# sleep when no pkts incomming
+# unit: microseconds
+
+# enabled port list
+#
+# EBNF grammar:
+#
+#    exp      ::= num_list {"," num_list}
+#    num_list ::= <num> | <range>
+#    range    ::= <num>"-"<num>
+#    num      ::= '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9'
+#
+# examples
+#    0-3       ports 0, 1,2,3 are enabled
+#    1-3,4,7   ports 1,2,3,4,7 are enabled
+port_list=0
+
+# Number of vdev.
+
+# Port config section
+# Correspond to dpdk.port_list's index: port0, port1...
+[port0]
+addr=192.168.1.2
+netmask=255.255.255.0
+broadcast=192.168.1.255
+gateway=192.168.1.1
+
+# lcore list used to handle this port
+# the format is same as port_list
+# lcore_list= 0
+
+# Packet capture path, this will hurt performance
+#pcap=./a.pcap
+
+# Vdev config section
+# orrespond to dpdk.nb_vdev's index: vdev0, vdev1...
+#    iface : Shouldn't set always.
+#    path : The vuser device path in container. Required.
+#    queues : The max queues of vuser. Optional, default 1, greater or equal to the number of processes.
+#    queue_size : Queue size.Optional, default 256.
+#    mac : The mac address of vuser. Optional, default random, if vhost use phy NIC, it should be set to the phy NIC's mac.
+#    cq : Optional, if queues = 1, default 0; if queues > 1 default 1.
+#[vdev0]
+##iface=/usr/local/var/run/openvswitch/vhost-user0
+#path=/var/run/openvswitch/vhost-user0
+#queues=1
+#queue_size=256
+#mac=00:00:00:00:00:01
+#cq=0
+
+# Kni config: if enabled and method=reject,
+# all packets that do not belong to the following tcp_port and udp_port
+# will transmit to kernel; if method=accept, all packets that belong to
+# the following tcp_port and udp_port will transmit to kernel.
+#[kni]
+#enable=1
+#method=reject
+# The format is same as port_list
+#tcp_port=80,443
+#udp_port=53
+
+# FreeBSD network performance tuning configurations.
+# Most native FreeBSD configurations are supported.
+[freebsd.boot]
+hz=100
+
+# Block out a range of descriptors to avoid overlap
+# with the kernel's descriptor space.
+# You can increase this value according to your app.
+fd_reserve=512
+
+kern.ipc.maxsockets=262144
+
+net.inet.tcp.syncache.hashsize=4096
+net.inet.tcp.syncache.bucketlimit=100
+
+net.inet.tcp.tcbhashsize=65536
+
+kern.ncallout=262144
+
+[freebsd.sysctl]
+kern.ipc.somaxconn=32768
+kern.ipc.maxsockbuf=16777216
+
+net.link.ether.inet.maxhold=5
+
+net.inet.tcp.fast_finwait2_recycle=1
+net.inet.tcp.sendspace=16384
+net.inet.tcp.recvspace=8192
+net.inet.tcp.nolocaltimewait=1
+net.inet.tcp.cc.algorithm=cubic
+net.inet.tcp.sendbuf_max=16777216
+net.inet.tcp.recvbuf_max=16777216
+net.inet.tcp.sendbuf_auto=1
+net.inet.tcp.recvbuf_auto=1
+net.inet.tcp.sendbuf_inc=16384
+net.inet.tcp.recvbuf_inc=524288
+net.inet.tcp.sack.enable=1
+net.inet.tcp.blackhole=1
+net.inet.tcp.msl=2000
+net.inet.tcp.delayed_ack=0
+
+net.inet.udp.blackhole=1
+net.inet.ip.redirect=0
diff --git a/stacks/fstack/config/module_config.json b/stacks/fstack/config/module_config.json
new file mode 100644
index 0000000..e3a7f22
--- /dev/null
+++ b/stacks/fstack/config/module_config.json
@@ -0,0 +1,28 @@
+{
+  "default_stack_name": "kernel",
+  "module_list": [
+    {
+      "stack_name": "kernel",
+	  "libname": "./",
+      "function_name": "kernel_stack_register",
+      "loadtype": "static",
+      "deploytype": "1",
+      "maxfd": "1024",                           /*the max fd supported*/
+      "minfd": "0",                              /*the min fd supported*/
+      "priorty": "1",                            /*priorty when executing, reserv*/
+
+      "stackid": "0"
+    },
+    {
+      "stack_name": "fstack",
+      "libname": "libdmm_fstack.so",
+      "function_name": "fstack_stack_register",
+      "loadtype": "dynmic",
+      "deploytype": "1",
+      "maxfd": "1024",                           /*the max fd supported*/
+      "minfd": "0",                              /*the min fd supported*/
+      "priorty": "1",                            /*priorty when executing, reserv*/
+      "stackid": "1"
+    }
+  ]
+}
diff --git a/stacks/fstack/config/rd_config.json b/stacks/fstack/config/rd_config.json
new file mode 100644
index 0000000..3c5268c
--- /dev/null
+++ b/stacks/fstack/config/rd_config.json
@@ -0,0 +1,10 @@
+{
+  "ip_route": [
+    {
+	  "subnet": "192.168.1.1/24",
+	  "type": "nstack-fstack"
+	}
+  ],
+  "prot_route": [
+  ]
+}
diff --git a/stacks/fstack/fstack.patch b/stacks/fstack/fstack.patch
new file mode 100644
index 0000000..40dfade
--- /dev/null
+++ b/stacks/fstack/fstack.patch
@@ -0,0 +1,12 @@
+diff --git a/lib/ff_syscall_wrapper.c b/lib/ff_syscall_wrapper.c
+index cfc407c..7b61aa6 100644
+--- a/lib/ff_syscall_wrapper.c
++++ b/lib/ff_syscall_wrapper.c
+@@ -710,6 +710,7 @@ ff_fcntl(int fd, int cmd, ...)
+     argp = va_arg(ap, uintptr_t);
+     va_end(ap);
+
++    curthread->td_retval[0] = 0;
+     if ((rc = kern_fcntl(curthread, fd, cmd, argp)))
+         goto kern_fail;
+     rc = curthread->td_retval[0];
diff --git a/stacks/fstack/run_fstack.txt b/stacks/fstack/run_fstack.txt
new file mode 100644
index 0000000..eaca7eb
--- /dev/null
+++ b/stacks/fstack/run_fstack.txt
@@ -0,0 +1,47 @@
+F-Stack RTC
+===========
+Currently a multi-socket application can only use F-Stack RTC and Kernel stacks together.
+[cannot use F-stack and rSocket together]
+
+
+Build F-Stack:
+-------------
+cd /DMM/scripts/
+sudo ./build.sh
+
+sudo apt-get install libnuma-dev
+sudo apt-get install libssl-dev
+
+cd /DMM/build/
+sudo make dmm_fstack
+
+
+Run F-Stack:
+------------
+sudo echo 1024 | sudo tee /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
+sudo mkdir /mnt/huge
+sudo mount -t hugetlbfs nodev /mnt/huge
+sudo echo 0 | sudo tee /proc/sys/kernel/randomize_va_space
+sudo modprobe uio
+sudo insmod /DMM/stacks/fstack/f-stack-1.12/dpdk/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko
+sudo insmod /DMM/stacks/fstack/f-stack-1.12/dpdk/x86_64-native-linuxapp-gcc/kmod/rte_kni.ko
+sudo python /DMM/stacks/fstack/f-stack-1.12/dpdk/usertools/dpdk-devbind.py --status
+sudo ifconfig enp0s8 down
+sudo python /DMM/stacks/fstack/f-stack-1.12/dpdk/usertools/dpdk-devbind.py --bind=igb_uio enp0s8
+
+go to app:
+----------
+cd /DMM/stacks/fstack/app
+sudo make
+
+cp -r /DMM/stacks/fstack/config/* /DMM/stacks/fstack/app
+edit config.ini and rd_config.json according to the requrired server/client ip
+
+#server command:
+---------------
+sudo FF_PATH=/DMM/stacks/fstack/f-stack-1.12/ FF_DPDK=/DMM/stacks/fstack/f-stack-1.12/dpdk/x86_64-native-linuxapp-gcc LD_LIBRARY_PATH=/DMM/release/lib64/ LD_PRELOAD=/DMM/release/lib64/libnStackAPI.so ./fstack_single_sock_server -f 172.28.128.13
+
+#client command:
+----------------
+curl http://172.28.128.13
+[172.28.128.13 - server ip]
diff --git a/stacks/fstack/src/fstack_adpt.c b/stacks/fstack/src/fstack_adpt.c
new file mode 100644
index 0000000..c25577c
--- /dev/null
+++ b/stacks/fstack/src/fstack_adpt.c
@@ -0,0 +1,241 @@
+/*
+*
+* Copyright (c) 2018 Huawei Technologies Co.,Ltd.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at:
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include <errno.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <dlfcn.h>
+#include "nstack_dmm_api.h"
+#include "ff_config.h"
+#include "ff_api.h"
+#include "ff_epoll.h"
+
+#define DMM_FSTACK_MAX_EP_EVENT 1024
+#define DMM_FSTACK_ENV_DEBUG     "DMM_FSTACK_DEBUG"
+#define DMM_FSTACK_ADPT_DEBUG dmm_fstack_debug
+#define DMM_FSTACK_ENV_PROC_TYPE "DMM_FSTACK_PROC_TYPE"
+#define DMM_FSTACK_ENV_PROC_ID "DMM_FSTACK_PROC_ID"
+#define DMM_FSTACK_ENV_CONF_FILE "DMM_FSTACK_CONF_FILE"
+
+static unsigned int dmm_fstack_debug;
+
+typedef struct dmm_fstack
+{
+  int epfd;
+  long unsigned int epoll_threadid;
+  nstack_event_cb regVal;
+  int (*p_epoll_create) (int size);
+  unsigned int (*p_epoll_ctl) (int epFD, int ctl_ops, int proFD,
+                               struct epoll_event * events);
+  unsigned int (*p_epoll_wait) (int epfd, struct epoll_event * events,
+                                int maxevents, int timeout);
+  int (*p_close) (int fd);
+} dmm_fstack_t;
+
+dmm_fstack_t g_dmm_fstack;
+
+unsigned int
+fstack_ep_ctl_ops (int epFD, int proFD, int ctl_ops,
+                   struct epoll_event *events, void *pdata)
+{
+  struct epoll_event tmpEvt;
+  int ret = 0;
+  int dmm_epfd;
+
+  tmpEvt.data.ptr = pdata;
+  tmpEvt.events = events->events;
+  tmpEvt.events |= (EPOLLIN | EPOLLOUT);
+
+  if (DMM_FSTACK_ADPT_DEBUG > 0)
+    fprintf (stdout, "DMM VCL ADPT<%d>: epfd=%d,fd=%d,ops=%d, events=%u",
+             getpid (), epFD, proFD, ctl_ops, events->events);
+
+  dmm_epfd = g_dmm_fstack.epfd;
+  switch (ctl_ops)
+    {
+    case nstack_ep_triggle_add:
+      ret =
+        g_dmm_fstack.p_epoll_ctl (dmm_epfd, EPOLL_CTL_ADD, proFD, &tmpEvt);
+      break;
+    case nstack_ep_triggle_mod:
+      ret =
+        g_dmm_fstack.p_epoll_ctl (dmm_epfd, EPOLL_CTL_MOD, proFD, &tmpEvt);
+      break;
+    case nstack_ep_triggle_del:
+      ret =
+        g_dmm_fstack.p_epoll_ctl (dmm_epfd, EPOLL_CTL_DEL, proFD, &tmpEvt);
+      break;
+    default:
+      ret = -1;
+      break;
+    }
+  return ret;
+}
+
+int
+fr_init (void)
+{
+  int ret;
+  char config_file[1024] = "--conf=config.ini";
+  char proc_type[1024] = "--proc-type=primary";
+  char proc_id[1024] = "--proc-id=0";
+
+  /* if env isn't set consider DPDK process type as primary */
+  if (getenv (DMM_FSTACK_ENV_CONF_FILE))
+    {
+      sprintf (proc_type, "--conf=%s", getenv (DMM_FSTACK_ENV_CONF_FILE));
+    }
+  if (getenv (DMM_FSTACK_ENV_PROC_TYPE))
+    {
+      sprintf (proc_type, "--proc-type=%s",
+               getenv (DMM_FSTACK_ENV_PROC_TYPE));
+    }
+  if (getenv (DMM_FSTACK_ENV_PROC_ID))
+    {
+      sprintf (proc_id, "--proc-id=%d",
+               atoi (getenv (DMM_FSTACK_ENV_PROC_ID)));
+    }
+
+  char *argv[] = {
+    "dmm-fstack",
+    config_file,
+    proc_type,
+    proc_id,
+    NULL
+  };
+
+  const int argc = sizeof (argv) / sizeof (argv[0]) - 1;
+
+  ret = ff_init (argc, argv);
+  if (ret)
+    return -1;
+
+  return 0;
+}
+
+static pid_t
+ff_fork (void)
+{
+  return -1;
+}
+
+int
+fr_run (void *loop)
+{
+  ff_run (loop, NULL);
+  return 0;
+}
+
+static int
+ff_accept4 (int sockfd, struct sockaddr *addr, socklen_t * addrlen, int flags)
+{
+  int fd;
+
+  fd = ff_accept (sockfd, (struct linux_sockaddr *) addr, addrlen);
+  if (fd < 0)
+    return fd;
+
+  if (flags & SOCK_NONBLOCK)
+    {
+      (void) ff_ioctl (sockfd, FIONBIO, 1);
+    }
+
+  if (flags & SOCK_CLOEXEC)
+    {
+    }
+
+  return fd;
+}
+
+void
+dmm_fstack_epoll ()
+{
+  int num, i;
+
+  struct epoll_event events[DMM_FSTACK_MAX_EP_EVENT];
+
+  num =
+    g_dmm_fstack.p_epoll_wait (g_dmm_fstack.epfd, events,
+                               DMM_FSTACK_MAX_EP_EVENT, 0);
+  for (i = 0; i < num; ++i)
+    {
+      g_dmm_fstack.regVal.event_cb (events[i].data.ptr, events[i].events);
+    }
+
+  return;
+}
+
+int
+dmm_fstack_init ()
+{
+  char *env_var_str;
+  int rv = 0;
+
+  if (0 != fr_init ())
+    {
+      return -1;
+    }
+
+  env_var_str = getenv (DMM_FSTACK_ENV_DEBUG);
+  if (env_var_str)
+    {
+      unsigned int tmp;
+      if (sscanf (env_var_str, "%u", &tmp) != 1)
+        fprintf (stdout,
+                 "DMM_FSTACK_ADPT: WARNING: Invalid debug level specified "
+                 "in the environment variable " DMM_FSTACK_ENV_DEBUG
+                 " (%s)!\n", env_var_str);
+      else
+        {
+          dmm_fstack_debug = tmp;
+          if (DMM_FSTACK_ADPT_DEBUG > 0)
+            fprintf (stdout,
+                     "DMM_FSTACK_ADPT: configured DMM FSTACK ADPT debug (%u) from "
+                     "DMM_STACK_ENV_DEBUG ", dmm_fstack_debug);
+        }
+    }
+
+  g_dmm_fstack.epfd = g_dmm_fstack.p_epoll_create (50);
+  if (g_dmm_fstack.epfd < 0)
+    return g_dmm_fstack.epfd;
+
+  return 0;
+}
+
+int
+fstack_stack_register (nstack_proc_cb * proc_fun, nstack_event_cb * event_ops)
+{
+#define NSTACK_MK_DECL(ret, fn, args) proc_fun->socket_ops.pf##fn = (void*)ff_##fn;
+#include "declare_syscalls.h"
+#undef NSTACK_MK_DECL
+  g_dmm_fstack.p_epoll_ctl = dlsym (event_ops->handle, "ff_epoll_ctl");
+  g_dmm_fstack.p_epoll_create = dlsym (event_ops->handle, "ff_epoll_create");
+  g_dmm_fstack.p_epoll_wait = dlsym (event_ops->handle, "ff_epoll_wait");
+  g_dmm_fstack.p_close = dlsym (event_ops->handle, "ff_close");
+  g_dmm_fstack.regVal = *event_ops;
+
+  proc_fun->socket_ops.pfselect = NULL;
+
+  proc_fun->extern_ops.module_init = dmm_fstack_init;
+  proc_fun->extern_ops.ep_ctl = fstack_ep_ctl_ops;
+  proc_fun->extern_ops.ep_getevt = NULL;
+  proc_fun->extern_ops.run = fr_run;
+  proc_fun->extern_ops.ep_prewait = dmm_fstack_epoll;
+  return 0;
+}
-- 
cgit 1.2.3-korg