diff options
Diffstat (limited to 'stacks')
-rw-r--r-- | stacks/fstack/CMakeLists.txt | 59 | ||||
-rw-r--r-- | stacks/fstack/app/Makefile | 9 | ||||
-rwxr-xr-x | stacks/fstack/app/dmm_start.sh | 65 | ||||
-rw-r--r-- | stacks/fstack/app/fstack_client.c | 49 | ||||
-rw-r--r-- | stacks/fstack/app/fstack_server.c | 78 | ||||
-rw-r--r-- | stacks/fstack/app/multi_sock_tcp_epoll.c | 322 | ||||
-rw-r--r-- | stacks/fstack/app/single_sock_tcp_epoll.c | 217 | ||||
-rw-r--r-- | stacks/fstack/config/config.ini | 124 | ||||
-rw-r--r-- | stacks/fstack/config/module_config.json | 28 | ||||
-rw-r--r-- | stacks/fstack/config/rd_config.json | 10 | ||||
-rw-r--r-- | stacks/fstack/fstack.patch | 12 | ||||
-rw-r--r-- | stacks/fstack/run_fstack.txt | 47 | ||||
-rw-r--r-- | stacks/fstack/src/fstack_adpt.c | 241 |
13 files changed, 1261 insertions, 0 deletions
diff --git a/stacks/fstack/CMakeLists.txt b/stacks/fstack/CMakeLists.txt new file mode 100644 index 0000000..a058749 --- /dev/null +++ b/stacks/fstack/CMakeLists.txt @@ -0,0 +1,59 @@ +######################################################################### +# +# Copyright (c) 2018 Huawei Technologies Co.,Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +######################################################################### + +SET(fstack_dir f-stack-1.12) +SET(dpdk_target x86_64-native-linuxapp-gcc) +SET(dmm_inc_dir ${DMM_REL_INC_DIR}) + + +SET(fstack_url https://github.com/F-Stack/f-stack/archive/v1.12.tar.gz) + +INCLUDE(ExternalProject) +ExternalProject_Add( + fstack + URL ${fstack_url} + TLS_VERIFY false + SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/${fstack_dir} + DOWNLOAD_DIR ${CMAKE_CURRENT_LIST_DIR} + BUILD_IN_SOURCE 1 + PATCH_COMMAND patch -p1 -i ../fstack.patch + CONFIGURE_COMMAND make -C dpdk config T=${dpdk_target} + build O=${dpdk_target} EXTRA_CFLAGS=-fPIC + BUILD_COMMAND make -C lib "CONF_CFLAGS=-fPIC -g" + INSTALL_COMMAND "" +) +set_target_properties(fstack PROPERTIES EXCLUDE_FROM_ALL TRUE) + +##################################################################### + +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -g -fPIC -m64 -pthread") + +LINK_DIRECTORIES(${fstack_dir}/lib ${fstack_dir}/dpdk/${dpdk_target}/lib) +ADD_LIBRARY(dmm_fstack SHARED src/fstack_adpt.c) + +ADD_DEFINITIONS(-D_GNU_SOURCE) + +INCLUDE_DIRECTORIES(${DMM_REL_INC_DIR}) +INCLUDE_DIRECTORIES(${fstack_dir}/lib) + +TARGET_LINK_LIBRARIES(dmm_fstack + -Wl,--whole-archive -lfstack -ldpdk + -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -pthread -lnuma +) + +ADD_DEPENDENCIES(dmm_fstack fstack post-compile) + +set_target_properties(dmm_fstack PROPERTIES EXCLUDE_FROM_ALL TRUE) diff --git a/stacks/fstack/app/Makefile b/stacks/fstack/app/Makefile new file mode 100644 index 0000000..42f5274 --- /dev/null +++ b/stacks/fstack/app/Makefile @@ -0,0 +1,9 @@ +LIBS+= -L../../../release/lib64 -Wl,--whole-archive,-lnStackAPI,--no-whole-archive +TARGET="fstack" +all: + cc -O -gdwarf-2 -Wl,--rpath,../../../release/lib64 -I../../../src/nSocket/include -o ${TARGET}_multi_sock_epoll multi_sock_tcp_epoll.c ${LIBS} + cc -O -gdwarf-2 -Wl,--rpath,../../../release/lib64 -I../../../src/nSocket/include -o ${TARGET}_server_no_epoll fstack_server.c ${LIBS} + cc -O -gdwarf-2 -Wl,--rpath,../../../release/lib64 -I../../../src/nSocket/include -o ${TARGET}_single_sock_server single_sock_tcp_epoll.c ${LIBS} +.PHONY: clean +clean: + rm -f *.o ${TARGET}_multi_sock_epoll ${TARGET}_single_sock_server ${TARGET}_server_no_epoll diff --git a/stacks/fstack/app/dmm_start.sh b/stacks/fstack/app/dmm_start.sh new file mode 100755 index 0000000..1fe99ee --- /dev/null +++ b/stacks/fstack/app/dmm_start.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +function usage() { + echo "F-Stack app start tool" + echo "Options:" + echo " -c [conf] Path of config file" + echo " -b [N] Path of binary" + echo " -o [N] Other ARGs for app" + echo " -h show this help" + exit +} + +conf=config.ini +bin=fstack_multi_proc + +while getopts "c:b:o:h" args +do + case $args in + c) + conf=$OPTARG + ;; + b) + bin=$OPTARG + ;; + o) + others=$OPTARG + ;; + h) + usage + exit 0 + ;; + esac +done + +if ! type "bc" > /dev/null 2>&1; then + echo "please install bc" + exit +fi + +allcmask0x=`cat ${conf}|grep lcore_mask|awk -F '=' '{print $2}'` +((allcmask=16#$allcmask0x)) + +num_procs=0 +PROCESSOR=$(grep 'processor' /proc/cpuinfo |sort |uniq |wc -l) +for((i=0;i<${PROCESSOR};++i)) +do + mask=`echo "2^$i"|bc` + ((result=${allcmask} & ${mask})) + if [ ${result} != 0 ] + then + ((num_procs++)); + fi +done + +for((proc_id=0; proc_id<${num_procs}; ++proc_id)) +do + if ((proc_id == 0)) + then + echo "number of processes ${num_procs}" + sudo LD_LIBRARY_PATH=../../../release/lib64/ LD_PRELOAD=../../../release/lib64/libnStackAPI.so F_PROC_ID=${proc_id} F_PROC_TYPE=primary FF_PATH=../f-stack-1.12/ FF_DPDK=../f-stack-1.12/dpdk/x86_64-native-linuxapp-gcc ./${bin} & + sleep 5 + else + sudo LD_LIBRARY_PATH=../../../release/lib64/ LD_PRELOAD=../../../release/lib64/libnStackAPI.so F_PROC_ID=${proc_id} F_PROC_TYPE=secondary FF_PATH=../f-stack-1.12/ FF_DPDK=../f-stack-1.12/dpdk/x86_64-native-linuxapp-gcc ./${bin} & + fi +done diff --git a/stacks/fstack/app/fstack_client.c b/stacks/fstack/app/fstack_client.c new file mode 100644 index 0000000..a4b378c --- /dev/null +++ b/stacks/fstack/app/fstack_client.c @@ -0,0 +1,49 @@ +// Client side C/C++ program to demonstrate Socket programming +#include <stdio.h> +#include <sys/socket.h> +#include <stdlib.h> +#include <netinet/in.h> +#include <string.h> +#include <arpa/inet.h> +#include <unistd.h> +#define PORT 8080 + +int +main (int argc, char const *argv[]) +{ + struct sockaddr_in address; + int sock = 0, valread; + struct sockaddr_in serv_addr; + char *hello = "Hello from client"; + char buffer[1024] = { 0 }; + if ((sock = socket (AF_INET, SOCK_STREAM, 0)) < 0) + { + printf ("\n Socket creation error \n"); + return -1; + } + + memset (&serv_addr, '0', sizeof (serv_addr)); + + serv_addr.sin_family = AF_INET; + serv_addr.sin_port = htons (PORT); + + // Convert IPv4 and IPv6 addresses from text to binary form + //if(inet_pton(AF_INET, "192.28.128.8", &serv_addr.sin_addr)<=0) + if (inet_pton (AF_INET, "172.28.128.13", &serv_addr.sin_addr) <= 0) + { + printf ("\nInvalid address/ Address not supported \n"); + return -1; + } + + if (connect (sock, (struct sockaddr *) &serv_addr, sizeof (serv_addr)) < 0) + { + printf ("\nConnection Failed \n"); + return -1; + } + send (sock, hello, strlen (hello), 0); + printf ("Hello message sent\n"); + valread = read (sock, buffer, 1024); + printf ("%s\n", buffer); + close (sock); + return 0; +} diff --git a/stacks/fstack/app/fstack_server.c b/stacks/fstack/app/fstack_server.c new file mode 100644 index 0000000..241284b --- /dev/null +++ b/stacks/fstack/app/fstack_server.c @@ -0,0 +1,78 @@ +#define _GNU_SOURCE +#include <unistd.h> +#include <stdio.h> +#include <sys/socket.h> +#include <stdlib.h> +#include <netinet/in.h> +#include <string.h> +#include <arpa/inet.h> +#include <sys/types.h> +#include <fcntl.h> +#include <sys/ioctl.h> +#include "dmm_pub_api.h" +#define PORT 8080 + +int server_fd; +struct sockaddr_in address; +int +loop (void *arg) +{ + int new_socket, valread; + int addrlen = sizeof (address); + char buffer[1024] = { 0 }; + char hello[1024] = { 0 }; + sprintf (hello, "Hello from server\t PROC_ID : %d", getpid ()); + printf ("before accept\n"); + if ((new_socket = accept (server_fd, (struct sockaddr *) &address, + (socklen_t *) & addrlen)) < 0) + { + printf ("accept failed"); + return -1; + } + + printf ("accept success\n"); + valread = read (new_socket, buffer, 1024); + printf ("%s\n", buffer); + send (new_socket, hello, strlen (hello), 0); + printf ("Hello message sent\n"); + return 0; +} + +int +main (int argc, char *argv[]) +{ + + int opt = 1; + int addrlen = sizeof (address); +#if 0 + if (-1 == nstack_conf (argc, argv)) + { + printf ("nstack_conf failed\n"); + return -1; + } +#endif + if ((server_fd = socket (AF_INET, SOCK_STREAM, 0)) == 0) + { + printf ("socket failed\n"); + exit (EXIT_FAILURE); + } + address.sin_family = AF_INET; + address.sin_addr.s_addr = inet_addr ("172.28.128.13"); + address.sin_port = htons (PORT); + + // Forcefully attaching socket to the port 8080 + if (bind (server_fd, (struct sockaddr *) &address, sizeof (address)) < 0) + { + printf ("bind failed"); + exit (EXIT_FAILURE); + } + printf ("bind success\n"); + if (listen (server_fd, 3) < 0) + { + printf ("listen"); + exit (EXIT_FAILURE); + } + printf ("listen success\n"); + nstack_run ((void *) loop); + return 0; +} diff --git a/stacks/fstack/app/multi_sock_tcp_epoll.c b/stacks/fstack/app/multi_sock_tcp_epoll.c new file mode 100644 index 0000000..56e6374 --- /dev/null +++ b/stacks/fstack/app/multi_sock_tcp_epoll.c @@ -0,0 +1,322 @@ +#define _GNU_SOURCE +#include <unistd.h> +#include <stdio.h> +#include <sys/ioctl.h> +#include <stdlib.h> +#include <stdint.h> +#include <string.h> +#include <strings.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <arpa/inet.h> +#include <errno.h> +#include <assert.h> +#include <sys/epoll.h> +#include "dmm_pub_api.h" +#include <pthread.h> +#include <fcntl.h> + +#define MAX_EVENTS 512 + +int gepfd; +int fstack_listen_fd, kernel_listen_fd, i, kernel_accept_fd, fstack_accept_fd, + readlen; +char buffer[1024] = { 0 }; + +char fstack_html[] = + "HTTP/1.1 200 OK\r\n" + "Server: F-Stack\r\n" + "Date: Sat, 25 Feb 2017 09:26:33 GMT\r\n" + "Content-Type: text/html\r\n" + "Content-Length: 439\r\n" + "Last-Modified: Tue, 21 Feb 2017 09:44:03 GMT\r\n" + "Connection: keep-alive\r\n" + "Accept-Ranges: bytes\r\n" + "\r\n" + "<!DOCTYPE html>\r\n" + "<html>\r\n" + "<head>\r\n" + "<title>Welcome to F-Stack!</title>\r\n" + "<style>\r\n" + " body { \r\n" + " width: 35em;\r\n" + " margin: 0 auto; \r\n" + " font-family: Tahoma, Verdana, Arial, sans-serif;\r\n" + " }\r\n" + "</style>\r\n" + "</head>\r\n" + "<body>\r\n" + "<h1>Welcome to F-Stack!</h1>\r\n" + "\r\n" + "<p>For online documentation and support please refer to\r\n" + "<a href=\"http://F-Stack.org/\">F-Stack.org</a>.<br/>\r\n" + "\r\n" + "<p><em>Thank you for using F-Stack.</em></p>\r\n" "</body>\r\n" "</html>"; +char kernel_html[] = + "HTTP/1.1 200 OK\r\n" + "Server: Kernel-Stack\r\n" + "Date: Sat, 25 Feb 2017 09:26:33 GMT\r\n" + "Content-Type: text/html\r\n" + "Content-Length: 439\r\n" + "Last-Modified: Tue, 21 Feb 2017 09:44:03 GMT\r\n" + "Connection: keep-alive\r\n" + "Accept-Ranges: bytes\r\n" + "\r\n" + "<!DOCTYPE html>\r\n" + "<html>\r\n" + "<head>\r\n" + "<title>You are using Kernel stack!</title>\r\n" + "<style>\r\n" + " body { \r\n" + " width: 35em;\r\n" + " margin: 0 auto; \r\n" + " font-family: Tahoma, Verdana, Arial, sans-serif;\r\n" + " }\r\n" + "</style>\r\n" + "</head>\r\n" + "<body>\r\n" + "<h1>You are using Kernel stack!</h1>\r\n" + "\r\n" + "<p>For online documentation and support please refer to\r\n" + "<a href=\"https://www.kernel.org/doc//\">kernel.org</a>.<br/>\r\n" + "\r\n" + "<p><em>Thank you for using kernel.</em></p>\r\n" "</body>\r\n" "</html>"; +struct epoll_event ev; +struct epoll_event events[MAX_EVENTS]; + +static struct sockaddr_in g_k_ser; +static struct sockaddr_in g_f_ser; + +static void +setArgsDefault () +{ + + memset (&g_k_ser, 0, sizeof (g_k_ser)); + g_k_ser.sin_family = AF_INET; + g_k_ser.sin_addr.s_addr = inet_addr ("127.0.0.1"); + g_k_ser.sin_port = htons (80); + bzero (&(g_k_ser.sin_zero), 8); + + memset (&g_f_ser, 0, sizeof (g_f_ser)); + g_f_ser.sin_family = AF_INET; + g_f_ser.sin_addr.s_addr = inet_addr ("0.0.0.0"); + g_f_ser.sin_port = htons (80); + bzero (&(g_f_ser.sin_zero), 8); +} + +static int +process_arg (int argc, char *argv[]) +{ + int opt = 0; + const char *optstring = "f:k:"; + + if (argc < 2) + { + printf + ("Param info :-f fstack server address; -k kernel server address; \n"); + return 0; + } + while ((opt = getopt (argc, argv, optstring)) != -1) + { + switch (opt) + { + case 'f': + g_f_ser.sin_addr.s_addr = inet_addr (optarg); + break; + case 'k': + g_k_ser.sin_addr.s_addr = inet_addr (optarg); + break; + } + } + return 1; +} + +void +clear_fd (int clearFd) +{ + epoll_ctl (gepfd, EPOLL_CTL_DEL, clearFd, NULL); + close (clearFd); + if (clearFd == kernel_accept_fd) + { + kernel_accept_fd = 0; + } + else if (clearFd == fstack_accept_fd) + { + fstack_accept_fd = 0; + } + + printf ("%d fd has been deleted from the epoll list\n\n\n", clearFd); +} + +int +loop (void *arg) +{ + /* Wait for events to happen */ + (void) nstack_epoll_prewait (); + int nevents = epoll_wait (gepfd, events, MAX_EVENTS, 0); + + for (i = 0; i < nevents; ++i) + { + /* Handle new connect */ + if ((events[i].data.fd == fstack_listen_fd) + || (events[i].data.fd == kernel_listen_fd)) + { + while (1) + { + int acceptFd = accept (events[i].data.fd, NULL, NULL); + if (acceptFd < 0) + { + break; + } + + if (events[i].data.fd == kernel_listen_fd) + { + kernel_accept_fd = acceptFd; + printf ("kernel listen fd = %d accept fd = %d\n", + kernel_listen_fd, kernel_accept_fd); + } + else + { + fstack_accept_fd = acceptFd; + printf ("f-stack listen fd = %d accept fd = %d\n", + kernel_listen_fd, fstack_accept_fd); + } + + ev.data.fd = acceptFd; + ev.events = EPOLLIN; + + /* Add to event list */ + if (epoll_ctl (gepfd, EPOLL_CTL_ADD, acceptFd, &ev) != 0) + { + printf ("epoll_ctl failed:%d, %s\n", errno, + strerror (errno)); + break; + } + } + } + else + { + if (events[i].events & EPOLLERR) + { + /* Simply close socket */ + clear_fd (events[i].data.fd); + } + else if (events[i].events & EPOLLIN) + { + readlen = recv (events[i].data.fd, buffer, sizeof (buffer), 0); + printf ("read event on fd = %d readlen %d\n", events[i].data.fd, + readlen); + if (readlen > 0) + { + printf ("received below buffer from client\n %s\n", buffer); + if (events[i].data.fd == kernel_accept_fd) + { + if (strlen (kernel_html) != + send (events[i].data.fd, kernel_html, + strlen (kernel_html), 0)) + { + printf ("kernel send failed\n"); + } + printf ("kernel send fd = %d\n", events[i].data.fd); + } + else + { + if (sizeof (fstack_html) != + send (events[i].data.fd, fstack_html, + sizeof (fstack_html), 0)) + { + printf ("fstack send failed\n"); + } + printf ("fstack send fd = %d\n", events[i].data.fd); + } + } + else + { + clear_fd (events[i].data.fd); + } + } + else + { + printf ("unknown event: %8.8X\n", events[i].events); + } + } + } +} + +void +main (int argc, char *argv[]) +{ + int on = 1, ret = 0; + + setArgsDefault (); + ret = process_arg (argc, argv); + if (ret != 1) + { + printf ("The param error\n"); + return; + } + + // Creating socket file descriptor + if ((kernel_listen_fd = socket (AF_INET, SOCK_STREAM, 0)) == 0) + { + printf ("kernel socket failed\n"); + exit (EXIT_FAILURE); + } + + printf ("kernel socket success\n"); + ioctl (kernel_listen_fd, FIONBIO, &on); + + // Forcefully attaching socket to the port 8080 + if (bind (kernel_listen_fd, (struct sockaddr *) &g_k_ser, + sizeof (g_k_ser)) < 0) + { + printf ("kernel bind failed\n"); + exit (EXIT_FAILURE); + } + + printf ("kernel bind success\n"); + if (listen (kernel_listen_fd, 3) < 0) + { + printf ("kernel listen failed \n"); + exit (EXIT_FAILURE); + } + printf ("kernel listen success \n"); + + assert ((gepfd = epoll_create (100)) > 0); + ev.data.fd = kernel_listen_fd; + ev.events = EPOLLIN; + + epoll_ctl (gepfd, EPOLL_CTL_ADD, kernel_listen_fd, &ev); + + /* fstack */ + fstack_listen_fd = socket (AF_INET, SOCK_STREAM, 0); + printf ("sockfd:%d\n", fstack_listen_fd); + if (fstack_listen_fd < 0) + { + printf ("socket failed\n"); + exit (1); + } + + ioctl (fstack_listen_fd, FIONBIO, &on); + + ret = + bind (fstack_listen_fd, (struct sockaddr *) &g_f_ser, sizeof (g_f_ser)); + if (ret < 0) + { + printf ("bind failed\n"); + exit (1); + } + + ret = listen (fstack_listen_fd, MAX_EVENTS); + if (ret < 0) + { + printf ("listen failed\n"); + exit (1); + } + + ev.data.fd = fstack_listen_fd; + ev.events = EPOLLIN; + epoll_ctl (gepfd, EPOLL_CTL_ADD, fstack_listen_fd, &ev); + nstack_run ((void *) loop); + return; +} diff --git a/stacks/fstack/app/single_sock_tcp_epoll.c b/stacks/fstack/app/single_sock_tcp_epoll.c new file mode 100644 index 0000000..2facd67 --- /dev/null +++ b/stacks/fstack/app/single_sock_tcp_epoll.c @@ -0,0 +1,217 @@ +#define _GNU_SOURCE +#include <unistd.h> +#include <stdio.h> +#include <sys/ioctl.h> +#include <stdlib.h> +#include <stdint.h> +#include <string.h> +#include <strings.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <arpa/inet.h> +#include <errno.h> +#include <assert.h> +#include <sys/epoll.h> +#include "dmm_pub_api.h" +#include <pthread.h> +#include <fcntl.h> + +#define MAX_EVENTS 512 + +int gepfd; +int fstack_listen_fd, i, readlen; +char buffer[1024] = { 0 }; + +char fstack_html[] = + "HTTP/1.1 200 OK\r\n" + "Server: F-Stack\r\n" + "Date: Sat, 25 Feb 2017 09:26:33 GMT\r\n" + "Content-Type: text/html\r\n" + "Content-Length: 439\r\n" + "Last-Modified: Tue, 21 Feb 2017 09:44:03 GMT\r\n" + "Connection: keep-alive\r\n" + "Accept-Ranges: bytes\r\n" + "\r\n" + "<!DOCTYPE html>\r\n" + "<html>\r\n" + "<head>\r\n" + "<title>Welcome to F-Stack!</title>\r\n" + "<style>\r\n" + " body { \r\n" + " width: 35em;\r\n" + " margin: 0 auto; \r\n" + " font-family: Tahoma, Verdana, Arial, sans-serif;\r\n" + " }\r\n" + "</style>\r\n" + "</head>\r\n" + "<body>\r\n" + "<h1>Welcome to F-Stack!</h1>\r\n" + "\r\n" + "<p>For online documentation and support please refer to\r\n" + "<a href=\"http://F-Stack.org/\">F-Stack.org</a>.<br/>\r\n" + "\r\n" + "<p><em>Thank you for using F-Stack.</em></p>\r\n" "</body>\r\n" "</html>"; +struct epoll_event ev; +struct epoll_event events[MAX_EVENTS]; + +static struct sockaddr_in g_f_ser; + +static void +setArgsDefault () +{ + memset (&g_f_ser, 0, sizeof (g_f_ser)); + g_f_ser.sin_family = AF_INET; + g_f_ser.sin_addr.s_addr = inet_addr ("0.0.0.0"); + g_f_ser.sin_port = htons (80); + bzero (&(g_f_ser.sin_zero), 8); +} + +static int +process_arg (int argc, char *argv[]) +{ + int opt = 0; + const char *optstring = "f:"; + + if (argc < 1) + { + printf ("Param info :-f fstack server address;\n"); + return 0; + } + while ((opt = getopt (argc, argv, optstring)) != -1) + { + switch (opt) + { + case 'f': + g_f_ser.sin_addr.s_addr = inet_addr (optarg); + break; + } + } + return 1; +} + +void +clear_fd (int clearFd) +{ + epoll_ctl (gepfd, EPOLL_CTL_DEL, clearFd, NULL); + close (clearFd); + printf ("%d fd has been deleted from the epoll list\n\n\n", clearFd); +} + +int +loop (void *arg) +{ + /* Wait for events to happen */ + (void) nstack_epoll_prewait (); + int nevents = epoll_wait (gepfd, events, MAX_EVENTS, 0); + + for (i = 0; i < nevents; ++i) + { + /* Handle new connect */ + if (events[i].data.fd == fstack_listen_fd) + { + while (1) + { + int fstack_accept_fd = accept (events[i].data.fd, NULL, NULL); + if (fstack_accept_fd < 0) + { + break; + } + + printf ("f-stack listen fd = %d f-stack accept fd = %d\n", + fstack_listen_fd, fstack_accept_fd); + + ev.data.fd = fstack_accept_fd; + ev.events = EPOLLIN; + + /* Add to event list */ + if (epoll_ctl (gepfd, EPOLL_CTL_ADD, fstack_accept_fd, &ev) != + 0) + { + printf ("epoll_ctl failed:%d, %s\n", errno, + strerror (errno)); + break; + } + } + } + else + { + if (events[i].events & EPOLLERR) + { + /* Simply close socket */ + clear_fd (events[i].data.fd); + } + else if (events[i].events & EPOLLIN) + { + readlen = recv (events[i].data.fd, buffer, sizeof (buffer), 0); + printf ("read event on fd = %d readlen %d\n", events[i].data.fd, + readlen); + if (readlen > 0) + { + printf ("received below buffer from client\n %s\n", buffer); + if (sizeof (fstack_html) != + send (events[i].data.fd, fstack_html, + sizeof (fstack_html), 0)) + { + printf ("fstack send failed\n"); + } + printf ("fstack send fd = %d\n", events[i].data.fd); + } + else + { + clear_fd (events[i].data.fd); + } + } + else + { + printf ("unknown event: %8.8X\n", events[i].events); + } + } + } +} + +void +main (int argc, char *argv[]) +{ + int on = 1, ret = 0; + + setArgsDefault (); + ret = process_arg (argc, argv); + if (ret != 1) + { + printf ("The param error\n"); + return; + } + + /* fstack */ + fstack_listen_fd = socket (AF_INET, SOCK_STREAM, 0); + printf ("sockfd:%d\n", fstack_listen_fd); + if (fstack_listen_fd < 0) + { + printf ("socket failed\n"); + exit (1); + } + + ioctl (fstack_listen_fd, FIONBIO, &on); + + ret = + bind (fstack_listen_fd, (struct sockaddr *) &g_f_ser, sizeof (g_f_ser)); + if (ret < 0) + { + printf ("bind failed\n"); + exit (1); + } + + ret = listen (fstack_listen_fd, MAX_EVENTS); + if (ret < 0) + { + printf ("listen failed\n"); + exit (1); + } + + assert ((gepfd = epoll_create (100)) > 0); + ev.data.fd = fstack_listen_fd; + ev.events = EPOLLIN; + epoll_ctl (gepfd, EPOLL_CTL_ADD, fstack_listen_fd, &ev); + nstack_run ((void *) loop); + return; +} diff --git a/stacks/fstack/config/config.ini b/stacks/fstack/config/config.ini new file mode 100644 index 0000000..19a7e75 --- /dev/null +++ b/stacks/fstack/config/config.ini @@ -0,0 +1,124 @@ +[dpdk] +# Hexadecimal bitmask of cores to run on. +lcore_mask=1 + +# Number of memory channels. +channel=4 + +# Specify base virtual address to map. +#base_virtaddr=0x7f0000000000 + +# Promiscuous mode of nic, defualt: enabled. +promiscuous=1 +numa_on=1 + +# TCP segment offload, default: disabled. +tso=0 + +# HW vlan strip, default: enabled. +vlan_strip=1 + +# sleep when no pkts incomming +# unit: microseconds + +# enabled port list +# +# EBNF grammar: +# +# exp ::= num_list {"," num_list} +# num_list ::= <num> | <range> +# range ::= <num>"-"<num> +# num ::= '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' +# +# examples +# 0-3 ports 0, 1,2,3 are enabled +# 1-3,4,7 ports 1,2,3,4,7 are enabled +port_list=0 + +# Number of vdev. + +# Port config section +# Correspond to dpdk.port_list's index: port0, port1... +[port0] +addr=192.168.1.2 +netmask=255.255.255.0 +broadcast=192.168.1.255 +gateway=192.168.1.1 + +# lcore list used to handle this port +# the format is same as port_list +# lcore_list= 0 + +# Packet capture path, this will hurt performance +#pcap=./a.pcap + +# Vdev config section +# orrespond to dpdk.nb_vdev's index: vdev0, vdev1... +# iface : Shouldn't set always. +# path : The vuser device path in container. Required. +# queues : The max queues of vuser. Optional, default 1, greater or equal to the number of processes. +# queue_size : Queue size.Optional, default 256. +# mac : The mac address of vuser. Optional, default random, if vhost use phy NIC, it should be set to the phy NIC's mac. +# cq : Optional, if queues = 1, default 0; if queues > 1 default 1. +#[vdev0] +##iface=/usr/local/var/run/openvswitch/vhost-user0 +#path=/var/run/openvswitch/vhost-user0 +#queues=1 +#queue_size=256 +#mac=00:00:00:00:00:01 +#cq=0 + +# Kni config: if enabled and method=reject, +# all packets that do not belong to the following tcp_port and udp_port +# will transmit to kernel; if method=accept, all packets that belong to +# the following tcp_port and udp_port will transmit to kernel. +#[kni] +#enable=1 +#method=reject +# The format is same as port_list +#tcp_port=80,443 +#udp_port=53 + +# FreeBSD network performance tuning configurations. +# Most native FreeBSD configurations are supported. +[freebsd.boot] +hz=100 + +# Block out a range of descriptors to avoid overlap +# with the kernel's descriptor space. +# You can increase this value according to your app. +fd_reserve=512 + +kern.ipc.maxsockets=262144 + +net.inet.tcp.syncache.hashsize=4096 +net.inet.tcp.syncache.bucketlimit=100 + +net.inet.tcp.tcbhashsize=65536 + +kern.ncallout=262144 + +[freebsd.sysctl] +kern.ipc.somaxconn=32768 +kern.ipc.maxsockbuf=16777216 + +net.link.ether.inet.maxhold=5 + +net.inet.tcp.fast_finwait2_recycle=1 +net.inet.tcp.sendspace=16384 +net.inet.tcp.recvspace=8192 +net.inet.tcp.nolocaltimewait=1 +net.inet.tcp.cc.algorithm=cubic +net.inet.tcp.sendbuf_max=16777216 +net.inet.tcp.recvbuf_max=16777216 +net.inet.tcp.sendbuf_auto=1 +net.inet.tcp.recvbuf_auto=1 +net.inet.tcp.sendbuf_inc=16384 +net.inet.tcp.recvbuf_inc=524288 +net.inet.tcp.sack.enable=1 +net.inet.tcp.blackhole=1 +net.inet.tcp.msl=2000 +net.inet.tcp.delayed_ack=0 + +net.inet.udp.blackhole=1 +net.inet.ip.redirect=0 diff --git a/stacks/fstack/config/module_config.json b/stacks/fstack/config/module_config.json new file mode 100644 index 0000000..e3a7f22 --- /dev/null +++ b/stacks/fstack/config/module_config.json @@ -0,0 +1,28 @@ +{ + "default_stack_name": "kernel", + "module_list": [ + { + "stack_name": "kernel", + "libname": "./", + "function_name": "kernel_stack_register", + "loadtype": "static", + "deploytype": "1", + "maxfd": "1024", /*the max fd supported*/ + "minfd": "0", /*the min fd supported*/ + "priorty": "1", /*priorty when executing, reserv*/ + + "stackid": "0" + }, + { + "stack_name": "fstack", + "libname": "libdmm_fstack.so", + "function_name": "fstack_stack_register", + "loadtype": "dynmic", + "deploytype": "1", + "maxfd": "1024", /*the max fd supported*/ + "minfd": "0", /*the min fd supported*/ + "priorty": "1", /*priorty when executing, reserv*/ + "stackid": "1" + } + ] +} diff --git a/stacks/fstack/config/rd_config.json b/stacks/fstack/config/rd_config.json new file mode 100644 index 0000000..3c5268c --- /dev/null +++ b/stacks/fstack/config/rd_config.json @@ -0,0 +1,10 @@ +{ + "ip_route": [ + { + "subnet": "192.168.1.1/24", + "type": "nstack-fstack" + } + ], + "prot_route": [ + ] +} diff --git a/stacks/fstack/fstack.patch b/stacks/fstack/fstack.patch new file mode 100644 index 0000000..40dfade --- /dev/null +++ b/stacks/fstack/fstack.patch @@ -0,0 +1,12 @@ +diff --git a/lib/ff_syscall_wrapper.c b/lib/ff_syscall_wrapper.c +index cfc407c..7b61aa6 100644 +--- a/lib/ff_syscall_wrapper.c ++++ b/lib/ff_syscall_wrapper.c +@@ -710,6 +710,7 @@ ff_fcntl(int fd, int cmd, ...) + argp = va_arg(ap, uintptr_t); + va_end(ap); + ++ curthread->td_retval[0] = 0; + if ((rc = kern_fcntl(curthread, fd, cmd, argp))) + goto kern_fail; + rc = curthread->td_retval[0]; diff --git a/stacks/fstack/run_fstack.txt b/stacks/fstack/run_fstack.txt new file mode 100644 index 0000000..eaca7eb --- /dev/null +++ b/stacks/fstack/run_fstack.txt @@ -0,0 +1,47 @@ +F-Stack RTC +=========== +Currently a multi-socket application can only use F-Stack RTC and Kernel stacks together. +[cannot use F-stack and rSocket together] + + +Build F-Stack: +------------- +cd /DMM/scripts/ +sudo ./build.sh + +sudo apt-get install libnuma-dev +sudo apt-get install libssl-dev + +cd /DMM/build/ +sudo make dmm_fstack + + +Run F-Stack: +------------ +sudo echo 1024 | sudo tee /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages +sudo mkdir /mnt/huge +sudo mount -t hugetlbfs nodev /mnt/huge +sudo echo 0 | sudo tee /proc/sys/kernel/randomize_va_space +sudo modprobe uio +sudo insmod /DMM/stacks/fstack/f-stack-1.12/dpdk/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko +sudo insmod /DMM/stacks/fstack/f-stack-1.12/dpdk/x86_64-native-linuxapp-gcc/kmod/rte_kni.ko +sudo python /DMM/stacks/fstack/f-stack-1.12/dpdk/usertools/dpdk-devbind.py --status +sudo ifconfig enp0s8 down +sudo python /DMM/stacks/fstack/f-stack-1.12/dpdk/usertools/dpdk-devbind.py --bind=igb_uio enp0s8 + +go to app: +---------- +cd /DMM/stacks/fstack/app +sudo make + +cp -r /DMM/stacks/fstack/config/* /DMM/stacks/fstack/app +edit config.ini and rd_config.json according to the requrired server/client ip + +#server command: +--------------- +sudo FF_PATH=/DMM/stacks/fstack/f-stack-1.12/ FF_DPDK=/DMM/stacks/fstack/f-stack-1.12/dpdk/x86_64-native-linuxapp-gcc LD_LIBRARY_PATH=/DMM/release/lib64/ LD_PRELOAD=/DMM/release/lib64/libnStackAPI.so ./fstack_single_sock_server -f 172.28.128.13 + +#client command: +---------------- +curl http://172.28.128.13 +[172.28.128.13 - server ip] diff --git a/stacks/fstack/src/fstack_adpt.c b/stacks/fstack/src/fstack_adpt.c new file mode 100644 index 0000000..c25577c --- /dev/null +++ b/stacks/fstack/src/fstack_adpt.c @@ -0,0 +1,241 @@ +/* +* +* Copyright (c) 2018 Huawei Technologies Co.,Ltd. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at: +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#include <errno.h> +#include <stdlib.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/ioctl.h> +#include <sys/socket.h> +#include <dlfcn.h> +#include "nstack_dmm_api.h" +#include "ff_config.h" +#include "ff_api.h" +#include "ff_epoll.h" + +#define DMM_FSTACK_MAX_EP_EVENT 1024 +#define DMM_FSTACK_ENV_DEBUG "DMM_FSTACK_DEBUG" +#define DMM_FSTACK_ADPT_DEBUG dmm_fstack_debug +#define DMM_FSTACK_ENV_PROC_TYPE "DMM_FSTACK_PROC_TYPE" +#define DMM_FSTACK_ENV_PROC_ID "DMM_FSTACK_PROC_ID" +#define DMM_FSTACK_ENV_CONF_FILE "DMM_FSTACK_CONF_FILE" + +static unsigned int dmm_fstack_debug; + +typedef struct dmm_fstack +{ + int epfd; + long unsigned int epoll_threadid; + nstack_event_cb regVal; + int (*p_epoll_create) (int size); + unsigned int (*p_epoll_ctl) (int epFD, int ctl_ops, int proFD, + struct epoll_event * events); + unsigned int (*p_epoll_wait) (int epfd, struct epoll_event * events, + int maxevents, int timeout); + int (*p_close) (int fd); +} dmm_fstack_t; + +dmm_fstack_t g_dmm_fstack; + +unsigned int +fstack_ep_ctl_ops (int epFD, int proFD, int ctl_ops, + struct epoll_event *events, void *pdata) +{ + struct epoll_event tmpEvt; + int ret = 0; + int dmm_epfd; + + tmpEvt.data.ptr = pdata; + tmpEvt.events = events->events; + tmpEvt.events |= (EPOLLIN | EPOLLOUT); + + if (DMM_FSTACK_ADPT_DEBUG > 0) + fprintf (stdout, "DMM VCL ADPT<%d>: epfd=%d,fd=%d,ops=%d, events=%u", + getpid (), epFD, proFD, ctl_ops, events->events); + + dmm_epfd = g_dmm_fstack.epfd; + switch (ctl_ops) + { + case nstack_ep_triggle_add: + ret = + g_dmm_fstack.p_epoll_ctl (dmm_epfd, EPOLL_CTL_ADD, proFD, &tmpEvt); + break; + case nstack_ep_triggle_mod: + ret = + g_dmm_fstack.p_epoll_ctl (dmm_epfd, EPOLL_CTL_MOD, proFD, &tmpEvt); + break; + case nstack_ep_triggle_del: + ret = + g_dmm_fstack.p_epoll_ctl (dmm_epfd, EPOLL_CTL_DEL, proFD, &tmpEvt); + break; + default: + ret = -1; + break; + } + return ret; +} + +int +fr_init (void) +{ + int ret; + char config_file[1024] = "--conf=config.ini"; + char proc_type[1024] = "--proc-type=primary"; + char proc_id[1024] = "--proc-id=0"; + + /* if env isn't set consider DPDK process type as primary */ + if (getenv (DMM_FSTACK_ENV_CONF_FILE)) + { + sprintf (proc_type, "--conf=%s", getenv (DMM_FSTACK_ENV_CONF_FILE)); + } + if (getenv (DMM_FSTACK_ENV_PROC_TYPE)) + { + sprintf (proc_type, "--proc-type=%s", + getenv (DMM_FSTACK_ENV_PROC_TYPE)); + } + if (getenv (DMM_FSTACK_ENV_PROC_ID)) + { + sprintf (proc_id, "--proc-id=%d", + atoi (getenv (DMM_FSTACK_ENV_PROC_ID))); + } + + char *argv[] = { + "dmm-fstack", + config_file, + proc_type, + proc_id, + NULL + }; + + const int argc = sizeof (argv) / sizeof (argv[0]) - 1; + + ret = ff_init (argc, argv); + if (ret) + return -1; + + return 0; +} + +static pid_t +ff_fork (void) +{ + return -1; +} + +int +fr_run (void *loop) +{ + ff_run (loop, NULL); + return 0; +} + +static int +ff_accept4 (int sockfd, struct sockaddr *addr, socklen_t * addrlen, int flags) +{ + int fd; + + fd = ff_accept (sockfd, (struct linux_sockaddr *) addr, addrlen); + if (fd < 0) + return fd; + + if (flags & SOCK_NONBLOCK) + { + (void) ff_ioctl (sockfd, FIONBIO, 1); + } + + if (flags & SOCK_CLOEXEC) + { + } + + return fd; +} + +void +dmm_fstack_epoll () +{ + int num, i; + + struct epoll_event events[DMM_FSTACK_MAX_EP_EVENT]; + + num = + g_dmm_fstack.p_epoll_wait (g_dmm_fstack.epfd, events, + DMM_FSTACK_MAX_EP_EVENT, 0); + for (i = 0; i < num; ++i) + { + g_dmm_fstack.regVal.event_cb (events[i].data.ptr, events[i].events); + } + + return; +} + +int +dmm_fstack_init () +{ + char *env_var_str; + int rv = 0; + + if (0 != fr_init ()) + { + return -1; + } + + env_var_str = getenv (DMM_FSTACK_ENV_DEBUG); + if (env_var_str) + { + unsigned int tmp; + if (sscanf (env_var_str, "%u", &tmp) != 1) + fprintf (stdout, + "DMM_FSTACK_ADPT: WARNING: Invalid debug level specified " + "in the environment variable " DMM_FSTACK_ENV_DEBUG + " (%s)!\n", env_var_str); + else + { + dmm_fstack_debug = tmp; + if (DMM_FSTACK_ADPT_DEBUG > 0) + fprintf (stdout, + "DMM_FSTACK_ADPT: configured DMM FSTACK ADPT debug (%u) from " + "DMM_STACK_ENV_DEBUG ", dmm_fstack_debug); + } + } + + g_dmm_fstack.epfd = g_dmm_fstack.p_epoll_create (50); + if (g_dmm_fstack.epfd < 0) + return g_dmm_fstack.epfd; + + return 0; +} + +int +fstack_stack_register (nstack_proc_cb * proc_fun, nstack_event_cb * event_ops) +{ +#define NSTACK_MK_DECL(ret, fn, args) proc_fun->socket_ops.pf##fn = (void*)ff_##fn; +#include "declare_syscalls.h" +#undef NSTACK_MK_DECL + g_dmm_fstack.p_epoll_ctl = dlsym (event_ops->handle, "ff_epoll_ctl"); + g_dmm_fstack.p_epoll_create = dlsym (event_ops->handle, "ff_epoll_create"); + g_dmm_fstack.p_epoll_wait = dlsym (event_ops->handle, "ff_epoll_wait"); + g_dmm_fstack.p_close = dlsym (event_ops->handle, "ff_close"); + g_dmm_fstack.regVal = *event_ops; + + proc_fun->socket_ops.pfselect = NULL; + + proc_fun->extern_ops.module_init = dmm_fstack_init; + proc_fun->extern_ops.ep_ctl = fstack_ep_ctl_ops; + proc_fun->extern_ops.ep_getevt = NULL; + proc_fun->extern_ops.run = fr_run; + proc_fun->extern_ops.ep_prewait = dmm_fstack_epoll; + return 0; +} |