http: return more than data from server app

Server app could return headers in front of body/data buffer.
Offers apis for building and serialization of headers section.
HTTP layer now only add Date, Server and Content-Lengths headers,
rest is up to app. Well known header names are predefined.

Type: improvement

Change-Id: If778bdfc9acf6b0d11a48f0a745a3a56c96c2436
Signed-off-by: Matus Fabian <matfabia@cisco.com>

13 files changed, 532 insertions, 140 deletions

diff --git a/src/plugins/hs_apps/http_cli.c b/src/plugins/hs_apps/http_cli.c
index 4970da72012..dfe50c58092 100644
--- a/src/plugins/hs_apps/http_cli.c
+++ b/src/plugins/hs_apps/http_cli.c
@@ -17,6 +17,8 @@
 #include <vnet/session/application_interface.h>
 #include <vnet/session/session.h>
 #include <http/http.h>
+#include <http/http_header_names.h>
+#include <http/http_content_types.h>
 
 #define HCS_DEBUG 0
 
@@ -43,6 +45,7 @@ typedef struct
   u8 *tx_buf;
   u32 tx_offset;
   u32 vpp_session_index;
+  http_header_t *resp_headers;
 } hcs_session_t;
 
 typedef struct
@@ -148,24 +151,45 @@ hcs_cli_output (uword arg, u8 *buffer, uword buffer_bytes)
 }
 
 static void
-start_send_data (hcs_session_t *hs, http_status_code_t status,
-		 http_content_type_t type)
+start_send_data (hcs_session_t *hs, http_status_code_t status)
 {
   http_msg_t msg;
   session_t *ts;
+  u8 *headers_buf = 0;
   int rv;
 
+  if (vec_len (hs->resp_headers))
+    {
+      headers_buf = http_serialize_headers (hs->resp_headers);
+      vec_free (hs->resp_headers);
+      msg.data.headers_offset = 0;
+      msg.data.headers_len = vec_len (headers_buf);
+    }
+  else
+    {
+      msg.data.headers_offset = 0;
+      msg.data.headers_len = 0;
+    }
+
   msg.type = HTTP_MSG_REPLY;
   msg.code = status;
-  msg.content_type = type;
   msg.data.type = HTTP_MSG_DATA_INLINE;
-  msg.data.len = vec_len (hs->tx_buf);
+  msg.data.body_len = vec_len (hs->tx_buf);
+  msg.data.body_offset = msg.data.headers_len;
+  msg.data.len = msg.data.body_len + msg.data.headers_len;
 
   ts = session_get (hs->vpp_session_index, hs->thread_index);
   rv = svm_fifo_enqueue (ts->tx_fifo, sizeof (msg), (u8 *) &msg);
   ASSERT (rv == sizeof (msg));
 
-  if (!msg.data.len)
+  if (msg.data.headers_len)
+    {
+      rv = svm_fifo_enqueue (ts->tx_fifo, vec_len (headers_buf), headers_buf);
+      ASSERT (rv == msg.data.headers_len);
+      vec_free (headers_buf);
+    }
+
+  if (!msg.data.body_len)
     goto done;
 
   rv = svm_fifo_enqueue (ts->tx_fifo, vec_len (hs->tx_buf), hs->tx_buf);
@@ -203,7 +227,12 @@ send_data_to_http (void *rpc_args)
   hs->tx_buf = args->buf;
   if (args->plain_text)
     type = HTTP_CONTENT_TEXT_PLAIN;
-  start_send_data (hs, HTTP_STATUS_OK, type);
+
+  http_add_header (&hs->resp_headers,
+		   http_header_name_token (HTTP_HEADER_CONTENT_TYPE),
+		   http_content_type_token (type));
+
+  start_send_data (hs, HTTP_STATUS_OK);
 
 cleanup:
 
@@ -325,6 +354,7 @@ hcs_ts_rx_callback (session_t *ts)
 
   hs = hcs_session_get (ts->thread_index, ts->opaque);
   hs->tx_buf = 0;
+  hs->resp_headers = 0;
 
   /* Read the http message header */
   rv = svm_fifo_dequeue (ts->rx_fifo, sizeof (msg), (u8 *) &msg);
@@ -332,16 +362,17 @@ hcs_ts_rx_callback (session_t *ts)
 
   if (msg.type != HTTP_MSG_REQUEST || msg.method_type != HTTP_REQ_GET)
     {
-      start_send_data (hs, HTTP_STATUS_METHOD_NOT_ALLOWED,
-		       HTTP_CONTENT_TEXT_HTML);
+      http_add_header (&hs->resp_headers,
+		       http_header_name_token (HTTP_HEADER_ALLOW),
+		       http_token_lit ("GET"));
+      start_send_data (hs, HTTP_STATUS_METHOD_NOT_ALLOWED);
       goto done;
     }
 
   if (msg.data.target_path_len == 0 ||
       msg.data.target_form != HTTP_TARGET_ORIGIN_FORM)
     {
-      hs->tx_buf = 0;
-      start_send_data (hs, HTTP_STATUS_BAD_REQUEST, HTTP_CONTENT_TEXT_HTML);
+      start_send_data (hs, HTTP_STATUS_BAD_REQUEST);
       goto done;
     }
 
@@ -353,7 +384,7 @@ hcs_ts_rx_callback (session_t *ts)
   HCS_DBG ("%v", args.buf);
   if (http_validate_abs_path_syntax (args.buf, &is_encoded))
     {
-      start_send_data (hs, HTTP_STATUS_BAD_REQUEST, HTTP_CONTENT_TEXT_HTML);
+      start_send_data (hs, HTTP_STATUS_BAD_REQUEST);
       vec_free (args.buf);
       goto done;
     }
@@ -374,13 +405,13 @@ hcs_ts_rx_callback (session_t *ts)
       ASSERT (rv == msg.data.headers_len);
       if (http_parse_headers (headers, &ht))
 	{
-	  start_send_data (hs, HTTP_STATUS_BAD_REQUEST,
-			   HTTP_CONTENT_TEXT_HTML);
+	  start_send_data (hs, HTTP_STATUS_BAD_REQUEST);
 	  vec_free (args.buf);
 	  vec_free (headers);
 	  goto done;
 	}
-      const char *accept_value = http_get_header (ht, HTTP_HEADER_ACCEPT);
+      const char *accept_value =
+	http_get_header (ht, http_header_name_str (HTTP_HEADER_ACCEPT));
       if (accept_value)
 	{
 	  HCS_DBG ("client accept: %s", accept_value);
diff --git a/src/plugins/hs_apps/http_tps.c b/src/plugins/hs_apps/http_tps.c
index 3a086501f86..9cc592cd746 100644
--- a/src/plugins/hs_apps/http_tps.c
+++ b/src/plugins/hs_apps/http_tps.c
@@ -17,6 +17,8 @@
 #include <vnet/session/application_interface.h>
 #include <vnet/session/session.h>
 #include <http/http.h>
+#include <http/http_header_names.h>
+#include <http/http_content_types.h>
 
 typedef struct
 {
@@ -34,6 +36,7 @@ typedef struct
     u32 close_rate;
   };
   u8 *uri;
+  http_header_t *resp_headers;
 } hts_session_t;
 
 typedef struct hts_listen_cfg_
@@ -223,19 +226,41 @@ hts_start_send_data (hts_session_t *hs, http_status_code_t status)
 {
   http_msg_t msg;
   session_t *ts;
+  u8 *headers_buf = 0;
   int rv;
 
+  if (vec_len (hs->resp_headers))
+    {
+      headers_buf = http_serialize_headers (hs->resp_headers);
+      vec_free (hs->resp_headers);
+      msg.data.headers_offset = 0;
+      msg.data.headers_len = vec_len (headers_buf);
+    }
+  else
+    {
+      msg.data.headers_offset = 0;
+      msg.data.headers_len = 0;
+    }
+
   msg.type = HTTP_MSG_REPLY;
   msg.code = status;
-  msg.content_type = HTTP_CONTENT_APP_OCTET_STREAM;
   msg.data.type = HTTP_MSG_DATA_INLINE;
-  msg.data.len = hs->data_len;
+  msg.data.body_len = hs->data_len;
+  msg.data.body_offset = msg.data.headers_len;
+  msg.data.len = msg.data.body_len + msg.data.headers_len;
 
   ts = session_get (hs->vpp_session_index, hs->thread_index);
   rv = svm_fifo_enqueue (ts->tx_fifo, sizeof (msg), (u8 *) &msg);
   ASSERT (rv == sizeof (msg));
 
-  if (!msg.data.len)
+  if (msg.data.headers_len)
+    {
+      rv = svm_fifo_enqueue (ts->tx_fifo, vec_len (headers_buf), headers_buf);
+      ASSERT (rv == msg.data.headers_len);
+      vec_free (headers_buf);
+    }
+
+  if (!msg.data.body_len)
     {
       if (svm_fifo_set_event (ts->tx_fifo))
 	session_send_io_evt_to_thread (ts->tx_fifo, SESSION_IO_EVT_TX);
@@ -286,6 +311,10 @@ try_test_file (hts_session_t *hs, u8 *target)
 	}
     }
 
+  http_add_header (&hs->resp_headers,
+		   http_header_name_token (HTTP_HEADER_CONTENT_TYPE),
+		   http_content_type_token (HTTP_CONTENT_APP_OCTET_STREAM));
+
   hts_start_send_data (hs, HTTP_STATUS_OK);
 
 done:
@@ -304,6 +333,8 @@ hts_ts_rx_callback (session_t *ts)
   int rv;
 
   hs = hts_session_get (ts->thread_index, ts->opaque);
+  hs->data_len = 0;
+  hs->resp_headers = 0;
 
   /* Read the http message header */
   rv = svm_fifo_dequeue (ts->rx_fifo, sizeof (msg), (u8 *) &msg);
@@ -311,6 +342,9 @@ hts_ts_rx_callback (session_t *ts)
 
   if (msg.type != HTTP_MSG_REQUEST || msg.method_type != HTTP_REQ_GET)
     {
+      http_add_header (&hs->resp_headers,
+		       http_header_name_token (HTTP_HEADER_ALLOW),
+		       http_token_lit ("GET"));
       hts_start_send_data (hs, HTTP_STATUS_METHOD_NOT_ALLOWED);
       goto done;
     }
diff --git a/src/plugins/http/http.c b/src/plugins/http/http.c
index c5e2cc1df7d..db101539ac4 100644
--- a/src/plugins/http/http.c
+++ b/src/plugins/http/http.c
@@ -36,12 +36,6 @@ const char *http_status_code_str[] = {
 #undef _
 };
 
-const char *http_content_type_str[] = {
-#define _(s, ext, str) str,
-  foreach_http_content_type
-#undef _
-};
-
 const http_buffer_type_t msg_to_buf_type[] = {
   [HTTP_MSG_DATA_INLINE] = HTTP_BUFFER_FIFO,
   [HTTP_MSG_DATA_PTR] = HTTP_BUFFER_PTR,
@@ -374,22 +368,17 @@ http_ts_reset_callback (session_t *ts)
  */
 static const char *http_error_template = "HTTP/1.1 %s\r\n"
 					 "Date: %U GMT\r\n"
-					 "Content-Type: text/html\r\n"
 					 "Connection: close\r\n"
-					 "Pragma: no-cache\r\n"
 					 "Content-Length: 0\r\n\r\n";
 
-static const char *http_redirect_template = "HTTP/1.1 %s\r\n";
-
 /**
  * http response boilerplate
  */
 static const char *http_response_template = "HTTP/1.1 %s\r\n"
 					    "Date: %U GMT\r\n"
-					    "Expires: %U GMT\r\n"
 					    "Server: %v\r\n"
-					    "Content-Type: %s\r\n"
-					    "Content-Length: %lu\r\n\r\n";
+					    "Content-Length: %u\r\n"
+					    "%s";
 
 static const char *http_request_template = "GET %s HTTP/1.1\r\n"
 					   "User-Agent: %v\r\n"
@@ -1004,49 +993,54 @@ http_state_wait_app_reply (http_conn_t *hc, transport_send_params_t *sp)
       goto error;
     }
 
-  http_buffer_init (&hc->tx_buf, msg_to_buf_type[msg.data.type], as->tx_fifo,
-		    msg.data.len);
+  if (msg.code >= HTTP_N_STATUS)
+    {
+      clib_warning ("unsupported status code: %d", msg.code);
+      return HTTP_SM_ERROR;
+    }
 
   /*
-   * Add headers. For now:
+   * Add "protocol layer" headers:
    * - current time
-   * - expiration time
    * - server name
-   * - content type
    * - data length
    */
   now = clib_timebase_now (&hm->timebase);
-
-  switch (msg.code)
-    {
-    case HTTP_STATUS_NOT_FOUND:
-    case HTTP_STATUS_METHOD_NOT_ALLOWED:
-    case HTTP_STATUS_BAD_REQUEST:
-    case HTTP_STATUS_INTERNAL_ERROR:
-    case HTTP_STATUS_FORBIDDEN:
-    case HTTP_STATUS_OK:
-      header =
-	format (0, http_response_template, http_status_code_str[msg.code],
-		/* Date */
-		format_clib_timebase_time, now,
-		/* Expires */
-		format_clib_timebase_time, now + 600.0,
-		/* Server */
-		hc->app_name,
-		/* Content type */
-		http_content_type_str[msg.content_type],
-		/* Length */
-		msg.data.len);
-      break;
-    case HTTP_STATUS_MOVED:
-      header =
-	format (0, http_redirect_template, http_status_code_str[msg.code]);
-      /* Location: http(s)://new-place already queued up as data */
-      break;
-    default:
-      clib_warning ("unsupported status code: %d", msg.code);
-      return HTTP_SM_ERROR;
+  header = format (0, http_response_template, http_status_code_str[msg.code],
+		   /* Date */
+		   format_clib_timebase_time, now,
+		   /* Server */
+		   hc->app_name,
+		   /* Length */
+		   msg.data.body_len,
+		   /* Any headers from app? */
+		   msg.data.headers_len ? "" : "\r\n");
+
+  /* Add headers from app (if any) */
+  if (msg.data.headers_len)
+    {
+      HTTP_DBG (0, "got headers from app, len %d", msg.data.headers_len);
+      if (msg.data.type == HTTP_MSG_DATA_PTR)
+	{
+	  uword app_headers_ptr;
+	  rv = svm_fifo_dequeue (as->tx_fifo, sizeof (app_headers_ptr),
+				 (u8 *) &app_headers_ptr);
+	  ASSERT (rv == sizeof (app_headers_ptr));
+	  vec_append (header, uword_to_pointer (app_headers_ptr, u8 *));
+	}
+      else
+	{
+	  u32 orig_len = vec_len (header);
+	  vec_resize (header, msg.data.headers_len);
+	  u8 *p = header + orig_len;
+	  rv = svm_fifo_dequeue (as->tx_fifo, msg.data.headers_len, p);
+	  ASSERT (rv == msg.data.headers_len);
+	}
     }
+  HTTP_DBG (0, "%v", header);
+
+  http_buffer_init (&hc->tx_buf, msg_to_buf_type[msg.data.type], as->tx_fifo,
+		    msg.data.body_len);
 
   offset = http_send_data (hc, header, vec_len (header), 0);
   if (offset != vec_len (header))
diff --git a/src/plugins/http/http.h b/src/plugins/http/http.h
index debdebcf050..07d5472346e 100644
--- a/src/plugins/http/http.h
+++ b/src/plugins/http/http.h
@@ -51,6 +51,14 @@ typedef struct http_conn_id_
 STATIC_ASSERT (sizeof (http_conn_id_t) <= TRANSPORT_CONN_ID_LEN,
 	       "ctx id must be less than TRANSPORT_CONN_ID_LEN");
 
+typedef struct
+{
+  char *base;
+  uword len;
+} http_token_t;
+
+#define http_token_lit(s) (s), sizeof (s) - 1
+
 typedef enum http_conn_state_
 {
   HTTP_CONN_STATE_LISTEN,
@@ -235,50 +243,100 @@ typedef enum http_status_code_
     HTTP_N_STATUS
 } http_status_code_t;
 
-#define HTTP_HEADER_ACCEPT		      "Accept"
-#define HTTP_HEADER_ACCEPT_CHARSET	      "Accept-Charset"
-#define HTTP_HEADER_ACCEPT_ENCODING	      "Accept-Encoding"
-#define HTTP_HEADER_ACCEPT_LANGUAGE	      "Accept-Language"
-#define HTTP_HEADER_ACCEPT_RANGES	      "Accept-Ranges"
-#define HTTP_HEADER_ALLOW		      "Allow"
-#define HTTP_HEADER_AUTHENTICATION_INFO	      "Authentication-Info"
-#define HTTP_HEADER_AUTHORIZATION	      "Authorization"
-#define HTTP_HEADER_CLOSE		      "Close"
-#define HTTP_HEADER_CONNECTION		      "Connection"
-#define HTTP_HEADER_CONTENT_ENCODING	      "Content-Encoding"
-#define HTTP_HEADER_CONTENT_LANGUAGE	      "Content-Language"
-#define HTTP_HEADER_CONTENT_LENGTH	      "Content-Length"
-#define HTTP_HEADER_CONTENT_LOCATION	      "Content-Location"
-#define HTTP_HEADER_CONTENT_RANGE	      "Content-Range"
-#define HTTP_HEADER_CONTENT_TYPE	      "Content-Type"
-#define HTTP_HEADER_DATE		      "Date"
-#define HTTP_HEADER_ETAG		      "ETag"
-#define HTTP_HEADER_EXPECT		      "Expect"
-#define HTTP_HEADER_FROM		      "From"
-#define HTTP_HEADER_HOST		      "Host"
-#define HTTP_HEADER_IF_MATCH		      "If-Match"
-#define HTTP_HEADER_IF_MODIFIED_SINCE	      "If-Modified-Since"
-#define HTTP_HEADER_IF_NONE_MATCH	      "If-None-Match"
-#define HTTP_HEADER_IF_RANGE		      "If-Range"
-#define HTTP_HEADER_IF_UNMODIFIED_SINCE	      "If-Unmodified-Since"
-#define HTTP_HEADER_LAST_MODIFIED	      "Last-Modified"
-#define HTTP_HEADER_LOCATION		      "Location"
-#define HTTP_HEADER_MAX_FORWARDS	      "Max-Forwards"
-#define HTTP_HEADER_PROXY_AUTHENTICATE	      "Proxy-Authenticate"
-#define HTTP_HEADER_PROXY_AUTHENTICATION_INFO "Proxy-Authentication-Info"
-#define HTTP_HEADER_PROXY_AUTHORIZATION	      "Proxy-Authorization"
-#define HTTP_HEADER_RANGE		      "Range"
-#define HTTP_HEADER_REFERER		      "Referer"
-#define HTTP_HEADER_RETRY_AFTER		      "Retry-After"
-#define HTTP_HEADER_SERVER		      "Server"
-#define HTTP_HEADER_TE			      "TE"
-#define HTTP_HEADER_TRAILER		      "Trailer"
-#define HTTP_HEADER_TRANSFER_ENCODING	      "Transfer-Encoding"
-#define HTTP_HEADER_UPGRADE		      "Upgrade"
-#define HTTP_HEADER_USER_AGENT		      "User-Agent"
-#define HTTP_HEADER_VARY		      "Vary"
-#define HTTP_HEADER_VIA			      "Via"
-#define HTTP_HEADER_WWW_AUTHENTICATE	      "WWW-Authenticate"
+#define foreach_http_header_name                                              \
+  _ (ACCEPT, "Accept")                                                        \
+  _ (ACCEPT_CHARSET, "Accept-Charset")                                        \
+  _ (ACCEPT_ENCODING, "Accept-Encoding")                                      \
+  _ (ACCEPT_LANGUAGE, "Accept-Language")                                      \
+  _ (ACCEPT_RANGES, "Accept-Ranges")                                          \
+  _ (ACCESS_CONTROL_ALLOW_CREDENTIALS, "Access-Control-Allow-Credentials")    \
+  _ (ACCESS_CONTROL_ALLOW_HEADERS, "Access-Control-Allow-Headers")            \
+  _ (ACCESS_CONTROL_ALLOW_METHODS, "Access-Control-Allow-Methods")            \
+  _ (ACCESS_CONTROL_ALLOW_ORIGIN, "Access-Control-Allow-Origin")              \
+  _ (ACCESS_CONTROL_EXPOSE_HEADERS, "Access-Control-Expose-Headers")          \
+  _ (ACCESS_CONTROL_MAX_AGE, "Access-Control-Max-Age")                        \
+  _ (ACCESS_CONTROL_REQUEST_HEADERS, "Access-Control-Request-Headers")        \
+  _ (ACCESS_CONTROL_REQUEST_METHOD, "Access-Control-Request-Method")          \
+  _ (AGE, "Age")                                                              \
+  _ (ALLOW, "Allow")                                                          \
+  _ (ALPN, "ALPN")                                                            \
+  _ (ALT_SVC, "Alt-Svc")                                                      \
+  _ (ALT_USED, "Alt-Used")                                                    \
+  _ (ALTERNATES, "Alternates")                                                \
+  _ (AUTHENTICATION_CONTROL, "Authentication-Control")                        \
+  _ (AUTHENTICATION_INFO, "Authentication-Info")                              \
+  _ (AUTHORIZATION, "Authorization")                                          \
+  _ (CACHE_CONTROL, "Cache-Control")                                          \
+  _ (CACHE_STATUS, "Cache-Status")                                            \
+  _ (CAPSULE_PROTOCOL, "Capsule-Protocol")                                    \
+  _ (CDN_CACHE_CONTROL, "CDN-Cache-Control")                                  \
+  _ (CDN_LOOP, "CDN-Loop")                                                    \
+  _ (CLIENT_CERT, "Client-Cert")                                              \
+  _ (CLIENT_CERT_CHAIN, "Client-Cert-Chain")                                  \
+  _ (CLOSE, "Close")                                                          \
+  _ (CONNECTION, "Connection")                                                \
+  _ (CONTENT_DIGEST, "Content-Digest")                                        \
+  _ (CONTENT_DISPOSITION, "Content-Disposition")                              \
+  _ (CONTENT_ENCODING, "Content-Encoding")                                    \
+  _ (CONTENT_LANGUAGE, "Content-Language")                                    \
+  _ (CONTENT_LENGTH, "Content-Length")                                        \
+  _ (CONTENT_LOCATION, "Content-Location")                                    \
+  _ (CONTENT_RANGE, "Content-Range")                                          \
+  _ (CONTENT_TYPE, "Content-Type")                                            \
+  _ (COOKIE, "Cookie")                                                        \
+  _ (DATE, "Date")                                                            \
+  _ (DIGEST, "Digest")                                                        \
+  _ (DPOP, "DPoP")                                                            \
+  _ (DPOP_NONCE, "DPoP-Nonce")                                                \
+  _ (EARLY_DATA, "Early-Data")                                                \
+  _ (ETAG, "ETag")                                                            \
+  _ (EXPECT, "Expect")                                                        \
+  _ (EXPIRES, "Expires")                                                      \
+  _ (FORWARDED, "Forwarded")                                                  \
+  _ (FROM, "From")                                                            \
+  _ (HOST, "Host")                                                            \
+  _ (IF_MATCH, "If-Match")                                                    \
+  _ (IF_MODIFIED_SINCE, "If-Modified-Since")                                  \
+  _ (IF_NONE_MATCH, "If-None-Match")                                          \
+  _ (IF_RANGE, "If-Range")                                                    \
+  _ (IF_UNMODIFIED_SINCE, "If-Unmodified-Since")                              \
+  _ (KEEP_ALIVE, "Keep-Alive")                                                \
+  _ (LAST_MODIFIED, "Last-Modified")                                          \
+  _ (LINK, "Link")                                                            \
+  _ (LOCATION, "Location")                                                    \
+  _ (MAX_FORWARDS, "Max-Forwards")                                            \
+  _ (ORIGIN, "Origin")                                                        \
+  _ (PRIORITY, "Priority")                                                    \
+  _ (PROXY_AUTHENTICATE, "Proxy-Authenticate")                                \
+  _ (PROXY_AUTHENTICATION_INFO, "Proxy-Authentication-Info")                  \
+  _ (PROXY_AUTHORIZATION, "Proxy-Authorization")                              \
+  _ (PROXY_STATUS, "Proxy-Status")                                            \
+  _ (RANGE, "Range")                                                          \
+  _ (REFERER, "Referer")                                                      \
+  _ (REPR_DIGEST, "Repr-Digest")                                              \
+  _ (SET_COOKIE, "Set-Cookie")                                                \
+  _ (SIGNATURE, "Signature")                                                  \
+  _ (SIGNATURE_INPUT, "Signature-Input")                                      \
+  _ (STRICT_TRANSPORT_SECURITY, "Strict-Transport-Security")                  \
+  _ (RETRY_AFTER, "Retry-After")                                              \
+  _ (SERVER, "Server")                                                        \
+  _ (TE, "TE")                                                                \
+  _ (TRAILER, "Trailer")                                                      \
+  _ (TRANSFER_ENCODING, "Transfer-Encoding")                                  \
+  _ (UPGRADE, "Upgrade")                                                      \
+  _ (USER_AGENT, "User-Agent")                                                \
+  _ (VARY, "Vary")                                                            \
+  _ (VIA, "Via")                                                              \
+  _ (WANT_CONTENT_DIGEST, "Want-Content-Digest")                              \
+  _ (WANT_REPR_DIGEST, "Want-Repr-Digest")                                    \
+  _ (WWW_AUTHENTICATE, "WWW-Authenticate")
+
+typedef enum http_header_name_
+{
+#define _(sym, str) HTTP_HEADER_##sym,
+  foreach_http_header_name
+#undef _
+} http_header_name_t;
 
 typedef enum http_msg_data_type_
 {
@@ -669,11 +727,17 @@ typedef struct
 {
   u8 *name;
   u8 *value;
+} http_header_ht_t;
+
+typedef struct
+{
+  http_token_t name;
+  http_token_t value;
 } http_header_t;
 
 typedef struct
 {
-  http_header_t *headers;
+  http_header_ht_t *headers;
   uword *value_by_name;
 } http_header_table_t;
 
@@ -685,7 +749,7 @@ typedef struct
 always_inline void
 http_free_header_table (http_header_table_t *ht)
 {
-  http_header_t *header;
+  http_header_ht_t *header;
   vec_foreach (header, ht->headers)
     {
       vec_free (header->name);
@@ -713,7 +777,7 @@ http_parse_headers (u8 *headers, http_header_table_t **header_table)
   u8 *pos, *end, *name_start, *value_start, *name;
   u32 name_len, value_len;
   int rv;
-  http_header_t *header;
+  http_header_ht_t *header;
   http_header_table_t *ht;
   uword *p;
 
@@ -779,7 +843,7 @@ always_inline const char *
 http_get_header (http_header_table_t *header_table, const char *name)
 {
   uword *p;
-  http_header_t *header;
+  http_header_ht_t *header;
 
   p = hash_get_mem (header_table->value_by_name, name);
   if (p)
@@ -791,6 +855,67 @@ http_get_header (http_header_table_t *header_table, const char *name)
   return 0;
 }
 
+/**
+ * Add header to the list.
+ *
+ * @param headers Header list.
+ * @param name Pointer to header's name buffer.
+ * @param name_len Length of the name.
+ * @param value Pointer to header's value buffer.
+ * @param value_len Length of the value.
+ *
+ * @note Headers added at protocol layer: Date, Server, Content-Length
+ */
+always_inline void
+http_add_header (http_header_t **headers, const char *name, uword name_len,
+		 const char *value, uword value_len)
+{
+  http_header_t *header;
+  vec_add2 (*headers, header, 1);
+  header->name.base = (char *) name;
+  header->name.len = name_len;
+  header->value.base = (char *) value;
+  header->value.len = value_len;
+}
+
+/**
+ * Serialize the header list.
+ *
+ * @param headers Header list to serialize.
+ *
+ * @return New vector with serialized headers.
+ *
+ * The caller is always responsible to free the returned vector.
+ */
+always_inline u8 *
+http_serialize_headers (http_header_t *headers)
+{
+  u8 *headers_buf = 0, *dst;
+  u32 headers_buf_len = 2;
+  http_header_t *header;
+
+  vec_foreach (header, headers)
+    headers_buf_len += header->name.len + header->value.len + 4;
+
+  vec_validate (headers_buf, headers_buf_len - 1);
+  dst = headers_buf;
+
+  vec_foreach (header, headers)
+    {
+      clib_memcpy (dst, header->name.base, header->name.len);
+      dst += header->name.len;
+      *dst++ = ':';
+      *dst++ = ' ';
+      clib_memcpy (dst, header->value.base, header->value.len);
+      dst += header->value.len;
+      *dst++ = '\r';
+      *dst++ = '\n';
+    }
+  *dst++ = '\r';
+  *dst = '\n';
+  return headers_buf;
+}
+
 #endif /* SRC_PLUGINS_HTTP_HTTP_H_ */
 
 /*
diff --git a/src/plugins/http/http_content_types.h b/src/plugins/http/http_content_types.h
new file mode 100644
index 00000000000..ddc02566db7
--- /dev/null
+++ b/src/plugins/http/http_content_types.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: Apache-2.0
+ * Copyright(c) 2024 Cisco Systems, Inc.
+ */
+
+#ifndef SRC_PLUGINS_HTTP_HTTP_CONTENT_TYPES_H_
+#define SRC_PLUGINS_HTTP_HTTP_CONTENT_TYPES_H_
+
+#include <http/http.h>
+
+static http_token_t http_content_types[] = {
+#define _(s, ext, str) { http_token_lit (str) },
+  foreach_http_content_type
+#undef _
+};
+
+#define http_content_type_token(e)                                            \
+  http_content_types[e].base, http_content_types[e].len
+
+#endif /* SRC_PLUGINS_HTTP_HTTP_CONTENT_TYPES_H_ */
diff --git a/src/plugins/http/http_header_names.h b/src/plugins/http/http_header_names.h
new file mode 100644
index 00000000000..99acac786db
--- /dev/null
+++ b/src/plugins/http/http_header_names.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: Apache-2.0
+ * Copyright(c) 2024 Cisco Systems, Inc.
+ */
+
+#ifndef SRC_PLUGINS_HTTP_HTTP_HEADER_NAMES_H_
+#define SRC_PLUGINS_HTTP_HTTP_HEADER_NAMES_H_
+
+#include <http/http.h>
+
+static http_token_t http_header_names[] = {
+#define _(sym, str) { http_token_lit (str) },
+  foreach_http_header_name
+#undef _
+};
+
+#define http_header_name_token(e)                                             \
+  http_header_names[e].base, http_header_names[e].len
+
+#define http_header_name_str(e) http_header_names[e].base
+
+#endif /* SRC_PLUGINS_HTTP_HTTP_HEADER_NAMES_H_ */
diff --git a/src/plugins/http/http_plugin.rst b/src/plugins/http/http_plugin.rst
index c4c4d2c8234..2f7a58ef9bf 100644
--- a/src/plugins/http/http_plugin.rst
+++ b/src/plugins/http/http_plugin.rst
@@ -121,6 +121,7 @@ Following example shows how to parse headers:
 
 .. code-block:: C
 
+  #include <http/http_header_names.h>
   if (msg.data.headers_len)
     {
       u8 *headers = 0;
@@ -134,7 +135,7 @@ Following example shows how to parse headers:
           /* your error handling */
         }
       /* get Accept header */
-      const char *accept_value = http_get_header (ht, HTTP_HEADER_ACCEPT);
+      const char *accept_value = http_get_header (ht, http_header_name_str (HTTP_HEADER_ACCEPT));
       if (accept_value)
         {
           /* do something interesting */
@@ -154,3 +155,97 @@ Finally application reads body:
       rv = svm_fifo_peek (ts->rx_fifo, msg.data.body_offset, msg.data.body_len, body);
       ASSERT (rv == msg.data.body_len);
     }
+
+Sending data
+""""""""""""""
+
+When server application sends response back to HTTP layer it starts with message metadata, followed by optional serialized headers and finally body (if any).
+
+Application should set following items:
+
+* Status code
+* target form
+* header section offset and length
+* body offset and length
+
+Application could pass headers back to HTTP layer. Header list is created dynamically as vector of ``http_header_t``,
+where we store only pointers to buffers (zero copy).
+Well known header names are predefined.
+The list is serialized just before you send buffer to HTTP layer.
+
+.. note::
+    Following headers are added at protocol layer and **MUST NOT** be set by application: Date, Server, Content-Length
+
+Following example shows how to create headers section:
+
+.. code-block:: C
+
+  #include <http/http.h>
+  #include <http/http_header_names.h>
+  #include <http/http_content_types.h>
+  http_header_t *resp_headers = 0;
+  u8 *headers_buf = 0;
+  http_add_header (resp_headers,
+		   http_header_name_token (HTTP_HEADER_CONTENT_TYPE),
+		   http_content_type_token (HTTP_CONTENT_TEXT_HTML));
+  http_add_header (resp_headers,
+		   http_header_name_token (HTTP_HEADER_CACHE_CONTROL),
+		   http_token_lit ("max-age=600"));
+  http_add_header (&hs->resp_headers,
+		   http_header_name_token (HTTP_HEADER_LOCATION),
+		   (const char *) redirect, vec_len (redirect));
+  headers_buf = http_serialize_headers (hs->resp_headers);
+
+The example below show how to create and send response HTTP message metadata:
+
+.. code-block:: C
+
+  http_msg_t msg;
+  msg.type = HTTP_MSG_REPLY;
+  msg.code = HTTP_STATUS_MOVED
+  msg.data.headers_offset = 0;
+  msg.data.headers_len = vec_len (headers_buf);
+  msg.data.type = HTTP_MSG_DATA_INLINE;
+  msg.data.body_len = vec_len (tx_buf);
+  msg.data.body_offset = msg.data.headers_len;
+  msg.data.len = msg.data.body_len + msg.data.headers_len;
+  ts = session_get (hs->vpp_session_index, hs->thread_index);
+  rv = svm_fifo_enqueue (ts->tx_fifo, sizeof (msg), (u8 *) &msg);
+  ASSERT (rv == sizeof (msg));
+
+Next you will send your serialized headers:
+
+.. code-block:: C
+
+  rv = svm_fifo_enqueue (ts->tx_fifo, vec_len (headers_buf), headers_buf);
+  ASSERT (rv == msg.data.headers_len);
+  vec_free (headers_buf);
+
+Finally application sends response body:
+
+.. code-block:: C
+
+  rv = svm_fifo_enqueue (ts->tx_fifo, vec_len (tx_buf), tx_buf);
+  if (rv != vec_len (hs->tx_buf))
+    {
+      hs->tx_offset = rv;
+      svm_fifo_add_want_deq_ntf (ts->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF);
+    }
+  else
+    {
+      vec_free (tx_buf);
+    }
+  if (svm_fifo_set_event (ts->tx_fifo))
+    session_send_io_evt_to_thread (ts->tx_fifo, SESSION_IO_EVT_TX);
+
+Example above shows how to send body data by copy, alternatively you could pass it as pointer:
+
+.. code-block:: C
+
+  msg.data.type = HTTP_MSG_DATA_PTR;
+  /* code omitted for brevity */
+  uword data = pointer_to_uword (tx_buf);
+  rv = svm_fifo_enqueue (ts->tx_fifo, sizeof (data), (u8 *) &data);
+  ASSERT (rv == sizeof (data));
+
+In this case you need to free data when you receive next request or when session is closed.
diff --git a/src/plugins/http_static/builtinurl/json_urls.c b/src/plugins/http_static/builtinurl/json_urls.c
index 8578be147f3..19c5245e4b2 100644
--- a/src/plugins/http_static/builtinurl/json_urls.c
+++ b/src/plugins/http_static/builtinurl/json_urls.c
@@ -45,6 +45,7 @@ handle_get_version (hss_url_handler_args_t *args)
 
   args->data = s;
   args->data_len = vec_len (s);
+  args->ct = HTTP_CONTENT_APP_JSON;
   args->free_vec_data = 1;
   return HSS_URL_HANDLER_OK;
 }
@@ -117,6 +118,7 @@ handle_get_interface_stats (hss_url_handler_args_t *args)
 out:
   args->data = s;
   args->data_len = vec_len (s);
+  args->ct = HTTP_CONTENT_APP_JSON;
   args->free_vec_data = 1;
   vec_free (sw_if_indices);
   vec_free (stats);
@@ -157,6 +159,7 @@ handle_get_interface_list (hss_url_handler_args_t *args)
 
   args->data = s;
   args->data_len = vec_len (s);
+  args->ct = HTTP_CONTENT_APP_JSON;
   args->free_vec_data = 1;
   return HSS_URL_HANDLER_OK;
 }
diff --git a/src/plugins/http_static/http_cache.c b/src/plugins/http_static/http_cache.c
index 8b9751b7f78..7a069dace00 100644
--- a/src/plugins/http_static/http_cache.c
+++ b/src/plugins/http_static/http_cache.c
@@ -421,19 +421,19 @@ format_hss_cache (u8 *s, va_list *args)
     {
       s = format (s, "cache size %lld bytes, limit %lld bytes, evictions %lld",
 		  hc->cache_size, hc->cache_limit, hc->cache_evictions);
-      return 0;
+      return s;
     }
 
   vm = vlib_get_main ();
   now = vlib_time_now (vm);
 
-  s = format (s, "%U", format_hss_cache_entry, 0 /* header */, now);
+  s = format (s, "%U\n", format_hss_cache_entry, 0 /* header */, now);
 
   for (index = hc->first_index; index != ~0;)
     {
       ce = pool_elt_at_index (hc->cache_pool, index);
       index = ce->next_index;
-      s = format (s, "%U", format_hss_cache_entry, ce, now);
+      s = format (s, "%U\n", format_hss_cache_entry, ce, now);
     }
 
   s = format (s, "%40s%12lld", "Total Size", hc->cache_size);
diff --git a/src/plugins/http_static/http_static.h b/src/plugins/http_static/http_static.h
index 8f336e83d58..4c22e1d7ed7 100644
--- a/src/plugins/http_static/http_static.h
+++ b/src/plugins/http_static/http_static.h
@@ -50,8 +50,10 @@ typedef struct
   int free_data;
   /** File cache pool index */
   u32 cache_pool_index;
-  /** Content type, e.g. text, text/javascript, etc. */
-  http_content_type_t content_type;
+  /** Response header list */
+  http_header_t *resp_headers;
+  /** Serialized headers to send */
+  u8 *headers_buf;
 } hss_session_t;
 
 typedef struct hss_session_handle_
@@ -91,6 +93,7 @@ typedef struct hss_url_handler_args_
       uword data_len;
       u8 free_vec_data;
       http_status_code_t sc;
+      http_content_type_t ct;
     };
   };
 } hss_url_handler_args_t;
diff --git a/src/plugins/http_static/static_server.c b/src/plugins/http_static/static_server.c
index 26224989332..bc18875626e 100644
--- a/src/plugins/http_static/static_server.c
+++ b/src/plugins/http_static/static_server.c
@@ -19,6 +19,9 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include <http/http_header_names.h>
+#include <http/http_content_types.h>
+
 /** @file static_server.c
  *  Static http server, sufficient to serve .html / .css / .js content.
  */
@@ -83,34 +86,65 @@ start_send_data (hss_session_t *hs, http_status_code_t status)
 {
   http_msg_t msg;
   session_t *ts;
+  u8 *headers_buf = 0;
   int rv;
 
   ts = session_get (hs->vpp_session_index, hs->thread_index);
 
+  if (vec_len (hs->resp_headers))
+    {
+      headers_buf = http_serialize_headers (hs->resp_headers);
+      vec_free (hs->resp_headers);
+      msg.data.headers_offset = 0;
+      msg.data.headers_len = vec_len (headers_buf);
+    }
+  else
+    {
+      msg.data.headers_offset = 0;
+      msg.data.headers_len = 0;
+    }
+
   msg.type = HTTP_MSG_REPLY;
   msg.code = status;
-  msg.content_type = hs->content_type;
-  msg.data.len = hs->data_len;
+  msg.data.body_len = hs->data_len;
+  msg.data.len = msg.data.body_len + msg.data.headers_len;
 
-  if (hs->data_len > hss_main.use_ptr_thresh)
+  if (msg.data.len > hss_main.use_ptr_thresh)
     {
       msg.data.type = HTTP_MSG_DATA_PTR;
       rv = svm_fifo_enqueue (ts->tx_fifo, sizeof (msg), (u8 *) &msg);
       ASSERT (rv == sizeof (msg));
 
+      if (msg.data.headers_len)
+	{
+	  hs->headers_buf = headers_buf;
+	  uword headers = pointer_to_uword (hs->headers_buf);
+	  rv =
+	    svm_fifo_enqueue (ts->tx_fifo, sizeof (headers), (u8 *) &headers);
+	  ASSERT (rv == sizeof (headers));
+	}
+
       uword data = pointer_to_uword (hs->data);
       rv = svm_fifo_enqueue (ts->tx_fifo, sizeof (data), (u8 *) &data);
-      ASSERT (rv == sizeof (sizeof (data)));
+      ASSERT (rv == sizeof (data));
 
       goto done;
     }
 
   msg.data.type = HTTP_MSG_DATA_INLINE;
+  msg.data.body_offset = msg.data.headers_len;
 
   rv = svm_fifo_enqueue (ts->tx_fifo, sizeof (msg), (u8 *) &msg);
   ASSERT (rv == sizeof (msg));
 
-  if (!msg.data.len)
+  if (msg.data.headers_len)
+    {
+      rv = svm_fifo_enqueue (ts->tx_fifo, vec_len (headers_buf), headers_buf);
+      ASSERT (rv == msg.data.headers_len);
+      vec_free (headers_buf);
+    }
+
+  if (!msg.data.body_len)
     goto done;
 
   rv = svm_fifo_enqueue (ts->tx_fifo, hs->data_len, hs->data);
@@ -142,6 +176,15 @@ hss_session_send_data (hss_url_handler_args_t *args)
   hs->data = args->data;
   hs->data_len = args->data_len;
   hs->free_data = args->free_vec_data;
+
+  /* Set content type only if we have some response data */
+  if (hs->data_len)
+    {
+      http_add_header (&hs->resp_headers,
+		       http_header_name_token (HTTP_HEADER_CONTENT_TYPE),
+		       http_content_type_token (args->ct));
+    }
+
   start_send_data (hs, args->sc);
 }
 
@@ -217,7 +260,6 @@ try_url_handler (hss_main_t *hsm, hss_session_t *hs, http_req_method_t rt,
   http_status_code_t sc = HTTP_STATUS_OK;
   hss_url_handler_args_t args = {};
   uword *p, *url_table;
-  http_content_type_t type;
   int rv;
 
   if (!hsm->enable_url_handlers || !target_path)
@@ -229,8 +271,6 @@ try_url_handler (hss_main_t *hsm, hss_session_t *hs, http_req_method_t rt,
       target_path = format (target_path, "index.html");
     }
 
-  type = content_type_from_request (target_path);
-
   /* Look for built-in GET / POST handlers */
   url_table =
     (rt == HTTP_REQ_GET) ? hsm->get_url_handlers : hsm->post_url_handlers;
@@ -263,17 +303,24 @@ try_url_handler (hss_main_t *hsm, hss_session_t *hs, http_req_method_t rt,
     {
       clib_warning ("builtin handler %llx hit on %s '%s' but failed!", p[0],
 		    (rt == HTTP_REQ_GET) ? "GET" : "POST", target_path);
-      sc = HTTP_STATUS_NOT_FOUND;
+      sc = HTTP_STATUS_BAD_GATEWAY;
     }
 
   hs->data = args.data;
   hs->data_len = args.data_len;
   hs->free_data = args.free_vec_data;
-  hs->content_type = type;
+
+  /* Set content type only if we have some response data */
+  if (hs->data_len)
+    {
+      http_add_header (&hs->resp_headers,
+		       http_header_name_token (HTTP_HEADER_CONTENT_TYPE),
+		       http_content_type_token (args.ct));
+    }
 
   start_send_data (hs, sc);
 
-  if (!hs->data)
+  if (!hs->data_len)
     hss_session_disconnect_transport (hs);
 
   return 0;
@@ -337,18 +384,20 @@ try_index_file (hss_main_t *hsm, hss_session_t *hs, u8 *path)
     }
 
   redirect =
-    format (0,
-	    "Location: http%s://%U%s%s\r\n\r\n",
-	    proto == TRANSPORT_PROTO_TLS ? "s" : "", format_ip46_address,
-	    &endpt.ip, endpt.is_ip4, print_port ? port_str : (u8 *) "", path);
+    format (0, "http%s://%U%s%s", proto == TRANSPORT_PROTO_TLS ? "s" : "",
+	    format_ip46_address, &endpt.ip, endpt.is_ip4,
+	    print_port ? port_str : (u8 *) "", path);
 
   if (hsm->debug_level > 0)
     clib_warning ("redirect: %s", redirect);
 
   vec_free (port_str);
 
-  hs->data = redirect;
-  hs->data_len = vec_len (redirect);
+  http_add_header (&hs->resp_headers,
+		   http_header_name_token (HTTP_HEADER_LOCATION),
+		   (const char *) redirect, vec_len (redirect));
+  hs->data = redirect; /* TODO: find better way  */
+  hs->data_len = 0;
   hs->free_data = 1;
 
   return HTTP_STATUS_MOVED;
@@ -367,8 +416,6 @@ try_file_handler (hss_main_t *hsm, hss_session_t *hs, http_req_method_t rt,
   if (!hsm->www_root)
     return -1;
 
-  type = content_type_from_request (target);
-
   /* Remove dot segments to prevent path traversal */
   sanitized_path = http_path_remove_dot_segments (target);
 
@@ -420,11 +467,23 @@ try_file_handler (hss_main_t *hsm, hss_session_t *hs, http_req_method_t rt,
   hs->path = path;
   hs->cache_pool_index = ce_index;
 
+  /* Set following headers only for happy path:
+   * Content-Type
+   * Cache-Control max-age
+   */
+  type = content_type_from_request (target);
+  http_add_header (&hs->resp_headers,
+		   http_header_name_token (HTTP_HEADER_CONTENT_TYPE),
+		   http_content_type_token (type));
+  /* TODO configurable max-age value */
+  http_add_header (&hs->resp_headers,
+		   http_header_name_token (HTTP_HEADER_CACHE_CONTROL),
+		   http_token_lit ("max-age=600"));
+
 done:
   vec_free (sanitized_path);
-  hs->content_type = type;
   start_send_data (hs, sc);
-  if (!hs->data)
+  if (!hs->data_len)
     hss_session_disconnect_transport (hs);
 
   return 0;
@@ -459,6 +518,8 @@ hss_ts_rx_callback (session_t *ts)
   if (hs->free_data)
     vec_free (hs->data);
   hs->data = 0;
+  hs->resp_headers = 0;
+  vec_free (hs->headers_buf);
 
   /* Read the http message header */
   rv = svm_fifo_dequeue (ts->rx_fifo, sizeof (msg), (u8 *) &msg);
@@ -467,6 +528,9 @@ hss_ts_rx_callback (session_t *ts)
   if (msg.type != HTTP_MSG_REQUEST ||
       (msg.method_type != HTTP_REQ_GET && msg.method_type != HTTP_REQ_POST))
     {
+      http_add_header (&hs->resp_headers,
+		       http_header_name_token (HTTP_HEADER_ALLOW),
+		       http_token_lit ("GET, POST"));
       start_send_data (hs, HTTP_STATUS_METHOD_NOT_ALLOWED);
       goto done;
     }
@@ -648,6 +712,7 @@ hss_ts_cleanup (session_t *s, session_cleanup_ntf_t ntf)
   hs->data = 0;
   hs->data_offset = 0;
   hs->free_data = 0;
+  vec_free (hs->headers_buf);
   vec_free (hs->path);
 
   hss_session_free (hs);
diff --git a/src/plugins/mactime/builtins.c b/src/plugins/mactime/builtins.c
index c487d0375bf..f726d3c03ed 100644
--- a/src/plugins/mactime/builtins.c
+++ b/src/plugins/mactime/builtins.c
@@ -147,6 +147,7 @@ handle_get_mactime (hss_url_handler_args_t *args)
 
   args->data = s;
   args->data_len = vec_len (s);
+  args->ct = HTTP_CONTENT_APP_JSON;
   args->free_vec_data = 1;
   return HSS_URL_HANDLER_OK;
 }
diff --git a/src/plugins/prom/prom.c b/src/plugins/prom/prom.c
index 934e8480d3c..76899a2fd12 100644
--- a/src/plugins/prom/prom.c
+++ b/src/plugins/prom/prom.c
@@ -191,6 +191,7 @@ send_data_to_hss (hss_session_handle_t sh)
   args.sh = sh;
   args.data = vec_dup (pm->stats);
   args.data_len = vec_len (pm->stats);
+  args.ct = HTTP_CONTENT_TEXT_PLAIN;
   args.sc = HTTP_STATUS_OK;
   args.free_vec_data = 1;