http_static: cleanup file handler and cache

Type: refactor

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: I7aff3a02315f9f92039dd2e3af1cbd8312aec662

6 files changed, 674 insertions, 493 deletions

diff --git a/src/plugins/http_static/CMakeLists.txt b/src/plugins/http_static/CMakeLists.txt
index cf9e41e30e5..5e51704dc96 100644
--- a/src/plugins/http_static/CMakeLists.txt
+++ b/src/plugins/http_static/CMakeLists.txt
@@ -14,9 +14,10 @@
 
 add_vpp_plugin(http_static
   SOURCES
+  http_cache.c
+  http_cache.h
   http_static.c
   static_server.c
-  http_static.h
   builtinurl/json_urls.c
 
   API_FILES
diff --git a/src/plugins/http_static/http_cache.c b/src/plugins/http_static/http_cache.c
new file mode 100644
index 00000000000..8b9751b7f78
--- /dev/null
+++ b/src/plugins/http_static/http_cache.c
@@ -0,0 +1,450 @@
+/*
+ * Copyright (c) 2022 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <http_static/http_cache.h>
+#include <vppinfra/bihash_template.c>
+#include <vppinfra/unix.h>
+#include <vlib/vlib.h>
+
+static void
+hss_cache_lock (hss_cache_t *hc)
+{
+  clib_spinlock_lock (&hc->cache_lock);
+}
+
+static void
+hss_cache_unlock (hss_cache_t *hc)
+{
+  clib_spinlock_unlock (&hc->cache_lock);
+}
+
+/** \brief Sanity-check the forward and reverse LRU lists
+ */
+static inline void
+lru_validate (hss_cache_t *hc)
+{
+#if CLIB_DEBUG > 0
+  f64 last_timestamp;
+  u32 index;
+  int i;
+  hss_cache_entry_t *ce;
+
+  last_timestamp = 1e70;
+  for (i = 1, index = hc->first_index; index != ~0;)
+    {
+      ce = pool_elt_at_index (hc->cache_pool, index);
+      /* Timestamps should be smaller (older) as we walk the fwd list */
+      if (ce->last_used > last_timestamp)
+	{
+	  clib_warning ("%d[%d]: last used %.6f, last_timestamp %.6f", index,
+			i, ce->last_used, last_timestamp);
+	}
+      index = ce->next_index;
+      last_timestamp = ce->last_used;
+      i++;
+    }
+
+  last_timestamp = 0.0;
+  for (i = 1, index = hc->last_index; index != ~0;)
+    {
+      ce = pool_elt_at_index (hc->cache_pool, index);
+      /* Timestamps should be larger (newer) as we walk the rev list */
+      if (ce->last_used < last_timestamp)
+	{
+	  clib_warning ("%d[%d]: last used %.6f, last_timestamp %.6f", index,
+			i, ce->last_used, last_timestamp);
+	}
+      index = ce->prev_index;
+      last_timestamp = ce->last_used;
+      i++;
+    }
+#endif
+}
+
+/** \brief Remove a data cache entry from the LRU lists
+ */
+static inline void
+lru_remove (hss_cache_t *hc, hss_cache_entry_t *ce)
+{
+  hss_cache_entry_t *next_ep, *prev_ep;
+  u32 ce_index;
+
+  lru_validate (hc);
+
+  ce_index = ce - hc->cache_pool;
+
+  /* Deal with list heads */
+  if (ce_index == hc->first_index)
+    hc->first_index = ce->next_index;
+  if (ce_index == hc->last_index)
+    hc->last_index = ce->prev_index;
+
+  /* Fix next->prev */
+  if (ce->next_index != ~0)
+    {
+      next_ep = pool_elt_at_index (hc->cache_pool, ce->next_index);
+      next_ep->prev_index = ce->prev_index;
+    }
+  /* Fix prev->next */
+  if (ce->prev_index != ~0)
+    {
+      prev_ep = pool_elt_at_index (hc->cache_pool, ce->prev_index);
+      prev_ep->next_index = ce->next_index;
+    }
+  lru_validate (hc);
+}
+
+/** \brief Add an entry to the LRU lists, tag w/ supplied timestamp
+ */
+static inline void
+lru_add (hss_cache_t *hc, hss_cache_entry_t *ce, f64 now)
+{
+  hss_cache_entry_t *next_ce;
+  u32 ce_index;
+
+  lru_validate (hc);
+
+  ce_index = ce - hc->cache_pool;
+
+  /*
+   * Re-add at the head of the forward LRU list,
+   * tail of the reverse LRU list
+   */
+  if (hc->first_index != ~0)
+    {
+      next_ce = pool_elt_at_index (hc->cache_pool, hc->first_index);
+      next_ce->prev_index = ce_index;
+    }
+
+  ce->prev_index = ~0;
+
+  /* ep now the new head of the LRU forward list */
+  ce->next_index = hc->first_index;
+  hc->first_index = ce_index;
+
+  /* single session case: also the tail of the reverse LRU list */
+  if (hc->last_index == ~0)
+    hc->last_index = ce_index;
+  ce->last_used = now;
+
+  lru_validate (hc);
+}
+
+/** \brief Remove and re-add a cache entry from/to the LRU lists
+ */
+static inline void
+lru_update (hss_cache_t *hc, hss_cache_entry_t *ep, f64 now)
+{
+  lru_remove (hc, ep);
+  lru_add (hc, ep, now);
+}
+
+static void
+hss_cache_attach_entry (hss_cache_t *hc, u32 ce_index, u8 **data,
+			u64 *data_len)
+{
+  hss_cache_entry_t *ce;
+
+  /* Expect ce_index to be validated outside */
+  ce = pool_elt_at_index (hc->cache_pool, ce_index);
+  ce->inuse++;
+  *data = ce->data;
+  *data_len = vec_len (ce->data);
+
+  /* Update the cache entry, mark it in-use */
+  lru_update (hc, ce, vlib_time_now (vlib_get_main ()));
+
+  if (hc->debug_level > 1)
+    clib_warning ("index %d refcnt now %d", ce_index, ce->inuse);
+}
+
+/** \brief Detach cache entry from session
+ */
+void
+hss_cache_detach_entry (hss_cache_t *hc, u32 ce_index)
+{
+  hss_cache_entry_t *ce;
+
+  hss_cache_lock (hc);
+
+  ce = pool_elt_at_index (hc->cache_pool, ce_index);
+  ce->inuse--;
+
+  if (hc->debug_level > 1)
+    clib_warning ("index %d refcnt now %d", ce_index, ce->inuse);
+
+  hss_cache_unlock (hc);
+}
+
+static u32
+hss_cache_lookup (hss_cache_t *hc, u8 *path)
+{
+  BVT (clib_bihash_kv) kv;
+  int rv;
+
+  kv.key = (u64) path;
+  kv.value = ~0;
+
+  /* Value updated only if lookup succeeds */
+  rv = BV (clib_bihash_search) (&hc->name_to_data, &kv, &kv);
+  ASSERT (!rv || kv.value == ~0);
+
+  if (hc->debug_level > 1)
+    clib_warning ("lookup '%s' %s", kv.key, kv.value == ~0 ? "fail" : "found");
+
+  return kv.value;
+}
+
+u32
+hss_cache_lookup_and_attach (hss_cache_t *hc, u8 *path, u8 **data,
+			     u64 *data_len)
+{
+  u32 ce_index;
+
+  /* Make sure nobody removes the entry while we look it up */
+  hss_cache_lock (hc);
+
+  ce_index = hss_cache_lookup (hc, path);
+  if (ce_index != ~0)
+    hss_cache_attach_entry (hc, ce_index, data, data_len);
+
+  hss_cache_unlock (hc);
+
+  return ce_index;
+}
+
+static void
+hss_cache_do_evictions (hss_cache_t *hc)
+{
+  BVT (clib_bihash_kv) kv;
+  hss_cache_entry_t *ce;
+  u32 free_index;
+
+  free_index = hc->last_index;
+
+  while (free_index != ~0)
+    {
+      /* pick the LRU */
+      ce = pool_elt_at_index (hc->cache_pool, free_index);
+      /* Which could be in use... */
+      if (ce->inuse)
+	{
+	  if (hc->debug_level > 1)
+	    clib_warning ("index %d in use refcnt %d", free_index, ce->inuse);
+	}
+      free_index = ce->prev_index;
+      kv.key = (u64) (ce->filename);
+      kv.value = ~0ULL;
+      if (BV (clib_bihash_add_del) (&hc->name_to_data, &kv, 0 /* is_add */) <
+	  0)
+	{
+	  clib_warning ("LRU delete '%s' FAILED!", ce->filename);
+	}
+      else if (hc->debug_level > 1)
+	clib_warning ("LRU delete '%s' ok", ce->filename);
+
+      lru_remove (hc, ce);
+      hc->cache_size -= vec_len (ce->data);
+      hc->cache_evictions++;
+      vec_free (ce->filename);
+      vec_free (ce->data);
+
+      if (hc->debug_level > 1)
+	clib_warning ("pool put index %d", ce - hc->cache_pool);
+
+      pool_put (hc->cache_pool, ce);
+      if (hc->cache_size < hc->cache_limit)
+	break;
+    }
+}
+
+u32
+hss_cache_add_and_attach (hss_cache_t *hc, u8 *path, u8 **data, u64 *data_len)
+{
+  BVT (clib_bihash_kv) kv;
+  hss_cache_entry_t *ce;
+  clib_error_t *error;
+  u8 *file_data;
+  u32 ce_index;
+
+  hss_cache_lock (hc);
+
+  /* Need to recycle one (or more cache) entries? */
+  if (hc->cache_size > hc->cache_limit)
+    hss_cache_do_evictions (hc);
+
+  /* Read the file */
+  error = clib_file_contents ((char *) path, &file_data);
+  if (error)
+    {
+      clib_warning ("Error reading '%s'", path);
+      clib_error_report (error);
+      return ~0;
+    }
+
+  /* Create a cache entry for it */
+  pool_get_zero (hc->cache_pool, ce);
+  ce->filename = vec_dup (path);
+  ce->data = file_data;
+
+  /* Attach cache entry without additional lock */
+  ce->inuse++;
+  *data = file_data;
+  *data_len = vec_len (file_data);
+  lru_add (hc, ce, vlib_time_now (vlib_get_main ()));
+
+  hc->cache_size += vec_len (ce->data);
+  ce_index = ce - hc->cache_pool;
+
+  if (hc->debug_level > 1)
+    clib_warning ("index %d refcnt now %d", ce_index, ce->inuse);
+
+  /* Add to the lookup table */
+
+  kv.key = (u64) vec_dup (path);
+  kv.value = ce_index;
+
+  if (hc->debug_level > 1)
+    clib_warning ("add '%s' value %lld", kv.key, kv.value);
+
+  if (BV (clib_bihash_add_del) (&hc->name_to_data, &kv, 1 /* is_add */) < 0)
+    {
+      clib_warning ("BUG: add failed!");
+    }
+
+  hss_cache_unlock (hc);
+
+  return ce_index;
+}
+
+u32
+hss_cache_clear (hss_cache_t *hc)
+{
+  u32 free_index, busy_items = 0;
+  hss_cache_entry_t *ce;
+  BVT (clib_bihash_kv) kv;
+
+  hss_cache_lock (hc);
+
+  /* Walk the LRU list to find active entries */
+  free_index = hc->last_index;
+  while (free_index != ~0)
+    {
+      ce = pool_elt_at_index (hc->cache_pool, free_index);
+      free_index = ce->prev_index;
+      /* Which could be in use... */
+      if (ce->inuse)
+	{
+	  busy_items++;
+	  free_index = ce->next_index;
+	  continue;
+	}
+      kv.key = (u64) (ce->filename);
+      kv.value = ~0ULL;
+      if (BV (clib_bihash_add_del) (&hc->name_to_data, &kv, 0 /* is_add */) <
+	  0)
+	{
+	  clib_warning ("BUG: cache clear delete '%s' FAILED!", ce->filename);
+	}
+
+      lru_remove (hc, ce);
+      hc->cache_size -= vec_len (ce->data);
+      hc->cache_evictions++;
+      vec_free (ce->filename);
+      vec_free (ce->data);
+      if (hc->debug_level > 1)
+	clib_warning ("pool put index %d", ce - hc->cache_pool);
+      pool_put (hc->cache_pool, ce);
+      free_index = hc->last_index;
+    }
+
+  hss_cache_unlock (hc);
+
+  return busy_items;
+}
+
+void
+hss_cache_init (hss_cache_t *hc, uword cache_size, u8 debug_level)
+{
+  clib_spinlock_init (&hc->cache_lock);
+
+  /* Init path-to-cache hash table */
+  BV (clib_bihash_init) (&hc->name_to_data, "http cache", 128, 32 << 20);
+
+  hc->cache_limit = cache_size;
+  hc->debug_level = debug_level;
+  hc->first_index = hc->last_index = ~0;
+}
+
+/** \brief format a file cache entry
+ */
+static u8 *
+format_hss_cache_entry (u8 *s, va_list *args)
+{
+  hss_cache_entry_t *ep = va_arg (*args, hss_cache_entry_t *);
+  f64 now = va_arg (*args, f64);
+
+  /* Header */
+  if (ep == 0)
+    {
+      s = format (s, "%40s%12s%20s", "File", "Size", "Age");
+      return s;
+    }
+  s = format (s, "%40s%12lld%20.2f", ep->filename, vec_len (ep->data),
+	      now - ep->last_used);
+  return s;
+}
+
+u8 *
+format_hss_cache (u8 *s, va_list *args)
+{
+  hss_cache_t *hc = va_arg (*args, hss_cache_t *);
+  u32 verbose = va_arg (*args, u32);
+  hss_cache_entry_t *ce;
+  vlib_main_t *vm;
+  u32 index;
+  f64 now;
+
+  if (verbose == 0)
+    {
+      s = format (s, "cache size %lld bytes, limit %lld bytes, evictions %lld",
+		  hc->cache_size, hc->cache_limit, hc->cache_evictions);
+      return 0;
+    }
+
+  vm = vlib_get_main ();
+  now = vlib_time_now (vm);
+
+  s = format (s, "%U", format_hss_cache_entry, 0 /* header */, now);
+
+  for (index = hc->first_index; index != ~0;)
+    {
+      ce = pool_elt_at_index (hc->cache_pool, index);
+      index = ce->next_index;
+      s = format (s, "%U", format_hss_cache_entry, ce, now);
+    }
+
+  s = format (s, "%40s%12lld", "Total Size", hc->cache_size);
+
+  return s;
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/http_static/http_cache.h b/src/plugins/http_static/http_cache.h
new file mode 100644
index 00000000000..a89ed5e7e94
--- /dev/null
+++ b/src/plugins/http_static/http_cache.h
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2022 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SRC_PLUGINS_HTTP_STATIC_HTTP_CACHE_H_
+#define SRC_PLUGINS_HTTP_STATIC_HTTP_CACHE_H_
+
+#include <vppinfra/bihash_vec8_8.h>
+
+typedef struct hss_cache_entry_
+{
+  /** Name of the file */
+  u8 *filename;
+  /** Contents of the file, as a u8 * vector */
+  u8 *data;
+  /** Last time the cache entry was used */
+  f64 last_used;
+  /** Cache LRU links */
+  u32 next_index;
+  u32 prev_index;
+  /** Reference count, so we don't recycle while referenced */
+  int inuse;
+} hss_cache_entry_t;
+
+typedef struct hss_cache_
+{
+  /** Unified file data cache pool */
+  hss_cache_entry_t *cache_pool;
+  /** Hash table which maps file name to file data */
+  BVT (clib_bihash) name_to_data;
+
+  /** Session pool lock */
+  clib_spinlock_t cache_lock;
+
+  /** Current cache size */
+  u64 cache_size;
+  /** Max cache size in bytes */
+  u64 cache_limit;
+  /** Number of cache evictions */
+  u64 cache_evictions;
+
+  /** Cache LRU listheads */
+  u32 first_index;
+  u32 last_index;
+
+  u8 debug_level;
+} hss_cache_t;
+
+u32 hss_cache_lookup_and_attach (hss_cache_t *hc, u8 *path, u8 **data,
+				 u64 *data_len);
+u32 hss_cache_add_and_attach (hss_cache_t *hc, u8 *path, u8 **data,
+			      u64 *data_len);
+void hss_cache_detach_entry (hss_cache_t *hc, u32 ce_index);
+u32 hss_cache_clear (hss_cache_t *hc);
+void hss_cache_init (hss_cache_t *hc, uword cache_size, u8 debug_level);
+
+u8 *format_hss_cache (u8 *s, va_list *args);
+
+#endif /* SRC_PLUGINS_HTTP_STATIC_HTTP_CACHE_H_ */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/http_static/http_static.c b/src/plugins/http_static/http_static.c
index 4afe42291db..005eefc731c 100644
--- a/src/plugins/http_static/http_static.c
+++ b/src/plugins/http_static/http_static.c
@@ -73,7 +73,7 @@ hss_enable_api (u32 fifo_size, u32 cache_limit, u32 prealloc_fifos,
   int rv;
 
   hsm->fifo_size = fifo_size;
-  hsm->cache_limit = cache_limit;
+  hsm->cache_size = cache_limit;
   hsm->prealloc_fifos = prealloc_fifos;
   hsm->private_segment_size = private_segment_size;
   hsm->www_root = format (0, "%s%c", www_root, 0);
diff --git a/src/plugins/http_static/http_static.h b/src/plugins/http_static/http_static.h
index d3fa29b068d..c754337e303 100644
--- a/src/plugins/http_static/http_static.h
+++ b/src/plugins/http_static/http_static.h
@@ -21,7 +21,7 @@
 
 #include <vppinfra/hash.h>
 #include <vppinfra/error.h>
-#include <vppinfra/bihash_vec8_8.h>
+#include <http_static/http_cache.h>
 
 /** @file http_static.h
  * Static http server definitions
@@ -43,7 +43,7 @@ typedef struct
   /** Data to send */
   u8 *data;
   /** Data length */
-  u32 data_len;
+  u64 data_len;
   /** Current data send offset */
   u32 data_offset;
   /** Need to free data in detach_cache_entry */
@@ -67,22 +67,6 @@ typedef struct hss_session_handle_
 
 STATIC_ASSERT_SIZEOF (hss_session_handle_t, sizeof (u64));
 
-/** \brief In-memory file data cache entry
- */
-typedef struct
-{
-  /** Name of the file */
-  u8 *filename;
-  /** Contents of the file, as a u8 * vector */
-  u8 *data;
-  /** Last time the cache entry was used */
-  f64 last_used;
-  /** Cache LRU links */
-  u32 next_index;
-  u32 prev_index;
-  /** Reference count, so we don't recycle while referenced */
-  int inuse;
-} hss_cache_entry_t;
 
 typedef struct hss_url_handler_args_
 {
@@ -125,28 +109,12 @@ typedef struct
 {
   /** Per thread vector of session pools */
   hss_session_t **sessions;
-  /** Session pool reader writer lock */
-  clib_spinlock_t cache_lock;
-
-  /** Unified file data cache pool */
-  hss_cache_entry_t *cache_pool;
-  /** Hash table which maps file name to file data */
-    BVT (clib_bihash) name_to_data;
 
   /** Hash tables for built-in GET and POST handlers */
   uword *get_url_handlers;
   uword *post_url_handlers;
 
-  /** Current cache size */
-  u64 cache_size;
-  /** Max cache size in bytes */
-  u64 cache_limit;
-  /** Number of cache evictions */
-  u64 cache_evictions;
-
-  /** Cache LRU listheads */
-  u32 first_index;
-  u32 last_index;
+  hss_cache_t cache;
 
   /** root path to be served */
   u8 *www_root;
@@ -180,6 +148,8 @@ typedef struct
   u32 use_ptr_thresh;
   /** Enable the use of builtinurls */
   u8 enable_url_handlers;
+  /** Max cache size before LRU occurs */
+  u64 cache_size;
 } hss_main_t;
 
 extern hss_main_t hss_main;
diff --git a/src/plugins/http_static/static_server.c b/src/plugins/http_static/static_server.c
index 0a1eb356fe8..a71136db7d9 100644
--- a/src/plugins/http_static/static_server.c
+++ b/src/plugins/http_static/static_server.c
@@ -14,8 +14,6 @@
  */
 
 #include <http_static/http_static.h>
-#include <vppinfra/bihash_template.c>
-#include <vppinfra/unix.h>
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -30,18 +28,6 @@
 
 hss_main_t hss_main;
 
-static void
-hss_cache_lock (void)
-{
-  clib_spinlock_lock (&hss_main.cache_lock);
-}
-
-static void
-hss_cache_unlock (void)
-{
-  clib_spinlock_unlock (&hss_main.cache_lock);
-}
-
 static hss_session_t *
 hss_session_alloc (u32 thread_index)
 {
@@ -81,36 +67,6 @@ hss_session_free (hss_session_t *hs)
     }
 }
 
-/** \brief Detach cache entry from session
- */
-static void
-hss_detach_cache_entry (hss_session_t *hs)
-{
-  hss_main_t *hsm = &hss_main;
-  hss_cache_entry_t *ep;
-
-  /*
-   * Decrement cache pool entry reference count
-   * Note that if e.g. a file lookup fails, the cache pool index
-   * won't be set
-   */
-  if (hs->cache_pool_index != ~0)
-    {
-      ep = pool_elt_at_index (hsm->cache_pool, hs->cache_pool_index);
-      ep->inuse--;
-      if (hsm->debug_level > 1)
-	clib_warning ("index %d refcnt now %d", hs->cache_pool_index,
-		      ep->inuse);
-    }
-  hs->cache_pool_index = ~0;
-  if (hs->free_data)
-    vec_free (hs->data);
-  hs->data = 0;
-  hs->data_offset = 0;
-  hs->free_data = 0;
-  vec_free (hs->path);
-}
-
 /** \brief Disconnect a session
  */
 static void
@@ -122,129 +78,6 @@ hss_session_disconnect_transport (hss_session_t *hs)
   vnet_disconnect_session (a);
 }
 
-/** \brief Sanity-check the forward and reverse LRU lists
- */
-static inline void
-lru_validate (hss_main_t *hsm)
-{
-#if CLIB_DEBUG > 0
-  f64 last_timestamp;
-  u32 index;
-  int i;
-  hss_cache_entry_t *ep;
-
-  last_timestamp = 1e70;
-  for (i = 1, index = hsm->first_index; index != ~0;)
-    {
-      ep = pool_elt_at_index (hsm->cache_pool, index);
-      index = ep->next_index;
-      /* Timestamps should be smaller (older) as we walk the fwd list */
-      if (ep->last_used > last_timestamp)
-	{
-	  clib_warning ("%d[%d]: last used %.6f, last_timestamp %.6f",
-			ep - hsm->cache_pool, i,
-			ep->last_used, last_timestamp);
-	}
-      last_timestamp = ep->last_used;
-      i++;
-    }
-
-  last_timestamp = 0.0;
-  for (i = 1, index = hsm->last_index; index != ~0;)
-    {
-      ep = pool_elt_at_index (hsm->cache_pool, index);
-      index = ep->prev_index;
-      /* Timestamps should be larger (newer) as we walk the rev list */
-      if (ep->last_used < last_timestamp)
-	{
-	  clib_warning ("%d[%d]: last used %.6f, last_timestamp %.6f",
-			ep - hsm->cache_pool, i,
-			ep->last_used, last_timestamp);
-	}
-      last_timestamp = ep->last_used;
-      i++;
-    }
-#endif
-}
-
-/** \brief Remove a data cache entry from the LRU lists
- */
-static inline void
-lru_remove (hss_main_t *hsm, hss_cache_entry_t *ep)
-{
-  hss_cache_entry_t *next_ep, *prev_ep;
-  u32 ep_index;
-
-  lru_validate (hsm);
-
-  ep_index = ep - hsm->cache_pool;
-
-  /* Deal with list heads */
-  if (ep_index == hsm->first_index)
-    hsm->first_index = ep->next_index;
-  if (ep_index == hsm->last_index)
-    hsm->last_index = ep->prev_index;
-
-  /* Fix next->prev */
-  if (ep->next_index != ~0)
-    {
-      next_ep = pool_elt_at_index (hsm->cache_pool, ep->next_index);
-      next_ep->prev_index = ep->prev_index;
-    }
-  /* Fix prev->next */
-  if (ep->prev_index != ~0)
-    {
-      prev_ep = pool_elt_at_index (hsm->cache_pool, ep->prev_index);
-      prev_ep->next_index = ep->next_index;
-    }
-  lru_validate (hsm);
-}
-
-/** \brief Add an entry to the LRU lists, tag w/ supplied timestamp
- */
-static inline void
-lru_add (hss_main_t *hsm, hss_cache_entry_t *ep, f64 now)
-{
-  hss_cache_entry_t *next_ep;
-  u32 ep_index;
-
-  lru_validate (hsm);
-
-  ep_index = ep - hsm->cache_pool;
-
-  /*
-   * Re-add at the head of the forward LRU list,
-   * tail of the reverse LRU list
-   */
-  if (hsm->first_index != ~0)
-    {
-      next_ep = pool_elt_at_index (hsm->cache_pool, hsm->first_index);
-      next_ep->prev_index = ep_index;
-    }
-
-  ep->prev_index = ~0;
-
-  /* ep now the new head of the LRU forward list */
-  ep->next_index = hsm->first_index;
-  hsm->first_index = ep_index;
-
-  /* single session case: also the tail of the reverse LRU list */
-  if (hsm->last_index == ~0)
-    hsm->last_index = ep_index;
-  ep->last_used = now;
-
-  lru_validate (hsm);
-}
-
-/** \brief Remove and re-add a cache entry from/to the LRU lists
- */
-static inline void
-lru_update (hss_main_t *hsm, hss_cache_entry_t *ep, f64 now)
-{
-  lru_remove (hsm, ep);
-  lru_add (hsm, ep, now);
-}
-
 static void
 start_send_data (hss_session_t *hs, http_status_code_t status)
 {
@@ -367,24 +200,94 @@ try_url_handler (hss_main_t *hsm, hss_session_t *hs, http_req_method_t rt,
   return 0;
 }
 
-static int
-handle_request (hss_session_t *hs, http_req_method_t rt, u8 *request)
+static u8
+file_path_is_valid (u8 *path)
 {
-  http_status_code_t sc = HTTP_STATUS_OK;
-  hss_main_t *hsm = &hss_main;
   struct stat _sb, *sb = &_sb;
-  clib_error_t *error;
-  u8 *path;
 
-  if (!try_url_handler (hsm, hs, rt, request))
+  if (stat ((char *) path, sb) < 0 /* can't stat the file */
+      || (sb->st_mode & S_IFMT) != S_IFREG /* not a regular file */)
     return 0;
 
-  if (!hsm->www_root)
+  return 1;
+}
+
+static u32
+try_index_file (hss_main_t *hsm, hss_session_t *hs, u8 *path)
+{
+  u8 *port_str = 0, *redirect;
+  transport_endpoint_t endpt;
+  transport_proto_t proto;
+  int print_port = 0;
+  u16 local_port;
+  session_t *ts;
+  u32 plen;
+
+  /* Remove the trailing space */
+  _vec_len (path) -= 1;
+  plen = vec_len (path);
+
+  /* Append "index.html" */
+  if (path[plen - 1] != '/')
+    path = format (path, "/index.html%c", 0);
+  else
+    path = format (path, "index.html%c", 0);
+
+  if (hsm->debug_level > 0)
+    clib_warning ("trying to find index: %s", path);
+
+  if (!file_path_is_valid (path))
+    return HTTP_STATUS_NOT_FOUND;
+
+  /*
+   * We found an index.html file, build a redirect
+   */
+  vec_delete (path, vec_len (hsm->www_root) - 1, 0);
+
+  ts = session_get (hs->vpp_session_index, hs->thread_index);
+  session_get_endpoint (ts, &endpt, 1 /* is_local */);
+
+  local_port = clib_net_to_host_u16 (endpt.port);
+  proto = session_type_transport_proto (ts->session_type);
+
+  if ((proto == TRANSPORT_PROTO_TCP && local_port != 80) ||
+      (proto == TRANSPORT_PROTO_TLS && local_port != 443))
     {
-      sc = HTTP_STATUS_NOT_FOUND;
-      goto done;
+      print_port = 1;
+      port_str = format (0, ":%u", (u32) local_port);
     }
 
+  redirect =
+    format (0,
+	    "HTTP/1.1 301 Moved Permanently\r\n"
+	    "Location: http%s://%U%s%s\r\n\r\n",
+	    proto == TRANSPORT_PROTO_TLS ? "s" : "", format_ip46_address,
+	    &endpt.ip, endpt.is_ip4, print_port ? port_str : (u8 *) "", path);
+
+  if (hsm->debug_level > 0)
+    clib_warning ("redirect: %s", redirect);
+
+  vec_free (port_str);
+
+  hs->data = redirect;
+  hs->data_len = vec_len (redirect);
+  hs->free_data = 1;
+
+  return HTTP_STATUS_OK;
+}
+
+static int
+try_file_handler (hss_main_t *hsm, hss_session_t *hs, http_req_method_t rt,
+		  u8 *request)
+{
+  http_status_code_t sc = HTTP_STATUS_OK;
+  u8 *path;
+  u32 ce_index;
+
+  /* Feature not enabled */
+  if (!hsm->www_root)
+    return -1;
+
   /*
    * Construct the file to open
    * Browsers are capable of sporadically including a leading '/'
@@ -399,199 +302,57 @@ handle_request (hss_session_t *hs, http_req_method_t rt, u8 *request)
   if (hsm->debug_level > 0)
     clib_warning ("%s '%s'", (rt == HTTP_REQ_GET) ? "GET" : "POST", path);
 
-  /* Try to find the file. 2x special cases to find index.html */
-  if (stat ((char *) path, sb) < 0	/* cant even stat the file */
-      || sb->st_size < 20	/* file too small */
-      || (sb->st_mode & S_IFMT) != S_IFREG /* not a regular file */ )
-    {
-      u32 save_length = vec_len (path) - 1;
-      /* Try appending "index.html"... */
-      _vec_len (path) -= 1;
-      path = format (path, "index.html%c", 0);
-      if (stat ((char *) path, sb) < 0	/* cant even stat the file */
-	  || sb->st_size < 20	/* file too small */
-	  || (sb->st_mode & S_IFMT) != S_IFREG /* not a regular file */ )
-	{
-	  _vec_len (path) = save_length;
-	  path = format (path, "/index.html%c", 0);
+  if (hs->data && hs->free_data)
+    vec_free (hs->data);
 
-	  /* Send a redirect, otherwise the browser will confuse itself */
-	  if (stat ((char *) path, sb) < 0	/* cant even stat the file */
-	      || sb->st_size < 20	/* file too small */
-	      || (sb->st_mode & S_IFMT) != S_IFREG /* not a regular file */ )
-	    {
-	      sc = HTTP_STATUS_NOT_FOUND;
-	      goto done;
-	    }
-	  else
-	    {
-	      transport_endpoint_t endpoint;
-	      transport_proto_t proto;
-	      u16 local_port;
-	      int print_port = 0;
-	      u8 *port_str = 0;
-	      session_t *ts;
-
-	      /*
-	       * To make this bit work correctly, we need to know our local
-	       * IP address, etc. and send it in the redirect...
-	       */
-	      u8 *redirect;
-
-	      vec_delete (path, vec_len (hsm->www_root) - 1, 0);
-
-	      ts = session_get (hs->vpp_session_index, hs->thread_index);
-	      session_get_endpoint (ts, &endpoint, 1 /* is_local */);
-
-	      local_port = clib_net_to_host_u16 (endpoint.port);
-
-	      proto = session_type_transport_proto (ts->session_type);
-
-	      if ((proto == TRANSPORT_PROTO_TCP && local_port != 80)
-		  || (proto == TRANSPORT_PROTO_TLS && local_port != 443))
-		{
-		  print_port = 1;
-		  port_str = format (0, ":%u", (u32) local_port);
-		}
-
-	      redirect = format (0, "HTTP/1.1 301 Moved Permanently\r\n"
-				 "Location: http%s://%U%s%s\r\n\r\n",
-				 proto == TRANSPORT_PROTO_TLS ? "s" : "",
-				 format_ip46_address, &endpoint.ip,
-				 endpoint.is_ip4,
-				 print_port ? port_str : (u8 *) "", path);
-	      if (hsm->debug_level > 0)
-		clib_warning ("redirect: %s", redirect);
-
-	      vec_free (port_str);
-
-	      hs->data = redirect;
-	      goto done;
-	    }
-	}
-    }
+  hs->path = path;
+  hs->data_offset = 0;
 
-  /* find or read the file if we haven't done so yet. */
-  if (hs->data == 0)
+  ce_index =
+    hss_cache_lookup_and_attach (&hsm->cache, path, &hs->data, &hs->data_len);
+  if (ce_index == ~0)
     {
-      BVT (clib_bihash_kv) kv;
-      hss_cache_entry_t *ce;
-
-      hs->path = path;
-
-      /* First, try the cache */
-      kv.key = (u64) hs->path;
-      if (BV (clib_bihash_search) (&hsm->name_to_data, &kv, &kv) == 0)
+      if (!file_path_is_valid (path))
 	{
-	  if (hsm->debug_level > 1)
-	    clib_warning ("lookup '%s' returned %lld", kv.key, kv.value);
-
-	  hss_cache_lock ();
-
-	  /* found the data.. */
-	  ce = pool_elt_at_index (hsm->cache_pool, kv.value);
-	  hs->data = ce->data;
-	  /* Update the cache entry, mark it in-use */
-	  lru_update (hsm, ce, vlib_time_now (vlib_get_main ()));
-	  hs->cache_pool_index = ce - hsm->cache_pool;
-	  ce->inuse++;
-	  if (hsm->debug_level > 1)
-	    clib_warning ("index %d refcnt now %d", hs->cache_pool_index,
-			  ce->inuse);
-
-	  hss_cache_unlock ();
+	  sc = try_index_file (hsm, hs, path);
+	  goto done;
 	}
-      else
+      ce_index =
+	hss_cache_add_and_attach (&hsm->cache, path, &hs->data, &hs->data_len);
+      if (ce_index == ~0)
 	{
-	  hss_cache_lock ();
-
-	  if (hsm->debug_level > 1)
-	    clib_warning ("lookup '%s' failed", kv.key, kv.value);
-	  /* Need to recycle one (or more cache) entries? */
-	  if (hsm->cache_size > hsm->cache_limit)
-	    {
-	      int free_index = hsm->last_index;
-
-	      while (free_index != ~0)
-		{
-		  /* pick the LRU */
-		  ce = pool_elt_at_index (hsm->cache_pool, free_index);
-		  free_index = ce->prev_index;
-		  /* Which could be in use... */
-		  if (ce->inuse)
-		    {
-		      if (hsm->debug_level > 1)
-			clib_warning ("index %d in use refcnt %d",
-				      ce - hsm->cache_pool, ce->inuse);
-		    }
-		  kv.key = (u64) (ce->filename);
-		  kv.value = ~0ULL;
-		  if (BV (clib_bihash_add_del) (&hsm->name_to_data, &kv,
-						0 /* is_add */ ) < 0)
-		    {
-		      clib_warning ("LRU delete '%s' FAILED!", ce->filename);
-		    }
-		  else if (hsm->debug_level > 1)
-		    clib_warning ("LRU delete '%s' ok", ce->filename);
-
-		  lru_remove (hsm, ce);
-		  hsm->cache_size -= vec_len (ce->data);
-		  hsm->cache_evictions++;
-		  vec_free (ce->filename);
-		  vec_free (ce->data);
-		  if (hsm->debug_level > 1)
-		    clib_warning ("pool put index %d", ce - hsm->cache_pool);
-		  pool_put (hsm->cache_pool, ce);
-		  if (hsm->cache_size < hsm->cache_limit)
-		    break;
-		}
-	    }
-
-	  /* Read the file */
-	  error = clib_file_contents ((char *) (hs->path), &hs->data);
-	  if (error)
-	    {
-	      clib_warning ("Error reading '%s'", hs->path);
-	      clib_error_report (error);
-	      sc = HTTP_STATUS_INTERNAL_ERROR;
-	      hss_cache_unlock ();
-	      goto done;
-	    }
-	  /* Create a cache entry for it */
-	  pool_get_zero (hsm->cache_pool, ce);
-	  ce->filename = vec_dup (hs->path);
-	  ce->data = hs->data;
-	  hs->cache_pool_index = ce - hsm->cache_pool;
-	  ce->inuse++;
-	  if (hsm->debug_level > 1)
-	    clib_warning ("index %d refcnt now %d", hs->cache_pool_index,
-			  ce->inuse);
-	  lru_add (hsm, ce, vlib_time_now (vlib_get_main ()));
-	  kv.key = (u64) vec_dup (hs->path);
-	  kv.value = ce - hsm->cache_pool;
-	  /* Add to the lookup table */
-	  if (hsm->debug_level > 1)
-	    clib_warning ("add '%s' value %lld", kv.key, kv.value);
-
-	  if (BV (clib_bihash_add_del) (&hsm->name_to_data, &kv,
-					1 /* is_add */ ) < 0)
-	    {
-	      clib_warning ("BUG: add failed!");
-	    }
-	  hsm->cache_size += vec_len (ce->data);
-
-	  hss_cache_unlock ();
+	  sc = HTTP_STATUS_INTERNAL_ERROR;
+	  goto done;
 	}
-      hs->data_offset = 0;
     }
 
+  hs->cache_pool_index = ce_index;
+
 done:
 
   start_send_data (hs, sc);
   if (!hs->data)
     hss_session_disconnect_transport (hs);
 
-  return sc;
+  return 0;
+}
+
+static int
+handle_request (hss_session_t *hs, http_req_method_t rt, u8 *request)
+{
+  hss_main_t *hsm = &hss_main;
+
+  if (!try_url_handler (hsm, hs, rt, request))
+    return 0;
+
+  if (!try_file_handler (hsm, hs, rt, request))
+    return 0;
+
+  /* Handler did not find anything return 404 */
+  start_send_data (hs, HTTP_STATUS_NOT_FOUND);
+  hss_session_disconnect_transport (hs);
+
+  return 0;
 }
 
 static int
@@ -732,6 +493,7 @@ hss_add_segment_callback (u32 client_index, u64 segment_handle)
 static void
 hss_ts_cleanup (session_t *s, session_cleanup_ntf_t ntf)
 {
+  hss_main_t *hsm = &hss_main;
   hss_session_t *hs;
 
   if (ntf == SESSION_CLEANUP_TRANSPORT)
@@ -741,7 +503,19 @@ hss_ts_cleanup (session_t *s, session_cleanup_ntf_t ntf)
   if (!hs)
     return;
 
-  hss_detach_cache_entry (hs);
+  if (hs->cache_pool_index != ~0)
+    {
+      hss_cache_detach_entry (&hsm->cache, hs->cache_pool_index);
+      hs->cache_pool_index = ~0;
+    }
+
+  if (hs->free_data)
+    vec_free (hs->data);
+  hs->data = 0;
+  hs->data_offset = 0;
+  hs->free_data = 0;
+  vec_free (hs->path);
+
   hss_session_free (hs);
 }
 
@@ -873,8 +647,6 @@ hss_create (vlib_main_t *vm)
   num_threads = 1 /* main thread */  + vtm->n_threads;
   vec_validate (hsm->sessions, num_threads - 1);
 
-  clib_spinlock_init (&hsm->cache_lock);
-
   if (hss_attach ())
     {
       clib_warning ("failed to attach server");
@@ -886,8 +658,8 @@ hss_create (vlib_main_t *vm)
       return -1;
     }
 
-  /* Init path-to-cache hash table */
-  BV (clib_bihash_init) (&hsm->name_to_data, "http cache", 128, 32 << 20);
+  if (hsm->www_root)
+    hss_cache_init (&hsm->cache, hsm->cache_size, hsm->debug_level);
 
   if (hsm->enable_url_handlers)
     hss_url_handlers_init (hsm);
@@ -911,8 +683,7 @@ hss_create_command_fn (vlib_main_t *vm, unformat_input_t *input,
   hsm->prealloc_fifos = 0;
   hsm->private_segment_size = 0;
   hsm->fifo_size = 0;
-  /* 10mb cache limit, before LRU occurs */
-  hsm->cache_limit = 10 << 20;
+  hsm->cache_size = 10 << 20;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -931,7 +702,7 @@ hss_create_command_fn (vlib_main_t *vm, unformat_input_t *input,
       else if (unformat (line_input, "fifo-size %d", &hsm->fifo_size))
 	hsm->fifo_size <<= 10;
       else if (unformat (line_input, "cache-size %U", unformat_memory_size,
-			 &hsm->cache_limit))
+			 &hsm->cache_size))
 	;
       else if (unformat (line_input, "uri %s", &hsm->uri))
 	;
@@ -964,7 +735,7 @@ no_input:
       goto done;
     }
 
-  if (hsm->cache_limit < (128 << 10))
+  if (hsm->cache_size < (128 << 10))
     {
       error = clib_error_return (0, "cache-size must be at least 128kb");
       vec_free (hsm->www_root);
@@ -1005,25 +776,6 @@ VLIB_CLI_COMMAND (hss_create_command, static) = {
   .function = hss_create_command_fn,
 };
 
-/** \brief format a file cache entry
- */
-static u8 *
-format_hss_cache_entry (u8 *s, va_list *args)
-{
-  hss_cache_entry_t *ep = va_arg (*args, hss_cache_entry_t *);
-  f64 now = va_arg (*args, f64);
-
-  /* Header */
-  if (ep == 0)
-    {
-      s = format (s, "%40s%12s%20s", "File", "Size", "Age");
-      return s;
-    }
-  s = format (s, "%40s%12lld%20.2f", ep->filename, vec_len (ep->data),
-	      now - ep->last_used);
-  return s;
-}
-
 static u8 *
 format_hss_session (u8 *s, va_list *args)
 {
@@ -1040,13 +792,8 @@ static clib_error_t *
 hss_show_command_fn (vlib_main_t *vm, unformat_input_t *input,
 		     vlib_cli_command_t *cmd)
 {
+  int verbose = 0, show_cache = 0, show_sessions = 0;
   hss_main_t *hsm = &hss_main;
-  hss_cache_entry_t *ep, **entries = 0;
-  int verbose = 0;
-  int show_cache = 0;
-  int show_sessions = 0;
-  u32 index;
-  f64 now;
 
   if (hsm->www_root == 0)
     return clib_error_return (0, "Static server disabled");
@@ -1069,32 +816,7 @@ hss_show_command_fn (vlib_main_t *vm, unformat_input_t *input,
     return clib_error_return (0, "specify one or more of cache, sessions");
 
   if (show_cache)
-    {
-      if (verbose == 0)
-	{
-	  vlib_cli_output
-	    (vm, "www_root %s, cache size %lld bytes, limit %lld bytes, "
-	     "evictions %lld",
-	     hsm->www_root, hsm->cache_size, hsm->cache_limit,
-	     hsm->cache_evictions);
-	  return 0;
-	}
-
-      now = vlib_time_now (vm);
-
-      vlib_cli_output (vm, "%U", format_hss_cache_entry, 0 /* header */, now);
-
-      for (index = hsm->first_index; index != ~0;)
-	{
-	  ep = pool_elt_at_index (hsm->cache_pool, index);
-	  index = ep->next_index;
-	  vlib_cli_output (vm, "%U", format_hss_cache_entry, ep, now);
-	}
-
-      vlib_cli_output (vm, "%40s%12lld", "Total Size", hsm->cache_size);
-
-      vec_free (entries);
-    }
+    vlib_cli_output (vm, "%U", format_hss_cache, &hsm->cache, verbose);
 
   if (show_sessions)
     {
@@ -1102,14 +824,11 @@ hss_show_command_fn (vlib_main_t *vm, unformat_input_t *input,
       hss_session_t *hs;
       int i, j;
 
-      hss_cache_lock ();
 
       for (i = 0; i < vec_len (hsm->sessions); i++)
 	{
 	  pool_foreach (hs, hsm->sessions[i])
-           {
             vec_add1 (session_indices, hs - hsm->sessions[i]);
-          }
 
 	  for (j = 0; j < vec_len (session_indices); j++)
 	    {
@@ -1120,7 +839,6 @@ hss_show_command_fn (vlib_main_t *vm, unformat_input_t *input,
 	    }
 	  vec_reset_length (session_indices);
 	}
-      hss_cache_unlock ();
       vec_free (session_indices);
     }
   return 0;
@@ -1147,48 +865,13 @@ hss_clear_cache_command_fn (vlib_main_t *vm, unformat_input_t *input,
 			    vlib_cli_command_t *cmd)
 {
   hss_main_t *hsm = &hss_main;
-  hss_cache_entry_t *ce;
-  u32 free_index;
   u32 busy_items = 0;
-  BVT (clib_bihash_kv) kv;
 
   if (hsm->www_root == 0)
     return clib_error_return (0, "Static server disabled");
 
-  hss_cache_lock ();
+  busy_items = hss_cache_clear (&hsm->cache);
 
-  /* Walk the LRU list to find active entries */
-  free_index = hsm->last_index;
-  while (free_index != ~0)
-    {
-      ce = pool_elt_at_index (hsm->cache_pool, free_index);
-      free_index = ce->prev_index;
-      /* Which could be in use... */
-      if (ce->inuse)
-	{
-	  busy_items++;
-	  free_index = ce->next_index;
-	  continue;
-	}
-      kv.key = (u64) (ce->filename);
-      kv.value = ~0ULL;
-      if (BV (clib_bihash_add_del) (&hsm->name_to_data, &kv,
-				    0 /* is_add */ ) < 0)
-	{
-	  clib_warning ("BUG: cache clear delete '%s' FAILED!", ce->filename);
-	}
-
-      lru_remove (hsm, ce);
-      hsm->cache_size -= vec_len (ce->data);
-      hsm->cache_evictions++;
-      vec_free (ce->filename);
-      vec_free (ce->data);
-      if (hsm->debug_level > 1)
-	clib_warning ("pool put index %d", ce - hsm->cache_pool);
-      pool_put (hsm->cache_pool, ce);
-      free_index = hsm->last_index;
-    }
-  hss_cache_unlock ();
   if (busy_items > 0)
     vlib_cli_output (vm, "Note: %d busy items still in cache...", busy_items);
   else
@@ -1220,7 +903,6 @@ hss_main_init (vlib_main_t *vm)
 
   hsm->app_index = ~0;
   hsm->vlib_main = vm;
-  hsm->first_index = hsm->last_index = ~0;
 
   return 0;
 }