WATCHDOG - disable monitors when done to avoid crash when joining on

other threads

6 files changed, 43 insertions, 10 deletions

diff --git a/src/latency.cpp b/src/latency.cpp
index fd2a5b5a..acbe26d4 100644
--- a/src/latency.cpp
+++ b/src/latency.cpp
@@ -811,6 +811,11 @@ void  CLatencyManager::start(int iter, TrexWatchDog *watchdog) {
         m_rx_check_manager.tw_drain();
     }
 
+    /* disable the monitor */
+    if (m_watchdog) {
+        m_watchdog->disable_monitor(m_watchdog_handle);
+    }
+
 }
 
 void  CLatencyManager::stop(){
diff --git a/src/main_dpdk.cpp b/src/main_dpdk.cpp
index 0de646b7..c72af57a 100644
--- a/src/main_dpdk.cpp
+++ b/src/main_dpdk.cpp
@@ -4074,6 +4074,9 @@ int CGlobalTRex::run_in_core(virtual_thread_id_t virt_core_id){
         lpt->start_generate_stateful(CGlobalInfo::m_options.out_file,*lp);
     }
 
+    /* done - remove this from the watchdog (we might wait on join for a long time) */
+    lpt->m_watchdog->disable_monitor(lpt->m_watchdog_handle);
+
     m_signal[virt_core_id]=1;
     return (0);
 }
diff --git a/src/rpc-server/trex_rpc_req_resp_server.cpp b/src/rpc-server/trex_rpc_req_resp_server.cpp
index d36753d4..033f265c 100644
--- a/src/rpc-server/trex_rpc_req_resp_server.cpp
+++ b/src/rpc-server/trex_rpc_req_resp_server.cpp
@@ -100,6 +100,9 @@ void TrexRpcServerReqRes::_rpc_thread_cb() {
 
     /* must be done from the same thread */
     zmq_close(m_socket);
+
+    /* done */
+    m_watchdog->disable_monitor(m_watchdog_handle);
 }
 
 bool
diff --git a/src/stateless/rx/trex_stateless_rx_core.cpp b/src/stateless/rx/trex_stateless_rx_core.cpp
index 95fcc1b0..8f2f5d64 100644
--- a/src/stateless/rx/trex_stateless_rx_core.cpp
+++ b/src/stateless/rx/trex_stateless_rx_core.cpp
@@ -182,6 +182,8 @@ void CRxCoreStateless::start(TrexWatchDog &watchdog) {
         count += try_rx();
     }
     rte_pause();
+
+    m_watchdog->disable_monitor(m_watchdog_handle);
 }
 
 void CRxCoreStateless::handle_rx_pkt(CLatencyManagerPerPortStl *lp, rte_mbuf_t *m) {
diff --git a/src/trex_watchdog.cpp b/src/trex_watchdog.cpp
index d38809fc..e78e8e6d 100644
--- a/src/trex_watchdog.cpp
+++ b/src/trex_watchdog.cpp
@@ -36,6 +36,8 @@ limitations under the License.
 #include <iostream>
 #include  <stdexcept>
 
+#define DISABLE_WATCHDOG_ON_GDB
+
 static TrexWatchDog::monitor_st *global_monitor;
 
 const char *get_exe_name();
@@ -166,6 +168,7 @@ int TrexWatchDog::register_monitor(const std::string &name, double timeout_sec)
     /* cannot add monitors while active */
     assert(m_active == false);
 
+    monitor.active       = true;   
     monitor.tid          = pthread_self();
     monitor.name         = name;
     monitor.timeout_sec  = timeout_sec;
@@ -197,16 +200,21 @@ int TrexWatchDog::register_monitor(const std::string &name, double timeout_sec)
 }
 
 /**
+ * will disable the monitor - it will no longer be watched
+ * 
+ */
+void TrexWatchDog::disable_monitor(int handle) {
+    assert(handle < m_monitors.size());
+
+    m_monitors[handle].active = false;
+}
+
+/**
  * thread safe function
  * 
  */
 void TrexWatchDog::tickle(int handle) {
 
-    /* ignore ticks if not active */
-    if (!m_active) {
-        return;
-    }
-
     assert(handle < m_monitors.size());
 
     /* not nesscary but write gets cache invalidate for nothing */
@@ -245,13 +253,13 @@ void TrexWatchDog::start() {
     assert(m_pending == 0);
 
     /* under GDB - disable the watchdog */
+    #ifdef DISABLE_WATCHDOG_ON_GDB
     if (ptrace(PTRACE_TRACEME, 0, NULL, 0) == -1) {
         printf("\n\n*** GDB detected - disabling watchdog... ***\n\n");
         return;
     }
+    #endif
 
-    register_signal();
-   
     m_active = true;
     m_thread = new std::thread(&TrexWatchDog::_main, this);
     if (!m_thread) {
@@ -267,8 +275,6 @@ void TrexWatchDog::stop() {
         delete m_thread;
         m_thread = NULL;
     }
-
-    m_monitors.clear();
 }
 
 
@@ -290,6 +296,12 @@ void TrexWatchDog::_main() {
         dsec_t now = now_sec();
 
         for (auto &monitor : m_monitors) {
+
+            /* skip non active monitors */
+            if (!monitor.active) {
+                continue;
+            }
+
             /* if its own - turn it off and write down the time */
             if (monitor.tickled) {
                 monitor.tickled = false;
diff --git a/src/trex_watchdog.h b/src/trex_watchdog.h
index 53a48217..63255180 100644
--- a/src/trex_watchdog.h
+++ b/src/trex_watchdog.h
@@ -75,6 +75,13 @@ public:
 
 
     /**
+     * disable a monitor - it will no longer be watched
+     * 
+     */
+    void disable_monitor(int handle);
+
+
+    /**
      * should be called by each thread on it's handle
      * 
      * @author imarom (31-May-16)
@@ -101,6 +108,7 @@ public:
     /* should be cache aligned to avoid false sharing */
     struct monitor_st {
         /* write fields are first */
+        volatile bool   active;
         volatile bool   tickled;
         dsec_t          ts;
 
@@ -110,7 +118,7 @@ public:
         std::string     name;
 
         /* for for a full cacheline */
-        uint8_t       pad[16];
+        uint8_t       pad[15];
     };