Tests: Raise exception if API cli_inband command fails.

* Configure tests to raise exception if cli_inband fails.
* Fix failing tests.
* Add filename detail to pcap.stat clib_error_return for debugging.

Note: this change identifies spurious issues with packet-generator such as:
  CliFailedCommandError: packet-generator capture: pcap file
  '/tmp/vpp-unittest-Test6RD-v09RPA/pg0_out.pcap' does not exist.

These issues resolve themselves on remaining test passes.

Change-Id: Iecbd09daee954d892306d11baff3864a43c5b603
Signed-off-by: Paul Vinciguerra <pvinci@vinciconsulting.com>

7 files changed, 29 insertions, 18 deletions

diff --git a/test/template_ipsec.py b/test/template_ipsec.py
index c623d6a4d9b..61ac0b5cd22 100644
--- a/test/template_ipsec.py
+++ b/test/template_ipsec.py
@@ -442,7 +442,7 @@ class IpsecTra4(object):
                     raise
         finally:
             self.logger.info(self.vapi.ppcli("show error"))
-            self.logger.info(self.vapi.ppcli("show ipsec"))
+            self.logger.info(self.vapi.ppcli("show ipsec all"))
 
         pkts = p.tra_sa_in.get_stats()['packets']
         self.assertEqual(pkts, count,
@@ -495,7 +495,7 @@ class IpsecTra6(object):
                     raise
         finally:
             self.logger.info(self.vapi.ppcli("show error"))
-            self.logger.info(self.vapi.ppcli("show ipsec"))
+            self.logger.info(self.vapi.ppcli("show ipsec all"))
 
         pkts = p.tra_sa_in.get_stats()['packets']
         self.assertEqual(pkts, count,
@@ -598,7 +598,7 @@ class IpsecTun4(object):
 
         finally:
             self.logger.info(self.vapi.ppcli("show error"))
-            self.logger.info(self.vapi.ppcli("show ipsec"))
+            self.logger.info(self.vapi.ppcli("show ipsec all"))
 
         self.verify_counters(p, count)
 
@@ -636,7 +636,7 @@ class IpsecTun4(object):
                     raise
         finally:
             self.logger.info(self.vapi.ppcli("show error"))
-            self.logger.info(self.vapi.ppcli("show ipsec"))
+            self.logger.info(self.vapi.ppcli("show ipsec all"))
 
         self.verify_counters(p, count)
 
@@ -705,7 +705,7 @@ class IpsecTun6(object):
                     raise
         finally:
             self.logger.info(self.vapi.ppcli("show error"))
-            self.logger.info(self.vapi.ppcli("show ipsec"))
+            self.logger.info(self.vapi.ppcli("show ipsec all"))
         self.verify_counters(p, count)
 
     def verify_tun_46(self, p, count=1):
@@ -744,7 +744,7 @@ class IpsecTun6(object):
                     raise
         finally:
             self.logger.info(self.vapi.ppcli("show error"))
-            self.logger.info(self.vapi.ppcli("show ipsec"))
+            self.logger.info(self.vapi.ppcli("show ipsec all"))
         self.verify_counters(p, count)
 
 
diff --git a/test/test_gbp.py b/test/test_gbp.py
index 33de9b574bb..e386a31e039 100644
--- a/test/test_gbp.py
+++ b/test/test_gbp.py
@@ -2438,7 +2438,7 @@ class TestGBP(VppTestCase):
     def test_gbp_learn_l3(self):
         """ GBP L3 Endpoint Learning """
 
-        self.vapi.cli("set logging class gbp debug")
+        self.vapi.cli("set logging class gbp level debug")
 
         ep_flags = VppEnum.vl_api_gbp_endpoint_flags_t
         routed_dst_mac = "00:0c:0c:0c:0c:0c"
@@ -2877,7 +2877,7 @@ class TestGBP(VppTestCase):
     def test_gbp_redirect(self):
         """ GBP Endpoint Redirect """
 
-        self.vapi.cli("set logging class gbp debug")
+        self.vapi.cli("set logging class gbp level debug")
 
         ep_flags = VppEnum.vl_api_gbp_endpoint_flags_t
         routed_dst_mac = "00:0c:0c:0c:0c:0c"
@@ -3449,7 +3449,7 @@ class TestGBP(VppTestCase):
         """ GBP L3 Out """
 
         ep_flags = VppEnum.vl_api_gbp_endpoint_flags_t
-        self.vapi.cli("set logging class gbp debug")
+        self.vapi.cli("set logging class gbp level debug")
 
         routed_dst_mac = "00:0c:0c:0c:0c:0c"
         routed_src_mac = "00:22:bd:f8:19:ff"
diff --git a/test/test_ipsec_nat.py b/test/test_ipsec_nat.py
index d97fb791cf5..3209def213b 100644
--- a/test/test_ipsec_nat.py
+++ b/test/test_ipsec_nat.py
@@ -57,7 +57,7 @@ class IPSecNATTestCase(TemplateIpsec):
 
         p = self.ipv4_params
         self.config_esp_tun(p)
-        self.logger.info(self.vapi.ppcli("show ipsec"))
+        self.logger.info(self.vapi.ppcli("show ipsec all"))
 
         d = DpoProto.DPO_PROTO_IP6 if p.is_ipv6 else DpoProto.DPO_PROTO_IP4
         VppIpRoute(self,  p.remote_tun_if_host, p.addr_len,
diff --git a/test/test_l2bd_arp_term.py b/test/test_l2bd_arp_term.py
index a3feb99a9c7..fa67acc18ec 100644
--- a/test/test_l2bd_arp_term.py
+++ b/test/test_l2bd_arp_term.py
@@ -70,7 +70,10 @@ class TestL2bdArpTerm(VppTestCase):
 
     def show_commands_at_teardown(self):
         self.logger.info(self.vapi.ppcli("show l2fib verbose"))
-        self.logger.info(self.vapi.ppcli("show bridge-domain 1 detail"))
+        # many tests delete bridge-domain 1 as the last task.  don't output
+        # the details of a non-existent bridge-domain.
+        if self.vapi.l2_fib_table_dump(bd_id=1):
+            self.logger.info(self.vapi.ppcli("show bridge-domain 1 detail"))
 
     def add_del_arp_term_hosts(self, entries, bd_id=1, is_add=1, is_ipv6=0):
         for e in entries:
diff --git a/test/test_punt.py b/test/test_punt.py
index 5dec066a8a0..f68a38f7515 100644
--- a/test/test_punt.py
+++ b/test/test_punt.py
@@ -846,8 +846,6 @@ class TestPunt(VppTestCase):
         self.logger.info(self.vapi.cli("show punt stats"))
         self.logger.info(self.vapi.cli("show punt db"))
 
-        self.vapi.cli("test punt clear")
-
 
 if __name__ == '__main__':
     unittest.main(testRunner=VppTestRunner)
diff --git a/test/test_vcl.py b/test/test_vcl.py
index 121b972d954..3e10764c1e3 100644
--- a/test/test_vcl.py
+++ b/test/test_vcl.py
@@ -257,8 +257,8 @@ class LDPCutThruTestCase(VCLTestCase):
                                               self.server_port]
 
     def tearDown(self):
-        self.cut_thru_tear_down()
         super(LDPCutThruTestCase, self).tearDown()
+        self.cut_thru_tear_down()
 
     def show_commands_at_teardown(self):
         self.logger.debug(self.vapi.cli("show session verbose 2"))
@@ -695,9 +695,8 @@ class LDPIpv6CutThruTestCase(VCLTestCase):
                                                    self.server_port]
 
     def tearDown(self):
-        self.cut_thru_tear_down()
-
         super(LDPIpv6CutThruTestCase, self).tearDown()
+        self.cut_thru_tear_down()
 
     def test_ldp_ipv6_cut_thru_echo(self):
         """ run LDP IPv6 cut thru echo test """
@@ -774,9 +773,8 @@ class VCLIpv6CutThruTestCase(VCLTestCase):
                                                    self.server_port]
 
     def tearDown(self):
-        self.cut_thru_tear_down()
-
         super(VCLIpv6CutThruTestCase, self).tearDown()
+        self.cut_thru_tear_down()
 
     def test_vcl_ipv6_cut_thru_echo(self):
         """ run VCL IPv6 cut thru echo test """
diff --git a/test/vpp_papi_provider.py b/test/vpp_papi_provider.py
index 291a904dd6f..ad6d6bcf574 100644
--- a/test/vpp_papi_provider.py
+++ b/test/vpp_papi_provider.py
@@ -170,6 +170,14 @@ defaultmapping = {
 }
 
 
+class CliFailedCommandError(Exception):
+    """ cli command failed."""
+
+
+class CliSyntaxError(Exception):
+    """ cli command had a syntax error."""
+
+
 class UnexpectedApiReturnValueError(Exception):
     """ exception raised when the API return value is unexpected """
     pass
@@ -366,6 +374,10 @@ class VppPapiProvider(object):
         cli += '\n'
         r = self.papi.cli_inband(cmd=cli)
         self.hook.after_cli(cli)
+        if r.retval == -156:
+            raise CliSyntaxError(r.reply)
+        if r.retval != 0:
+            raise CliFailedCommandError(r.reply)
         if hasattr(r, 'reply'):
             return r.reply