CSIT-PERF: New trigger design

- CSIT-1128
- This patch is implementing processing of TAGS specified
as part of commit message

Change-Id: Iad0923d22fae369dfe923a917e1c0d47d5cbf656
Signed-off-by: Peter Mikus <pmikus@cisco.com>

4 files changed, 260 insertions, 90 deletions

diff --git a/bootstrap-verify-perf-DPDK.sh b/bootstrap-verify-perf-DPDK.sh
index b75caebefc..f15a38c2f4 100755
--- a/bootstrap-verify-perf-DPDK.sh
+++ b/bootstrap-verify-perf-DPDK.sh
@@ -128,19 +128,54 @@ case "$TEST_TAG" in
              'mrrANDimixAND2t2c'
              'mrrANDimixAND4t4c')
         ;;
-    VERIFY-PERF-MRR )
-        TAGS=('mrrAND1t1c'
-              'mrrAND2t2c')
+    VERIFY-PERF-PATCH )
+        if [[ -z "$TEST_TAG_STRING" ]]; then
+            # If nothing is specified, we will run pre-selected tests by
+            # following tags. Items of array will be concatenated by OR in Robot
+            # Framework.
+            TEST_TAG_ARRAY=('mrrANDnic_intel-x710AND1t1cAND64bANDip4base'
+                            'mrrANDnic_intel-x710AND1t1cAND78bANDip6base'
+                            'mrrANDnic_intel-x710AND1t1cAND64bANDl2bdbase')
+        else
+            # If trigger contains tags, split them into array.
+            TEST_TAG_ARRAY=(${TEST_TAG_STRING//:/ })
+        fi
+
+        TAGS=()
+
+        for TAG in "${TEST_TAG_ARRAY[@]}"; do
+            if [[ ${TAG} == "!"* ]]; then
+                # Exclude tags are not prefixed.
+                TAGS+=("${TAG}")
+            else
+                # We will prefix with perftest to prevent running other tests
+                # (e.g. Functional).
+                prefix="perftestAND"
+                if [[ ${JOB_NAME} == vpp-* ]] ; then
+                    # Automatic prefixing for VPP jobs to limit the NIC used and
+                    # traffic evaluation to MRR.
+                    prefix="${prefix}mrrANDnic_intel-x710AND"
+                fi
+                TAGS+=("$prefix${TAG}")
+            fi
+        done
         ;;
     * )
         TAGS=('perftest')
 esac
 
-# Catenate TAG selections by 'OR'
-printf -v INCLUDES " --include %s " "${TAGS[@]}"
+# Catenate TAG selections
+EXPANDED_TAGS=()
+for TAG in "${TAGS[@]}"; do
+    if [[ ${TAG} == "!"* ]]; then
+        EXPANDED_TAGS+=(" --exclude ${TAG#$"!"} ")
+    else
+        EXPANDED_TAGS+=(" --include ${TAG} ")
+    fi
+done
 
 # Execute the test
-pybot ${PYBOT_ARGS}${INCLUDES} tests/
+pybot ${PYBOT_ARGS}${EXPANDED_TAGS[@]} tests/
 RETURN_STATUS=$(echo $?)
 
 # Archive JOB artifacts in jenkins
diff --git a/bootstrap-verify-perf-ligato.sh b/bootstrap-verify-perf-ligato.sh
index e72673d4f8..e3b130d3c2 100644
--- a/bootstrap-verify-perf-ligato.sh
+++ b/bootstrap-verify-perf-ligato.sh
@@ -258,97 +258,54 @@ case "$TEST_TAG" in
              'mrrANDimixAND2t2cANDmemif'
              'mrrANDimixAND4t4cANDmemif')
         ;;
-    VERIFY-PERF-NDRDISC )
-        TAGS=('ndrdiscAND1t1c'
-              'ndrdiscAND2t2c')
-        ;;
-    VERIFY-PERF-PDRDISC )
-        TAGS=('pdrdiscAND1t1c'
-              'pdrdiscAND2t2c')
-        ;;
-    VERIFY-PERF-MRR )
-        TAGS=('mrrAND1t1c'
-              'mrrAND2t2c')
-        ;;
-    VERIFY-PERF-IP4 )
-        TAGS=('mrrANDnic_intel-x520-da2AND1t1cANDip4base'
-              'mrrANDnic_intel-x520-da2AND1t1cANDip4fwdANDfib_2m')
-        ;;
-    VERIFY-PERF-IP6 )
-        TAGS=('mrrANDnic_intel-x520-da2AND1t1cANDip6base'
-              'mrrANDnic_intel-x520-da2AND1t1cANDip6fwdANDfib_2m')
-        ;;
-    VERIFY-PERF-L2 )
-        TAGS=('mrrANDnic_intel-x520-da2AND1t1cANDl2xcbase'
-              'mrrANDnic_intel-x520-da2AND1t1cANDl2bdbase')
-        ;;
-    VERIFY-PERF-LISP )
-        TAGS=('pdrchkANDnic_intel-x520-da2AND1t1cANDlisp')
-        ;;
-    VERIFY-PERF-VXLAN )
-        TAGS=('pdrchkANDnic_intel-x520-da2AND1t1cANDvxlan')
-        ;;
-    VERIFY-PERF-VHOST )
-        TAGS=('pdrdiscANDnic_intel-x520-da2AND1t1cANDvhost')
-        ;;
-    VERIFY-PERF-MEMIF )
-        TAGS=('pdrdiscANDnic_intel-x520-da2AND1t1cANDmemif'
-              'pdrdiscANDnic_intel-x520-da2AND2t2cANDmemif'
-              'mrrANDnic_intel-x520-da2AND1t1cANDmemif'
-              'mrrANDnic_intel-x520-da2AND2t2cANDmemif')
-        ;;
-    VERIFY-PERF-IPSECHW )
-        TAGS=('pdrdiscANDnic_intel-xl710AND1t1cANDipsechw'
-              'pdrdiscANDnic_intel-xl710AND2t2cANDipsechw'
-              'mrrANDnic_intel-xl710AND1t1cANDipsechw'
-              'mrrANDnic_intel-xl710AND2t2cANDipsechw')
-        ;;
-    VPP-VERIFY-PERF-IP4 )
-        TAGS=('mrrANDnic_intel-x520-da2AND1t1cANDip4base'
-              'mrrANDnic_intel-x520-da2AND1t1cANDip4fwdANDfib_2m')
-        ;;
-    VPP-VERIFY-PERF-IP6 )
-        TAGS=('mrrANDnic_intel-x520-da2AND1t1cANDip6base'
-              'mrrANDnic_intel-x520-da2AND1t1cANDip6fwdANDfib_2m')
-        ;;
-    VPP-VERIFY-PERF-L2 )
-        TAGS=('mrrANDnic_intel-x520-da2AND1t1cANDl2xcbase'
-              'mrrANDnic_intel-x520-da2AND1t1cANDl2bdbase')
-        ;;
-    VPP-VERIFY-PERF-LISP )
-        TAGS=('pdrchkANDnic_intel-x520-da2AND1t1cANDlisp')
-        ;;
-    VPP-VERIFY-PERF-VXLAN )
-        TAGS=('pdrchkANDnic_intel-x520-da2AND1t1cANDvxlan')
-        ;;
-    VPP-VERIFY-PERF-VHOST )
-        TAGS=('pdrdiscANDnic_intel-x520-da2AND1t1cANDvhost')
-        ;;
-    VPP-VERIFY-PERF-MEMIF )
-        TAGS=('pdrdiscANDnic_intel-x520-da2AND1t1cANDmemif'
-              'pdrdiscANDnic_intel-x520-da2AND2t2cANDmemif'
-              'mrrANDnic_intel-x520-da2AND1t1cANDmemif'
-              'mrrANDnic_intel-x520-da2AND2t2cANDmemif')
-        ;;
-    VPP-VERIFY-PERF-ACL )
-        TAGS=('pdrdiscANDnic_intel-x520-da2AND1t1cANDacl'
-              'pdrdiscANDnic_intel-x520-da2AND2t2cANDacl')
-        ;;
-    VPP-VERIFY-PERF-IPSECHW )
-        TAGS=('pdrdiscANDnic_intel-xl710AND1t1cANDipsechw'
-              'pdrdiscANDnic_intel-xl710AND2t2cANDipsechw'
-              'mrrANDnic_intel-xl710AND1t1cANDipsechw'
-              'mrrANDnic_intel-xl710AND2t2cANDipsechw')
+    VERIFY-PERF-PATCH )
+        if [[ -z "$TEST_TAG_STRING" ]]; then
+            # If nothing is specified, we will run pre-selected tests by
+            # following tags. Items of array will be concatenated by OR in Robot
+            # Framework.
+            TEST_TAG_ARRAY=('mrrANDnic_intel-x710AND1t1cAND64bANDip4base'
+                            'mrrANDnic_intel-x710AND1t1cAND78bANDip6base'
+                            'mrrANDnic_intel-x710AND1t1cAND64bANDl2bdbase')
+        else
+            # If trigger contains tags, split them into array.
+            TEST_TAG_ARRAY=(${TEST_TAG_STRING//:/ })
+        fi
+
+        TAGS=()
+
+        for TAG in "${TEST_TAG_ARRAY[@]}"; do
+            if [[ ${TAG} == "!"* ]]; then
+                # Exclude tags are not prefixed.
+                TAGS+=("${TAG}")
+            else
+                # We will prefix with perftest to prevent running other tests
+                # (e.g. Functional).
+                prefix="perftestAND"
+                if [[ ${JOB_NAME} == vpp-* ]] ; then
+                    # Automatic prefixing for VPP jobs to limit the NIC used and
+                    # traffic evaluation to MRR.
+                    prefix="${prefix}mrrANDnic_intel-x710AND"
+                fi
+                TAGS+=("$prefix${TAG}")
+            fi
+        done
         ;;
     * )
         TAGS=('perftest')
 esac
 
-# Catenate TAG selections by 'OR'
-printf -v INCLUDES " --include %s " "${TAGS[@]}"
+# Catenate TAG selections
+EXPANDED_TAGS=()
+for TAG in "${TAGS[@]}"; do
+    if [[ ${TAG} == "!"* ]]; then
+        EXPANDED_TAGS+=(" --exclude ${TAG#$"!"} ")
+    else
+        EXPANDED_TAGS+=(" --include ${TAG} ")
+    fi
+done
 
 # Execute the test
-pybot ${PYBOT_ARGS}${INCLUDES} tests/
+pybot ${PYBOT_ARGS}${EXPANDED_TAGS[@]} tests/
 RETURN_STATUS=$(echo $?)
 
 # Archive JOB artifacts in jenkins
diff --git a/bootstrap-verify-perf.sh b/bootstrap-verify-perf.sh
index 7b1aa26e69..f2e4df58de 100755
--- a/bootstrap-verify-perf.sh
+++ b/bootstrap-verify-perf.sh
@@ -276,6 +276,38 @@ case "$TEST_TAG" in
         TAGS=('mrrANDsrv6AND1t1c'
               'mrrANDsrv6AND2t2c')
         ;;
+    VERIFY-PERF-PATCH )
+        if [[ -z "$TEST_TAG_STRING" ]]; then
+            # If nothing is specified, we will run pre-selected tests by
+            # following tags. Items of array will be concatenated by OR in Robot
+            # Framework.
+            TEST_TAG_ARRAY=('mrrANDnic_intel-x710AND1t1cAND64bANDip4base'
+                            'mrrANDnic_intel-x710AND1t1cAND78bANDip6base'
+                            'mrrANDnic_intel-x710AND1t1cAND64bANDl2bdbase')
+        else
+            # If trigger contains tags, split them into array.
+            TEST_TAG_ARRAY=(${TEST_TAG_STRING//:/ })
+        fi
+
+        TAGS=()
+
+        for TAG in "${TEST_TAG_ARRAY[@]}"; do
+            if [[ ${TAG} == "!"* ]]; then
+                # Exclude tags are not prefixed.
+                TAGS+=("${TAG}")
+            else
+                # We will prefix with perftest to prevent running other tests
+                # (e.g. Functional).
+                prefix="perftestAND"
+                if [[ ${JOB_NAME} == vpp-* ]] ; then
+                    # Automatic prefixing for VPP jobs to limit the NIC used and
+                    # traffic evaluation to MRR.
+                    prefix="${prefix}mrrANDnic_intel-x710AND"
+                fi
+                TAGS+=("$prefix${TAG}")
+            fi
+        done
+        ;;
     * )
         TAGS=('perftest')
 esac
diff --git a/docs/perf-triggers-design.rst b/docs/perf-triggers-design.rst
new file mode 100644
index 0000000000..4a0232cde0
--- /dev/null
+++ b/docs/perf-triggers-design.rst
@@ -0,0 +1,146 @@
+Introduction
+------------
+
+Previous gerrit triggers for performance tests (
+*vpp-csit-verify-hw-perf-{branch}* and *csit-vpp-verify-hw-perf-{branch}*) are
+listed in [1]_ with jjb definition in [2]_. Mapping of triggers to CSIT test RF
+tags [4]_ driving the selection of test cases for execution is listed in [3]_.
+
+Previous mappings of trigger to RF tags
+---------------------------------------
+
+(vpp-csit job) vpp-verify-perf-{**keyword**} OR (csit-vpp job) verify-perf-{**keyword**}
+  - **acl**:
+     - 'mrrANDnic_intel-x520-da2AND1t1cANDacl'
+     - 'mrrANDnic_intel-x520-da2AND2t2cANDacl'
+  - **ip4**:
+     - 'mrrANDnic_intel-x520-da2AND1t1cANDip4base'
+     - 'mrrANDnic_intel-x520-da2AND1t1cANDip4fwdANDfib_2m'
+  - **ip6**
+     - 'mrrANDnic_intel-x520-da2AND1t1cANDip6base'
+     - 'mrrANDnic_intel-x520-da2AND1t1cANDip6fwdANDfib_2m'
+  - **ipsechw**
+     - 'pdrdiscANDnic_intel-xl710AND1t1cANDipsechw'
+     - 'pdrdiscANDnic_intel-xl710AND2t2cANDipsechw'
+     - 'mrrANDnic_intel-xl710AND1t1cANDipsechw'
+     - 'mrrANDnic_intel-xl710AND2t2cANDipsechw'
+  - **l2**
+     - 'mrrANDnic_intel-x520-da2AND1t1cANDl2xcbase'
+     - 'mrrANDnic_intel-x520-da2AND1t1cANDl2bdbase'
+     - 'mrrANDnic_intel-x520-da2AND1t1cANDdot1q'
+     - '!lbond_dpdk'
+  - **lisp**
+     - 'mrrANDnic_intel-x520-da2AND1t1cANDlisp'
+  - **memif**
+     - 'pdrdiscANDnic_intel-x520-da2AND1t1cANDmemif'
+     - 'pdrdiscANDnic_intel-x520-da2AND2t2cANDmemif'
+     - 'mrrANDnic_intel-x520-da2AND1t1cANDmemif'
+     - 'mrrANDnic_intel-x520-da2AND2t2cANDmemif'
+  - **vhost**
+     - 'mrrANDnic_intel-x520-da2AND1t1cANDvhost'
+     - '!lbond_dpdk'
+  - **vxlan**
+     - 'mrrANDnic_intel-x520-da2AND1t1cANDvxlan'
+  - **srv6**
+     - 'mrrANDsrv6AND1t1c'
+     - 'mrrANDsrv6AND2t2c'
+
+Proposal for mapping triggers to RF tags
+----------------------------------------
+
+*Goal*
+  make it simpler to use, scalable, parametrize and prepare for full CI/CD
+  automation.
+
+*Syntax*
+  trigger_keyword [{tag1} {tag2}AND{tag3} !{tag4} !{tag5}]
+
+*Inputs*
+  - trigger_keyword for vpp-* jobs: 'perftest'
+  - trigger_keyword for csit-* jobs: 'csit-perftest'
+  - tags: existing CSIT tags [4]_ i.e. ip4base, ip6base, iacldst, memif
+
+Set of default tags appended to user input, under control by CSIT
+  - always-on for vpp-csit*.job: 'mrr' 'nic_intel_x710-da2' '1t1c'
+  - if input with no tags, following set applied:
+     - 'mrrANDnic_intel-x710AND1t1cAND64bANDip4base'
+     - 'mrrANDnic_intel-x710AND1t1cAND78bANDip6base'
+     - 'mrrANDnic_intel-x710AND1t1cAND64bANDl2bdbase'
+
+Examples
+  input: 'perftest'
+    expanded: 'mrrANDnic_intel_x710-da2AND1t1cAND64bANDl2bdbase mrrANDnic_intel_x710-da2AND1t1cAND64bANDip4base mrrANDnic_intel_x710-da2AND1t1cAND78bANDip6base'
+  input: 'perftest l2bdbase l2xcbase'
+    expanded: 'mrrANDnic_intel_x710-da2ANDl2bdbase mrrANDnic_intel_x710-da2ANDl2xcbase'
+  input: 'perftest ip4base !feature'
+    expanded: 'mrrANDnic_intel_x710-da2ANDip4base' not 'feature'
+  input: 'perftest ip4base !feature !lbond_dpdk'
+    expanded: 'mrrANDnic_intel_x710-da2ANDip4base' not 'feature' not 'lbond_dpdk'
+  input: 'perftestxyx ip4base !feature !lbond_dpdk'
+    invalid: detected as error
+  input: 'perftestip4base !feature !lbond_dpdk'
+    invalid: detected as error
+  input: 'perftest ip4base!feature!lbond_dpdk'
+    invalid expand: 'mrrANDnic_intel_x710-da2ANDip4base!feature!lbond_dpdk'
+    execution of RobotFramework will fail
+
+Constrains
+  Trigger keyword must be different for every job to avoid running multiple jobs
+  at once. Trigger keyword must not be substring of job name or any other
+  message printed by JJB bach to gerrit message which can lead to recursive
+  execution.
+
+Implementation
+--------------
+
+Implementation is separated into two projects.
+
+CI-MANGEMENT
+~~~~~~~~~~~~
+
+https://gerrit.fd.io/r/#/c/13027/
+
+Implementing new gerrit keyword `csit-perftest` in JJB for
+*csit-vpp-perf-verify-{stream}* and `perftest` for
+*vpp-csit-verify-hw-perf-{stream}* performance jobs.
+
+::
+
+  if [[ ${GERRIT_EVENT_TYPE} == 'comment-added' ]]; then
+      TRIGGER=`echo ${GERRIT_EVENT_COMMENT_TEXT} \
+          | grep -oE '(perftest$|perftest[[:space:]].+$)'`
+  else
+      TRIGGER=''
+  fi
+  # Export test type
+  export TEST_TAG="VERIFY-PERF-PATCH"
+  # Export test tags as string
+  export TEST_TAG_STRING=${TRIGGER#$"perftest"}
+
+Code is automatically detecting trigger type and parse the gerrit comment
+massage. Stripped TAGs are exported as bash variable `$TEST_TAG_STRING` together
+with `$TEST_TAG`.
+
+CSIT
+~~~~
+
+https://gerrit.fd.io/r/#/c/13025/
+
+Implementing `$TEST_TAG_STRING` variable post processing. String of TAGs is
+automatically converted into array to be able to loop the items. If variable is
+empty default set of TAGs is applied.
+
+Array is then converted into Robot Framework parameter notation where every word
+means new `--include` parameter. Having multiple words (multiple includes) means
+logical OR in selection of test cases and could be applied to add additional
+test cases that have no common more specific match. See exmaples section for
+more details. Script also detects an exclamation mark before TAG that is
+translated to as a `--exclude` parameter.
+
+References
+----------
+
+.. [1] https://wiki.fd.io/view/CSIT/Jobs
+.. [2] https://git.fd.io/ci-management/tree/jjb/vpp/vpp.yaml#n762
+.. [3] https://git.fd.io/csit/tree/bootstrap-verify-perf.sh#n235
+.. [4] https://git.fd.io/csit/tree/docs/tag_documentation.rst
\ No newline at end of file