Trending: Memory consumption

Change-Id: I26a7d9e7ee0b1a64666c3ffbcfc424f04a52d046
Signed-off-by: Tibor Frank <tifrank@cisco.com>

5 files changed, 75 insertions, 10 deletions

diff --git a/resources/tools/presentation/generator_CPTA.py b/resources/tools/presentation/generator_CPTA.py
index 1e7719153f..a9c4d472ec 100644
--- a/resources/tools/presentation/generator_CPTA.py
+++ b/resources/tools/presentation/generator_CPTA.py
@@ -138,7 +138,8 @@ def generate_cpta(spec, data):
             css_file:
         css_file.write(THEME_OVERRIDES)
 
-    archive_input_data(spec)
+    if spec.configuration.get("archive-inputs", True):
+        archive_input_data(spec)
 
     logging.info("Done.")
 
diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py
index 119a3102e0..892dfe2a07 100644
--- a/resources/tools/presentation/input_data_files.py
+++ b/resources/tools/presentation/input_data_files.py
@@ -219,14 +219,15 @@ def download_and_unzip_data_file(spec, job, build, pid, log):
     else:
         return False
 
-    if spec.input["file-name"].endswith(".gz"):
-        if "docs.fd.io" in url:
-            execute_command("gzip --decompress --keep --force {0}".
-                            format(new_name))
-        else:
-            rename(new_name, new_name[:-3])
-            execute_command("gzip --keep {0}".format(new_name[:-3]))
-        build["file-name"] = new_name[:-3]
+    if spec.configuration.get("archive-inputs", True):
+        if spec.input["file-name"].endswith(".gz"):
+            if "docs.fd.io" in url:
+                execute_command("gzip --decompress --keep --force {0}".
+                                format(new_name))
+            else:
+                rename(new_name, new_name[:-3])
+                execute_command("gzip --keep {0}".format(new_name[:-3]))
+            build["file-name"] = new_name[:-3]
 
     if new_name.endswith(".zip"):
         if is_zipfile(new_name):
diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py
index 762afc4180..37532c83d2 100644
--- a/resources/tools/presentation/input_data_parser.py
+++ b/resources/tools/presentation/input_data_parser.py
@@ -22,6 +22,8 @@
 import multiprocessing
 import os
 import re
+import resource
+import objgraph
 import pandas as pd
 import logging
 
@@ -1329,6 +1331,11 @@ class InputData(object):
 
                 self._cfg.set_input_state(job, build_nr, result["state"])
 
+                logging.info("ru_maxrss = {0}".format(
+                    resource.getrusage(resource.RUSAGE_SELF).ru_maxrss))
+
+                logging.info(objgraph.most_common_types())
+
         logging.info("Done.")
 
     @staticmethod
diff --git a/resources/tools/presentation/requirements.txt b/resources/tools/presentation/requirements.txt
index 66a993df79..6ada198b70 100644
--- a/resources/tools/presentation/requirements.txt
+++ b/resources/tools/presentation/requirements.txt
@@ -8,3 +8,4 @@ numpy==1.16.4
 pandas==0.24.2
 plotly==3.3.0
 PTable==0.9.2
+objgraph==3.4.1
diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index a00fd52c2b..c98a51ae28 100644
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -75,6 +75,8 @@
 -
   type: "configuration"
 
+  archive-inputs: False
+
   mapping-file: "mapping_tc_names.yaml"
 
   ignore-list: "ignored_tcs.yaml"
@@ -206,19 +208,27 @@
           - 787
           - 788
           - 789
+          - 790 #
           - 791 #
+          - 792 #
           - 793
+          - 794 #
           - 795
           - 796
           - 797
           - 798
           - 799
+          - 800 #
           - 801 #
+          - 802 #
           - 803 #
+          - 804 #
           - 805
           - 806
+          - 807 #
           - 808 #
           - 809 #
+          - 810 #
           - 815
       csit-dpdk-perf-mrr-weekly-master:
         start: 73
@@ -263,19 +273,27 @@
           - 787
           - 788
           - 789
+          - 790 #
           - 791 #
+          - 792 #
           - 793
+          - 794 #
           - 795
           - 796
           - 797
           - 798
           - 799
+          - 800 #
           - 801 #
+          - 802 #
           - 803 #
+          - 804 #
           - 805
           - 806
+          - 807 #
           - 808 #
           - 809 #
+          - 810 #
           - 815
 
     plot-performance-trending-dpdk-3n-hsw:
@@ -318,19 +336,30 @@
           - 532
           - 533
           - 534
+          - 536
+          - 537
+          - 538
+          - 539
+          - 540 #
           - 541 #
-          - 542
+          - 542 #
           - 543 #
+          - 544 #
           - 545
           - 546
           - 547
           - 548
           - 549
+          - 550 #
           - 551 #
+          - 552 #
           - 553 #
+          - 554 #
           - 555
           - 556
+          - 557 #
           - 558 #
+          - 559 #
           - 560 #
           - 562 #
       csit-dpdk-perf-mrr-weekly-master-3n-skx:
@@ -371,19 +400,30 @@
           - 532
           - 533
           - 534
+          - 536
+          - 537
+          - 538
+          - 539
+          - 540 #
           - 541 #
           - 542 #
           - 543 #
+          - 544 #
           - 545
           - 546
           - 547
           - 548
           - 549
+          - 550 #
           - 551 #
+          - 552 #
           - 553 #
+          - 554 #
           - 555
           - 556
+          - 557 #
           - 558 #
+          - 559 #
           - 560 #
           - 562 #
 
@@ -623,19 +663,27 @@
         - 787
         - 788
         - 789
+        - 790 #
         - 791 #
+        - 792 #
         - 793
+        - 794 #
         - 795
         - 796
         - 797
         - 798
         - 799
+        - 800 #
         - 801 #
+        - 802 #
         - 803 #
+        - 804 #
         - 805
         - 806
+        - 807 #
         - 808 #
         - 809 #
+        - 810 #
         - 815
     csit-dpdk-perf-mrr-weekly-master:
       start: 73
@@ -679,19 +727,26 @@
         - 537
         - 538
         - 539
+        - 540 #
         - 541 #
         - 542 #
         - 543 #
+        - 544 #
         - 545
         - 546
         - 547
         - 548
         - 549
+        - 550 #
         - 551 #
+        - 552 #
         - 553 #
+        - 554 #
         - 555
         - 556
+        - 557 #
         - 558 #
+        - 559 #
         - 560 #
         - 562 #
     csit-dpdk-perf-mrr-weekly-master-3n-skx: