summaryrefslogtreecommitdiffstats
path: root/doxygen/siphon-process
blob: 698da8828e1d80508346155a0e7711ef58c5f51e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
#!/usr/bin/env python
# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Filter for .siphon files that are generated by other filters.
# The idea is to siphon off certain initializers so that we can better
# auto-document the contents of that initializer.

import os, sys, argparse, logging
import siphon

DEFAULT_LOGFILE = None
DEFAULT_LOGLEVEL = "info"
DEFAULT_SIPHON ="clicmd"
DEFAULT_FORMAT = "markdown"
DEFAULT_OUTPUT = None
DEFAULT_TEMPLATES = os.path.dirname(__file__) + "/siphon_templates"

ap = argparse.ArgumentParser()
ap.add_argument("--log-file", default=DEFAULT_LOGFILE,
        help="Log file [%s]" % DEFAULT_LOGFILE)
ap.add_argument("--log-level", default=DEFAULT_LOGLEVEL,
        choices=["debug", "info", "warning", "error", "critical"],
        help="Logging level [%s]" % DEFAULT_LOGLEVEL)

ap.add_argument("--type", '-t', metavar="siphon_type", default=DEFAULT_SIPHON,
        choices=siphon.process.siphons.keys(),
        help="Siphon type to process [%s]" % DEFAULT_SIPHON)
ap.add_argument("--format", '-f', default=DEFAULT_FORMAT,
        choices=siphon.process.formats.keys(),
        help="Output format to generate [%s]" % DEFAULT_FORMAT)
ap.add_argument("--output", '-o', metavar="file", default=DEFAULT_OUTPUT,
        help="Output file (uses stdout if not defined) [%s]" % DEFAULT_OUTPUT)
ap.add_argument("--templates", metavar="directory", default=DEFAULT_TEMPLATES,
        help="Path to render templates directory [%s]" % DEFAULT_TEMPLATES)
ap.add_argument("input", nargs='+', metavar="input_file",
        help="Input .siphon files")
args = ap.parse_args()

logging.basicConfig(filename=args.log_file,
        level=getattr(logging, args.log_level.upper(), None))
log = logging.getLogger("siphon_process")

# Determine where to send the generated output
if args.output is None:
    out = sys.stdout
else:
    out = open(args.output, "w+")

# Get our processor
klass = siphon.process.siphons[args.type]
processor = klass(template_directory=args.templates, format=args.format)

# Load the input files
processor.load_json(args.input)

# Process the data
processor.process(out=out)

# All done
lor: #008800; font-weight: bold } /* Keyword.Reserved */ .highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */ .highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */ .highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */ .highlight .na { color: #336699 } /* Name.Attribute */ .highlight .nb { color: #003388 } /* Name.Builtin */ .highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */ .highlight .no { color: #003366; font-weight: bold } /* Name.Constant */ .highlight .nd { color: #555555 } /* Name.Decorator */ .highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */ .highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */ .highlight .nl { color: #336699; font-style: italic } /* Name.Label */ .highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */ .highlight .py { color: #336699; font-weight: bold } /* Name.Property */ .highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */ .highlight .nv { color: #336699 } /* Name.Variable */ .highlight .ow { color: #008800 } /* Operator.Word */ .highlight .w { color: #bbbbbb } /* Text.Whitespace */ .highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */ .highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */ .highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */ .highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */ .highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */ .highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */ .highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */ .highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */ .highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */ .highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */ .highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */ .highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */ .highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */ .highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */ .highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */ .highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */ .highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */ .highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */ .highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */ .highlight .vc { color: #336699 } /* Name.Variable.Class */ .highlight .vg { color: #dd7700 } /* Name.Variable.Global */ .highlight .vi { color: #3333bb } /* Name.Variable.Instance */ .highlight .vm { color: #336699 } /* Name.Variable.Magic */ .highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */ }
################################################################################
###                          E N V I R O N M E N T                           ###
################################################################################

- type: "environment"

  spec-files:
    - "specifications/report/data_sets.yaml"
    - "specifications/report/layouts.yaml"
    - "specifications/report/elements.yaml"

  paths:
    # Top level directories:
    ## Working directory
    DIR[WORKING]: "_tmp"
    ## Build directories
    DIR[BUILD,HTML]: "_build"
    DIR[BUILD,LATEX]: "_build_latex"

    # Static .rst files
    DIR[RST]: "../../../docs/report"

    # Working directories
    ## Input data files (.zip, .xml)
    DIR[WORKING,DATA]: "{DIR[WORKING]}/data"
    ## Static source files from git
    DIR[WORKING,SRC]: "{DIR[WORKING]}/src"
    DIR[WORKING,SRC,STATIC]: "{DIR[WORKING,SRC]}/_static"

    # Static html content
    DIR[STATIC]: "{DIR[BUILD,HTML]}/_static"
    DIR[STATIC,VPP]: "{DIR[STATIC]}/vpp"
    DIR[STATIC,DPDK]: "{DIR[STATIC]}/dpdk"
    DIR[STATIC,TREX]: "{DIR[STATIC]}/trex"
    DIR[STATIC,ARCH]: "{DIR[STATIC]}/archive"

    # Detailed test results
    DIR[DTR]: "{DIR[WORKING,SRC]}/detailed_test_results"
    DIR[DTR,PERF,DPDK,2N,ICX]: "{DIR[DTR]}/dpdk_performance_results_2n_icx"
    DIR[DTR,PERF,DPDK,3N,ICX]: "{DIR[DTR]}/dpdk_performance_results_3n_icx"
    DIR[DTR,PERF,DPDK,3N,SKX]: "{DIR[DTR]}/dpdk_performance_results_3n_skx"
    DIR[DTR,PERF,DPDK,2N,SKX]: "{DIR[DTR]}/dpdk_performance_results_2n_skx"
    DIR[DTR,PERF,DPDK,2N,ZN2]: "{DIR[DTR]}/dpdk_performance_results_2n_zn2"
    DIR[DTR,PERF,DPDK,2N,CLX]: "{DIR[DTR]}/dpdk_performance_results_2n_clx"
    DIR[DTR,PERF,DPDK,2N,DNV]: "{DIR[DTR]}/dpdk_performance_results_2n_dnv"
    DIR[DTR,PERF,DPDK,3N,DNV]: "{DIR[DTR]}/dpdk_performance_results_3n_dnv"
    DIR[DTR,PERF,DPDK,3N,TSH]: "{DIR[DTR]}/dpdk_performance_results_3n_tsh"
    DIR[DTR,PERF,DPDK,2N,TX2]: "{DIR[DTR]}/dpdk_performance_results_2n_tx2"
    DIR[DTR,PERF,TREX,2N,SKX]: "{DIR[DTR]}/trex_performance_results_2n_skx"
    DIR[DTR,PERF,VPP,ALL]: "{DIR[DTR]}/vpp_performance_results"
    DIR[DTR,PERF,VPP,3N,SKX]: "{DIR[DTR]}/vpp_performance_results_3n_skx"
    DIR[DTR,PERF,VPP,2N,SKX]: "{DIR[DTR]}/vpp_performance_results_2n_skx"
    DIR[DTR,PERF,VPP,2N,ZN2]: "{DIR[DTR]}/vpp_performance_results_2n_zn2"
    DIR[DTR,PERF,VPP,2N,CLX]: "{DIR[DTR]}/vpp_performance_results_2n_clx"
    DIR[DTR,PERF,VPP,2N,ICX]: "{DIR[DTR]}/vpp_performance_results_2n_icx"
    DIR[DTR,PERF,VPP,3N,ICX]: "{DIR[DTR]}/vpp_performance_results_3n_icx"
    DIR[DTR,PERF,VPP,3N,TSH]: "{DIR[DTR]}/vpp_performance_results_3n_tsh"
    DIR[DTR,PERF,VPP,2N,TX2]: "{DIR[DTR]}/vpp_performance_results_2n_tx2"
    DIR[DTR,PERF,VPP,3N,DNV]: "{DIR[DTR]}/vpp_performance_results_3n_dnv"
    DIR[DTR,PERF,VPP,2N,DNV]: "{DIR[DTR]}/vpp_performance_results_2n_dnv"
    DIR[DTR,PERF,VPP,3N,AWS]: "{DIR[DTR]}/vpp_performance_results_3n_aws"
    DIR[DTR,PERF,VPP,2N,AWS]: "{DIR[DTR]}/vpp_performance_results_2n_aws"
    DIR[DTR,MRR,VPP,ALL]: "{DIR[DTR]}/vpp_mrr_results"
    DIR[DTR,MRR,VPP,3N,SKX]: "{DIR[DTR]}/vpp_mrr_results_3n_skx"
    DIR[DTR,MRR,VPP,2N,SKX]: "{DIR[DTR]}/vpp_mrr_results_2n_skx"
    DIR[DTR,MRR,VPP,2N,ZN2]: "{DIR[DTR]}/vpp_mrr_results_2n_zn2"
    DIR[DTR,MRR,VPP,2N,CLX]: "{DIR[DTR]}/vpp_mrr_results_2n_clx"
    DIR[DTR,MRR,VPP,2N,ICX]: "{DIR[DTR]}/vpp_mrr_results_2n_icx"
    DIR[DTR,MRR,VPP,3N,ICX]: "{DIR[DTR]}/vpp_mrr_results_3n_icx"
    DIR[DTR,MRR,VPP,3N,TSH]: "{DIR[DTR]}/vpp_mrr_results_3n_tsh"
    DIR[DTR,MRR,VPP,2N,TX2]: "{DIR[DTR]}/vpp_mrr_results_2n_tx2"
    DIR[DTR,MRR,VPP,3N,DNV]: "{DIR[DTR]}/vpp_mrr_results_3n_dnv"
    DIR[DTR,MRR,VPP,2N,DNV]: "{DIR[DTR]}/vpp_mrr_results_2n_dnv"
    DIR[DTR,MRR,VPP,3N,AWS]: "{DIR[DTR]}/vpp_mrr_results_3n_aws"
    DIR[DTR,MRR,VPP,2N,AWS]: "{DIR[DTR]}/vpp_mrr_results_2n_aws"
    DIR[DTR,FUNC,VPP,DEVICE,UBUNTU]: "{DIR[DTR]}/vpp_device_results_ubuntu"

    # Detailed test configurations
    DIR[DTC]: "{DIR[WORKING,SRC]}/test_configuration"
    DIR[DTC,PERF,VPP,ALL]: "{DIR[DTC]}/vpp_performance_configuration"
    DIR[DTC,PERF,VPP,3N,SKX]: "{DIR[DTC]}/vpp_performance_configuration_3n_skx"
    DIR[DTC,PERF,VPP,2N,SKX]: "{DIR[DTC]}/vpp_performance_configuration_2n_skx"
    DIR[DTC,PERF,VPP,2N,ZN2]: "{DIR[DTC]}/vpp_performance_configuration_2n_zn2"
    DIR[DTC,PERF,VPP,2N,CLX]: "{DIR[DTC]}/vpp_performance_configuration_2n_clx"
    DIR[DTC,PERF,VPP,2N,ICX]: "{DIR[DTC]}/vpp_performance_configuration_2n_icx"
    DIR[DTC,PERF,VPP,3N,ICX]: "{DIR[DTC]}/vpp_performance_configuration_3n_icx"
    DIR[DTC,PERF,VPP,3N,TSH]: "{DIR[DTC]}/vpp_performance_configuration_3n_tsh"
    DIR[DTC,PERF,VPP,2N,TX2]: "{DIR[DTC]}/vpp_performance_configuration_2n_tx2"
    DIR[DTC,PERF,VPP,3N,DNV]: "{DIR[DTC]}/vpp_performance_configuration_3n_dnv"
    DIR[DTC,PERF,VPP,2N,DNV]: "{DIR[DTC]}/vpp_performance_configuration_2n_dnv"
    DIR[DTC,PERF,VPP,3N,AWS]: "{DIR[DTC]}/vpp_performance_configuration_3n_aws"
    DIR[DTC,PERF,VPP,2N,AWS]: "{DIR[DTC]}/vpp_performance_configuration_2n_aws"
    DIR[DTC,MRR,VPP,ALL]: "{DIR[DTC]}/vpp_mrr_configuration"
    DIR[DTC,MRR,VPP,3N,SKX]: "{DIR[DTC]}/vpp_mrr_configuration_3n_skx"
    DIR[DTC,MRR,VPP,2N,SKX]: "{DIR[DTC]}/vpp_mrr_configuration_2n_skx"
    DIR[DTC,MRR,VPP,2N,ZN2]: "{DIR[DTC]}/vpp_mrr_configuration_2n_zn2"
    DIR[DTC,MRR,VPP,2N,CLX]: "{DIR[DTC]}/vpp_mrr_configuration_2n_clx"
    DIR[DTC,MRR,VPP,2N,ICX]: "{DIR[DTC]}/vpp_mrr_configuration_2n_icx"
    DIR[DTC,MRR,VPP,3N,ICX]: "{DIR[DTC]}/vpp_mrr_configuration_3n_icx"
    DIR[DTC,MRR,VPP,3N,TSH]: "{DIR[DTC]}/vpp_mrr_configuration_3n_tsh"
    DIR[DTC,MRR,VPP,2N,TX2]: "{DIR[DTC]}/vpp_mrr_configuration_2n_tx2"
    DIR[DTC,MRR,VPP,3N,DNV]: "{DIR[DTC]}/vpp_mrr_configuration_3n_dnv"
    DIR[DTC,MRR,VPP,2N,DNV]: "{DIR[DTC]}/vpp_mrr_configuration_2n_dnv"
    DIR[DTC,MRR,VPP,3N,AWS]: "{DIR[DTC]}/vpp_mrr_configuration_3n_aws"
    DIR[DTC,MRR,VPP,2N,AWS]: "{DIR[DTC]}/vpp_mrr_configuration_2n_aws"
    DIR[DTC,FUNC,VPP,DEVICE,UBUNTU]: "{DIR[DTC]}/vpp_device_configuration_ubuntu"

    # Detailed tests operational data
    DIR[DTO]: "{DIR[WORKING,SRC]}/test_operational_data"
    DIR[DTO,PERF,VPP,ALL]: "{DIR[DTO]}/vpp_performance_operational_data"
    DIR[DTO,PERF,VPP,3N,SKX]: "{DIR[DTO]}/vpp_performance_operational_data_3n_skx"
    DIR[DTO,PERF,VPP,2N,SKX]: "{DIR[DTO]}/vpp_performance_operational_data_2n_skx"
    DIR[DTO,PERF,VPP,2N,ZN2]: "{DIR[DTO]}/vpp_performance_operational_data_2n_zn2"
    DIR[DTO,PERF,VPP,2N,CLX]: "{DIR[DTO]}/vpp_performance_operational_data_2n_clx"
    DIR[DTO,PERF,VPP,2N,ICX]: "{DIR[DTO]}/vpp_performance_operational_data_2n_icx"
    DIR[DTO,PERF,VPP,3N,ICX]: "{DIR[DTO]}/vpp_performance_operational_data_3n_icx"
    DIR[DTO,PERF,VPP,3N,TSH]: "{DIR[DTO]}/vpp_performance_operational_data_3n_tsh"
    DIR[DTO,PERF,VPP,2N,TX2]: "{DIR[DTO]}/vpp_performance_operational_data_2n_tx2"
    DIR[DTO,PERF,VPP,3N,DNV]: "{DIR[DTO]}/vpp_performance_operational_data_3n_dnv"
    DIR[DTO,PERF,VPP,2N,DNV]: "{DIR[DTO]}/vpp_performance_operational_data_2n_dnv"
    DIR[DTO,PERF,VPP,3N,AWS]: "{DIR[DTO]}/vpp_performance_operational_data_3n_aws"
    DIR[DTO,PERF,VPP,2N,AWS]: "{DIR[DTO]}/vpp_performance_operational_data_2n_aws"

    # Latencies:
    DIR[LAT,VPP]: "{DIR[WORKING,SRC]}/vpp_performance_tests/hdrh_packet_latency_by_percentile_graphs"
    DIR[LAT,DPDK]: "{DIR[WORKING,SRC]}/dpdk_performance_tests/hdrh_packet_latency_by_percentile_graphs"
    DIR[LAT,TREX]: "{DIR[WORKING,SRC]}/trex_performance_tests/hdrh_packet_latency_by_percentile_graphs"

    # .css patch file to fix tables generated by Sphinx
    DIR[CSS_PATCH_FILE]: "{DIR[STATIC]}/theme_overrides.css"
    DIR[CSS_PATCH_FILE2]: "{DIR[WORKING,SRC,STATIC]}/theme_overrides.css"

  # Data sources are used in this order:
  data-sources:
    # XML
    - type: "xml"
      url: "https://s3-logs.fd.io/vex-yul-rot-jenkins-1"
      path: "{job}/{build}/{filename}"
      file-name: "output_info.xml.gz"
      file-format: ".gz"
      enabled: True
    - type: "xml"
      url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
      path: "{job}/{build}/archives/{filename}"
      file-name: "output_info.xml.gz"
      file-format: ".gz"
      enabled: True
    - type: "xml"
      url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
      path: "{job}/{build}/{filename}"
      file-name: "output_info.xml.gz"
      file-format: ".gz"
      enabled: True
    # To be deprecated soon:
    - type: "xml"
      url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
      path: "{job}/{build}/{filename}"
      file-name: "output_info.xml.gz"
      file-format: ".gz"
      enabled: True
    # Deprecated:
    - type: "xml"
      url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
      path: "{job}/{build}/archives/{filename}"
      file-name: "output_info.xml.gz"
      file-format: ".gz"
      enabled: True
    - type: "xml"
      url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
      path: "{job}/{build}/archives/{filename}"
      file-name: "output.xml.gz"
      file-format: ".gz"
      enabled: True
    - type: "xml"
      url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
      path: "{job}/{build}/{filename}"
      file-name: "output.xml.gz"
      file-format: ".gz"
      enabled: True
    - type: "xml"
      url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
      path: "{job}/{build}/archives/{filename}"
      file-name: "output.xml.gz"
      file-format: ".gz"
      enabled: True
    - type: "xml"
      url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
      path: "{job}/{build}/{filename}"
      file-name: "output.xml.gz"
      file-format: ".gz"
      enabled: True
    # XML from docs.nexus
    - type: "xml-docs"
      url: "https://docs.fd.io/csit"
      path: "report/_static/archive"
      file-name: "output_info.xml.gz"
      file-format: ".gz"
      enabled: True
    - type: "xml-docs"
      url: "https://docs.fd.io/csit"
      path: "report/_static/archive"
      file-name: "output.xml.gz"
      file-format: ".gz"
      enabled: True
    - type: "xml-docs"
      url: "https://docs.fd.io/csit"
      path: "report/_static/archive"
      file-name: "robot-plugin.zip"
      file-format: ".zip"
      enabled: False
    # JSON from S3
    - type: "json"
      url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
      path: "{job}/{build}/{filename}"
      file-name: "output.json.gz"
      file-format: ".gz"
      enabled: False

  archive-inputs: False

  mapping-file: ""

  ignore-list: ""

  make-dirs:
  # List the directories which are created while preparing the environment.
  # All directories MUST be defined in "paths" section.
    - "DIR[WORKING,DATA]"
    - "DIR[STATIC,VPP]"
    - "DIR[STATIC,DPDK]"
    - "DIR[STATIC,TREX]"
    - "DIR[STATIC,ARCH]"
    - "DIR[BUILD,LATEX]"
    - "DIR[WORKING,SRC]"
    - "DIR[WORKING,SRC,STATIC]"

  remove-dirs:
  # List the directories which are deleted while cleaning the environment.
  # All directories MUST be defined in "paths" section.
  #  - "DIR[BUILD,HTML]"
    - "DIR[WORKING,DATA]"

  build-dirs:
  # List the directories where the results (build) is stored.
  # All directories MUST be defined in "paths" section.
    - "DIR[BUILD,HTML]"
    - "DIR[BUILD,LATEX]"

  reverse-input: False  # Needed for trending, not important for the report

  limits:
    nic:
      x520: 24460000
      x553: 29761905
      x710: 35800000
      xxv710: 35800000
      xl710: 35800000
      cx556a: 148809523.81
      e810cq: 148809523.81

    link:
      10ge: 14880952.38
      25ge: 37202380.95
      40ge: 59523809.52
      100ge: 148809523.81

    pci:
      pci-g3-x8: 74404761.90
      pci-g3-x16: 148809523.81

################################################################################
###                               S T A T I C                                ###
################################################################################

- type: "static"
  src-path: "{DIR[RST]}"
  dst-path: "{DIR[WORKING,SRC]}"

################################################################################
###                               O U T P U T                                ###
################################################################################

- type: "output"
  arch-file-format:
    - ".gz"
    - ".zip"
  output: "report"
  format:
    html:
      - full
    # pdf:
    #   - minimal