aboutsummaryrefslogtreecommitdiffstats
path: root/src/plugins/rdma
AgeCommit message (Expand)AuthorFilesLines
2020-05-07rdma: fix alignment issueDamjan Marion1-2/+2
2020-05-06rdma: fix clang buildBenoît Ganne1-2/+9
2020-04-27rdma: tx: interleave prefetchesBenoît Ganne1-24/+13
2020-03-31rdma: print device info from PCI VPD in 'show hardware' outputDamjan Marion1-0/+14
2020-03-20rdma: ring db only once per vector on txBenoît Ganne1-28/+44
2020-03-16rdma: optimize tx wqe_initDamjan Marion3-13/+33
2020-03-16rdma: introduce direct verb for Cx4/5 txBenoît Ganne4-73/+549
2020-03-16rdma: add Mellanox mlx5 Direct Verbs receive supportDamjan Marion6-37/+843
2020-03-02rdma: improve rx loopDamjan Marion1-38/+16
2020-03-02rdma: improve refillDamjan Marion1-65/+57
2020-03-02rdma: use per-thread buffer templateDamjan Marion3-5/+25
2020-02-18rdma: fix bug related to ring bufferElias Rudberg1-2/+2
2020-01-08rdma: api: prepare support for direct verbBenoît Ganne7-10/+79
2020-01-08rdma: add FEATURE.yamlBenoît Ganne1-0/+8
2019-12-12rdma: add explicit types in apiOle Troan1-2/+3
2019-11-08rdma: fix next node rx redirectBenoît Ganne1-8/+2
2019-11-08rdma: fix name auto-generation on createBenoît Ganne1-1/+5
2019-10-23rdma: build: fix ibverb compilation testBenoît Ganne1-5/+3
2019-10-16rdma: add rdma APIBenoît Ganne8-29/+401
2019-10-07rdma: fix crash when failing to read pci addr on createBenoît Ganne1-0/+3
2019-10-07rdma: fix crash when failing to detect numa node on createBenoît Ganne1-0/+5
2019-10-06rdma: prevent loopback of broadcast packetsBenoît Ganne1-4/+9
2019-10-05rdma: make sure pci subsystem is initializedBenoît Ganne1-1/+6
2019-10-01rdma: fix non-NULL terminated C-string overflowBenoît Ganne2-3/+3
2019-09-24vlib: add flag to explicitelly mark nodes which can init per-node packet traceDamjan Marion1-0/+1
2019-08-27rdma: fix pending packets check on txBenoît Ganne1-1/+1
2019-08-26rdma: use rings for buffers managementBenoît Ganne5-203/+243
2019-08-19rdma: add support for MAC changesBenoît Ganne2-1/+35
2019-08-19rdma: add support for input feature arcsBenoît Ganne1-29/+35
2019-08-07rdma: add missing symbolsBenoît Ganne1-2/+4
2019-08-06rdma: fix double-free in rdma-txBenoît Ganne2-9/+10
2019-07-23api: binary api cleanupDave Barach1-1/+1
2019-07-18vlib: convert frame_index into real pointersAndreas Schultz1-1/+1
2019-05-24rdma: add support for promiscuous modeBenoît Ganne2-71/+145
2019-05-14rdma: update MAINTAINERS and add docBenoît Ganne1-0/+51
2019-05-03plugins: clean up plugin descriptionsDave Wallace1-1/+1
2019-04-09rdma: add multiqueue supportBenoît Ganne4-76/+170
2019-04-05rdma: more batching, compile rdma-core in release modeBenoît Ganne3-101/+299
2019-04-03rdma: tx: fix stats and add batchingBenoît Ganne1-61/+88
2019-03-29rdma: add option to specify inteface nameDamjan Marion4-2/+12
2019-03-28cmake: add vpp_plugin_find_library macroDamjan Marion1-10/+10
2019-03-28Add RDMA ibverb driver pluginBenoît Ganne8-0/+1401
ack) > 0: parents, sdf = stack.pop() for column_name, column_type in sdf.dtypes: if column_type[:6] == "struct": projected_sdf = sdf.select(column_name + ".*") stack.append((parents + (column_name,), projected_sdf)) else: columns.append( col(".".join(parents + (column_name,))) \ .alias("_".join(parents + (column_name,))) ) return nested_sdf.select(columns) def process_json_to_dataframe(schema_name, paths): """Processes JSON to Spark DataFrame. :param schema_name: Schema name. :type schema_name: string :param paths: S3 paths to process. :type paths: list :returns: Spark DataFrame. :rtype: DataFrame """ drop_subset = [ "dut_type", "dut_version", "passed", "test_name_long", "test_name_short", "test_type", "version" ] # load schemas with open(f"trending_{schema_name}.json", "r", encoding="UTF-8") as f_schema: schema = StructType.fromJson(load(f_schema)) # create empty DF out of schemas sdf = spark.createDataFrame([], schema) # filter list filtered = [path for path in paths if schema_name in path] # select for path in filtered: print(path) sdf_loaded = spark \ .read \ .option("multiline", "true") \ .schema(schema) \ .json(path) \ .withColumn("job", lit(path.split("/")[4])) \ .withColumn("build", lit(path.split("/")[5])) sdf = sdf.unionByName(sdf_loaded, allowMissingColumns=True) # drop rows with all nulls and drop rows with null in critical frames sdf = sdf.na.drop(how="all") sdf = sdf.na.drop(how="any", thresh=None, subset=drop_subset) # flatten frame sdf = flatten_frame(sdf) return sdf # create SparkContext and GlueContext spark_context = SparkContext.getOrCreate() spark_context.setLogLevel("WARN") glue_context = GlueContext(spark_context) spark = glue_context.spark_session # files of interest paths = wr.s3.list_objects( path=PATH, suffix=SUFFIX, last_modified_begin=LAST_MODIFIED_BEGIN, last_modified_end=LAST_MODIFIED_END, ignore_suffix=IGNORE_SUFFIX, ignore_empty=True ) filtered_paths = [path for path in paths if "daily" in path or "weekly" in path] out_sdf = process_json_to_dataframe("mrr", filtered_paths) out_sdf.show(truncate=False) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ .withColumn("month", lit(datetime.now().month)) \ .withColumn("day", lit(datetime.now().day)) \ .repartition(1) try: boto3_session = session.Session( aws_access_key_id=environ["OUT_AWS_ACCESS_KEY_ID"], aws_secret_access_key=environ["OUT_AWS_SECRET_ACCESS_KEY"], region_name=environ["OUT_AWS_DEFAULT_REGION"] ) except KeyError: boto3_session = session.Session() try: wr.s3.to_parquet( df=out_sdf.toPandas(), path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/trending", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", use_threads=True, mode="overwrite_partitions", boto3_session=boto3_session ) except EmptyDataFrame: pass