diff options
Diffstat (limited to 'libtransport/src/protocols')
82 files changed, 5459 insertions, 1883 deletions
diff --git a/libtransport/src/protocols/CMakeLists.txt b/libtransport/src/protocols/CMakeLists.txt index b763e95e2..51879a9ed 100644 --- a/libtransport/src/protocols/CMakeLists.txt +++ b/libtransport/src/protocols/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2019 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -52,9 +52,15 @@ list(APPEND SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/errors.cc ) -set(RAAQM_CONFIG_INSTALL_PREFIX - ${CMAKE_INSTALL_FULL_SYSCONFDIR}/hicn -) +if (${CMAKE_SYSTEM_NAME} MATCHES Darwin OR ${CMAKE_SYSTEM_NAME} MATCHES Linux) + set(RAAQM_CONFIG_INSTALL_PREFIX + ${CMAKE_INSTALL_FULL_SYSCONFDIR}/hicn + ) +else() + set(RAAQM_CONFIG_INSTALL_PREFIX + ${CMAKE_INSTALL_PREFIX}/etc/hicn + ) +endif() set(raaqm_config_path ${RAAQM_CONFIG_INSTALL_PREFIX}/consumer.conf @@ -67,7 +73,7 @@ set(TRANSPORT_CONFIG install( FILES ${TRANSPORT_CONFIG} - DESTINATION ${CMAKE_INSTALL_FULL_SYSCONFDIR}/hicn + DESTINATION ${RAAQM_CONFIG_INSTALL_PREFIX} COMPONENT ${LIBTRANSPORT_COMPONENT} ) diff --git a/libtransport/src/protocols/byte_stream_reassembly.cc b/libtransport/src/protocols/byte_stream_reassembly.cc index ac36d4e61..3278595b7 100644 --- a/libtransport/src/protocols/byte_stream_reassembly.cc +++ b/libtransport/src/protocols/byte_stream_reassembly.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/byte_stream_reassembly.h b/libtransport/src/protocols/byte_stream_reassembly.h index 278740bd3..bfcac3181 100644 --- a/libtransport/src/protocols/byte_stream_reassembly.h +++ b/libtransport/src/protocols/byte_stream_reassembly.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/cbr.cc b/libtransport/src/protocols/cbr.cc index 1548cc68d..446ea8b99 100644 --- a/libtransport/src/protocols/cbr.cc +++ b/libtransport/src/protocols/cbr.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -37,11 +37,11 @@ void CbrTransportProtocol::afterDataUnsatisfied(uint64_t segment) {} void CbrTransportProtocol::afterContentReception( const Interest &interest, const ContentObject &content_object) { auto segment = content_object.getName().getSuffix(); - auto now = utils::SteadyClock::now(); - auto rtt = std::chrono::duration_cast<utils::Microseconds>( - now - interest_timepoints_[segment & mask]); + auto now = utils::SteadyTime::Clock::now(); + auto rtt = utils::SteadyTime::getDurationMs( + interest_timepoints_[segment & mask], now); // Update stats - updateStats(segment, rtt.count(), now); + updateStats(segment, rtt, now); } } // end namespace protocol diff --git a/libtransport/src/protocols/cbr.h b/libtransport/src/protocols/cbr.h index 41cdbc98c..c178dbf60 100644 --- a/libtransport/src/protocols/cbr.h +++ b/libtransport/src/protocols/cbr.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/congestion_window_protocol.h b/libtransport/src/protocols/congestion_window_protocol.h index 36ac6eb17..f9ff208cc 100644 --- a/libtransport/src/protocols/congestion_window_protocol.h +++ b/libtransport/src/protocols/congestion_window_protocol.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/data_processing_events.h b/libtransport/src/protocols/data_processing_events.h index 28732502b..182de3ed8 100644 --- a/libtransport/src/protocols/data_processing_events.h +++ b/libtransport/src/protocols/data_processing_events.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/datagram_reassembly.cc b/libtransport/src/protocols/datagram_reassembly.cc index 069873a52..3a32c81f5 100644 --- a/libtransport/src/protocols/datagram_reassembly.cc +++ b/libtransport/src/protocols/datagram_reassembly.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/datagram_reassembly.h b/libtransport/src/protocols/datagram_reassembly.h index de294df06..0def32dd2 100644 --- a/libtransport/src/protocols/datagram_reassembly.h +++ b/libtransport/src/protocols/datagram_reassembly.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/errors.cc b/libtransport/src/protocols/errors.cc index 183fcc574..a7dd26e16 100644 --- a/libtransport/src/protocols/errors.cc +++ b/libtransport/src/protocols/errors.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/errors.h b/libtransport/src/protocols/errors.h index 58dadae5a..33d5fbee4 100644 --- a/libtransport/src/protocols/errors.h +++ b/libtransport/src/protocols/errors.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/fec/CMakeLists.txt b/libtransport/src/protocols/fec/CMakeLists.txt index 6d61ae043..8ae0a7360 100644 --- a/libtransport/src/protocols/fec/CMakeLists.txt +++ b/libtransport/src/protocols/fec/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2019 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/fec/fec.cc b/libtransport/src/protocols/fec/fec.cc index 16a04cb98..912e7a40f 100644 --- a/libtransport/src/protocols/fec/fec.cc +++ b/libtransport/src/protocols/fec/fec.cc @@ -44,6 +44,7 @@ #include "fec.h" +#include <assert.h> #include <hicn/transport/portability/platform.h> #include <stdio.h> #include <stdlib.h> @@ -60,72 +61,6 @@ #endif /* - * compatibility stuff - */ -#ifdef MSDOS /* but also for others, e.g. sun... */ -#define NEED_BCOPY -#define bcmp(a, b, n) memcmp(a, b, n) -#endif - -#ifdef ANDROID -#define bcmp(a, b, n) memcmp(a, b, n) -#endif - -#ifdef NEED_BCOPY -#define bcopy(s, d, siz) memcpy((d), (s), (siz)) -#define bzero(d, siz) memset((d), '\0', (siz)) -#endif - -/* - * stuff used for testing purposes only - */ - -#ifdef TEST -#define DEB(x) -#define DDB(x) x -#define DEBUG 0 /* minimal debugging */ -#ifdef MSDOS -#include <time.h> -struct timeval { - unsigned long ticks; -}; -#define gettimeofday(x, dummy) \ - { (x)->ticks = clock(); } -#define DIFF_T(a, b) (1 + 1000000 * (a.ticks - b.ticks) / CLOCKS_PER_SEC) -typedef unsigned long u_long; -typedef unsigned short u_short; -#else /* typically, unix systems */ -#include <sys/time.h> -#define DIFF_T(a, b) \ - (1 + 1000000 * (a.tv_sec - b.tv_sec) + (a.tv_usec - b.tv_usec)) -#endif - -#define TICK(t) \ - { \ - struct timeval x; \ - gettimeofday(&x, NULL); \ - t = x.tv_usec + 1000000 * (x.tv_sec & 0xff); \ - } -#define TOCK(t) \ - { \ - u_long t1; \ - TICK(t1); \ - if (t1 < t) \ - t = 256000000 + t1 - t; \ - else \ - t = t1 - t; \ - if (t == 0) t = 1; \ - } - -u_long ticks[10]; /* vars for timekeeping */ -#else -#define DEB(x) -#define DDB(x) -#define TICK(x) -#define TOCK(x) -#endif /* TEST */ - -/* * You should not need to change anything beyond this point. * The first part of the file implements linear algebra in GF. * @@ -402,31 +337,17 @@ static void matmul(gf *a, gf *b, gf *c, int n, int k, int m) { } } -#ifdef DEBUGG -/* - * returns 1 if the square matrix is identiy - * (only for test) - */ -static int is_identity(gf *m, int k) { - int row, col; - for (row = 0; row < k; row++) - for (col = 0; col < k; col++) - if ((row == col && *m != 1) || (row != col && *m != 0)) - return 0; - else - m++; - return 1; -} -#endif /* debug */ - /* * invert_mat() takes a matrix and produces its inverse * k is the size of the matrix. * (Gauss-Jordan, adapted from Numerical Recipes in C) * Return non-zero if singular. */ -DEB(int pivloops = 0; int pivswaps = 0; /* diagnostic */) +int pivloops = 0; +int pivswaps = 0; /* diagnostic */ static int invert_mat(gf *src, int k) { + assert(k > 0); + gf c, *p; int irow, icol, row, col, i, ix; @@ -436,9 +357,9 @@ static int invert_mat(gf *src, int k) { int *ipiv = (int *)my_malloc(k * sizeof(int), "ipiv"); gf *id_row = NEW_GF_MATRIX(1, k); gf *temp_row = NEW_GF_MATRIX(1, k); - - bzero(id_row, k * sizeof(gf)); - DEB(pivloops = 0; pivswaps = 0; /* diagnostic */) + memset(id_row, '\0', k * sizeof(gf)); + pivloops = 0; + pivswaps = 0; /* diagnostic */ /* * ipiv marks elements already used as pivots. */ @@ -459,7 +380,7 @@ static int invert_mat(gf *src, int k) { for (row = 0; row < k; row++) { if (ipiv[row] != 1) { for (ix = 0; ix < k; ix++) { - DEB(pivloops++;) + pivloops++; if (ipiv[ix] == 0) { if (src[row * k + ix] != 0) { irow = row; @@ -497,12 +418,9 @@ static int invert_mat(gf *src, int k) { fprintf(stderr, "singular matrix 2\n"); goto fail; } - if (c != 1) { /* otherwhise this is a NOP */ - /* - * this is done often , but optimizing is not so - * fruitful, at least in the obvious ways (unrolling) - */ - DEB(pivswaps++;) + + if (c != 1) { + pivswaps++; c = inverse[c]; pivot_row[icol] = 1; for (ix = 0; ix < k; ix++) pivot_row[ix] = gf_mul(c, pivot_row[ix]); @@ -515,7 +433,7 @@ static int invert_mat(gf *src, int k) { * we can optimize the addmul). */ id_row[icol] = 1; - if (bcmp(pivot_row, id_row, k * sizeof(gf)) != 0) { + if (memcmp(pivot_row, id_row, k * sizeof(gf)) != 0) { for (p = src, ix = 0; ix < k; ix++, p += k) { if (ix != icol) { c = p[icol]; @@ -560,6 +478,8 @@ fail: */ int invert_vdm(gf *src, int k) { + assert(k > 0); + int i, j, row, col; gf *b, *c, *p; gf t, xx; @@ -614,14 +534,8 @@ int invert_vdm(gf *src, int k) { static int fec_initialized = 0; static void init_fec() { - TICK(ticks[0]); generate_gf(); - TOCK(ticks[0]); - DDB(fprintf(stderr, "generate_gf took %ldus\n", ticks[0]);) - TICK(ticks[0]); init_mul_table(); - TOCK(ticks[0]); - DDB(fprintf(stderr, "init_mul_table took %ldus\n", ticks[0]);) fec_initialized = 1; } @@ -680,19 +594,14 @@ struct fec_parms *fec_new(int k, int n) { * k*k vandermonde matrix, multiply right the bottom n-k rows * by the inverse, and construct the identity matrix at the top. */ - TICK(ticks[3]); invert_vdm(tmp_m, k); /* much faster than invert_mat */ matmul(tmp_m + k * k, tmp_m, retval->enc_matrix + k * k, n - k, k, k); /* * the upper matrix is I so do not bother with a slow multiply */ - bzero(retval->enc_matrix, k * k * sizeof(gf)); + memset(retval->enc_matrix, '\0', k * k * sizeof(gf)); for (p = retval->enc_matrix, col = 0; col < k; col++, p += k + 1) *p = 1; free(tmp_m); - TOCK(ticks[3]); - - DDB(fprintf(stderr, "--- %ld us to build encoding matrix\n", ticks[3]);) - DEB(pr_matrix(retval->enc_matrix, n, k, "encoding_matrix");) return retval; } @@ -708,10 +617,10 @@ void fec_encode(struct fec_parms *code, gf *src[], gf *fec, int index, int sz) { if (GF_BITS > 8) sz /= 2; if (index < k) - bcopy(src[index], fec, sz * sizeof(gf)); + memcpy(fec, src[index], sz * sizeof(gf)); else if (index < code->n) { p = &(code->enc_matrix[index * k]); - bzero(fec, sz * sizeof(gf)); + memset(fec, '\0', sz * sizeof(gf)); for (i = 0; i < k; i++) addmul(fec, src[i], p[i], sz); } else fprintf(stderr, "Invalid index %d (max %d)\n", index, code->n - 1); @@ -733,22 +642,13 @@ static int shuffle(gf *pkt[], int index[], int k) { int c = index[i]; if (index[c] == c) { - DEB(fprintf(stderr, "\nshuffle, error at %d\n", i);) + fprintf(stderr, "\nshuffle, error at %d\n", i); return 1; } SWAP(index[i], index[c], int); SWAP(pkt[i], pkt[c], gf *); } } - DEB(/* just test that it works... */ - for (i = 0; i < k; i++) { - if (index[i] < k && index[i] != i) { - fprintf(stderr, "shuffle: after\n"); - for (i = 0; i < k; i++) fprintf(stderr, "%3d ", index[i]); - fprintf(stderr, "\n"); - return 1; - } - }) return 0; } @@ -757,20 +657,16 @@ static int shuffle(gf *pkt[], int index[], int k) { * indexes. The matrix must be already allocated as * a vector of k*k elements, in row-major order */ -static gf *build_decode_matrix(struct fec_parms *code, gf *pkt[], int index[]) { +static gf *build_decode_matrix(struct fec_parms *code, int index[]) { int i, k = code->k; gf *p, *matrix = NEW_GF_MATRIX(k, k); - TICK(ticks[9]); for (i = 0, p = matrix; i < k; i++, p += k) { -#if 1 /* this is simply an optimization, not very useful indeed */ if (index[i] < k) { - bzero(p, k * sizeof(gf)); + memset(p, '\0', k * sizeof(gf)); p[i] = 1; - } else -#endif - if (index[i] < code->n) - bcopy(&(code->enc_matrix[index[i] * k]), p, k * sizeof(gf)); + } else if (index[i] < code->n) + memcpy(p, &(code->enc_matrix[index[i] * k]), k * sizeof(gf)); else { fprintf(stderr, "decode: invalid index %d (max %d)\n", index[i], code->n - 1); @@ -778,12 +674,10 @@ static gf *build_decode_matrix(struct fec_parms *code, gf *pkt[], int index[]) { return NULL; } } - TICK(ticks[9]); if (invert_mat(matrix, k)) { free(matrix); matrix = NULL; } - TOCK(ticks[9]); return matrix; } @@ -800,39 +694,29 @@ static gf *build_decode_matrix(struct fec_parms *code, gf *pkt[], int index[]) { */ int fec_decode(struct fec_parms *code, gf *pkt[], int index[], int sz) { gf *m_dec; - gf **new_pkt; + gf **new_pkt = nullptr; int row, col, k = code->k; + int i = 0; if (GF_BITS > 8) sz /= 2; if (shuffle(pkt, index, k)) /* error if true */ return 1; - m_dec = build_decode_matrix(code, pkt, index); + m_dec = build_decode_matrix(code, index); if (m_dec == NULL) return 1; /* error */ /* * do the actual decoding */ - new_pkt = (gf **)my_malloc(k * sizeof(gf *), "new pkt pointers"); + new_pkt = pkt + k; for (row = 0; row < k; row++) { if (index[row] >= k) { - new_pkt[row] = (gf *)my_malloc(sz * sizeof(gf), "new pkt buffer"); - bzero(new_pkt[row], sz * sizeof(gf)); + memset(new_pkt[i], '\0', sz * sizeof(gf)); for (col = 0; col < k; col++) - addmul(new_pkt[row], pkt[col], m_dec[row * k + col], sz); - } - } - /* - * move pkts to their final destination - */ - for (row = 0; row < k; row++) { - if (index[row] >= k) { - bcopy(new_pkt[row], pkt[row], sz * sizeof(gf)); - free(new_pkt[row]); + addmul(new_pkt[i], pkt[col], m_dec[row * k + col], sz); + i++; } } - free(new_pkt); free(m_dec); - return 0; } diff --git a/libtransport/src/protocols/fec/rely.cc b/libtransport/src/protocols/fec/rely.cc index 7a30a62e2..d4d98a90b 100644 --- a/libtransport/src/protocols/fec/rely.cc +++ b/libtransport/src/protocols/fec/rely.cc @@ -23,38 +23,39 @@ namespace transport { namespace protocol { namespace fec { -RelyEncoder::RelyEncoder(uint32_t k, uint32_t n, uint32_t seq_offset) +RelyEncoder::RelyEncoder(uint32_t k, uint32_t n, uint32_t /* seq_offset */) : RelyBase(k, n) { configure(kmtu, ktimeout, kmax_stream_size); set_repair_trigger(k_, n_ - k_, n_ - k_); } void RelyEncoder::onPacketProduced(core::ContentObject &content_object, - uint32_t offset) { + uint32_t offset, uint32_t metadata) { // Get pointer to payload, leaving space to insert FEC header. // TODO Check if this additional header is really needed. - auto data = content_object.writableData() + offset - sizeof(fec_header); - auto length = content_object.length() - offset + sizeof(fec_header); + auto data = content_object.writableData() + offset - sizeof(fec_metadata); + auto length = content_object.length() - offset + sizeof(fec_metadata); // Check packet length does not exceed maximum length supported by the // encoder (otherwise segmentation would take place). - assert(length < max_packet_bytes()); + DCHECK(length < max_packet_bytes()); DLOG_IF(INFO, VLOG_IS_ON(4)) - << "Encoding packet of length " << length - sizeof(fec_header); + << "Encoding packet of length " << length - sizeof(fec_metadata); - // Get the suffix. With rely we need to write it in the fec_header in order to - // be able to recognize the seq number upon recovery. + // Get the suffix. With rely we need to write it in the fec_metadata in order + // to be able to recognize the seq number upon recovery. auto suffix = content_object.getName().getSuffix(); DLOG_IF(INFO, VLOG_IS_ON(4)) << "Producing packet " << suffix << " (index == " << current_index_ << ")"; - // Consume payload. Add fec_header in front before feeding payload to encoder, - // and copy original content of packet - fec_header *h = reinterpret_cast<fec_header *>(data); - fec_header copy = *h; + // Consume payload. Add fec_metadata in front before feeding payload to + // encoder, and copy original content of packet + fec_metadata *h = reinterpret_cast<fec_metadata *>(data); + fec_metadata copy = *h; h->setSeqNumberBase(suffix); + h->setMetadataBase(metadata); auto packets = consume(data, length, getCurrentTime()); - assert(packets == 1); + DCHECK(packets == 1); // Update packet counter current_index_ += packets; @@ -62,7 +63,7 @@ void RelyEncoder::onPacketProduced(core::ContentObject &content_object, // Restore original packet content and increment data pointer to the correct // position *h = copy; - data += sizeof(fec_header); + data += sizeof(fec_metadata); // Check position of this packet inside N size block auto i = current_index_ % n_; @@ -74,24 +75,24 @@ void RelyEncoder::onPacketProduced(core::ContentObject &content_object, // TODO Optimize it by copying only the RELY header // Be sure encoder can produce - assert(can_produce()); + DCHECK(can_produce()); // Check new payload size and make sure it fits in packet buffer auto new_payload_size = produce_bytes(); int difference = new_payload_size - length; - assert(difference > 0); - assert(content_object.ensureCapacity(difference)); + DCHECK(difference > 0); + DCHECK(content_object.ensureCapacity(difference)); // Update length DLOG_IF(INFO, VLOG_IS_ON(4)) << "The packet length will be incremented by " - << difference + sizeof(fec_header); - content_object.append(difference + sizeof(fec_header)); + << difference + sizeof(fec_metadata); + content_object.append(difference + sizeof(fec_metadata)); content_object.updateLength(); // Make sure we got a source packet, otherwise we would put a repair symbol // in a source packet - assert(rely::packet_is_systematic(produce_data())); + DCHECK(rely::packet_is_systematic(produce_data())); // Copy rely packet replacing old source packet. std::memcpy(data, produce_data(), new_payload_size); @@ -111,7 +112,7 @@ void RelyEncoder::onPacketProduced(core::ContentObject &content_object, while (can_produce()) { // The current index MUST be k_, because we enforce n - k repair to be // produced after k sources - assert(current_index_ == k_); + DCHECK(current_index_ == k_); buffer packet; if (!buffer_callback_) { @@ -130,7 +131,7 @@ void RelyEncoder::onPacketProduced(core::ContentObject &content_object, std::memcpy(packet->writableData(), produce_data(), produce_bytes()); // Push symbol in repair_packets - packets_.emplace_back(0, std::move(packet)); + packets_.emplace_back(0, metadata, std::move(packet)); // Advance the encoder produce_next(); @@ -143,7 +144,7 @@ void RelyEncoder::onPacketProduced(core::ContentObject &content_object, // If we have generated repair symbols, let's notify caller via the installed // callback if (packets_.size()) { - assert(packets_.size() == n_ - k_); + DCHECK(packets_.size() == n_ - k_); fec_callback_(packets_); packets_.clear(); current_index_ = 0; @@ -156,7 +157,7 @@ RelyDecoder::RelyDecoder(uint32_t k, uint32_t n, uint32_t seq_offset) } void RelyDecoder::onDataPacket(core::ContentObject &content_object, - uint32_t offset) { + uint32_t offset, uint32_t metadata) { // Adjust pointers to point to packet payload auto data = content_object.writableData() + offset; auto size = content_object.length() - offset; @@ -164,30 +165,38 @@ void RelyDecoder::onDataPacket(core::ContentObject &content_object, // Pass payload to decoder consume(data, size, getCurrentTime()); + producePackets(); +} + +void RelyDecoder::producePackets() { // Drain decoder if possible while (can_produce()) { - // Get size of decoded packet - auto size = produce_bytes(); - - // Get buffer to copy packet in - auto packet = core::PacketManager<>::getInstance().getMemBuf(); + auto fec_header_size = sizeof(fec_metadata); + auto payload_size = produce_bytes() - sizeof(fec_metadata); - // Copy buffer - packet->append(size); - std::memcpy(packet->writableData(), produce_data(), size); + buffer packet; + if (!buffer_callback_) { + packet = core::PacketManager<>::getInstance().getMemBuf(); + packet->append(payload_size); + } else { + packet = buffer_callback_(payload_size); + } // Read seq number - fec_header *h = reinterpret_cast<fec_header *>(packet->writableData()); + const fec_metadata *h = + reinterpret_cast<const fec_metadata *>(produce_data()); uint32_t index = h->getSeqNumberBase(); + uint32_t metadata = h->getMetadataBase(); DLOG_IF(INFO, VLOG_IS_ON(4)) << "The index written in the packet is " << index; - // Remove FEC header - packet->trimStart(sizeof(fec_header)); + // Copy payload + std::memcpy(packet->writableData(), produce_data() + fec_header_size, + payload_size); // Save packet in buffer - packets_.emplace_back(index, std::move(packet)); + packets_.emplace_back(index, metadata, std::move(packet)); // Advance to next packet produce_next(); @@ -198,8 +207,28 @@ void RelyDecoder::onDataPacket(core::ContentObject &content_object, fec_callback_(packets_); packets_.clear(); } + + flushOutOfOrder(); +} + +void RelyDecoder::flushOutOfOrder() { + if (flush_timer_ == nullptr) return; + flush_timer_->cancel(); + + if (has_upcoming_flush()) { + flush_timer_->expires_from_now(std::chrono::milliseconds( + std::max((int64_t)0, upcoming_flush(getCurrentTime())))); + + flush_timer_->async_wait([this](const std::error_code &ec) { + if (ec) return; + if (has_upcoming_flush()) { + flush(getCurrentTime()); + producePackets(); + } + }); + } } } // namespace fec } // namespace protocol -} // namespace transport
\ No newline at end of file +} // namespace transport diff --git a/libtransport/src/protocols/fec/rely.h b/libtransport/src/protocols/fec/rely.h index bfbdb30bc..001a26002 100644 --- a/libtransport/src/protocols/fec/rely.h +++ b/libtransport/src/protocols/fec/rely.h @@ -33,8 +33,12 @@ namespace fec { * @brief Table of used codes. */ #define foreach_rely_fec_type \ + _(Rely, 1, 2) \ _(Rely, 1, 3) \ + _(Rely, 1, 4) \ _(Rely, 2, 3) \ + _(Rely, 2, 6) \ + _(Rely, 3, 9) \ _(Rely, 4, 5) \ _(Rely, 4, 6) \ _(Rely, 4, 7) \ @@ -74,11 +78,17 @@ class RelyBase : public virtual FECBase { * decoding operations. It may be removed once we know the meaning of the * fields in the rely header. */ - struct fec_header { - uint32_t seq_number; - + class fec_metadata { + public: void setSeqNumberBase(uint32_t suffix) { seq_number = htonl(suffix); } - uint32_t getSeqNumberBase() { return ntohl(seq_number); } + uint32_t getSeqNumberBase() const { return ntohl(seq_number); } + + void setMetadataBase(uint32_t value) { metadata = htonl(value); } + uint32_t getMetadataBase() const { return ntohl(metadata); } + + private: + uint32_t seq_number; + uint32_t metadata; }; /** @@ -112,21 +122,20 @@ class RelyBase : public virtual FECBase { #if RELY_DEBUG return time_++; #else - auto _time = utils::SteadyClock::now().time_since_epoch(); - auto time = std::chrono::duration_cast<utils::Milliseconds>(_time).count(); - return time; + return utils::SteadyTime::nowMs().count(); #endif } - protected: uint32_t k_; uint32_t n_; + std::uint32_t seq_offset_; + /** * @brief Vector of packets to be passed to caller callbacks. For encoder it * will contain the repair packets, for decoder the recovered sources. */ - std::vector<std::pair<uint32_t, buffer>> packets_; + BufferArray packets_; /** * @brief Current index to be used for local packet count. @@ -142,50 +151,54 @@ class RelyBase : public virtual FECBase { * @brief The Rely Encoder implementation. * */ -class RelyEncoder : private RelyBase, - private rely::encoder, - public ProducerFEC { +class RelyEncoder : RelyBase, rely::encoder, public ProducerFEC { public: RelyEncoder(uint32_t k, uint32_t n, uint32_t seq_offset = 0); /** * Producers will call this function when they produce a data packet. */ - void onPacketProduced(core::ContentObject &content_object, - uint32_t offset) override; + void onPacketProduced(core::ContentObject &content_object, uint32_t offset, + uint32_t metadata = FECBase::INVALID_METADATA) override; /** * @brief Get the fec header size, if added to source packets */ std::size_t getFecHeaderSize() override { - return header_bytes() + sizeof(fec_header) + 4; + return header_bytes() + sizeof(fec_metadata) + 4; } - void reset() override {} + void reset() override { + // Nothing to do here + } }; -class RelyDecoder : private RelyBase, - private rely::decoder, - public ConsumerFEC { +class RelyDecoder : RelyBase, rely::decoder, public ConsumerFEC { public: RelyDecoder(uint32_t k, uint32_t n, uint32_t seq_offset = 0); /** * Consumers will call this function when they receive a data packet */ - void onDataPacket(core::ContentObject &content_object, - uint32_t offset) override; + void onDataPacket(core::ContentObject &content_object, uint32_t offset, + uint32_t metadata = FECBase::INVALID_METADATA) override; /** * @brief Get the fec header size, if added to source packets */ std::size_t getFecHeaderSize() override { - return header_bytes() + sizeof(fec_header); + return header_bytes() + sizeof(fec_metadata); + } + + void reset() override { + // Nothing to do here } - void reset() override {} + private: + void producePackets(); + void flushOutOfOrder(); }; } // namespace fec } // namespace protocol -} // namespace transport
\ No newline at end of file +} // namespace transport diff --git a/libtransport/src/protocols/fec/rs.cc b/libtransport/src/protocols/fec/rs.cc index 2c23d515d..9c0a3d4fb 100644 --- a/libtransport/src/protocols/fec/rs.cc +++ b/libtransport/src/protocols/fec/rs.cc @@ -46,23 +46,26 @@ bool BlockCode::addRepairSymbol(const fec::buffer &packet, uint32_t i, to_decode_ = true; DLOG_IF(INFO, VLOG_IS_ON(4)) << "Adding symbol of size " << packet->length(); return addSymbol(packet, i, offset, - packet->length() - sizeof(fec_header) - offset); + packet->length() - sizeof(fec_header) - offset, + FECBase::INVALID_METADATA); } bool BlockCode::addSourceSymbol(const fec::buffer &packet, uint32_t i, - uint32_t offset) { + uint32_t offset, uint32_t metadata) { DLOG_IF(INFO, VLOG_IS_ON(4)) << "Adding source symbol of size " << packet->length() << ", offset " << offset; - return addSymbol(packet, i, offset, packet->length() - offset); + return addSymbol(packet, i, offset, packet->length() - offset, metadata); } bool BlockCode::addSymbol(const fec::buffer &packet, uint32_t i, - uint32_t offset, std::size_t size) { + uint32_t offset, std::size_t size, + uint32_t metadata) { if (size > max_buffer_size_) { max_buffer_size_ = size; } - operator[](current_block_size_++) = std::make_tuple(i, packet, offset); + operator[](current_block_size_) = RSBufferInfo(offset, i, metadata, packet); + current_block_size_++; if (current_block_size_ >= k_) { if (to_decode_) { @@ -80,12 +83,13 @@ bool BlockCode::addSymbol(const fec::buffer &packet, uint32_t i, void BlockCode::encode() { gf *data[n_]; - uint32_t base = std::get<0>(operator[](0)); + uint32_t base = operator[](0).getIndex(); // Set packet length in first 2 bytes for (uint32_t i = 0; i < k_; i++) { - auto &packet = std::get<1>(operator[](i)); - auto offset = std::get<2>(operator[](i)); + auto &packet = operator[](i).getBuffer(); + auto offset = operator[](i).getOffset(); + auto metadata_base = operator[](i).getMetadata(); auto ret = packet->ensureCapacityAndFillUnused(max_buffer_size_ + offset, 0); @@ -98,10 +102,11 @@ void BlockCode::encode() { // Buffers should hold 2 *after* the padding, in order to be // able to set the length for the encoding operation. // packet->trimStart(offset); - uint16_t *length = reinterpret_cast<uint16_t *>(packet->writableData() + - max_buffer_size_ + offset); + fec_metadata *metadata = reinterpret_cast<fec_metadata *>( + packet->writableData() + max_buffer_size_ + offset); auto buffer_length = packet->length() - offset; - *length = htons(buffer_length); + metadata->setPacketLength(buffer_length); + metadata->setMetadataBase(metadata_base); DLOG_IF(INFO, VLOG_IS_ON(4)) << "Current buffer size: " << packet->length(); @@ -109,7 +114,7 @@ void BlockCode::encode() { } // Finish to fill source block with the buffers to hold the repair symbols - auto length = max_buffer_size_ + sizeof(fec_header) + LEN_SIZE_BYTES; + auto length = max_buffer_size_ + sizeof(fec_header) + METADATA_BYTES; for (uint32_t i = k_; i < n_; i++) { buffer packet; if (!params_.buffer_callback_) { @@ -133,19 +138,20 @@ void BlockCode::encode() { DLOG_IF(INFO, VLOG_IS_ON(4)) << "Current symbol size: " << packet->length(); data[i] = packet->writableData(); - operator[](i) = std::make_tuple(i, std::move(packet), uint32_t(0)); + operator[](i) = RSBufferInfo(uint32_t(0), i, FECBase::INVALID_METADATA, + std::move(packet)); } // Generate repair symbols and put them in corresponding buffers DLOG_IF(INFO, VLOG_IS_ON(4)) << "Calling encode with max_buffer_size_ = " << max_buffer_size_; for (uint32_t i = k_; i < n_; i++) { - fec_encode(code_, data, data[i], i, max_buffer_size_ + LEN_SIZE_BYTES); + fec_encode(code_, data, data[i], i, max_buffer_size_ + METADATA_BYTES); } // Re-include header in repair packets for (uint32_t i = k_; i < n_; i++) { - auto &packet = std::get<1>(operator[](i)); + auto &packet = operator[](i).getBuffer(); packet->prepend(sizeof(fec_header)); DLOG_IF(INFO, VLOG_IS_ON(4)) << "Produced repair symbol of size = " << packet->length(); @@ -153,16 +159,31 @@ void BlockCode::encode() { } void BlockCode::decode() { - gf *data[k_]; + gf *data[n_]; uint32_t index[k_]; + buffer aux_fec_packets[n_ - k_]; + // FEC packet number k0 + uint32_t k0 = 0; + + // Reorder block by index with in-place sorting + for (uint32_t i = 0; i < k_;) { + uint32_t idx = operator[](i).getIndex(); + if (idx >= k_ || idx == i) { + i++; + } else { + std::swap(operator[](i), operator[](idx)); + } + } for (uint32_t i = 0; i < k_; i++) { - auto &packet = std::get<1>(operator[](i)); - index[i] = std::get<0>(operator[](i)); - auto offset = std::get<2>(operator[](i)); + auto &packet = operator[](i).getBuffer(); + index[i] = operator[](i).getIndex(); + auto offset = operator[](i).getOffset(); + auto metadata_base = operator[](i).getMetadata(); sorted_index_[i] = index[i]; if (index[i] < k_) { + operator[](i).setReceived(); DLOG_IF(INFO, VLOG_IS_ON(4)) << "DECODE SOURCE - index " << index[i] << " - Current buffer size: " << packet->length(); @@ -173,49 +194,51 @@ void BlockCode::decode() { // able to set the length for the encoding operation packet->trimStart(offset); packet->ensureCapacityAndFillUnused(max_buffer_size_, 0); - uint16_t *length = reinterpret_cast<uint16_t *>( - packet->writableData() + max_buffer_size_ - LEN_SIZE_BYTES); - - *length = htons(packet->length()); + fec_metadata *metadata = reinterpret_cast<fec_metadata *>( + packet->writableData() + max_buffer_size_ - METADATA_BYTES); + metadata->setPacketLength(packet->length()); + metadata->setMetadataBase(metadata_base); } else { DLOG_IF(INFO, VLOG_IS_ON(4)) << "DECODE SYMBOL - index " << index[i] << " - Current buffer size: " << packet->length(); packet->trimStart(sizeof(fec_header) + offset); + aux_fec_packets[k0] = core::PacketManager<>::getInstance().getMemBuf(); + data[k_ + k0] = aux_fec_packets[k0]->writableData(); + k0++; } - data[i] = packet->writableData(); } - // We decode the source block DLOG_IF(INFO, VLOG_IS_ON(4)) << "Calling decode with max_buffer_size_ = " << max_buffer_size_; + fec_decode(code_, data, reinterpret_cast<int *>(index), max_buffer_size_); // Find the index in the block for recovered packets - for (uint32_t i = 0; i < k_; i++) { - if (index[i] != i) { - for (uint32_t j = 0; j < k_; j++) - if (sorted_index_[j] == uint32_t(index[i])) { - sorted_index_[j] = i; - } - } - } - - // Reorder block by index with in-place sorting - for (uint32_t i = 0; i < k_; i++) { - for (uint32_t j = sorted_index_[i]; j != i; j = sorted_index_[i]) { - std::swap(sorted_index_[j], sorted_index_[i]); - std::swap(operator[](j), operator[](i)); + for (uint32_t i = 0, j = 0; i < k_; i++) { + if (index[i] >= k_) { + operator[](i).setBuffer(aux_fec_packets[j++]); + operator[](i).setIndex(i); } } // Adjust length according to the one written in the source packet for (uint32_t i = 0; i < k_; i++) { - auto &packet = std::get<1>(operator[](i)); - uint16_t *length = reinterpret_cast<uint16_t *>( - packet->writableData() + max_buffer_size_ - LEN_SIZE_BYTES); - packet->setLength(ntohs(*length)); + auto &packet = operator[](i).getBuffer(); + fec_metadata *metadata = reinterpret_cast<fec_metadata *>( + packet->writableData() + max_buffer_size_ - METADATA_BYTES); + // Adjust buffer length + packet->setLength(metadata->getPacketLength()); + // Adjust metadata + operator[](i).setMetadata(metadata->getMetadataBase()); + + // reset the point to the beginning of the packets for all received packets + if (operator[](i).getReceived()) { + auto &packet = operator[](i).getBuffer(); + auto offset = operator[](i).getOffset(); + packet->prepend(offset); + } } } @@ -252,12 +275,11 @@ RSEncoder::RSEncoder(uint32_t k, uint32_t n, uint32_t seq_offset) source_block_(k_, n_, seq_offset_, current_code_, *this) {} void RSEncoder::consume(const fec::buffer &packet, uint32_t index, - uint32_t offset) { - if (!source_block_.addSourceSymbol(packet, index, offset)) { - std::vector<std::pair<uint32_t, buffer>> repair_packets; + uint32_t offset, uint32_t metadata) { + if (!source_block_.addSourceSymbol(packet, index, offset, metadata)) { + fec::BufferArray repair_packets; for (uint32_t i = k_; i < n_; i++) { - repair_packets.emplace_back(std::move(std::get<0>(source_block_[i])), - std::move(std::get<1>(source_block_[i]))); + repair_packets.emplace_back(std::move(source_block_[i])); } fec_callback_(repair_packets); @@ -265,9 +287,9 @@ void RSEncoder::consume(const fec::buffer &packet, uint32_t index, } void RSEncoder::onPacketProduced(core::ContentObject &content_object, - uint32_t offset) { + uint32_t offset, uint32_t metadata) { consume(content_object.shared_from_this(), - content_object.getName().getSuffix(), offset); + content_object.getName().getSuffix(), offset, metadata); } RSDecoder::RSDecoder(uint32_t k, uint32_t n, uint32_t seq_offset) @@ -276,10 +298,15 @@ RSDecoder::RSDecoder(uint32_t k, uint32_t n, uint32_t seq_offset) void RSDecoder::recoverPackets(SourceBlocks::iterator &src_block_it) { DLOG_IF(INFO, VLOG_IS_ON(4)) << "recoverPackets for " << k_; auto &src_block = src_block_it->second; - std::vector<std::pair<uint32_t, buffer>> source_packets(k_); + auto base_index = src_block_it->first; + BufferArray source_packets(k_); + + // Iterate over packets in the block and adjust indexed accordingly. This must + // be done because indexes are from 0 to (n - k - 1), but we need indexes from + // base_index to base_index + (n - k - 1) for (uint32_t i = 0; i < src_block.getK(); i++) { - source_packets[i] = std::make_pair(src_block_it->first + i, - std::move(std::get<1>(src_block[i]))); + src_block[i].setIndex(base_index + src_block[i].getIndex()); + source_packets[i] = FECBufferInfo(std::move(src_block[i])); } setProcessed(src_block_it->first); @@ -296,9 +323,9 @@ void RSDecoder::recoverPackets(SourceBlocks::iterator &src_block_it) { } void RSDecoder::consumeSource(const fec::buffer &packet, uint32_t index, - uint32_t offset) { + uint32_t offset, uint32_t metadata) { // Normalize index - assert(index >= seq_offset_); + DCHECK(index >= seq_offset_); auto i = (index - seq_offset_) % n_; // Get base @@ -324,16 +351,19 @@ void RSDecoder::consumeSource(const fec::buffer &packet, uint32_t index, // protocol that may come in the future. auto it = src_blocks_.find(base); if (it != src_blocks_.end()) { - auto ret = it->second.addSourceSymbol(packet, i, offset); + auto ret = it->second.addSourceSymbol(packet, i, offset, metadata); if (!ret) { recoverPackets(it); } } else { DLOG_IF(INFO, VLOG_IS_ON(4)) << "Adding to parked source packets"; - auto ret = parked_packets_.emplace( - base, std::vector<std::pair<buffer, uint32_t>>()); - ret.first->second.emplace_back(packet, i); + auto ret = parked_packets_.emplace(base, BufferInfoArray()); + ret.first->second.emplace_back(offset, i, metadata, packet); + /** + * If we reached k source packets, we do not have any missing packet to + * recover via FEC. Delete the block. + */ if (ret.first->second.size() >= k_) { setProcessed(ret.first->first); parked_packets_.erase(ret.first); @@ -356,8 +386,8 @@ void RSDecoder::consumeRepair(const fec::buffer &packet, uint32_t offset) { DLOG_IF(INFO, VLOG_IS_ON(4)) << "Decoder consume called for repair symbol. BASE = " << base - << ", index = " << base + i << " and i = " << i << ". K=" << k - << ", N=" << n; + << ", index = " << base + i << " and i = " << (int)i << ". K=" << (int)k + << ", N=" << (int)n; // check if a source block already exist for this symbol auto it = src_blocks_.find(base); @@ -380,8 +410,9 @@ void RSDecoder::consumeRepair(const fec::buffer &packet, uint32_t offset) { auto it2 = parked_packets_.find(base); if (it2 != parked_packets_.end()) { for (auto &packet_index : it2->second) { - auto ret = it->second.addSourceSymbol(packet_index.first, - packet_index.second, offset); + auto ret = it->second.addSourceSymbol( + packet_index.getBuffer(), packet_index.getIndex(), + packet_index.getOffset(), packet_index.getMetadata()); if (!ret) { recoverPackets(it); // Finish to delete packets in same source block that were @@ -399,7 +430,7 @@ void RSDecoder::consumeRepair(const fec::buffer &packet, uint32_t offset) { } void RSDecoder::onDataPacket(core::ContentObject &content_object, - uint32_t offset) { + uint32_t offset, uint32_t metadata) { DLOG_IF(INFO, VLOG_IS_ON(4)) << "Calling fec for data packet " << content_object.getName() << ". Offset: " << offset; @@ -409,7 +440,7 @@ void RSDecoder::onDataPacket(core::ContentObject &content_object, if (isSymbol(suffix)) { consumeRepair(content_object.shared_from_this(), offset); } else { - consumeSource(content_object.shared_from_this(), suffix, offset); + consumeSource(content_object.shared_from_this(), suffix, offset, metadata); } } diff --git a/libtransport/src/protocols/fec/rs.h b/libtransport/src/protocols/fec/rs.h index e159ad9f7..034c32bdc 100644 --- a/libtransport/src/protocols/fec/rs.h +++ b/libtransport/src/protocols/fec/rs.h @@ -34,10 +34,28 @@ namespace protocol { namespace fec { #define foreach_rs_fec_type \ + _(RS, 1, 2) \ _(RS, 1, 3) \ + _(RS, 1, 4) \ + _(RS, 2, 3) \ + _(RS, 2, 4) \ + _(RS, 2, 5) \ + _(RS, 2, 6) \ + _(RS, 3, 6) \ + _(RS, 3, 7) \ + _(RS, 3, 8) \ + _(RS, 3, 9) \ + _(RS, 3, 10) \ + _(RS, 3, 11) \ + _(RS, 3, 12) \ _(RS, 4, 5) \ _(RS, 4, 6) \ _(RS, 4, 7) \ + _(RS, 4, 8) \ + _(RS, 4, 9) \ + _(RS, 4, 10) \ + _(RS, 4, 11) \ + _(RS, 4, 12) \ _(RS, 6, 10) \ _(RS, 8, 10) \ _(RS, 8, 11) \ @@ -45,6 +63,8 @@ namespace fec { _(RS, 8, 14) \ _(RS, 8, 16) \ _(RS, 8, 32) \ + _(RS, 10, 20) \ + _(RS, 10, 25) \ _(RS, 10, 30) \ _(RS, 10, 40) \ _(RS, 10, 60) \ @@ -56,12 +76,24 @@ namespace fec { _(RS, 16, 27) \ _(RS, 17, 21) \ _(RS, 17, 34) \ + _(RS, 20, 45) \ + _(RS, 20, 50) \ + _(RS, 20, 60) \ + _(RS, 20, 70) \ + _(RS, 30, 70) \ + _(RS, 30, 75) \ + _(RS, 30, 85) \ + _(RS, 30, 95) \ _(RS, 32, 36) \ _(RS, 32, 41) \ _(RS, 32, 46) \ _(RS, 32, 54) \ _(RS, 34, 42) \ _(RS, 35, 70) \ + _(RS, 40, 95) \ + _(RS, 40, 100) \ + _(RS, 40, 110) \ + _(RS, 40, 120) \ _(RS, 52, 62) static const constexpr uint16_t MAX_SOURCE_BLOCK_SIZE = 128; @@ -73,9 +105,24 @@ static const constexpr uint16_t MAX_SOURCE_BLOCK_SIZE = 128; * std::array allows to be constructed in place, saving the allocation at the * price os knowing in advance its size. */ -using Packets = std::array<std::tuple</* index */ uint32_t, /* buffer */ buffer, - uint32_t /* offset */>, - MAX_SOURCE_BLOCK_SIZE>; +class RSBufferInfo : public FECBufferInfo { + public: + RSBufferInfo() : FECBufferInfo() {} + + RSBufferInfo(uint32_t offset, uint32_t index, uint32_t metadata, + buffer buffer) + : FECBufferInfo(index, metadata, buffer), offset_(offset) {} + + uint32_t getOffset() { return offset_; } + RSBufferInfo &setOffset(uint32_t offset) { + offset_ = offset; + return *this; + } + + private: + uint32_t offset_ = 0; +}; +using Packets = std::array<RSBufferInfo, MAX_SOURCE_BLOCK_SIZE>; /** * FEC Header, prepended to symbol packets. @@ -123,10 +170,32 @@ class rs; */ class BlockCode : public Packets { /** + * @brief Metadata to include when encoding the buffers. This does not need to + * be sent over the network, but just to be included in the FEC protected + * bytes. + * + */ + class __attribute__((__packed__)) fec_metadata { + public: + void setPacketLength(uint16_t length) { packet_length = htons(length); } + uint32_t getPacketLength() { return ntohs(packet_length); } + + void setMetadataBase(uint32_t value) { metadata = htonl(value); } + uint32_t getMetadataBase() { return ntohl(metadata); } + + private: + uint16_t packet_length; /* Used to get the real size of the packet after we + pad it */ + uint32_t + metadata; /* Caller may specify an integer for storing additional + metadata that can be used when recovering the packet. */ + }; + + /** * For variable length packet we need to prepend to the padded payload the * real length of the packet. This is *not* sent over the network. */ - static constexpr std::size_t LEN_SIZE_BYTES = 2; + static constexpr std::size_t METADATA_BYTES = sizeof(fec_metadata); public: BlockCode(uint32_t k, uint32_t n, uint32_t seq_offset, struct fec_parms *code, @@ -142,7 +211,8 @@ class BlockCode : public Packets { * Add a source symbol to the source block. */ bool addSourceSymbol(const fec::buffer &packet, uint32_t i, - uint32_t offset = 0); + uint32_t offset = 0, + uint32_t metadata = FECBase::INVALID_METADATA); /** * Get current length of source block. @@ -169,7 +239,7 @@ class BlockCode : public Packets { * Add symbol to source block **/ bool addSymbol(const fec::buffer &packet, uint32_t i, uint32_t offset, - std::size_t size); + std::size_t size, uint32_t metadata); /** * Starting from k source symbols, get the n - k repair symbols @@ -310,10 +380,11 @@ class RSEncoder : public rs, public ProducerFEC { /** * Always consume source symbols. */ - void consume(const fec::buffer &packet, uint32_t index, uint32_t offset = 0); + void consume(const fec::buffer &packet, uint32_t index, uint32_t offset = 0, + uint32_t metadata = FECBase::INVALID_METADATA); - void onPacketProduced(core::ContentObject &content_object, - uint32_t offset) override; + void onPacketProduced(core::ContentObject &content_object, uint32_t offset, + uint32_t metadata = FECBase::INVALID_METADATA) override; /** * @brief Get the fec header size, if added to source packets @@ -348,8 +419,8 @@ class RSDecoder : public rs, public ConsumerFEC { /** * Consume source symbol */ - void consumeSource(const fec::buffer &packet, uint32_t i, - uint32_t offset = 0); + void consumeSource(const fec::buffer &packet, uint32_t i, uint32_t offset = 0, + uint32_t metadata = FECBase::INVALID_METADATA); /** * Consume repair symbol @@ -359,8 +430,8 @@ class RSDecoder : public rs, public ConsumerFEC { /** * Consumers will call this function when they receive a data packet */ - void onDataPacket(core::ContentObject &content_object, - uint32_t offset) override; + void onDataPacket(core::ContentObject &content_object, uint32_t offset, + uint32_t metadata = FECBase::INVALID_METADATA) override; /** * @brief Get the fec header size, if added to source packets @@ -398,8 +469,8 @@ class RSDecoder : public rs, public ConsumerFEC { * not make any sense to build the source block, since we received all the * source packet of the block. */ - std::unordered_map<uint32_t, std::vector<std::pair<buffer, uint32_t>>> - parked_packets_; + using BufferInfoArray = std::vector<RSBufferInfo>; + std::unordered_map<uint32_t, BufferInfoArray> parked_packets_; }; } // namespace fec diff --git a/libtransport/src/protocols/fec_base.h b/libtransport/src/protocols/fec_base.h index a1929d85e..bda3ee756 100644 --- a/libtransport/src/protocols/fec_base.h +++ b/libtransport/src/protocols/fec_base.h @@ -15,6 +15,7 @@ #pragma once +#include <hicn/transport/core/asio_wrapper.h> #include <hicn/transport/core/content_object.h> #include <hicn/transport/errors/not_implemented_exception.h> @@ -25,12 +26,67 @@ namespace protocol { namespace fec { -using buffer = typename utils::MemBuf::Ptr; -using BufferArray = std::vector<std::pair<uint32_t, buffer>>; +using buffer = utils::MemBuf::Ptr; + +class FECBufferInfo { + public: + FECBufferInfo() + : index_(~0), metadata_(~0), received_(false), buffer_(nullptr) {} + + template <typename T> + FECBufferInfo(uint32_t index, uint32_t metadata, T &&buffer) + : index_(index), + metadata_(metadata), + received_(false), + buffer_(std::forward<T>(buffer)) {} + + // Getters + uint32_t getIndex() const { return index_; } + + uint32_t getMetadata() const { return metadata_; } + + bool getReceived() const { return received_; } + + buffer &getBuffer() { return buffer_; } + + // Setters + void setReceived() { received_ = true; } + + FECBufferInfo &setIndex(uint32_t index) { + index_ = index; + return *this; + } + + FECBufferInfo &setMetadata(uint32_t metadata) { + metadata_ = metadata; + return *this; + } + + FECBufferInfo &setBuffer(buffer &buffer) { + buffer_ = buffer; + return *this; + } + + FECBufferInfo &setBuffer(buffer &&buffer) { + buffer_ = std::move(buffer); + return *this; + } + + private: + uint32_t index_; + uint32_t metadata_; + bool received_; + buffer buffer_; +}; + +using BufferArray = typename std::vector<FECBufferInfo>; class FECBase { public: - virtual ~FECBase() = default; + static inline uint32_t INVALID_METADATA = ~0; + static inline uint32_t INVALID_INDEX = ~0; + + virtual ~FECBase() {} /** * Callback to be called after the encode or the decode operations. In the * former case it will contain the symbols, while in the latter the sources. @@ -64,11 +120,21 @@ class FECBase { buffer_callback_ = buffer_callback; } + /** + * Creates the timer to flush packets. So far needed only if using Rely and + * want to avoid expired packets blocked by missing pkts to wait for a new + * packet to arrive and trigger the flush + */ + void setIOService(asio::io_service &io_service) { + flush_timer_ = std::make_unique<asio::steady_timer>(io_service); + } + virtual void reset() = 0; protected: PacketsReady fec_callback_{0}; BufferRequested buffer_callback_{0}; + std::unique_ptr<asio::steady_timer> flush_timer_; }; /** @@ -80,8 +146,9 @@ class ProducerFEC : public virtual FECBase { /** * Producers will call this function upon production of a new packet. */ - virtual void onPacketProduced(core::ContentObject &content_object, - uint32_t offset) = 0; + virtual void onPacketProduced( + core::ContentObject &content_object, uint32_t offset, + uint32_t metadata = FECBase::INVALID_METADATA) = 0; }; /** @@ -95,9 +162,10 @@ class ConsumerFEC : public virtual FECBase { * Consumers will call this function when they receive a data packet */ virtual void onDataPacket(core::ContentObject &content_object, - uint32_t offset) = 0; + uint32_t offset, + uint32_t metadata = FECBase::INVALID_METADATA) = 0; }; } // namespace fec } // namespace protocol -} // namespace transport
\ No newline at end of file +} // namespace transport diff --git a/libtransport/src/protocols/incremental_indexer_bytestream.cc b/libtransport/src/protocols/incremental_indexer_bytestream.cc index cc302a98a..b94f229e5 100644 --- a/libtransport/src/protocols/incremental_indexer_bytestream.cc +++ b/libtransport/src/protocols/incremental_indexer_bytestream.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -29,9 +29,9 @@ void IncrementalIndexer::onContentObject(core::Interest &interest, DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received content " << content_object.getName(); - assert(reassembly_); + DCHECK(reassembly_); - if (TRANSPORT_EXPECT_FALSE(content_object.testRst())) { + if (TRANSPORT_EXPECT_FALSE(content_object.isLast())) { final_suffix_ = content_object.getName().getSuffix(); } diff --git a/libtransport/src/protocols/incremental_indexer_bytestream.h b/libtransport/src/protocols/incremental_indexer_bytestream.h index c6a669629..422e49ecd 100644 --- a/libtransport/src/protocols/incremental_indexer_bytestream.h +++ b/libtransport/src/protocols/incremental_indexer_bytestream.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -62,7 +62,7 @@ class IncrementalIndexer : public Indexer { next_reassembly_suffix_ = first_suffix_; } - virtual uint32_t checkNextSuffix() override { + virtual uint32_t checkNextSuffix() const override { return next_download_suffix_ <= final_suffix_ ? next_download_suffix_ : Indexer::invalid_index; } @@ -76,7 +76,7 @@ class IncrementalIndexer : public Indexer { first_suffix_ = suffix; } - uint32_t getFirstSuffix() override { return first_suffix_; } + uint32_t getFirstSuffix() const override { return first_suffix_; } virtual uint32_t jumpToIndex(uint32_t index) override { next_download_suffix_ = index; @@ -95,14 +95,14 @@ class IncrementalIndexer : public Indexer { return final_suffix_ != Indexer::invalid_index; } - virtual uint32_t getFinalSuffix() override { return final_suffix_; } + virtual uint32_t getFinalSuffix() const override { return final_suffix_; } void enableFec(fec::FECType fec_type) override {} void disableFec() override {} void setNFec(uint32_t n_fec) override {} - virtual uint32_t getNFec() override { return 0; } + virtual uint32_t getNFec() const override { return 0; } virtual void onContentObject(core::Interest &interest, core::ContentObject &content_object, diff --git a/libtransport/src/protocols/index_manager_bytestream.cc b/libtransport/src/protocols/index_manager_bytestream.cc index c78dc634d..952f36e0e 100644 --- a/libtransport/src/protocols/index_manager_bytestream.cc +++ b/libtransport/src/protocols/index_manager_bytestream.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/index_manager_bytestream.h b/libtransport/src/protocols/index_manager_bytestream.h index e14c8845b..7ea31dfa5 100644 --- a/libtransport/src/protocols/index_manager_bytestream.h +++ b/libtransport/src/protocols/index_manager_bytestream.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -40,7 +40,9 @@ class IndexManager : public IncrementalIndexer { indexer_->setFirstSuffix(suffix); } - uint32_t getFirstSuffix() override { return indexer_->getFirstSuffix(); } + uint32_t getFirstSuffix() const override { + return indexer_->getFirstSuffix(); + } uint32_t getNextReassemblySegment() override { return indexer_->getNextReassemblySegment(); @@ -50,22 +52,26 @@ class IndexManager : public IncrementalIndexer { return indexer_->isFinalSuffixDiscovered(); } - uint32_t getFinalSuffix() override { return indexer_->getFinalSuffix(); } + uint32_t getFinalSuffix() const override { + return indexer_->getFinalSuffix(); + } uint32_t jumpToIndex(uint32_t index) override { return indexer_->jumpToIndex(index); } void setNFec(uint32_t n_fec) override { return indexer_->setNFec(n_fec); } - uint32_t getNFec() override { return indexer_->getNFec(); } + uint32_t getNFec() const override { return indexer_->getNFec(); } void enableFec(fec::FECType fec_type) override { return indexer_->enableFec(fec_type); } - double getFecOverhead() override { return indexer_->getFecOverhead(); } + double getFecOverhead() const override { return indexer_->getFecOverhead(); } - double getMaxFecOverhead() override { return indexer_->getMaxFecOverhead(); } + double getMaxFecOverhead() const override { + return indexer_->getMaxFecOverhead(); + } void disableFec() override { return indexer_->disableFec(); } diff --git a/libtransport/src/protocols/indexer.cc b/libtransport/src/protocols/indexer.cc index 8d4cf04d7..41465755b 100644 --- a/libtransport/src/protocols/indexer.cc +++ b/libtransport/src/protocols/indexer.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -14,10 +14,10 @@ */ #include <implementation/socket_consumer.h> +#include <protocols/errors.h> #include <protocols/indexer.h> namespace transport { - namespace protocol { using namespace interface; @@ -36,6 +36,35 @@ void Indexer::setVerifier() { } } -} // end namespace protocol +void Indexer::applyPolicy(core::Interest &interest, + core::ContentObject &content_object, bool reassembly, + auth::VerificationPolicy policy) const { + DCHECK(reassembly_ != nullptr); + + switch (policy) { + case auth::VerificationPolicy::ACCEPT: { + if (reassembly) { + reassembly_->reassemble(content_object); + } + break; + } + case auth::VerificationPolicy::UNKNOWN: + if (reassembly && reassembly_->reassembleUnverified()) { + reassembly_->reassemble(content_object); + } + break; + case auth::VerificationPolicy::DROP: + transport_->onPacketDropped( + interest, content_object, + make_error_code(protocol_error::verification_failed)); + break; + case auth::VerificationPolicy::ABORT: { + transport_->onContentReassembled( + make_error_code(protocol_error::session_aborted)); + break; + } + } +} +} // end namespace protocol } // end namespace transport diff --git a/libtransport/src/protocols/indexer.h b/libtransport/src/protocols/indexer.h index 7e3a52fb0..1bacb13aa 100644 --- a/libtransport/src/protocols/indexer.h +++ b/libtransport/src/protocols/indexer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -15,6 +15,7 @@ #pragma once +#include <hicn/transport/auth/policies.h> #include <hicn/transport/core/content_object.h> #include <hicn/transport/core/interest.h> #include <protocols/fec_utils.h> @@ -44,7 +45,7 @@ class Indexer { /** * Suffix getters */ - virtual uint32_t checkNextSuffix() = 0; + virtual uint32_t checkNextSuffix() const = 0; virtual uint32_t getNextSuffix() = 0; virtual uint32_t getNextReassemblySegment() = 0; @@ -52,24 +53,24 @@ class Indexer { * Set first suffix from where to start. */ virtual void setFirstSuffix(uint32_t suffix) = 0; - virtual uint32_t getFirstSuffix() = 0; + virtual uint32_t getFirstSuffix() const = 0; /** * Functions to set/enable/disable fec */ virtual void setNFec(uint32_t n_fec) = 0; - virtual uint32_t getNFec() = 0; + virtual uint32_t getNFec() const = 0; virtual void enableFec(fec::FECType fec_type) = 0; virtual void disableFec() = 0; virtual bool isFec(uint32_t index) { return false; } - virtual double getFecOverhead() { return 0.0; } - virtual double getMaxFecOverhead() { return 0.0; } + virtual double getFecOverhead() const { return 0.0; } + virtual double getMaxFecOverhead() const { return 0.0; } /** * Final suffix helpers. */ virtual bool isFinalSuffixDiscovered() = 0; - virtual uint32_t getFinalSuffix() = 0; + virtual uint32_t getFinalSuffix() const = 0; /** * Set reassembly protocol @@ -84,9 +85,15 @@ class Indexer { virtual void setVerifier(); /** + * Apply a verification policy + */ + virtual void applyPolicy(core::Interest &interest, + core::ContentObject &content_object, bool reassembly, + auth::VerificationPolicy policy) const; + /** * Jump to suffix. This may be useful if, for any protocol dependent - * mechanism, we need to suddenly change current suffix. This does not modify - * the way suffixes re incremented/decremented (that's part of the + * mechanism, we need to suddenly change current suffix. This does not + * modify the way suffixes re incremented/decremented (that's part of the * implementation). */ virtual uint32_t jumpToIndex(uint32_t index) = 0; @@ -108,6 +115,7 @@ class Indexer { TransportProtocol *transport_; Reassembly *reassembly_; std::shared_ptr<auth::Verifier> verifier_; + auth::CryptoHashType manifest_hash_type_; }; } // end namespace protocol diff --git a/libtransport/src/protocols/manifest_incremental_indexer_bytestream.cc b/libtransport/src/protocols/manifest_incremental_indexer_bytestream.cc index 168aa57af..b5ab8184f 100644 --- a/libtransport/src/protocols/manifest_incremental_indexer_bytestream.cc +++ b/libtransport/src/protocols/manifest_incremental_indexer_bytestream.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -31,8 +31,7 @@ ManifestIncrementalIndexer::ManifestIncrementalIndexer( implementation::ConsumerSocket *icn_socket, TransportProtocol *transport) : IncrementalIndexer(icn_socket, transport), suffix_strategy_(utils::SuffixStrategyFactory::getSuffixStrategy( - NextSegmentCalculationStrategy::INCREMENTAL, next_download_suffix_, - 0)) {} + utils::NextSuffixStrategy::INCREMENTAL, next_download_suffix_)) {} void ManifestIncrementalIndexer::onContentObject( core::Interest &interest, core::ContentObject &content_object, @@ -59,12 +58,7 @@ void ManifestIncrementalIndexer::onContentObject( void ManifestIncrementalIndexer::onUntrustedManifest( core::Interest &interest, core::ContentObject &content_object, bool reassembly) { - auto manifest = - std::make_unique<ContentObjectManifest>(std::move(content_object)); - - auth::VerificationPolicy policy = verifier_->verifyPackets(manifest.get()); - - manifest->decode(); + auth::VerificationPolicy policy = verifier_->verifyPackets(&content_object); if (policy != auth::VerificationPolicy::ACCEPT) { transport_->onContentReassembled( @@ -72,6 +66,10 @@ void ManifestIncrementalIndexer::onUntrustedManifest( return; } + auto manifest = + std::make_unique<ContentObjectManifest>(std::move(content_object)); + manifest->decode(); + processTrustedManifest(interest, std::move(manifest), reassembly); } @@ -83,9 +81,10 @@ void ManifestIncrementalIndexer::processTrustedManifest( throw errors::RuntimeException("Received manifest with unknown version."); } - switch (manifest->getManifestType()) { + switch (manifest->getType()) { case core::ManifestType::INLINE_MANIFEST: { - suffix_strategy_->setFinalSuffix(manifest->getFinalBlockNumber()); + suffix_strategy_->setFinalSuffix( + manifest->getParamsBytestream().final_segment); // The packets to verify with the received manifest std::vector<auth::PacketPtr> packets; @@ -162,45 +161,15 @@ void ManifestIncrementalIndexer::onUntrustedContentObject( applyPolicy(interest, content_object, reassembly, policy); } -void ManifestIncrementalIndexer::applyPolicy( - core::Interest &interest, core::ContentObject &content_object, - bool reassembly, auth::VerificationPolicy policy) { - assert(reassembly_); - switch (policy) { - case auth::VerificationPolicy::ACCEPT: { - if (reassembly && !reassembly_->reassembleUnverified()) { - reassembly_->reassemble(content_object); - } - break; - } - case auth::VerificationPolicy::DROP: { - transport_->onPacketDropped( - interest, content_object, - make_error_code(protocol_error::verification_failed)); - break; - } - case auth::VerificationPolicy::ABORT: { - transport_->onContentReassembled( - make_error_code(protocol_error::session_aborted)); - break; - } - case auth::VerificationPolicy::UNKNOWN: { - if (reassembly && reassembly_->reassembleUnverified()) { - reassembly_->reassemble(content_object); - } - } - } -} - -uint32_t ManifestIncrementalIndexer::checkNextSuffix() { - return suffix_strategy_->getNextSuffix(); +uint32_t ManifestIncrementalIndexer::checkNextSuffix() const { + return suffix_strategy_->checkNextSuffix(); } uint32_t ManifestIncrementalIndexer::getNextSuffix() { auto ret = suffix_strategy_->getNextSuffix(); if (ret <= suffix_strategy_->getFinalSuffix() && - ret != utils::SuffixStrategy::INVALID_SUFFIX) { + ret != utils::SuffixStrategy::MAX_SUFFIX) { suffix_queue_.push(ret); return ret; } @@ -208,7 +177,7 @@ uint32_t ManifestIncrementalIndexer::getNextSuffix() { return Indexer::invalid_index; } -uint32_t ManifestIncrementalIndexer::getFinalSuffix() { +uint32_t ManifestIncrementalIndexer::getFinalSuffix() const { return suffix_strategy_->getFinalSuffix(); } diff --git a/libtransport/src/protocols/manifest_incremental_indexer_bytestream.h b/libtransport/src/protocols/manifest_incremental_indexer_bytestream.h index d8cf5892f..12876f35c 100644 --- a/libtransport/src/protocols/manifest_incremental_indexer_bytestream.h +++ b/libtransport/src/protocols/manifest_incremental_indexer_bytestream.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -39,8 +39,7 @@ class ManifestIncrementalIndexer : public IncrementalIndexer { ManifestIncrementalIndexer(IncrementalIndexer &&indexer) : IncrementalIndexer(std::move(indexer)), suffix_strategy_(utils::SuffixStrategyFactory::getSuffixStrategy( - core::NextSegmentCalculationStrategy::INCREMENTAL, - next_download_suffix_, 0)) { + utils::NextSuffixStrategy::INCREMENTAL, next_download_suffix_)) { for (uint32_t i = first_suffix_; i < next_download_suffix_; i++) { suffix_queue_.push(i); } @@ -54,7 +53,7 @@ class ManifestIncrementalIndexer : public IncrementalIndexer { core::ContentObject &content_object, bool reassembly) override; - uint32_t checkNextSuffix() override; + uint32_t checkNextSuffix() const override; uint32_t getNextSuffix() override; @@ -62,7 +61,7 @@ class ManifestIncrementalIndexer : public IncrementalIndexer { bool isFinalSuffixDiscovered() override; - uint32_t getFinalSuffix() override; + uint32_t getFinalSuffix() const override; protected: std::unique_ptr<utils::SuffixStrategy> suffix_strategy_; @@ -82,9 +81,6 @@ class ManifestIncrementalIndexer : public IncrementalIndexer { void onUntrustedContentObject(core::Interest &interest, core::ContentObject &content_object, bool reassembly); - void applyPolicy(core::Interest &interest, - core::ContentObject &content_object, bool reassembly, - auth::VerificationPolicy policy); }; } // end namespace protocol diff --git a/libtransport/src/protocols/prod_protocol_bytestream.cc b/libtransport/src/protocols/prod_protocol_bytestream.cc index f659cb37c..2a3ec07e1 100644 --- a/libtransport/src/protocols/prod_protocol_bytestream.cc +++ b/libtransport/src/protocols/prod_protocol_bytestream.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -29,12 +29,7 @@ ByteStreamProductionProtocol::ByteStreamProductionProtocol( implementation::ProducerSocket *icn_socket) : ProductionProtocol(icn_socket) {} -ByteStreamProductionProtocol::~ByteStreamProductionProtocol() { - stop(); - if (listening_thread_.joinable()) { - listening_thread_.join(); - } -} +ByteStreamProductionProtocol::~ByteStreamProductionProtocol() { stop(); } uint32_t ByteStreamProductionProtocol::produceDatagram( const Name &content_name, std::unique_ptr<utils::MemBuf> &&buffer) { @@ -68,16 +63,16 @@ uint32_t ByteStreamProductionProtocol::produceStream( return 0; } - Name name(content_name); - - // Get the atomic variables to ensure they keep the same value - // during the production - // Total size of the data packet uint32_t data_packet_size; socket_->getSocketOption(GeneralTransportOptions::DATA_PACKET_SIZE, data_packet_size); + // Maximum size of a segment + uint32_t max_segment_size; + socket_->getSocketOption(GeneralTransportOptions::MAX_SEGMENT_SIZE, + max_segment_size); + // Expiry time uint32_t content_object_expiry_time; socket_->getSocketOption(GeneralTransportOptions::CONTENT_OBJECT_EXPIRY_TIME, @@ -88,106 +83,90 @@ uint32_t ByteStreamProductionProtocol::produceStream( socket_->getSocketOption(GeneralTransportOptions::HASH_ALGORITHM, hash_algo); // Suffix calculation strategy - core::NextSegmentCalculationStrategy _suffix_strategy; + std::shared_ptr<utils::SuffixStrategy> suffix_strategy; socket_->getSocketOption(GeneralTransportOptions::SUFFIX_STRATEGY, - _suffix_strategy); - auto suffix_strategy = utils::SuffixStrategyFactory::getSuffixStrategy( - _suffix_strategy, start_offset); + suffix_strategy); + suffix_strategy->reset(start_offset); - auto buffer_size = buffer->length(); + // Default format + core::Packet::Format default_format; + socket_->getSocketOption(GeneralTransportOptions::PACKET_FORMAT, + default_format); + + Name name(content_name); + size_t buffer_size = buffer->length(); + size_t signature_length = signer_->getSignatureFieldSize(); + uint32_t final_block_number = start_offset; + + // Content-related + core::Packet::Format content_format; + uint32_t content_header_size; + uint64_t content_free_space; + uint32_t nb_segments; int bytes_segmented = 0; - std::size_t header_size; - std::size_t manifest_header_size = 0; - std::size_t signature_length = 0; - std::uint32_t final_block_number = start_offset; - uint64_t free_space_for_content = 0; - core::Packet::Format format; + // Manifest-related + core::Packet::Format manifest_format; + uint32_t manifest_header_size; + uint64_t manifest_free_space; + uint32_t nb_manifests; std::shared_ptr<core::ContentObjectManifest> manifest; + uint32_t manifest_capacity = making_manifest_; bool is_last_manifest = false; - - // TODO Manifest may still be used for indexing - if (making_manifest_ && !signer_) { - LOG(FATAL) << "Making manifests without setting producer identity."; - } - - core::Packet::Format hf_format = core::Packet::Format::HF_UNSPEC; - core::Packet::Format hf_format_ah = core::Packet::Format::HF_UNSPEC; - - if (name.getType() == HNT_CONTIGUOUS_V4 || name.getType() == HNT_IOV_V4) { - hf_format = core::Packet::Format::HF_INET_TCP; - hf_format_ah = core::Packet::Format::HF_INET_TCP_AH; - } else if (name.getType() == HNT_CONTIGUOUS_V6 || - name.getType() == HNT_IOV_V6) { - hf_format = core::Packet::Format::HF_INET6_TCP; - hf_format_ah = core::Packet::Format::HF_INET6_TCP_AH; - } else { - throw errors::RuntimeException("Unknown name format."); - } - - format = hf_format; - if (making_manifest_) { - manifest_header_size = core::Packet::getHeaderSizeFromFormat( - signer_ ? hf_format_ah : hf_format, - signer_ ? signer_->getSignatureFieldSize() : 0); - } else if (signer_) { - format = hf_format_ah; - signature_length = signer_->getSignatureFieldSize(); + ParamsBytestream transport_params; + + manifest_format = Packet::toAHFormat(default_format); + content_format = + !making_manifest_ ? Packet::toAHFormat(default_format) : default_format; + + content_header_size = + core::Packet::getHeaderSizeFromFormat(content_format, signature_length); + manifest_header_size = + core::Packet::getHeaderSizeFromFormat(manifest_format, signature_length); + content_free_space = + std::min(max_segment_size, data_packet_size - content_header_size); + manifest_free_space = + std::min(max_segment_size, data_packet_size - manifest_header_size); + + // Compute the number of segments the data will be split into + nb_segments = + uint32_t(std::ceil(double(buffer_size) / double(content_free_space))); + if (content_free_space * nb_segments < buffer_size) { + nb_segments++; } - header_size = core::Packet::getHeaderSizeFromFormat(format, signature_length); - free_space_for_content = data_packet_size - header_size; - uint32_t number_of_segments = - uint32_t(std::ceil(double(buffer_size) / double(free_space_for_content))); - if (free_space_for_content * number_of_segments < buffer_size) { - number_of_segments++; - } - - // TODO allocate space for all the headers if (making_manifest_) { - uint32_t segment_in_manifest = static_cast<uint32_t>( - std::floor(double(data_packet_size - manifest_header_size - - ContentObjectManifest::getManifestHeaderSize()) / - ContentObjectManifest::getManifestEntrySize()) - - 1.0); - uint32_t number_of_manifests = static_cast<uint32_t>( - std::ceil(float(number_of_segments) / segment_in_manifest)); - final_block_number += number_of_segments + number_of_manifests - 1; + nb_manifests = static_cast<uint32_t>( + std::ceil(float(nb_segments) / manifest_capacity)); + final_block_number += nb_segments + nb_manifests - 1; + transport_params.final_segment = + is_last ? final_block_number : utils::SuffixStrategy::MAX_SUFFIX; manifest.reset(ContentObjectManifest::createManifest( + manifest_format, name.setSuffix(suffix_strategy->getNextManifestSuffix()), core::ManifestVersion::VERSION_1, core::ManifestType::INLINE_MANIFEST, - hash_algo, is_last_manifest, name, _suffix_strategy, - signer_ ? signer_->getSignatureFieldSize() : 0)); - manifest->setLifetime(content_object_expiry_time); + is_last_manifest, name, hash_algo, signature_length)); - if (is_last) { - manifest->setFinalBlockNumber(final_block_number); - } else { - manifest->setFinalBlockNumber(utils::SuffixStrategy::INVALID_SUFFIX); - } + manifest->setLifetime(content_object_expiry_time); + manifest->setParamsBytestream(transport_params); } - for (unsigned int packaged_segments = 0; - packaged_segments < number_of_segments; packaged_segments++) { + auto self = shared_from_this(); + for (unsigned int packaged_segments = 0; packaged_segments < nb_segments; + packaged_segments++) { if (making_manifest_) { - if (manifest->estimateManifestSize(2) > - data_packet_size - manifest_header_size) { + if (manifest->estimateManifestSize(1) > manifest_free_space) { manifest->encode(); - - // If identity set, sign manifest - if (signer_) { - signer_->signPacket(manifest.get()); - } + signer_->signPacket(manifest.get()); // Send the current manifest - passContentObjectToCallbacks(manifest); - + passContentObjectToCallbacks(manifest, self); DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send manifest " << manifest->getName(); // Send content objects stored in the queue while (!content_queue_.empty()) { - passContentObjectToCallbacks(content_queue_.front()); + passContentObjectToCallbacks(content_queue_.front(), self); DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send content " << content_queue_.front()->getName(); content_queue_.pop(); @@ -195,86 +174,84 @@ uint32_t ByteStreamProductionProtocol::produceStream( // Create new manifest. The reference to the last manifest has been // acquired in the passContentObjectToCallbacks function, so we can - // safely release this reference + // safely release this reference. manifest.reset(ContentObjectManifest::createManifest( + manifest_format, name.setSuffix(suffix_strategy->getNextManifestSuffix()), core::ManifestVersion::VERSION_1, - core::ManifestType::INLINE_MANIFEST, hash_algo, is_last_manifest, - name, _suffix_strategy, - signer_ ? signer_->getSignatureFieldSize() : 0)); + core::ManifestType::INLINE_MANIFEST, is_last_manifest, name, + hash_algo, signature_length)); manifest->setLifetime(content_object_expiry_time); - manifest->setFinalBlockNumber( - is_last ? final_block_number - : utils::SuffixStrategy::INVALID_SUFFIX); + manifest->setParamsBytestream(transport_params); } } - auto content_suffix = suffix_strategy->getNextContentSuffix(); + // Create content object + uint32_t content_suffix = suffix_strategy->getNextContentSuffix(); auto content_object = std::make_shared<ContentObject>( - name.setSuffix(content_suffix), format, - signer_ && !making_manifest_ ? signer_->getSignatureFieldSize() : 0); + name.setSuffix(content_suffix), content_format, + !making_manifest_ ? signature_length : 0); content_object->setLifetime(content_object_expiry_time); auto b = buffer->cloneOne(); - b->trimStart(free_space_for_content * packaged_segments); + b->trimStart(content_free_space * packaged_segments); b->trimEnd(b->length()); - if (TRANSPORT_EXPECT_FALSE(packaged_segments == number_of_segments - 1)) { + // Segment the input data + if (TRANSPORT_EXPECT_FALSE(packaged_segments == nb_segments - 1)) { b->append(buffer_size - bytes_segmented); bytes_segmented += (int)(buffer_size - bytes_segmented); if (is_last && making_manifest_) { is_last_manifest = true; } else if (is_last) { - content_object->setRst(); + content_object->setLast(); } } else { - b->append(free_space_for_content); - bytes_segmented += (int)(free_space_for_content); + b->append(content_free_space); + bytes_segmented += (int)(content_free_space); } + // Set the segmented data as payload content_object->appendPayload(std::move(b)); + // Either we sign the content object or we save its hash into the current + // manifest if (making_manifest_) { - using namespace std::chrono_literals; auth::CryptoHash hash = content_object->computeDigest(hash_algo); manifest->addSuffixHash(content_suffix, hash); content_queue_.push(content_object); } else { - if (signer_) { - signer_->signPacket(content_object.get()); - } - passContentObjectToCallbacks(content_object); + signer_->signPacket(content_object.get()); + passContentObjectToCallbacks(content_object, self); DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send content " << content_object->getName(); } } + // We send the manifest that hasn't been fully filled yet if (making_manifest_) { if (is_last_manifest) { - manifest->setFinalManifest(is_last_manifest); + manifest->setIsLast(is_last_manifest); } manifest->encode(); + signer_->signPacket(manifest.get()); - if (signer_) { - signer_->signPacket(manifest.get()); - } - - passContentObjectToCallbacks(manifest); + passContentObjectToCallbacks(manifest, self); DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send manifest " << manifest->getName(); while (!content_queue_.empty()) { - passContentObjectToCallbacks(content_queue_.front()); + passContentObjectToCallbacks(content_queue_.front(), self); DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send content " << content_queue_.front()->getName(); content_queue_.pop(); } } - portal_->getIoService().post([this]() { + portal_->getThread().add([this, self]() { std::shared_ptr<ContentObject> co; while (object_queue_for_callbacks_.pop(co)) { if (*on_new_segment_) { @@ -296,7 +273,7 @@ uint32_t ByteStreamProductionProtocol::produceStream( } }); - portal_->getIoService().dispatch([this, buffer_size]() { + portal_->getThread().add([this, buffer_size, self]() { if (*on_content_produced_) { on_content_produced_->operator()(*socket_->getInterface(), std::make_error_code(std::errc(0)), @@ -307,9 +284,10 @@ uint32_t ByteStreamProductionProtocol::produceStream( return suffix_strategy->getTotalCount(); } -void ByteStreamProductionProtocol::scheduleSendBurst() { - portal_->getIoService().post([this]() { - std::shared_ptr<ContentObject> co; +void ByteStreamProductionProtocol::scheduleSendBurst( + const std::shared_ptr<ByteStreamProductionProtocol> &self) { + portal_->getThread().add([this, self]() { + ContentObject::Ptr co; for (uint32_t i = 0; i < burst_size; i++) { if (object_queue_for_callbacks_.pop(co)) { @@ -321,11 +299,15 @@ void ByteStreamProductionProtocol::scheduleSendBurst() { on_content_object_to_sign_->operator()(*socket_->getInterface(), *co); } + output_buffer_.insert(co); + if (*on_content_object_in_output_buffer_) { on_content_object_in_output_buffer_->operator()( *socket_->getInterface(), *co); } + portal_->sendContentObject(*co); + if (*on_content_object_output_) { on_content_object_output_->operator()(*socket_->getInterface(), *co); } @@ -337,13 +319,12 @@ void ByteStreamProductionProtocol::scheduleSendBurst() { } void ByteStreamProductionProtocol::passContentObjectToCallbacks( - const std::shared_ptr<ContentObject> &content_object) { - output_buffer_.insert(content_object); - portal_->sendContentObject(*content_object); + const std::shared_ptr<ContentObject> &content_object, + const std::shared_ptr<ByteStreamProductionProtocol> &self) { object_queue_for_callbacks_.push(std::move(content_object)); if (object_queue_for_callbacks_.size() >= burst_size) { - scheduleSendBurst(); + scheduleSendBurst(self); } } @@ -376,7 +357,5 @@ void ByteStreamProductionProtocol::onInterest(Interest &interest) { } } -void ByteStreamProductionProtocol::onError(std::error_code ec) {} - } // namespace protocol } // end namespace transport diff --git a/libtransport/src/protocols/prod_protocol_bytestream.h b/libtransport/src/protocols/prod_protocol_bytestream.h index cf36b90a5..809ad8d5c 100644 --- a/libtransport/src/protocols/prod_protocol_bytestream.h +++ b/libtransport/src/protocols/prod_protocol_bytestream.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -50,16 +50,19 @@ class ByteStreamProductionProtocol : public ProductionProtocol { uint32_t produceDatagram(const Name &content_name, const uint8_t *buffer, size_t buffer_size) override; + auto shared_from_this() { return utils::shared_from(this); } + protected: // Consumer Callback // void reset() override; void onInterest(core::Interest &i) override; - void onError(std::error_code ec) override; private: void passContentObjectToCallbacks( - const std::shared_ptr<ContentObject> &content_object); - void scheduleSendBurst(); + const std::shared_ptr<ContentObject> &content_object, + const std::shared_ptr<ByteStreamProductionProtocol> &self); + void scheduleSendBurst( + const std::shared_ptr<ByteStreamProductionProtocol> &self); private: // While manifests are being built, contents are stored in a queue diff --git a/libtransport/src/protocols/prod_protocol_rtc.cc b/libtransport/src/protocols/prod_protocol_rtc.cc index cdc882d81..242abd30d 100644 --- a/libtransport/src/protocols/prod_protocol_rtc.cc +++ b/libtransport/src/protocols/prod_protocol_rtc.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -13,9 +13,11 @@ * limitations under the License. */ +#include <glog/logging.h> #include <hicn/transport/core/global_object_pool.h> #include <implementation/socket_producer.h> #include <protocols/prod_protocol_rtc.h> +#include <protocols/rtc/probe_handler.h> #include <protocols/rtc/rtc_consts.h> #include <stdlib.h> #include <time.h> @@ -38,71 +40,89 @@ RTCProductionProtocol::RTCProductionProtocol( bytes_production_rate_(0), packets_production_rate_(0), fec_packets_production_rate_(0), - last_round_(std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count()), + last_produced_data_ts_(0), + last_round_(utils::SteadyTime::nowMs().count()), allow_delayed_nacks_(false), queue_timer_on_(false), consumer_in_sync_(false), - on_consumer_in_sync_(nullptr) { - srand((unsigned int)time(NULL)); - prod_label_ = rand() % 256; + on_consumer_in_sync_(nullptr), + pending_fec_pace_(false), + max_len_(0), + queue_len_(0), + data_aggregation_(true), + data_aggregation_timer_switch_(false) { + std::uniform_int_distribution<> dis(0, 255); + prod_label_ = dis(gen_); cache_label_ = (prod_label_ + 1) % 256; interests_queue_timer_ = - std::make_unique<asio::steady_timer>(portal_->getIoService()); - round_timer_ = std::make_unique<asio::steady_timer>(portal_->getIoService()); + std::make_unique<asio::steady_timer>(portal_->getThread().getIoService()); + round_timer_ = + std::make_unique<asio::steady_timer>(portal_->getThread().getIoService()); + fec_pacing_timer_ = + std::make_unique<asio::steady_timer>(portal_->getThread().getIoService()); + app_packets_timer_ = + std::make_unique<asio::steady_timer>(portal_->getThread().getIoService()); setOutputBufferSize(10000); - scheduleRoundTimer(); +} + +RTCProductionProtocol::~RTCProductionProtocol() {} + +void RTCProductionProtocol::setProducerParam() { + // Flow name: here we assume there is only one prefix registered in the portal + flow_name_ = portal_->getServedNamespaces().begin()->getName(); + + // Manifest + uint32_t making_manifest; + socket_->getSocketOption(interface::GeneralTransportOptions::MAKE_MANIFEST, + making_manifest); + + // Signer + std::shared_ptr<auth::Signer> signer; + socket_->getSocketOption(interface::GeneralTransportOptions::SIGNER, signer); + + // Default format + core::Packet::Format default_format; + socket_->getSocketOption(interface::GeneralTransportOptions::PACKET_FORMAT, + default_format); // FEC using namespace std::placeholders; enableFEC(std::bind(&RTCProductionProtocol::onFecPackets, this, _1), std::bind(&RTCProductionProtocol::getBuffer, this, _1)); -} -RTCProductionProtocol::~RTCProductionProtocol() {} + // Aggregated data + socket_->getSocketOption(interface::RtcTransportOptions::AGGREGATED_DATA, + data_aggregation_); -void RTCProductionProtocol::registerNamespaceWithNetwork( - const Prefix &producer_namespace) { - ProductionProtocol::registerNamespaceWithNetwork(producer_namespace); - - flow_name_ = producer_namespace.getName(); - auto family = flow_name_.getAddressFamily(); - - switch (family) { - case AF_INET6: - data_header_size_ = - signer_ && !making_manifest_ - ? (uint32_t)Packet::getHeaderSizeFromFormat( - HF_INET6_TCP_AH, signer_->getSignatureFieldSize()) - : (uint32_t)Packet::getHeaderSizeFromFormat(HF_INET6_TCP); - ; - break; - case AF_INET: - data_header_size_ = - signer_ && !making_manifest_ - ? (uint32_t)Packet::getHeaderSizeFromFormat( - HF_INET_TCP_AH, signer_->getSignatureFieldSize()) - : (uint32_t)Packet::getHeaderSizeFromFormat(HF_INET_TCP); - break; - default: - throw errors::RuntimeException("Unknown name format."); - } + size_t signature_size = signer->getSignatureFieldSize(); + data_header_format_ = { + !making_manifest ? Packet::toAHFormat(default_format) : default_format, + !making_manifest ? signature_size : 0}; + manifest_header_format_ = {Packet::toAHFormat(default_format), + signature_size}; + nack_header_format_ = {Packet::toAHFormat(default_format), signature_size}; + fec_header_format_ = {Packet::toAHFormat(default_format), signature_size}; + + // Schedule round timer + scheduleRoundTimer(); } void RTCProductionProtocol::scheduleRoundTimer() { round_timer_->expires_from_now( std::chrono::milliseconds(rtc::PRODUCER_STATS_INTERVAL)); - round_timer_->async_wait([this](std::error_code ec) { + std::weak_ptr<RTCProductionProtocol> self = shared_from_this(); + round_timer_->async_wait([self](const std::error_code &ec) { if (ec) return; - updateStats(); + + auto sp = self.lock(); + if (sp && sp->isRunning()) { + sp->updateStats(); + } }); } void RTCProductionProtocol::updateStats() { - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + uint64_t now = utils::SteadyTime::nowMs().count(); uint64_t duration = now - last_round_; if (duration == 0) duration = 1; double per_second = rtc::MILLI_IN_A_SEC / duration; @@ -125,11 +145,12 @@ void RTCProductionProtocol::updateStats() { if (max_packet_production_ < rtc::WIN_MIN) max_packet_production_ = rtc::WIN_MIN; - if (packets_production_rate_ != 0) { - allow_delayed_nacks_ = false; - } else if (prev_packets_production_rate == 0) { - // at least 2 rounds with production rate = 0 + if (packets_production_rate_ <= rtc::MIN_PRODUCTION_RATE || + prev_packets_production_rate <= rtc::MIN_PRODUCTION_RATE) { allow_delayed_nacks_ = true; + } else { + // at least 2 rounds with enough packets + allow_delayed_nacks_ = false; } // check if the production rate is decreased. if yes send nacks if needed @@ -170,42 +191,237 @@ uint32_t RTCProductionProtocol::produceDatagram( socket_->getSocketOption(interface::GeneralTransportOptions::DATA_PACKET_SIZE, data_packet_size); - if (TRANSPORT_EXPECT_FALSE((buffer_size + data_header_size_ + - rtc::DATA_HEADER_SIZE) > data_packet_size)) { + if (TRANSPORT_EXPECT_FALSE( + (Packet::getHeaderSizeFromFormat(data_header_format_.first, + data_header_format_.second) + + rtc::DATA_HEADER_SIZE + buffer_size) > data_packet_size)) { return 0; } + if (!data_aggregation_) { + // if data aggregation is off emptyQueue will always return doing nothing + emptyQueue(); + + sendManifest(content_name); + + // create content object + auto content_object = + core::PacketManager<>::getInstance().getPacket<ContentObject>( + data_header_format_.first, data_header_format_.second); + + // add rtc header to the payload + struct rtc::data_packet_t header; + content_object->appendPayload((const uint8_t *)&header, + rtc::DATA_HEADER_SIZE); + content_object->appendPayload(buffer->data(), buffer->length()); + + // schedule actual sending on internal thread + portal_->getThread().tryRunHandlerNow( + [this, content_object{std::move(content_object)}, + content_name]() mutable { + produceInternal(std::move(content_object), content_name); + }); + } else { + // XXX here we assume that all the packets that we push to the queue have + // the same name + auto app_pkt = utils::MemBuf::copyBuffer(buffer->data(), buffer->length()); + addPacketToQueue(std::move(app_pkt)); + } + + return 1; +} + +void RTCProductionProtocol::addPacketToQueue( + std::unique_ptr<utils::MemBuf> &&buffer) { + std::size_t buffer_size = buffer->length(); + if ((queue_len_ + buffer_size) > rtc::MAX_RTC_PAYLOAD_SIZE) { + emptyQueue(); // this should guaranty that the generated packet will never + // be larger than an MTU + } + + waiting_app_packets_.push(std::move(buffer)); + if (max_len_ < buffer_size) max_len_ = buffer_size; + queue_len_ += buffer_size; + + if (waiting_app_packets_.size() >= rtc::MAX_AGGREGATED_PACKETS) { + emptyQueue(); + } + + if (waiting_app_packets_.size() >= 1 && !data_aggregation_timer_switch_) { + data_aggregation_timer_switch_ = true; + app_packets_timer_->expires_from_now( + std::chrono::milliseconds(rtc::AGGREGATED_PACKETS_TIMER)); + std::weak_ptr<RTCProductionProtocol> self = shared_from_this(); + app_packets_timer_->async_wait([self](const std::error_code &ec) { + if (ec) return; + + auto ptr = self.lock(); + if (ptr && ptr->isRunning()) { + if (!ptr->data_aggregation_timer_switch_) return; + ptr->emptyQueue(); + } + }); + } +} + +void RTCProductionProtocol::emptyQueue() { + if (waiting_app_packets_.size() == 0) return; // queue is empty + + Name n(flow_name_); + + // cancel timer is scheduled + if (data_aggregation_timer_switch_) { + data_aggregation_timer_switch_ = false; + app_packets_timer_->cancel(); + } + + // send a manifest beforehand if the hash buffer if full + sendManifest(n); + + // create content object auto content_object = core::PacketManager<>::getInstance().getPacket<ContentObject>( - signer_ ? Format::HF_INET6_TCP_AH : Format::HF_INET6_TCP, - signer_ ? signer_->getSignatureFieldSize() : 0); + data_header_format_.first, data_header_format_.second); + // add rtc header to the payload struct rtc::data_packet_t header; content_object->appendPayload((const uint8_t *)&header, rtc::DATA_HEADER_SIZE); - content_object->appendPayload(buffer->data(), buffer->length()); - // schedule actual sending on internal thread - portal_->getIoService().dispatch([this, - content_object{std::move(content_object)}, - content_name]() mutable { - produceInternal(std::move(content_object), content_name); + // init aggregated header + rtc::AggrPktHeader hdr( + (uint8_t *)(content_object->getPayload()->data() + rtc::DATA_HEADER_SIZE), + max_len_, waiting_app_packets_.size()); + uint32_t header_size = hdr.getHeaderLen(); + content_object->append(header_size); // leave space for the aggregated header + + uint8_t index = 0; + while (waiting_app_packets_.size() != 0) { + std::unique_ptr<utils::MemBuf> pkt = + std::move(waiting_app_packets_.front()); + waiting_app_packets_.pop(); + // XXX for the moment we have a single name, so this works, otherwise we + // need to do something else + hdr.addPacketToHeader(index, pkt->length()); + // append packet + content_object->appendPayload(pkt->data(), pkt->length()); + index++; + } + + // reset queue values + max_len_ = 0; + queue_len_ = 0; + + // the packet is ready we need to send it + portal_->getThread().tryRunHandlerNow( + [this, content_object{std::move(content_object)}, n]() mutable { + produceInternal(std::move(content_object), n); + }); +} + +void RTCProductionProtocol::sendManifest(const Name &name) { + if (!making_manifest_) { + return; + } + + Name manifest_name(name); + + uint32_t data_packet_size; + socket_->getSocketOption(interface::GeneralTransportOptions::DATA_PACKET_SIZE, + data_packet_size); + + // The maximum number of entries a manifest can hold + uint32_t manifest_capacity = making_manifest_; + + // If there is not enough hashes to fill a manifest, return early + if (manifest_entries_.size() < manifest_capacity) { + return; + } + + // Create a new manifest + std::shared_ptr<core::ContentObjectManifest> manifest = + createManifest(manifest_name.setSuffix(current_seg_)); + + // Fill the manifest with packet hashes that were previously saved + uint32_t nb_entries; + for (nb_entries = 0; nb_entries < manifest_capacity; ++nb_entries) { + if (manifest_entries_.empty()) { + break; + } + std::pair<uint32_t, auth::CryptoHash> front = manifest_entries_.front(); + manifest->addSuffixHash(front.first, front.second); + manifest_entries_.pop(); + } + + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "Sending manifest " << manifest->getName().getSuffix() << " of size " + << nb_entries; + + // Encode and send the manifest + manifest->encode(); + portal_->getThread().tryRunHandlerNow( + [this, content_object{std::move(manifest)}, manifest_name]() mutable { + produceInternal(std::move(content_object), manifest_name); + }); +} + +std::shared_ptr<core::ContentObjectManifest> +RTCProductionProtocol::createManifest(const Name &content_name) const { + Name name(content_name); + + auth::CryptoHashType hash_algo; + socket_->getSocketOption(interface::GeneralTransportOptions::HASH_ALGORITHM, + hash_algo); + + uint64_t now = utils::SteadyTime::nowMs().count(); + + // Create a new manifest + std::shared_ptr<core::ContentObjectManifest> manifest( + ContentObjectManifest::createManifest( + manifest_header_format_.first, name, core::ManifestVersion::VERSION_1, + core::ManifestType::INLINE_MANIFEST, false, name, hash_algo, + manifest_header_format_.second)); + + // Set connection parameters + manifest->setParamsRTC(ParamsRTC{ + .timestamp = now, + .prod_rate = bytes_production_rate_, + .prod_seg = current_seg_, + .support_fec = false, }); - return 1; + return manifest; } void RTCProductionProtocol::produceInternal( std::shared_ptr<ContentObject> &&content_object, const Name &content_name, bool fec) { + uint64_t now = utils::SteadyTime::nowMs().count(); + + if (fec && (now - last_produced_data_ts_) < rtc::FEC_PACING_TIME) { + paced_fec_packets_.push(std::pair<uint64_t, ContentObject::Ptr>( + now, std::move(content_object))); + postponeFecPacket(); + } else { + // need to check if there are FEC packets waiting to be sent + flushFecPkts(current_seg_); + producePktInternal(std::move(content_object), content_name, fec); + } +} + +void RTCProductionProtocol::producePktInternal( + std::shared_ptr<ContentObject> &&content_object, const Name &content_name, + bool fec) { + bool is_manifest = content_object->getPayloadType() == PayloadType::MANIFEST; + uint64_t now = utils::SteadyTime::nowMs().count(); + // set rtc header - struct rtc::data_packet_t *data_pkt = - (struct rtc::data_packet_t *)content_object->getPayload()->data(); - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); - data_pkt->setTimestamp(now); - data_pkt->setProductionRate(bytes_production_rate_); + if (!is_manifest) { + struct rtc::data_packet_t *data_pkt = + (struct rtc::data_packet_t *)content_object->getPayload()->data(); + data_pkt->setTimestamp(now); + data_pkt->setProductionRate(bytes_production_rate_); + } // set hicn stuff Name n(content_name); @@ -213,11 +429,6 @@ void RTCProductionProtocol::produceInternal( content_object->setLifetime(500); // XXX this should be set by the APP content_object->setPathLabel(prod_label_); - // sign packet - if (signer_) { - signer_->signPacket(content_object.get()); - } - // update stats if (!fec) { produced_bytes_ += @@ -227,7 +438,7 @@ void RTCProductionProtocol::produceInternal( produced_fec_packets_++; } - if (produced_packets_ >= max_packet_production_) { + if (!data_aggregation_ && produced_packets_ >= max_packet_production_) { // in this case all the pending interests may be used to accomodate the // sudden increase in the production rate. calling the updateStats we will // notify all the clients @@ -240,8 +451,12 @@ void RTCProductionProtocol::produceInternal( // pass packet to FEC encoder if (fec_encoder_ && !fec) { - fec_encoder_->onPacketProduced( - *content_object, content_object->headerSize() + rtc::DATA_HEADER_SIZE); + uint32_t offset = + is_manifest ? content_object->headerSize() + : content_object->headerSize() + rtc::DATA_HEADER_SIZE; + uint32_t metadata = static_cast<uint32_t>(content_object->getPayloadType()); + + fec_encoder_->onPacketProduced(*content_object, offset, metadata); } output_buffer_.insert(content_object); @@ -253,7 +468,7 @@ void RTCProductionProtocol::produceInternal( auto seq_it = seqs_map_.find(current_seg_); if (seq_it != seqs_map_.end()) { - portal_->sendContentObject(*content_object); + sendContentObject(content_object, false, fec); } if (*on_content_object_output_) { @@ -264,6 +479,8 @@ void RTCProductionProtocol::produceInternal( // remove interests from the interest cache if it exists removeFromInterestQueue(current_seg_); + if (!fec) last_produced_data_ts_ = now; + // Update current segment current_seg_ = (current_seg_ + 1) % rtc::MIN_PROBE_SEQ; @@ -277,6 +494,55 @@ void RTCProductionProtocol::produceInternal( } } +void RTCProductionProtocol::flushFecPkts(uint32_t current_seq_num) { + // Currently we immediately send all the pending fec packets + // A pacing policy may be helpful, but we do not want to delay too much + // the packets at this moment. + while (paced_fec_packets_.size() > 0) { + producePktInternal(std::move(paced_fec_packets_.front().second), flow_name_, + true); + paced_fec_packets_.pop(); + } + fec_pacing_timer_->cancel(); + pending_fec_pace_ = false; + postponeFecPacket(); +} + +void RTCProductionProtocol::postponeFecPacket() { + if (paced_fec_packets_.size() == 0) return; + if (pending_fec_pace_) { + return; + } + + uint64_t produced_time = paced_fec_packets_.front().first; + uint64_t now = utils::SteadyTime::nowMs().count(); + + uint64_t wait_time = 0; + if ((produced_time + rtc::FEC_PACING_TIME) > now) + wait_time = produced_time + rtc::FEC_PACING_TIME - now; + + fec_pacing_timer_->expires_from_now(std::chrono::milliseconds(wait_time)); + pending_fec_pace_ = true; + + std::weak_ptr<RTCProductionProtocol> self = shared_from_this(); + fec_pacing_timer_->async_wait([self](const std::error_code &ec) { + if (ec) return; + + auto sp = self.lock(); + if (sp && sp->isRunning()) { + if (!sp->pending_fec_pace_) return; + + if (sp->paced_fec_packets_.size() > 0) { + sp->producePktInternal(std::move(sp->paced_fec_packets_.front().second), + sp->flow_name_, true); + sp->paced_fec_packets_.pop(); + } + sp->pending_fec_pace_ = false; + sp->postponeFecPacket(); + } + }); +} + void RTCProductionProtocol::onInterest(Interest &interest) { if (*on_interest_input_) { on_interest_input_->operator()(*socket_->getInterface(), interest); @@ -284,7 +550,7 @@ void RTCProductionProtocol::onInterest(Interest &interest) { auto suffix = interest.firstSuffix(); // numberOfSuffixes returns only the prefixes in the payalod - // we add + 1 to count anche the seq in the name + // we add + 1 to count also the seq in the name auto n_suffixes = interest.numberOfSuffixes() + 1; Name name = interest.getName(); bool prev_consumer_state = consumer_in_sync_; @@ -293,6 +559,7 @@ void RTCProductionProtocol::onInterest(Interest &interest) { if (i > 0) { name.setSuffix(*(suffix + (i - 1))); } + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received interest " << name; const std::shared_ptr<ContentObject> content_object = @@ -312,7 +579,7 @@ void RTCProductionProtocol::onInterest(Interest &interest) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send content %u (onInterest) " << content_object->getName(); content_object->setPathLabel(cache_label_); - portal_->sendContentObject(*content_object); + sendContentObject(content_object); } else { if (*on_interest_process_) { on_interest_process_->operator()(*socket_->getInterface(), interest); @@ -332,14 +599,19 @@ void RTCProductionProtocol::processInterest(uint32_t interest_seg, consumer_in_sync_ = false; } - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + uint64_t now = utils::SteadyTime::nowMs().count(); - if (interest_seg > rtc::MIN_PROBE_SEQ) { - DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received probe " << interest_seg; - sendNack(interest_seg); - return; + switch (rtc::ProbeHandler::getProbeType(interest_seg)) { + case rtc::ProbeType::INIT: + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received init probe " << interest_seg; + sendManifestProbe(interest_seg); + return; + case rtc::ProbeType::RTT: + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received RTT probe " << interest_seg; + sendNack(interest_seg); + return; + default: + break; } // if the production rate 0 use delayed nacks @@ -349,7 +621,7 @@ void RTCProductionProtocol::processInterest(uint32_t interest_seg, next_timer = timers_map_.begin()->first; } - uint64_t expiration = now + rtc::SENTINEL_TIMER_INTERVAL; + uint64_t expiration = now + rtc::NACK_DELAY; addToInterestQueue(interest_seg, expiration); // here we have at least one interest in the queue, we need to start or @@ -369,8 +641,8 @@ void RTCProductionProtocol::processInterest(uint32_t interest_seg, } if (queue_timer_on_) { - // the producer is producing. Send nacks to packets that will expire before - // the data production and remove the timer + // the producer is producing. Send nacks to packets that will expire + // before the data production and remove the timer queue_timer_on_ = false; interests_queue_timer_->cancel(); sendNacksForPendingInterests(); @@ -387,8 +659,8 @@ void RTCProductionProtocol::processInterest(uint32_t interest_seg, sendNack(interest_seg); } else { if (!consumer_in_sync_ && on_consumer_in_sync_) { - // we consider the remote consumer to be in sync as soon as it covers 70% - // of the production window with interests + // we consider the remote consumer to be in sync as soon as it covers + // 70% of the production window with interests uint32_t perc = ceil((double)max_gap * 0.7); if (interest_seg > (perc + current_seg_)) { consumer_in_sync_ = true; @@ -401,13 +673,18 @@ void RTCProductionProtocol::processInterest(uint32_t interest_seg, } } -void RTCProductionProtocol::onError(std::error_code ec) {} - void RTCProductionProtocol::scheduleQueueTimer(uint64_t wait) { interests_queue_timer_->expires_from_now(std::chrono::milliseconds(wait)); - interests_queue_timer_->async_wait([this](std::error_code ec) { - if (ec) return; - interestQueueTimer(); + std::weak_ptr<RTCProductionProtocol> self = shared_from_this(); + interests_queue_timer_->async_wait([self](const std::error_code &ec) { + if (ec) { + return; + } + + auto sp = self.lock(); + if (sp && sp->isRunning()) { + sp->interestQueueTimer(); + } }); } @@ -450,9 +727,7 @@ void RTCProductionProtocol::sendNacksForPendingInterests() { if (packets_production_rate_ != 0) packet_gap = ceil(rtc::MILLI_IN_A_SEC / (double)packets_production_rate_); - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + uint64_t now = utils::SteadyTime::nowMs().count(); for (auto it = seqs_map_.begin(); it != seqs_map_.end(); it++) { if (it->first > current_seg_) { @@ -486,9 +761,7 @@ void RTCProductionProtocol::removeFromInterestQueue(uint32_t interest_seg) { } void RTCProductionProtocol::interestQueueTimer() { - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + uint64_t now = utils::SteadyTime::nowMs().count(); for (auto it_timers = timers_map_.begin(); it_timers != timers_map_.end();) { uint64_t expire = it_timers->first; @@ -511,20 +784,37 @@ void RTCProductionProtocol::interestQueueTimer() { } } +void RTCProductionProtocol::sendManifestProbe(uint32_t sequence) { + Name manifest_name(flow_name_); + manifest_name.setSuffix(sequence); + + std::shared_ptr<core::ContentObjectManifest> manifest_probe = + createManifest(manifest_name); + + manifest_probe->setLifetime(0); + manifest_probe->setPathLabel(prod_label_); + manifest_probe->encode(); + + if (*on_content_object_output_) { + on_content_object_output_->operator()(*socket_->getInterface(), + *manifest_probe); + } + + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send init probe " << sequence; + sendContentObject(manifest_probe, true, false); +} + void RTCProductionProtocol::sendNack(uint32_t sequence) { auto nack = core::PacketManager<>::getInstance().getPacket<ContentObject>( - signer_ ? Format::HF_INET6_TCP_AH : Format::HF_INET6_TCP, - signer_ ? signer_->getSignatureFieldSize() : 0); - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + nack_header_format_.first, nack_header_format_.second); + uint64_t now = utils::SteadyTime::nowMs().count(); uint32_t next_packet = current_seg_; uint32_t prod_rate = bytes_production_rate_; struct rtc::nack_packet_t header; header.setTimestamp(now); header.setProductionRate(prod_rate); - header.setProductionSegement(next_packet); + header.setProductionSegment(next_packet); nack->appendPayload((const uint8_t *)&header, rtc::NACK_HEADER_SIZE); Name n(flow_name_); @@ -533,14 +823,16 @@ void RTCProductionProtocol::sendNack(uint32_t sequence) { nack->setLifetime(0); nack->setPathLabel(prod_label_); - if (signer_) { - signer_->signPacket(nack.get()); - } - if (!consumer_in_sync_ && on_consumer_in_sync_ && - sequence < rtc::MIN_PROBE_SEQ && sequence > next_packet) { + rtc::ProbeHandler::getProbeType(sequence) == rtc::ProbeType::NOT_PROBE && + sequence > next_packet) { consumer_in_sync_ = true; - auto interest = core::PacketManager<>::getInstance().getPacket<Interest>(); + Packet::Format format; + socket_->getSocketOption(interface::GeneralTransportOptions::PACKET_FORMAT, + format); + + auto interest = + core::PacketManager<>::getInstance().getPacket<Interest>(format); interest->setName(n); on_consumer_in_sync_(*socket_->getInterface(), *interest); } @@ -550,16 +842,37 @@ void RTCProductionProtocol::sendNack(uint32_t sequence) { } DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send nack " << sequence; - portal_->sendContentObject(*nack); + sendContentObject(nack, true, false); } -void RTCProductionProtocol::onFecPackets( - std::vector<std::pair<uint32_t, fec::buffer>> &packets) { +void RTCProductionProtocol::sendContentObject( + std::shared_ptr<ContentObject> content_object, bool nack, bool fec) { + bool is_ah = _is_ah(content_object->getFormat()); + + // Compute signature + if (is_ah) { + signer_->signPacket(content_object.get()); + } + + portal_->sendContentObject(*content_object); + + // Compute and save data packet digest + if (making_manifest_ && !is_ah) { + auth::CryptoHashType hash_algo; + socket_->getSocketOption(interface::GeneralTransportOptions::HASH_ALGORITHM, + hash_algo); + manifest_entries_.push({content_object->getName().getSuffix(), + content_object->computeDigest(hash_algo)}); + } +} + +void RTCProductionProtocol::onFecPackets(fec::BufferArray &packets) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Produced " << packets.size() << " FEC packets"; + for (auto &packet : packets) { auto content_object = - std::static_pointer_cast<ContentObject>(packet.second); + std::static_pointer_cast<ContentObject>(packet.getBuffer()); content_object->prepend(content_object->headerSize() + rtc::DATA_HEADER_SIZE); pending_fec_packets_.push(std::move(content_object)); @@ -569,19 +882,21 @@ void RTCProductionProtocol::onFecPackets( fec::buffer RTCProductionProtocol::getBuffer(std::size_t size) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Asked buffer for FEC symbol of size " << size; + auto ret = core::PacketManager<>::getInstance().getPacket<ContentObject>( - signer_ ? Format::HF_INET6_TCP_AH : Format::HF_INET6_TCP, - signer_ ? signer_->getSignatureFieldSize() : 0); + fec_header_format_.first, fec_header_format_.second); + ret->updateLength(rtc::DATA_HEADER_SIZE + size); ret->append(rtc::DATA_HEADER_SIZE + size); ret->trimStart(ret->headerSize() + rtc::DATA_HEADER_SIZE); + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Responding with buffer of length " << ret->length(); - assert(ret->length() >= size); + DCHECK(ret->length() >= size); return ret; } } // namespace protocol -} // end namespace transport +} // namespace transport diff --git a/libtransport/src/protocols/prod_protocol_rtc.h b/libtransport/src/protocols/prod_protocol_rtc.h index 96ad5673d..7f50a2505 100644 --- a/libtransport/src/protocols/prod_protocol_rtc.h +++ b/libtransport/src/protocols/prod_protocol_rtc.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -32,6 +32,7 @@ class RTCProductionProtocol : public ProductionProtocol { using ProductionProtocol::start; using ProductionProtocol::stop; + void setProducerParam() override; void produce(ContentObject &content_object) override; uint32_t produceStream(const Name &content_name, @@ -49,21 +50,31 @@ class RTCProductionProtocol : public ProductionProtocol { buffer, buffer_size, buffer_size)); } - void registerNamespaceWithNetwork(const Prefix &producer_namespace) override; - void setConsumerInSyncCallback( interface::ProducerInterestCallback &&callback) { on_consumer_in_sync_ = std::move(callback); } + auto shared_from_this() { return utils::shared_from(this); } + private: // packet handlers void onInterest(Interest &interest) override; - void onError(std::error_code ec) override; + void onError(const std::error_code &ec) override{}; void processInterest(uint32_t interest_seg, uint32_t lifetime); + void producePktInternal(std::shared_ptr<ContentObject> &&content_object, + const Name &content_name, bool fec = false); void produceInternal(std::shared_ptr<ContentObject> &&content_object, const Name &content_name, bool fec = false); void sendNack(uint32_t sequence); + void sendContentObject(std::shared_ptr<ContentObject> content_object, + bool nac = false, bool fec = false); + + // manifests + void sendManifestProbe(uint32_t sequence); + void sendManifest(const Name &content_name); + std::shared_ptr<core::ContentObjectManifest> createManifest( + const Name &name) const; // stats void updateStats(); @@ -77,15 +88,25 @@ class RTCProductionProtocol : public ProductionProtocol { void interestQueueTimer(); // FEC functions - void onFecPackets(std::vector<std::pair<uint32_t, fec::buffer>> &packets); + void onFecPackets(fec::BufferArray &packets); fec::buffer getBuffer(std::size_t size); + void postponeFecPacket(); + void dispatchFecPacket(); + void flushFecPkts(uint32_t current_seq_num); + // aggregated data functions + void emptyQueue(); + void addPacketToQueue(std::unique_ptr<utils::MemBuf> &&buffer); core::Name flow_name_; - uint32_t current_seg_; // seq id of the next packet produced - uint32_t prod_label_; // path lable of the producer - uint32_t cache_label_; // path lable for content from the producer cache - uint16_t data_header_size_; // hicn data header size + std::pair<core::Packet::Format, size_t> data_header_format_; + std::pair<core::Packet::Format, size_t> manifest_header_format_; + std::pair<core::Packet::Format, size_t> fec_header_format_; + std::pair<core::Packet::Format, size_t> nack_header_format_; + + uint32_t current_seg_; // seq id of the next packet produced + uint32_t prod_label_; // path label of the producer + uint32_t cache_label_; // path label for content from the producer cache uint32_t produced_bytes_; // bytes produced in the last round uint32_t produced_packets_; // packet produed in the last round @@ -98,7 +119,11 @@ class RTCProductionProtocol : public ProductionProtocol { uint32_t packets_production_rate_; // pps uint32_t fec_packets_production_rate_; // pps + uint64_t last_produced_data_ts_; // ms + std::unique_ptr<asio::steady_timer> round_timer_; + std::unique_ptr<asio::steady_timer> fec_pacing_timer_; + uint64_t last_round_; // delayed nacks are used by the producer to avoid to send too @@ -131,6 +156,21 @@ class RTCProductionProtocol : public ProductionProtocol { // Save FEC packets here before sending them std::queue<ContentObject::Ptr> pending_fec_packets_; + std::queue<std::pair<uint64_t, ContentObject::Ptr>> paced_fec_packets_; + bool pending_fec_pace_; + + // Save application packets if they are small + std::queue<std::unique_ptr<utils::MemBuf>> waiting_app_packets_; + uint16_t max_len_; // len of the largest packet + uint16_t queue_len_; // total size of all packet in the queue + bool data_aggregation_; // turns on/off data aggregation + // timer to check the queue len + std::unique_ptr<asio::steady_timer> app_packets_timer_; + bool data_aggregation_timer_switch_; // bool to check if the timer is on + + // Manifest + std::queue<std::pair<uint32_t, auth::CryptoHash>> + manifest_entries_; // map a packet suffix to a packet hash }; } // namespace protocol diff --git a/libtransport/src/protocols/production_protocol.cc b/libtransport/src/protocols/production_protocol.cc index 6b317d47d..8b781e38a 100644 --- a/libtransport/src/protocols/production_protocol.cc +++ b/libtransport/src/protocols/production_protocol.cc @@ -24,8 +24,8 @@ using namespace interface; ProductionProtocol::ProductionProtocol( implementation::ProducerSocket *icn_socket) - : socket_(icn_socket), - is_running_(false), + : Protocol(), + socket_(icn_socket), fec_encoder_(nullptr), on_interest_input_(VOID_HANDLER), on_interest_dropped_input_buffer_(VOID_HANDLER), @@ -38,101 +38,92 @@ ProductionProtocol::ProductionProtocol( on_content_object_output_(VOID_HANDLER), on_content_object_evicted_from_output_buffer_(VOID_HANDLER), on_content_produced_(VOID_HANDLER), + producer_callback_(VOID_HANDLER), fec_type_(fec::FECType::UNKNOWN) { socket_->getSocketOption(GeneralTransportOptions::PORTAL, portal_); // TODO add statistics for producer // socket_->getSocketOption(OtherOptions::STATISTICS, &stats_); } -ProductionProtocol::~ProductionProtocol() { - if (!is_async_ && is_running_) { - stop(); - } +ProductionProtocol::~ProductionProtocol() {} - if (listening_thread_.joinable()) { - listening_thread_.join(); +int ProductionProtocol::start() { + if (isRunning()) { + return -1; } -} -int ProductionProtocol::start() { - socket_->getSocketOption(ProducerCallbacksOptions::INTEREST_INPUT, - &on_interest_input_); - socket_->getSocketOption(ProducerCallbacksOptions::INTEREST_DROP, - &on_interest_dropped_input_buffer_); - socket_->getSocketOption(ProducerCallbacksOptions::INTEREST_PASS, - &on_interest_inserted_input_buffer_); - socket_->getSocketOption(ProducerCallbacksOptions::CACHE_HIT, - &on_interest_satisfied_output_buffer_); - socket_->getSocketOption(ProducerCallbacksOptions::CACHE_MISS, - &on_interest_process_); - socket_->getSocketOption(ProducerCallbacksOptions::NEW_CONTENT_OBJECT, - &on_new_segment_); - socket_->getSocketOption(ProducerCallbacksOptions::CONTENT_OBJECT_READY, - &on_content_object_in_output_buffer_); - socket_->getSocketOption(ProducerCallbacksOptions::CONTENT_OBJECT_OUTPUT, - &on_content_object_output_); - socket_->getSocketOption(ProducerCallbacksOptions::CONTENT_OBJECT_TO_SIGN, - &on_content_object_to_sign_); - socket_->getSocketOption(ProducerCallbacksOptions::CONTENT_PRODUCED, - &on_content_produced_); - - socket_->getSocketOption(GeneralTransportOptions::ASYNC_MODE, is_async_); - socket_->getSocketOption(GeneralTransportOptions::SIGNER, signer_); - socket_->getSocketOption(GeneralTransportOptions::MAKE_MANIFEST, - making_manifest_); - - bool first = true; - - for (core::Prefix &producer_namespace : served_namespaces_) { - if (first) { - core::BindConfig bind_config(producer_namespace, 1000); - portal_->bind(bind_config); - portal_->setProducerCallback(this); - first = !first; - } else { - portal_->registerRoute(producer_namespace); + portal_->getThread().addAndWaitForExecution([this]() { + socket_->getSocketOption(ProducerCallbacksOptions::INTEREST_INPUT, + &on_interest_input_); + socket_->getSocketOption(ProducerCallbacksOptions::INTEREST_DROP, + &on_interest_dropped_input_buffer_); + socket_->getSocketOption(ProducerCallbacksOptions::INTEREST_PASS, + &on_interest_inserted_input_buffer_); + socket_->getSocketOption(ProducerCallbacksOptions::CACHE_HIT, + &on_interest_satisfied_output_buffer_); + socket_->getSocketOption(ProducerCallbacksOptions::CACHE_MISS, + &on_interest_process_); + socket_->getSocketOption(ProducerCallbacksOptions::NEW_CONTENT_OBJECT, + &on_new_segment_); + socket_->getSocketOption(ProducerCallbacksOptions::CONTENT_OBJECT_READY, + &on_content_object_in_output_buffer_); + socket_->getSocketOption(ProducerCallbacksOptions::CONTENT_OBJECT_OUTPUT, + &on_content_object_output_); + socket_->getSocketOption(ProducerCallbacksOptions::CONTENT_OBJECT_TO_SIGN, + &on_content_object_to_sign_); + socket_->getSocketOption(ProducerCallbacksOptions::CONTENT_PRODUCED, + &on_content_produced_); + socket_->getSocketOption(ProducerCallbacksOptions::PRODUCER_CALLBACK, + &producer_callback_); + + socket_->getSocketOption(GeneralTransportOptions::ASYNC_MODE, is_async_); + socket_->getSocketOption(GeneralTransportOptions::SIGNER, signer_); + socket_->getSocketOption(GeneralTransportOptions::MAKE_MANIFEST, + making_manifest_); + + std::string fec_type_str = ""; + socket_->getSocketOption(GeneralTransportOptions::FEC_TYPE, fec_type_str); + if (fec_type_str != "") { + fec_type_ = fec::FECUtils::fecTypeFromString(fec_type_str.c_str()); } - } - is_running_ = true; + portal_->registerTransportCallback(this); + setProducerParam(); - if (!is_async_) { - listening_thread_ = std::thread([this]() { portal_->runEventsLoop(); }); - } + setRunning(); + }); return 0; } -void ProductionProtocol::stop() { - is_running_ = false; - - if (!is_async_) { - portal_->stopEventsLoop(); - } else { - portal_->clear(); - } -} - void ProductionProtocol::produce(ContentObject &content_object) { - if (*on_content_object_in_output_buffer_) { - on_content_object_in_output_buffer_->operator()(*socket_->getInterface(), - content_object); - } + auto content_object_ptr = content_object.shared_from_this(); + portal_->getThread().add([this, co = std::move(content_object_ptr)]() { + if (*on_content_object_in_output_buffer_) { + on_content_object_in_output_buffer_->operator()(*socket_->getInterface(), + *co); + } - output_buffer_.insert(std::static_pointer_cast<ContentObject>( - content_object.shared_from_this())); + output_buffer_.insert(co); - if (*on_content_object_output_) { - on_content_object_output_->operator()(*socket_->getInterface(), - content_object); - } + if (*on_content_object_output_) { + on_content_object_output_->operator()(*socket_->getInterface(), *co); + } - portal_->sendContentObject(content_object); + portal_->sendContentObject(*co); + }); } -void ProductionProtocol::registerNamespaceWithNetwork( - const Prefix &producer_namespace) { - served_namespaces_.push_back(producer_namespace); +void ProductionProtocol::sendMapme() { portal_->sendMapme(); } + +void ProductionProtocol::onError(const std::error_code &ec) { + // Stop production protocol + stop(); + + // Call error callback + if (producer_callback_) { + producer_callback_->produceError(ec); + } } } // namespace protocol diff --git a/libtransport/src/protocols/production_protocol.h b/libtransport/src/protocols/production_protocol.h index 7366311eb..8e10d2f40 100644 --- a/libtransport/src/protocols/production_protocol.h +++ b/libtransport/src/protocols/production_protocol.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -22,6 +22,7 @@ #include <implementation/socket.h> #include <protocols/fec_base.h> #include <protocols/fec_utils.h> +#include <protocols/protocol.h> #include <utils/content_store.h> #include <atomic> @@ -33,17 +34,20 @@ namespace protocol { using namespace core; -class ProductionProtocol : public Portal::ProducerCallback { +class ProductionProtocol + : public Protocol, + public std::enable_shared_from_this<ProductionProtocol> { public: ProductionProtocol(implementation::ProducerSocket *icn_socket); virtual ~ProductionProtocol(); - bool isRunning() { return is_running_; } - virtual int start(); - virtual void stop(); + using Protocol::stop; + + virtual void setProducerParam(){}; virtual void produce(ContentObject &content_object); + virtual void sendMapme(); virtual uint32_t produceStream(const Name &content_name, std::unique_ptr<utils::MemBuf> &&buffer, bool is_last = true, @@ -61,20 +65,18 @@ class ProductionProtocol : public Portal::ProducerCallback { void setOutputBufferSize(std::size_t size) { output_buffer_.setLimit(size); } std::size_t getOutputBufferSize() { return output_buffer_.getLimit(); } - virtual void registerNamespaceWithNetwork(const Prefix &producer_namespace); - const std::list<Prefix> &getNamespaces() const { return served_namespaces_; } - protected: // Producer callback virtual void onInterest(core::Interest &i) override = 0; - virtual void onError(std::error_code ec) override{}; + virtual void onError(const std::error_code &ec) override; template <typename FECHandler, typename AllocatorHandler> void enableFEC(FECHandler &&fec_handler, AllocatorHandler &&allocator_handler) { if (!fec_encoder_) { // Try to get FEC from environment - if (const char *fec_str = std::getenv("TRANSPORT_FEC_TYPE")) { + const char *fec_str = std::getenv("TRANSPORT_FEC_TYPE"); + if (fec_str && (fec_type_ == fec::FECType::UNKNOWN)) { LOG(INFO) << "Using FEC " << fec_str; fec_type_ = fec::FECUtils::fecTypeFromString(fec_str); } @@ -95,11 +97,6 @@ class ProductionProtocol : public Portal::ProducerCallback { // Thread pool responsible for IO operations (send data / receive interests) std::vector<utils::EventThread> io_threads_; - - // TODO remove this thread - std::thread listening_thread_; - std::shared_ptr<Portal> portal_; - std::atomic<bool> is_running_; interface::ProductionStatistics *stats_; std::unique_ptr<fec::ProducerFEC> fec_encoder_; @@ -119,15 +116,14 @@ class ProductionProtocol : public Portal::ProducerCallback { interface::ProducerContentCallback *on_content_produced_; + interface::ProducerSocket::Callback *producer_callback_; + // Output buffer utils::ContentStore output_buffer_; - // List ot routes served by current producer protocol - std::list<Prefix> served_namespaces_; - // Signature and manifest std::shared_ptr<auth::Signer> signer_; - bool making_manifest_; + uint32_t making_manifest_; bool is_async_; fec::FECType fec_type_; diff --git a/libtransport/src/protocols/protocol.h b/libtransport/src/protocols/protocol.h new file mode 100644 index 000000000..a9f929db9 --- /dev/null +++ b/libtransport/src/protocols/protocol.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <core/portal.h> +#include <hicn/transport/errors/runtime_exception.h> +#include <hicn/transport/utils/noncopyable.h> + +#include <random> + +namespace transport { + +namespace protocol { + +class Protocol : public core::Portal::TransportCallback, utils::NonCopyable { + public: + virtual void stop() { + portal_->getThread().addAndWaitForExecution([this]() { + unSetRunning(); + portal_->unregisterTransportCallback(); + portal_->clear(); + }); + } + + virtual void onInterest(core::Interest &i) { + throw errors::RuntimeException("Not implemented"); + } + + virtual void onContentObject(core::Interest &i, core::ContentObject &c) { + throw errors::RuntimeException("Not implemented"); + } + + virtual void onTimeout(core::Interest::Ptr &i, const core::Name &n) { + throw errors::RuntimeException("Not implemented"); + } + + virtual void onError(const std::error_code &ec) { + throw errors::RuntimeException("Not implemented"); + } + + bool isRunning() { return is_running_; } + void setRunning() { is_running_ = true; } + void unSetRunning() { is_running_ = false; } + + protected: + Protocol() : portal_(nullptr), is_running_(false), gen_(rd_()) {} + virtual ~Protocol() {} + + protected: + std::shared_ptr<core::Portal> portal_; + std::atomic_bool is_running_; + + // Random engine + std::random_device rd_; + std::mt19937 gen_; +}; + +} // namespace protocol + +} // namespace transport
\ No newline at end of file diff --git a/libtransport/src/protocols/raaqm.cc b/libtransport/src/protocols/raaqm.cc index 1247af400..131367d78 100644 --- a/libtransport/src/protocols/raaqm.cc +++ b/libtransport/src/protocols/raaqm.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -36,8 +36,9 @@ RaaqmTransportProtocol::RaaqmTransportProtocol( current_window_size_(1), interests_in_flight_(0), cur_path_(nullptr), - t0_(utils::SteadyClock::now()), + t0_(utils::SteadyTime::Clock::now()), rate_estimator_(nullptr), + dis_(0, 1.0), schedule_interests_(true) { init(); } @@ -60,7 +61,7 @@ void RaaqmTransportProtocol::reset() { // Reset protocol variables interests_in_flight_ = 0; - t0_ = utils::SteadyClock::now(); + t0_ = utils::SteadyTime::Clock::now(); // Optionally reset congestion window bool reset_window; @@ -389,6 +390,8 @@ void RaaqmTransportProtocol::sendInterest( uint32_t len) { interests_in_flight_++; interest_retransmissions_[interest_name.getSuffix() & mask]++; + interest_timepoints_[interest_name.getSuffix() & mask] = + utils::SteadyTime::Clock::now(); TransportProtocol::sendInterest(interest_name, additional_suffixes, len); } @@ -477,7 +480,7 @@ void RaaqmTransportProtocol::scheduleNextInterests() { } } -void RaaqmTransportProtocol::onContentReassembled(std::error_code ec) { +void RaaqmTransportProtocol::onContentReassembled(const std::error_code &ec) { rate_estimator_->onDownloadFinished(); TransportProtocol::onContentReassembled(ec); schedule_interests_ = false; @@ -487,18 +490,18 @@ void RaaqmTransportProtocol::updateRtt(uint64_t segment) { if (TRANSPORT_EXPECT_FALSE(!cur_path_)) { throw std::runtime_error("RAAQM ERROR: no current path found, exit"); } else { - auto now = utils::SteadyClock::now(); - utils::Microseconds rtt = std::chrono::duration_cast<utils::Microseconds>( - now - interest_timepoints_[segment & mask]); + auto now = utils::SteadyTime::Clock::now(); + utils::SteadyTime::Milliseconds rtt = utils::SteadyTime::getDurationMs( + interest_timepoints_[segment & mask], now); // Update stats - updateStats((uint32_t)segment, rtt.count(), now); + updateStats((uint32_t)segment, rtt, now); if (rate_estimator_) { - rate_estimator_->onRttUpdate((double)rtt.count()); + rate_estimator_->onRttUpdate(rtt); } - cur_path_->insertNewRtt(rtt.count(), now); + cur_path_->insertNewRtt(rtt, now); cur_path_->smoothTimer(); if (cur_path_->newPropagationDelayAvailable()) { @@ -510,27 +513,27 @@ void RaaqmTransportProtocol::updateRtt(uint64_t segment) { void RaaqmTransportProtocol::RAAQM() { if (!cur_path_) { throw errors::RuntimeException("ERROR: no current path found, exit"); - exit(EXIT_FAILURE); } else { // Change drop probability according to RTT statistics cur_path_->updateDropProb(); - double coin = ((double)rand() / (RAND_MAX)); + double coin = dis_(gen_); if (coin <= cur_path_->getDropProb()) { decreaseWindow(); } } } -void RaaqmTransportProtocol::updateStats(uint32_t suffix, uint64_t rtt, - utils::TimePoint &now) { +void RaaqmTransportProtocol::updateStats( + uint32_t suffix, const utils::SteadyTime::Milliseconds &rtt, + utils::SteadyTime::TimePoint &now) { // Update RTT statistics stats_->updateAverageRtt(rtt); stats_->updateAverageWindowSize(current_window_size_); // Call statistics callback if (*stats_summary_) { - auto dt = std::chrono::duration_cast<utils::Milliseconds>(now - t0_); + auto dt = utils::SteadyTime::getDurationMs(t0_, now); uint32_t timer_interval_milliseconds = 0; socket_->getSocketOption(GeneralTransportOptions::STATS_INTERVAL, diff --git a/libtransport/src/protocols/raaqm.h b/libtransport/src/protocols/raaqm.h index ffbb30d3a..ec344c23a 100644 --- a/libtransport/src/protocols/raaqm.h +++ b/libtransport/src/protocols/raaqm.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -23,6 +23,7 @@ #include <protocols/transport_protocol.h> #include <queue> +#include <random> #include <vector> namespace transport { @@ -55,8 +56,9 @@ class RaaqmTransportProtocol : public TransportProtocol, const ContentObject &content_object); virtual void afterDataUnsatisfied(uint64_t segment); - virtual void updateStats(uint32_t suffix, uint64_t rtt, - utils::TimePoint &now); + virtual void updateStats(uint32_t suffix, + const utils::SteadyTime::Milliseconds &rtt, + utils::SteadyTime::TimePoint &now); private: void init(); @@ -73,7 +75,7 @@ class RaaqmTransportProtocol : public TransportProtocol, *additional_suffixes = nullptr, uint32_t len = 0) override; - void onContentReassembled(std::error_code ec) override; + void onContentReassembled(const std::error_code &ec) override; void updateRtt(uint64_t segment); void RAAQM(); void updatePathTable(const ContentObject &content_object); @@ -81,13 +83,15 @@ class RaaqmTransportProtocol : public TransportProtocol, void checkForStalePaths(); void printRtt(); + auto shared_from_this() { return utils::shared_from(this); } + protected: // Congestion window management double current_window_size_; // Protocol management uint64_t interests_in_flight_; std::array<std::uint32_t, buffer_size> interest_retransmissions_; - std::array<utils::TimePoint, buffer_size> interest_timepoints_; + std::array<utils::SteadyTime::TimePoint, buffer_size> interest_timepoints_; std::queue<uint32_t> interest_to_retransmit_; private: @@ -102,13 +106,16 @@ class RaaqmTransportProtocol : public TransportProtocol, PathTable path_table_; // TimePoints for statistic - utils::TimePoint t0_; + utils::SteadyTime::TimePoint t0_; bool set_interest_filter_; // for rate-estimation at packet level IcnRateEstimator *rate_estimator_; + // Real distribution + std::uniform_real_distribution<> dis_; + // params for autotuning bool raaqm_autotune_; double default_beta_; diff --git a/libtransport/src/protocols/raaqm_data_path.cc b/libtransport/src/protocols/raaqm_data_path.cc index f2c21b9ef..d06fee918 100644 --- a/libtransport/src/protocols/raaqm_data_path.cc +++ b/libtransport/src/protocols/raaqm_data_path.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -23,16 +23,18 @@ namespace protocol { RaaqmDataPath::RaaqmDataPath(double drop_factor, double minimum_drop_probability, unsigned new_timer, unsigned int samples, - uint64_t new_rtt, uint64_t new_rtt_min, - uint64_t new_rtt_max, unsigned new_pd) + const utils::SteadyTime::Milliseconds &new_rtt, + const utils::SteadyTime::Milliseconds &new_rtt_min, + const utils::SteadyTime::Milliseconds &new_rtt_max, + unsigned new_pd) : drop_factor_(drop_factor), minimum_drop_probability_(minimum_drop_probability), timer_(new_timer), samples_(samples), - rtt_(new_rtt), - rtt_min_(new_rtt_min), - rtt_max_(new_rtt_max), + rtt_(new_rtt.count()), + rtt_min_(new_rtt_min.count()), + rtt_max_(new_rtt_max.count()), prop_delay_(new_pd), new_prop_delay_(false), drop_prob_(0), @@ -43,14 +45,15 @@ RaaqmDataPath::RaaqmDataPath(double drop_factor, raw_data_bytes_received_(0), last_raw_data_bytes_received_(0), rtt_samples_(samples_), - last_received_pkt_(utils::SteadyClock::now()), + last_received_pkt_(utils::SteadyTime::Clock::now()), average_rtt_(0), alpha_(ALPHA) {} -RaaqmDataPath &RaaqmDataPath::insertNewRtt(uint64_t new_rtt, - const utils::TimePoint &now) { - rtt_ = new_rtt; - rtt_samples_.pushBack(new_rtt); +RaaqmDataPath &RaaqmDataPath::insertNewRtt( + const utils::SteadyTime::Milliseconds &new_rtt, + const utils::SteadyTime::TimePoint &now) { + rtt_ = new_rtt.count(); + rtt_samples_.pushBack(rtt_); rtt_max_ = rtt_samples_.rBegin(); rtt_min_ = rtt_samples_.begin(); @@ -143,10 +146,8 @@ unsigned int RaaqmDataPath::getPropagationDelay() { } bool RaaqmDataPath::isStale() { - utils::TimePoint now = utils::SteadyClock::now(); - auto time = - std::chrono::duration_cast<utils::Microseconds>(now - last_received_pkt_) - .count(); + utils::SteadyTime::TimePoint now = utils::SteadyTime::Clock::now(); + auto time = utils::SteadyTime::getDurationUs(last_received_pkt_, now).count(); if (time > 2000000) { return true; } diff --git a/libtransport/src/protocols/raaqm_data_path.h b/libtransport/src/protocols/raaqm_data_path.h index c0b53a690..b6f7c5ac1 100644 --- a/libtransport/src/protocols/raaqm_data_path.h +++ b/libtransport/src/protocols/raaqm_data_path.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -34,8 +34,13 @@ class RaaqmDataPath { public: RaaqmDataPath(double drop_factor, double minimum_drop_probability, unsigned new_timer, unsigned int samples, - uint64_t new_rtt = 1000, uint64_t new_rtt_min = 1000, - uint64_t new_rtt_max = 1000, unsigned new_pd = UINT_MAX); + const utils::SteadyTime::Milliseconds &new_rtt = + utils::SteadyTime::Milliseconds(1000), + const utils::SteadyTime::Milliseconds &new_rtt_min = + utils::SteadyTime::Milliseconds(1000), + const utils::SteadyTime::Milliseconds &new_rtt_max = + utils::SteadyTime::Milliseconds(1000), + unsigned new_pd = UINT_MAX); public: /* @@ -44,7 +49,8 @@ class RaaqmDataPath { * max of RTT. * @param new_rtt is the value of the new RTT */ - RaaqmDataPath &insertNewRtt(uint64_t new_rtt, const utils::TimePoint &now); + RaaqmDataPath &insertNewRtt(const utils::SteadyTime::Milliseconds &new_rtt, + const utils::SteadyTime::TimePoint &now); /** * @brief Update the path statistics @@ -214,7 +220,7 @@ class RaaqmDataPath { /** * Time of the last call to the path reporter method */ - utils::TimePoint last_received_pkt_; + utils::SteadyTime::TimePoint last_received_pkt_; double average_rtt_; double alpha_; diff --git a/libtransport/src/protocols/rate_estimation.cc b/libtransport/src/protocols/rate_estimation.cc index 2337e18be..d834b53e6 100644 --- a/libtransport/src/protocols/rate_estimation.cc +++ b/libtransport/src/protocols/rate_estimation.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -14,6 +14,7 @@ */ #include <glog/logging.h> +#include <hicn/transport/errors/runtime_exception.h> #include <hicn/transport/interfaces/socket_options_default_values.h> #include <protocols/rate_estimation.h> @@ -92,8 +93,8 @@ InterRttEstimator::InterRttEstimator(double alpha_arg) { this->win_current_ = 1.0; pthread_mutex_init(&(this->mutex_), NULL); - this->start_time_ = std::chrono::steady_clock::now(); - this->begin_batch_ = std::chrono::steady_clock::now(); + this->start_time_ = utils::SteadyTime::now(); + this->begin_batch_ = utils::SteadyTime::now(); } InterRttEstimator::~InterRttEstimator() { @@ -105,33 +106,35 @@ InterRttEstimator::~InterRttEstimator() { pthread_mutex_destroy(&(this->mutex_)); } -void InterRttEstimator::onRttUpdate(double rtt) { +void InterRttEstimator::onRttUpdate( + const utils::SteadyTime::Milliseconds &rtt) { pthread_mutex_lock(&(this->mutex_)); - this->rtt_ = rtt; + this->rtt_ = rtt.count(); this->number_of_packets_++; - this->avg_rtt_ += rtt; + this->avg_rtt_ += this->rtt_; pthread_mutex_unlock(&(this->mutex_)); if (!thread_is_running_) { my_th_ = (pthread_t *)malloc(sizeof(pthread_t)); if (!my_th_) { - LOG(ERROR) << "Error allocating thread."; - my_th_ = NULL; + throw errors::RuntimeException("Error allocating thread."); } - if (/*int err = */ pthread_create(my_th_, NULL, transport::protocol::Timer, - (void *)this)) { - LOG(ERROR) << "Error creating the thread"; + + if (pthread_create(my_th_, NULL, transport::protocol::Timer, + (void *)this)) { + free(my_th_); my_th_ = NULL; + throw errors::RuntimeException("Error allocating thread."); } + thread_is_running_ = true; } } void InterRttEstimator::onWindowIncrease(double win_current) { - TimePoint end = std::chrono::steady_clock::now(); + auto end = utils::SteadyTime::now(); auto delay = - std::chrono::duration_cast<Microseconds>(end - this->begin_batch_) - .count(); + utils::SteadyTime::getDurationUs(this->begin_batch_, end).count(); pthread_mutex_lock(&(this->mutex_)); this->avg_win_ += this->win_current_ * delay; @@ -139,14 +142,13 @@ void InterRttEstimator::onWindowIncrease(double win_current) { this->win_change_ += delay; pthread_mutex_unlock(&(this->mutex_)); - this->begin_batch_ = std::chrono::steady_clock::now(); + this->begin_batch_ = utils::SteadyTime::now(); } void InterRttEstimator::onWindowDecrease(double win_current) { - TimePoint end = std::chrono::steady_clock::now(); + auto end = utils::SteadyTime::now(); auto delay = - std::chrono::duration_cast<Microseconds>(end - this->begin_batch_) - .count(); + utils::SteadyTime::getDurationUs(this->begin_batch_, end).count(); pthread_mutex_lock(&(this->mutex_)); this->avg_win_ += this->win_current_ * delay; @@ -154,25 +156,24 @@ void InterRttEstimator::onWindowDecrease(double win_current) { this->win_change_ += delay; pthread_mutex_unlock(&(this->mutex_)); - this->begin_batch_ = std::chrono::steady_clock::now(); + this->begin_batch_ = utils::SteadyTime::now(); } ALaTcpEstimator::ALaTcpEstimator() { this->estimation_ = 0.0; this->observer_ = NULL; - this->start_time_ = std::chrono::steady_clock::now(); + this->start_time_ = utils::SteadyTime::now(); this->totalSize_ = 0.0; } void ALaTcpEstimator::onStart() { this->totalSize_ = 0.0; - this->start_time_ = std::chrono::steady_clock::now(); + this->start_time_ = utils::SteadyTime::now(); } void ALaTcpEstimator::onDownloadFinished() { - TimePoint end = std::chrono::steady_clock::now(); - auto delay = - std::chrono::duration_cast<Microseconds>(end - this->start_time_).count(); + auto end = utils::SteadyTime::now(); + auto delay = utils::SteadyTime::getDurationUs(this->start_time_, end).count(); this->estimation_ = this->totalSize_ * 8 * 1000000 / delay; if (observer_) { observer_->notifyStats(this->estimation_); @@ -192,22 +193,21 @@ SimpleEstimator::SimpleEstimator(double alphaArg, int batching_param) { this->number_of_packets_ = 0; this->base_alpha_ = alphaArg; this->alpha_ = alphaArg; - this->start_time_ = std::chrono::steady_clock::now(); - this->begin_batch_ = std::chrono::steady_clock::now(); + this->start_time_ = utils::SteadyTime::now(); + this->begin_batch_ = utils::SteadyTime::now(); } void SimpleEstimator::onStart() { this->estimated_ = false; this->number_of_packets_ = 0; this->total_size_ = 0.0; - this->start_time_ = std::chrono::steady_clock::now(); - this->begin_batch_ = std::chrono::steady_clock::now(); + this->start_time_ = utils::SteadyTime::now(); + this->begin_batch_ = utils::SteadyTime::now(); } void SimpleEstimator::onDownloadFinished() { - TimePoint end = std::chrono::steady_clock::now(); - auto delay = - std::chrono::duration_cast<Microseconds>(end - this->start_time_).count(); + auto end = utils::SteadyTime::now(); + auto delay = utils::SteadyTime::getDurationUs(this->start_time_, end).count(); if (observer_) { observer_->notifyDownloadTime((double)delay); } @@ -229,8 +229,7 @@ void SimpleEstimator::onDownloadFinished() { } else { if (this->number_of_packets_ >= (int)(75.0 * (double)this->batching_param_ / 100.0)) { - delay = std::chrono::duration_cast<Microseconds>(end - this->begin_batch_) - .count(); + delay = utils::SteadyTime::getDurationUs(this->begin_batch_, end).count(); // Assuming all packets carry max_packet_size_ bytes of data // (8*max_packet_size_ bits); 1000000 factor to convert us to seconds if (this->estimation_) { @@ -249,22 +248,21 @@ void SimpleEstimator::onDownloadFinished() { } this->number_of_packets_ = 0; this->total_size_ = 0.0; - this->start_time_ = std::chrono::steady_clock::now(); - this->begin_batch_ = std::chrono::steady_clock::now(); + this->start_time_ = utils::SteadyTime::now(); + this->begin_batch_ = utils::SteadyTime::now(); } void SimpleEstimator::onDataReceived(int packet_size) { this->total_size_ += packet_size; } -void SimpleEstimator::onRttUpdate(double rtt) { +void SimpleEstimator::onRttUpdate(const utils::SteadyTime::Milliseconds &rtt) { this->number_of_packets_++; if (this->number_of_packets_ == this->batching_param_) { - TimePoint end = std::chrono::steady_clock::now(); + auto end = utils::SteadyTime::now(); auto delay = - std::chrono::duration_cast<Microseconds>(end - this->begin_batch_) - .count(); + utils::SteadyTime::getDurationUs(this->begin_batch_, end).count(); // Assuming all packets carry max_packet_size_ bytes of data // (8*max_packet_size_ bits); 1000000 factor to convert us to seconds if (this->estimation_) { @@ -280,7 +278,7 @@ void SimpleEstimator::onRttUpdate(double rtt) { this->alpha_ = this->base_alpha_; this->number_of_packets_ = 0; this->total_size_ = 0.0; - this->begin_batch_ = std::chrono::steady_clock::now(); + this->begin_batch_ = utils::SteadyTime::now(); } } @@ -297,13 +295,14 @@ BatchingPacketsEstimator::BatchingPacketsEstimator(double alpha_arg, this->max_packet_size_ = 0; this->estimation_ = 0.0; this->win_current_ = 1.0; - this->begin_batch_ = std::chrono::steady_clock::now(); - this->start_time_ = std::chrono::steady_clock::now(); + this->begin_batch_ = utils::SteadyTime::now(); + this->start_time_ = utils::SteadyTime::now(); } -void BatchingPacketsEstimator::onRttUpdate(double rtt) { +void BatchingPacketsEstimator::onRttUpdate( + const utils::SteadyTime::Milliseconds &rtt) { this->number_of_packets_++; - this->avg_rtt_ += rtt; + this->avg_rtt_ += rtt.count(); if (number_of_packets_ == this->batching_param_) { if (estimation_ == 0) { @@ -329,25 +328,23 @@ void BatchingPacketsEstimator::onRttUpdate(double rtt) { } void BatchingPacketsEstimator::onWindowIncrease(double win_current) { - TimePoint end = std::chrono::steady_clock::now(); + auto end = utils::SteadyTime::now(); auto delay = - std::chrono::duration_cast<Microseconds>(end - this->begin_batch_) - .count(); + utils::SteadyTime::getDurationUs(this->begin_batch_, end).count(); this->avg_win_ += this->win_current_ * delay; this->win_current_ = win_current; this->win_change_ += delay; - this->begin_batch_ = std::chrono::steady_clock::now(); + this->begin_batch_ = utils::SteadyTime::now(); } void BatchingPacketsEstimator::onWindowDecrease(double win_current) { - TimePoint end = std::chrono::steady_clock::now(); + auto end = utils::SteadyTime::now(); auto delay = - std::chrono::duration_cast<Microseconds>(end - this->begin_batch_) - .count(); + utils::SteadyTime::getDurationUs(this->begin_batch_, end).count(); this->avg_win_ += this->win_current_ * delay; this->win_current_ = win_current; this->win_change_ += delay; - this->begin_batch_ = std::chrono::steady_clock::now(); + this->begin_batch_ = utils::SteadyTime::now(); } } // end namespace protocol diff --git a/libtransport/src/protocols/rate_estimation.h b/libtransport/src/protocols/rate_estimation.h index 42ae74194..b71de12e4 100644 --- a/libtransport/src/protocols/rate_estimation.h +++ b/libtransport/src/protocols/rate_estimation.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -16,6 +16,7 @@ #pragma once #include <hicn/transport/interfaces/statistics.h> +#include <hicn/transport/utils/noncopyable.h> #include <protocols/raaqm_data_path.h> #include <chrono> @@ -24,16 +25,13 @@ namespace transport { namespace protocol { -class IcnRateEstimator { +class IcnRateEstimator : utils::NonCopyable { public: - using TimePoint = std::chrono::steady_clock::time_point; - using Microseconds = std::chrono::microseconds; - IcnRateEstimator(){}; virtual ~IcnRateEstimator(){}; - virtual void onRttUpdate(double rtt){}; + virtual void onRttUpdate(const utils::SteadyTime::Milliseconds &rtt){}; virtual void onDataReceived(int packetSize){}; @@ -48,9 +46,10 @@ class IcnRateEstimator { virtual void setObserver(interface::IcnObserver *observer) { this->observer_ = observer; }; + interface::IcnObserver *observer_; - TimePoint start_time_; - TimePoint begin_batch_; + utils::SteadyTime::TimePoint start_time_; + utils::SteadyTime::TimePoint begin_batch_; double base_alpha_; double alpha_; double estimation_; @@ -67,7 +66,7 @@ class InterRttEstimator : public IcnRateEstimator { ~InterRttEstimator(); - void onRttUpdate(double rtt); + void onRttUpdate(const utils::SteadyTime::Milliseconds &rtt); void onDataReceived(int packet_size) { if (packet_size > this->max_packet_size_) { @@ -102,7 +101,7 @@ class BatchingPacketsEstimator : public IcnRateEstimator { public: BatchingPacketsEstimator(double alpha_arg, int batchingParam); - void onRttUpdate(double rtt); + void onRttUpdate(const utils::SteadyTime::Milliseconds &rtt); void onDataReceived(int packet_size) { if (packet_size > this->max_packet_size_) { @@ -149,7 +148,7 @@ class SimpleEstimator : public IcnRateEstimator { public: SimpleEstimator(double alpha, int batching_param); - void onRttUpdate(double rtt); + void onRttUpdate(const utils::SteadyTime::Milliseconds &rtt); void onDataReceived(int packet_size); diff --git a/libtransport/src/protocols/reassembly.cc b/libtransport/src/protocols/reassembly.cc index ce24fce1b..065458f7e 100644 --- a/libtransport/src/protocols/reassembly.cc +++ b/libtransport/src/protocols/reassembly.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/reassembly.h b/libtransport/src/protocols/reassembly.h index e072ad123..b0879201d 100644 --- a/libtransport/src/protocols/reassembly.h +++ b/libtransport/src/protocols/reassembly.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -36,7 +36,7 @@ class Reassembly { public: class ContentReassembledCallback { public: - virtual void onContentReassembled(std::error_code ec) = 0; + virtual void onContentReassembled(const std::error_code &ec) = 0; }; Reassembly(implementation::ConsumerSocket *icn_socket, @@ -47,12 +47,12 @@ class Reassembly { virtual ~Reassembly() = default; /** - * Hanle reassembly of content object. + * Handle reassembly of content object. */ virtual void reassemble(core::ContentObject &content_object) = 0; /** - * Hanle reassembly of content object. + * Handle reassembly of content object. */ virtual void reassemble(utils::MemBuf &buffer, uint32_t suffix) = 0; diff --git a/libtransport/src/protocols/rtc/CMakeLists.txt b/libtransport/src/protocols/rtc/CMakeLists.txt index 873b345d0..be8e0189c 100644 --- a/libtransport/src/protocols/rtc/CMakeLists.txt +++ b/libtransport/src/protocols/rtc/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2019 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -16,22 +16,43 @@ list(APPEND HEADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/rtc.h ${CMAKE_CURRENT_SOURCE_DIR}/rtc_consts.h ${CMAKE_CURRENT_SOURCE_DIR}/rtc_data_path.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_forwarding_strategy.h ${CMAKE_CURRENT_SOURCE_DIR}/rtc_indexer.h ${CMAKE_CURRENT_SOURCE_DIR}/rtc_ldr.h ${CMAKE_CURRENT_SOURCE_DIR}/rtc_packet.h ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rc.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rc_congestion_detection.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rc_iat.h ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rc_queue.h ${CMAKE_CURRENT_SOURCE_DIR}/rtc_reassembly.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_recovery_strategy.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_delay.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_fec_only.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_low_rate.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_recovery_off.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_rtx_only.h ${CMAKE_CURRENT_SOURCE_DIR}/rtc_state.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_verifier.h ) list(APPEND SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/probe_handler.cc ${CMAKE_CURRENT_SOURCE_DIR}/rtc.cc ${CMAKE_CURRENT_SOURCE_DIR}/rtc_data_path.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_forwarding_strategy.cc ${CMAKE_CURRENT_SOURCE_DIR}/rtc_ldr.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rc_congestion_detection.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rc_iat.cc ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rc_queue.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_reassembly.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_recovery_strategy.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_delay.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_fec_only.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_low_rate.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_recovery_off.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_rtx_only.cc ${CMAKE_CURRENT_SOURCE_DIR}/rtc_state.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_verifier.cc ) set(SOURCE_FILES ${SOURCE_FILES} PARENT_SCOPE) diff --git a/libtransport/src/protocols/rtc/probe_handler.cc b/libtransport/src/protocols/rtc/probe_handler.cc index abaca6ad9..abb234757 100644 --- a/libtransport/src/protocols/rtc/probe_handler.cc +++ b/libtransport/src/protocols/rtc/probe_handler.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -13,6 +13,7 @@ * limitations under the License. */ +#include <hicn/transport/utils/chrono_typedefs.h> #include <protocols/rtc/probe_handler.h> #include <protocols/rtc/rtc_consts.h> @@ -27,6 +28,7 @@ ProbeHandler::ProbeHandler(SendProbeCallback &&send_callback, : probe_interval_(0), max_probes_(0), sent_probes_(0), + recv_probes_(0), probe_timer_(std::make_unique<asio::steady_timer>(io_service)), rand_eng_((std::random_device())()), distr_(MIN_RTT_PROBE_SEQ, MAX_RTT_PROBE_SEQ), @@ -39,17 +41,25 @@ uint64_t ProbeHandler::getRtt(uint32_t seq) { if (it == pending_probes_.end()) return 0; - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + uint64_t now = utils::SteadyTime::nowMs().count(); uint64_t rtt = now - it->second; if (rtt < 1) rtt = 1; pending_probes_.erase(it); + recv_probes_++; return rtt; } +double ProbeHandler::getProbeLossRate() { + return 1.0 - ((double)recv_probes_ / (double)sent_probes_); +} + +void ProbeHandler::setSuffixRange(uint32_t min, uint32_t max) { + assert(min <= max && min >= MIN_PROBE_SEQ); + distr_ = std::uniform_int_distribution<uint32_t>(min, max); +} + void ProbeHandler::setProbes(uint32_t probe_interval, uint32_t max_probes) { stopProbes(); probe_interval_ = probe_interval; @@ -60,6 +70,7 @@ void ProbeHandler::stopProbes() { probe_interval_ = 0; max_probes_ = 0; sent_probes_ = 0; + recv_probes_ = 0; probe_timer_->cancel(); } @@ -67,9 +78,7 @@ void ProbeHandler::sendProbes() { if (probe_interval_ == 0) return; if (max_probes_ != 0 && sent_probes_ >= max_probes_) return; - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + uint64_t now = utils::SteadyTime::nowMs().count(); uint32_t seq = distr_(rand_eng_); pending_probes_.insert(std::pair<uint32_t, uint64_t>(seq, now)); @@ -92,14 +101,25 @@ void ProbeHandler::sendProbes() { std::weak_ptr<ProbeHandler> self(shared_from_this()); probe_timer_->expires_from_now(std::chrono::microseconds(probe_interval_)); - probe_timer_->async_wait([self](std::error_code ec) { + probe_timer_->async_wait([self](const std::error_code &ec) { if (ec) return; - if (auto s = self.lock()) { + auto s = self.lock(); + if (s) { s->sendProbes(); } }); } +ProbeType ProbeHandler::getProbeType(uint32_t seq) { + if (MIN_INIT_PROBE_SEQ <= seq && seq <= MAX_INIT_PROBE_SEQ) { + return ProbeType::INIT; + } + if (MIN_RTT_PROBE_SEQ <= seq && seq <= MAX_RTT_PROBE_SEQ) { + return ProbeType::RTT; + } + return ProbeType::NOT_PROBE; +} + } // namespace rtc } // namespace protocol diff --git a/libtransport/src/protocols/rtc/probe_handler.h b/libtransport/src/protocols/rtc/probe_handler.h index e34b23df0..2de908176 100644 --- a/libtransport/src/protocols/rtc/probe_handler.h +++ b/libtransport/src/protocols/rtc/probe_handler.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -26,6 +26,12 @@ namespace protocol { namespace rtc { +enum class ProbeType { + NOT_PROBE, + INIT, + RTT, +}; + class ProbeHandler : public std::enable_shared_from_this<ProbeHandler> { public: using SendProbeCallback = std::function<void(uint32_t)>; @@ -35,31 +41,39 @@ class ProbeHandler : public std::enable_shared_from_this<ProbeHandler> { ~ProbeHandler(); - // if the function returns 0 the probe is not valaid + // If the function returns 0 the probe is not valid. uint64_t getRtt(uint32_t seq); - // reset the probes parameters. it stop the current probing. - // to restar call sendProbes. - // probe_interval = 0 means that no event will be scheduled - // max_probe = 0 means no limit to the number of probe to send + // this function may return a residual loss rate higher than the real one if + // we don't wait enough time for the probes to come back + double getProbeLossRate(); + + // Set the probe suffix range [min, max] + void setSuffixRange(uint32_t min, uint32_t max); + + // Reset the probes parameters and stops the current probing. + // probe_interval = 0 means that no event will be scheduled. + // max_probe = 0 means no limit to the number of probe to send. void setProbes(uint32_t probe_interval, uint32_t max_probes); - // stop to schedule probes void stopProbes(); void sendProbes(); + static ProbeType getProbeType(uint32_t seq); + private: uint32_t probe_interval_; // us uint32_t max_probes_; // packets uint32_t sent_probes_; // packets + uint32_t recv_probes_; // packets std::unique_ptr<asio::steady_timer> probe_timer_; - // map from seqnumber to timestamp + // Map from packet suffixes to timestamp std::unordered_map<uint32_t, uint64_t> pending_probes_; - // random generator + // Random generator std::default_random_engine rand_eng_; std::uniform_int_distribution<uint32_t> distr_; diff --git a/libtransport/src/protocols/rtc/rtc.cc b/libtransport/src/protocols/rtc/rtc.cc index 0cb4cda1d..df6522471 100644 --- a/libtransport/src/protocols/rtc/rtc.cc +++ b/libtransport/src/protocols/rtc/rtc.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -22,7 +22,7 @@ #include <protocols/rtc/rtc.h> #include <protocols/rtc/rtc_consts.h> #include <protocols/rtc/rtc_indexer.h> -#include <protocols/rtc/rtc_rc_queue.h> +#include <protocols/rtc/rtc_rc_congestion_detection.h> #include <algorithm> @@ -37,13 +37,15 @@ using namespace interface; RTCTransportProtocol::RTCTransportProtocol( implementation::ConsumerSocket *icn_socket) : TransportProtocol(icn_socket, new RtcIndexer<>(icn_socket, this), - new DatagramReassembly(icn_socket, this)), + new RtcReassembly(icn_socket, this)), number_(0) { icn_socket->getSocketOption(PORTAL, portal_); - round_timer_ = std::make_unique<asio::steady_timer>(portal_->getIoService()); + round_timer_ = + std::make_unique<asio::steady_timer>(portal_->getThread().getIoService()); scheduler_timer_ = - std::make_unique<asio::steady_timer>(portal_->getIoService()); - pacing_timer_ = std::make_unique<asio::steady_timer>(portal_->getIoService()); + std::make_unique<asio::steady_timer>(portal_->getThread().getIoService()); + pacing_timer_ = + std::make_unique<asio::steady_timer>(portal_->getThread().getIoService()); } RTCTransportProtocol::~RTCTransportProtocol() {} @@ -61,25 +63,52 @@ std::size_t RTCTransportProtocol::transportHeaderLength() { // private void RTCTransportProtocol::initParams() { TransportProtocol::reset(); + fwd_strategy_.setCallback(on_fwd_strategy_); - rc_ = std::make_shared<RTCRateControlQueue>(); + std::weak_ptr<RTCTransportProtocol> self = shared_from_this(); + + std::shared_ptr<auth::Verifier> verifier; + socket_->getSocketOption(GeneralTransportOptions::VERIFIER, verifier); + + uint32_t max_unverified_delay; + socket_->getSocketOption(GeneralTransportOptions::MAX_UNVERIFIED_TIME, + max_unverified_delay); + + rc_ = std::make_shared<RTCRateControlCongestionDetection>(); ldr_ = std::make_shared<RTCLossDetectionAndRecovery>( - indexer_verifier_.get(), - std::bind(&RTCTransportProtocol::sendRtxInterest, this, - std::placeholders::_1), - portal_->getIoService()); + indexer_verifier_.get(), portal_->getThread().getIoService(), + interface::RtcTransportRecoveryStrategies::RTX_ONLY, + [self](uint32_t seq) { + auto ptr = self.lock(); + if (ptr && ptr->isRunning()) { + ptr->sendRtxInterest(seq); + } + }, + on_rec_strategy_); + verifier_ = std::make_shared<RTCVerifier>(verifier, max_unverified_delay); state_ = std::make_shared<RTCState>( indexer_verifier_.get(), - std::bind(&RTCTransportProtocol::sendProbeInterest, this, - std::placeholders::_1), - std::bind(&RTCTransportProtocol::discoveredRtt, this), - portal_->getIoService()); + [self](uint32_t seq) { + auto ptr = self.lock(); + if (ptr && ptr->isRunning()) { + ptr->sendProbeInterest(seq); + } + }, + [self]() { + auto ptr = self.lock(); + if (ptr && ptr->isRunning()) { + ptr->discoveredRtt(); + } + }, + portal_->getThread().getIoService()); + state_->initParams(); rc_->setState(state_); - // TODO: for the moment we keep the congestion control disabled - // rc_->tunrOnRateControl(); - ldr_->setState(state_); + rc_->turnOnRateControl(); + ldr_->setState(state_.get()); + ldr_->setRateControl(rc_.get()); + verifier_->setState(state_); // protocol state start_send_interest_ = false; @@ -102,6 +131,10 @@ void RTCTransportProtocol::initParams() { } #else max_aggregated_interest_ = 1; + if (const char *max_aggr = std::getenv("MAX_AGGREGATED_INTERESTS")) { + LOG(INFO) << "Max Aggregated: " << max_aggr; + max_aggregated_interest_ = std::stoul(std::string(max_aggr)); + } #endif max_sent_int_ = @@ -131,6 +164,7 @@ void RTCTransportProtocol::initParams() { indexer_verifier_->setNFec(0); ldr_->setFecParams(fec::FECUtils::getBlockSymbols(fec_type_), fec::FECUtils::getSourceSymbols(fec_type_)); + fec_decoder_->setIOService(portal_->getThread().getIoService()); } else { indexer_verifier_->disableFec(); } @@ -162,61 +196,97 @@ void RTCTransportProtocol::inactiveProducer() { void RTCTransportProtocol::newRound() { round_timer_->expires_from_now(std::chrono::milliseconds(ROUND_LEN)); - // TODO pass weak_ptr here - round_timer_->async_wait([this, n{number_}](std::error_code ec) { - if (ec) return; - if (n != number_) { + std::weak_ptr<RTCTransportProtocol> self = shared_from_this(); + round_timer_->async_wait([self](const std::error_code &ec) { + if (ec) { return; } + auto ptr = self.lock(); + + if (!ptr || !ptr->isRunning()) { + return; + } + + auto &state = ptr->state_; + // saving counters that will be reset on new round - uint32_t sent_retx = state_->getSentRtxInRound(); - uint32_t received_bytes = state_->getReceivedBytesInRound(); - uint32_t sent_interest = state_->getSentInterestInRound(); - uint32_t lost_data = state_->getLostData(); - uint32_t definitely_lost = state_->getDefinitelyLostPackets(); - uint32_t recovered_losses = state_->getRecoveredLosses(); - uint32_t received_nacks = state_->getReceivedNacksInRound(); - uint32_t received_fec = state_->getReceivedFecPackets(); - - bool in_sync = (current_state_ == SyncState::in_sync); - ldr_->onNewRound(in_sync); - state_->onNewRound((double)ROUND_LEN, in_sync); - rc_->onNewRound((double)ROUND_LEN); + uint32_t sent_retx = state->getSentRtxInRound(); + uint32_t received_bytes = + (state->getReceivedBytesInRound() + // data packets received + state->getReceivedFecBytesInRound()); // fec packets received + uint32_t sent_interest = state->getSentInterestInRound(); + uint32_t lost_data = state->getLostData(); + uint32_t definitely_lost = state->getDefinitelyLostPackets(); + uint32_t recovered_losses = state->getRecoveredLosses(); + uint32_t received_nacks = state->getReceivedNacksInRound(); + uint32_t received_fec = state->getReceivedFecPackets(); + + bool in_sync = (ptr->current_state_ == SyncState::in_sync); + ptr->ldr_->onNewRound(in_sync); + ptr->state_->onNewRound((double)ROUND_LEN, in_sync); + ptr->rc_->onNewRound((double)ROUND_LEN); // update sync state if needed - if (current_state_ == SyncState::in_sync) { - double cache_rate = state_->getPacketFromCacheRatio(); + if (ptr->current_state_ == SyncState::in_sync) { + double cache_rate = state->getPacketFromCacheRatio(); if (cache_rate > MAX_DATA_FROM_CACHE) { - current_state_ = SyncState::catch_up; + ptr->current_state_ = SyncState::catch_up; } } else { - double target_rate = state_->getProducerRate() * PRODUCTION_RATE_FRACTION; - double received_rate = state_->getReceivedRate(); - uint32_t round_without_nacks = state_->getRoundsWithoutNacks(); - double cache_ratio = state_->getPacketFromCacheRatio(); + double target_rate = state->getProducerRate() * PRODUCTION_RATE_FRACTION; + double received_rate = + state->getReceivedRate() + state->getRecoveredFecRate(); + uint32_t round_without_nacks = state->getRoundsWithoutNacks(); + double cache_ratio = state->getPacketFromCacheRatio(); if (round_without_nacks >= ROUNDS_IN_SYNC_BEFORE_SWITCH && received_rate >= target_rate && cache_ratio < MAX_DATA_FROM_CACHE) { - current_state_ = SyncState::in_sync; + ptr->current_state_ = SyncState::in_sync; } } DLOG_IF(INFO, VLOG_IS_ON(3)) << "Calling updateSyncWindow in newRound function"; - updateSyncWindow(); + ptr->updateSyncWindow(); - sendStatsToApp(sent_retx, received_bytes, sent_interest, lost_data, - definitely_lost, recovered_losses, received_nacks, - received_fec); - newRound(); + ptr->sendStatsToApp(sent_retx, received_bytes, sent_interest, lost_data, + definitely_lost, recovered_losses, received_nacks, + received_fec); + ptr->fwd_strategy_.checkStrategy(); + ptr->newRound(); }); } void RTCTransportProtocol::discoveredRtt() { start_send_interest_ = true; - ldr_->turnOnRTX(); + uint32_t strategy; + socket_->getSocketOption(RtcTransportOptions::RECOVERY_STRATEGY, strategy); + ldr_->changeRecoveryStrategy( + (interface::RtcTransportRecoveryStrategies)strategy); + ldr_->turnOnRecovery(); ldr_->onNewRound(false); + + // set forwarding strategy switch if selected + Name *name = nullptr; + socket_->getSocketOption(GeneralTransportOptions::NETWORK_NAME, &name); + Prefix prefix(*name, 128); + if ((interface::RtcTransportRecoveryStrategies)strategy == + interface::RtcTransportRecoveryStrategies::LOW_RATE_AND_BESTPATH) { + fwd_strategy_.initFwdStrategy(portal_, prefix, state_.get(), + RTCForwardingStrategy::BEST_PATH); + } else if ((interface::RtcTransportRecoveryStrategies)strategy == + interface::RtcTransportRecoveryStrategies:: + LOW_RATE_AND_REPLICATION) { + fwd_strategy_.initFwdStrategy(portal_, prefix, state_.get(), + RTCForwardingStrategy::REPLICATION); + } else if ((interface::RtcTransportRecoveryStrategies)strategy == + interface::RtcTransportRecoveryStrategies:: + LOW_RATE_AND_ALL_FWD_STRATEGIES) { + fwd_strategy_.initFwdStrategy(portal_, prefix, state_.get(), + RTCForwardingStrategy::BOTH); + } + updateSyncWindow(); } @@ -244,7 +314,7 @@ void RTCTransportProtocol::computeMaxSyncWindow() { (production_rate * lifetime_ms * INTEREST_LIFETIME_REDUCTION_FACTOR) / packet_size); - max_sync_win_ = std::min(max_sync_win_, rc_->getCongesionWindow()); + max_sync_win_ = std::min(max_sync_win_, rc_->getCongestionWindow()); } void RTCTransportProtocol::updateSyncWindow() { @@ -259,25 +329,14 @@ void RTCTransportProtocol::updateSyncWindow() { } double prod_rate = state_->getProducerRate(); - double rtt = (double)state_->getRTT() / MILLI_IN_A_SEC; + double rtt = (double)state_->getMinRTT() / MILLI_IN_A_SEC; double packet_size = state_->getAveragePacketSize(); // if some of the info are not available do not update the current win if (prod_rate != 0.0 && rtt != 0.0 && packet_size != 0.0) { - double fec_interest_overhead = (double)state_->getPendingFecPackets() / - (double)(state_->getPendingInterestNumber() - - state_->getPendingFecPackets()); - - double fec_overhead = - std::max(indexer_verifier_->getFecOverhead(), fec_interest_overhead); - - prod_rate += (prod_rate * fec_overhead); - current_sync_win_ = (uint32_t)ceil(prod_rate * rtt / packet_size); uint32_t buffer = PRODUCER_BUFFER_MS; - if (rtt > 150) - buffer = buffer * 2; // if the RTT is too large we increase the - // the size of the buffer + current_sync_win_ += ceil(prod_rate * (buffer / MILLI_IN_A_SEC) / packet_size); @@ -285,8 +344,17 @@ void RTCTransportProtocol::updateSyncWindow() { current_sync_win_ = current_sync_win_ * CATCH_UP_WIN_INCREMENT; } + uint32_t min_win = WIN_MIN; + bool aggregated_data_on; + socket_->getSocketOption(RtcTransportOptions::AGGREGATED_DATA, + aggregated_data_on); + if (aggregated_data_on) { + min_win = WIN_MIN_WITH_AGGREGARED_DATA; + min_win += (min_win * (1 - (std::max(0.3, rtt) - rtt) / 0.3)); + } + current_sync_win_ = std::min(current_sync_win_, max_sync_win_); - current_sync_win_ = std::max(current_sync_win_, WIN_MIN); + current_sync_win_ = std::max(current_sync_win_, min_win); } scheduleNextInterests(); @@ -322,7 +390,7 @@ void RTCTransportProtocol::sendProbeInterest(uint32_t seq) { socket_->getSocketOption(GeneralTransportOptions::NETWORK_NAME, &interest_name); - DLOG_IF(INFO, VLOG_IS_ON(3)) << "send probe " << seq; + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send probe " << seq; interest_name->setSuffix(seq); sendInterest(*interest_name); } @@ -330,13 +398,18 @@ void RTCTransportProtocol::sendProbeInterest(uint32_t seq) { void RTCTransportProtocol::scheduleNextInterests() { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Schedule next interests"; - if (!isRunning() && !is_first_) return; + if (!isRunning() && !is_first_) { + return; + } - if (pacing_timer_on_) return; // wait pacing timer for the next send + if (pacing_timer_on_) { + return; // wait pacing timer for the next send + } - if (!start_send_interest_) + if (!start_send_interest_) { return; // RTT discovering phase is not finished so // do not start to send interests + } if (TRANSPORT_EXPECT_FALSE(!state_->isProducerActive())) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Inactive producer."; @@ -352,7 +425,7 @@ void RTCTransportProtocol::scheduleNextInterests() { &interest_name); uint32_t next_seg = 0; - DLOG_IF(INFO, VLOG_IS_ON(3)) << "send interest " << next_seg; + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send interest " << next_seg; interest_name->setSuffix(next_seg); if (portal_->interestIsPending(*interest_name)) { @@ -370,28 +443,37 @@ void RTCTransportProtocol::scheduleNextInterests() { << " -- current_sync_win_: " << current_sync_win_; uint32_t pending = state_->getPendingInterestNumber(); - if (pending >= current_sync_win_) return; // no space in the window + uint32_t pending_fec = state_->getPendingFecPackets(); + + if ((pending - pending_fec) >= current_sync_win_) + return; // no space in the window - if ((current_sync_win_ - pending) < max_aggregated_interest_) { + // XXX double check if aggregated interests are still working here + if ((current_sync_win_ - (pending - pending_fec)) < + max_aggregated_interest_) { if (scheduler_timer_on_) return; // timer already scheduled - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + uint64_t now = utils::SteadyTime::nowMs().count(); uint64_t time = now - last_interest_sent_time_; if (time < WAIT_FOR_INTEREST_BATCH) { uint64_t next = WAIT_FOR_INTEREST_BATCH - time; scheduler_timer_on_ = true; scheduler_timer_->expires_from_now(std::chrono::milliseconds(next)); - scheduler_timer_->async_wait([this](std::error_code ec) { + + std::weak_ptr<RTCTransportProtocol> self = shared_from_this(); + scheduler_timer_->async_wait([self](const std::error_code &ec) { if (ec) return; - if (!scheduler_timer_on_) return; - scheduler_timer_on_ = false; - scheduleNextInterests(); + auto ptr = self.lock(); + if (ptr && ptr->isRunning()) { + if (!ptr->scheduler_timer_on_) return; + + ptr->scheduler_timer_on_ = false; + ptr->scheduleNextInterests(); + } }); - return; // whait for the timer + return; // wait for the timer } } @@ -403,7 +485,7 @@ void RTCTransportProtocol::scheduleNextInterests() { indexer_verifier_->jumpToIndex(state_->getLastSeqNacked() + 1); } - // skipe received packets + // skip received packets if (indexer_verifier_->checkNextSuffix() <= state_->getHighestSeqReceivedInOrder()) { indexer_verifier_->jumpToIndex(state_->getHighestSeqReceivedInOrder() + 1); @@ -417,7 +499,8 @@ void RTCTransportProtocol::scheduleNextInterests() { socket_->getSocketOption(GeneralTransportOptions::NETWORK_NAME, &name); std::array<uint32_t, MAX_AGGREGATED_INTEREST> additional_suffixes; - while ((state_->getPendingInterestNumber() < current_sync_win_) && + while (((state_->getPendingInterestNumber() - + state_->getPendingFecPackets()) < current_sync_win_) && (sent_interests < max_sent_int_)) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "In while loop. Window size: " << current_sync_win_; @@ -428,19 +511,20 @@ void RTCTransportProtocol::scheduleNextInterests() { // send the packet only if: // 1) it is not pending yet (not true for rtx) - // 2) the packet is not received or lost + // 2) the packet is not received or def lost // 3) is not in the rtx list // 4) is fec and is not in order (!= last sent + 1) + PacketState packet_state = state_->getPacketState(next_seg); if (portal_->interestIsPending(*name) || - state_->isReceivedOrLost(next_seg) != PacketState::UNKNOWN || - ldr_->isRtx(next_seg) || + packet_state == PacketState::RECEIVED || + packet_state == PacketState::DEFINITELY_LOST || ldr_->isRtx(next_seg) || (indexer_verifier_->isFec(next_seg) && next_seg != last_interest_sent_seq_ + 1)) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "skip interest " << next_seg << " because: pending " - << portal_->interestIsPending(*name) << ", recv " - << (state_->isReceivedOrLost(next_seg) != PacketState::UNKNOWN) - << ", rtx " << (ldr_->isRtx(next_seg)) << ", is old fec " + << portal_->interestIsPending(*name) << ", recv or lost" + << (int)packet_state << ", rtx " << (ldr_->isRtx(next_seg)) + << ", is old fec " << ((indexer_verifier_->isFec(next_seg) && next_seg != last_interest_sent_seq_ + 1)); continue; @@ -462,10 +546,7 @@ void RTCTransportProtocol::scheduleNextInterests() { sent_packets++; sent_interests++; sendInterest(interest_name, &additional_suffixes, aggregated_counter - 1); - last_interest_sent_time_ = - std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + last_interest_sent_time_ = utils::SteadyTime::nowMs().count(); aggregated_counter = 0; } } @@ -473,25 +554,29 @@ void RTCTransportProtocol::scheduleNextInterests() { // exiting the while we may have some pending interest to send if (aggregated_counter != 0) { sent_packets++; - last_interest_sent_time_ = - std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + last_interest_sent_time_ = utils::SteadyTime::nowMs().count(); sendInterest(interest_name, &additional_suffixes, aggregated_counter - 1); } - if (state_->getPendingInterestNumber() < current_sync_win_) { + if ((state_->getPendingInterestNumber() - state_->getPendingFecPackets()) < + current_sync_win_) { // we still have space in the window but we already sent too many packets // wait PACING_WAIT to avoid drops in the kernel pacing_timer_on_ = true; pacing_timer_->expires_from_now(std::chrono::microseconds(PACING_WAIT)); - scheduler_timer_->async_wait([this](std::error_code ec) { + + std::weak_ptr<RTCTransportProtocol> self = shared_from_this(); + scheduler_timer_->async_wait([self](const std::error_code &ec) { if (ec) return; - if (!pacing_timer_on_) return; - pacing_timer_on_ = false; - scheduleNextInterests(); + auto ptr = self.lock(); + if (ptr && ptr->isRunning()) { + if (!ptr->pacing_timer_on_) return; + + ptr->pacing_timer_on_ = false; + ptr->scheduleNextInterests(); + } }); } } @@ -500,13 +585,13 @@ void RTCTransportProtocol::onInterestTimeout(Interest::Ptr &interest, const Name &name) { uint32_t segment_number = name.getSuffix(); - if (segment_number >= MIN_PROBE_SEQ) { + if (ProbeHandler::getProbeType(segment_number) != ProbeType::NOT_PROBE) { // this is a timeout on a probe, do nothing return; } - PacketState state = state_->isReceivedOrLost(segment_number); - if (state != PacketState::UNKNOWN) { + PacketState state = state_->getPacketState(segment_number); + if (state == PacketState::RECEIVED || state == PacketState::DEFINITELY_LOST) { // we may recover a packets using fec, ignore this timer return; } @@ -524,13 +609,18 @@ void RTCTransportProtocol::onInterestTimeout(Interest::Ptr &interest, DLOG_IF(INFO, VLOG_IS_ON(3)) << "handle timeout for packet " << segment_number << " using rtx"; if (ldr_->isRtxOn()) { - ldr_->onTimeout(segment_number); - if (indexer_verifier_->isFec(segment_number)) + if (indexer_verifier_->isFec(segment_number)) { + // if this is a fec packet we do not recover it with rtx so we consider + // the packet to be lost + ldr_->onTimeout(segment_number, true); state_->onTimeout(segment_number, true); - else + } else { + ldr_->onTimeout(segment_number, false); state_->onTimeout(segment_number, false); + } } else { // in this case we wil never recover the timeout + ldr_->onTimeout(segment_number, true); state_->onTimeout(segment_number, true); } scheduleNextInterests(); @@ -559,7 +649,7 @@ void RTCTransportProtocol::onInterestTimeout(Interest::Ptr &interest, void RTCTransportProtocol::onNack(const ContentObject &content_object) { struct nack_packet_t *nack = (struct nack_packet_t *)content_object.getPayload()->data(); - uint32_t production_seg = nack->getProductionSegement(); + uint32_t production_seg = nack->getProductionSegment(); uint32_t nack_segment = content_object.getName().getSuffix(); bool is_rtx = ldr_->isRtx(nack_segment); @@ -592,6 +682,8 @@ void RTCTransportProtocol::onNack(const ContentObject &content_object) { // remove the nack is it exists if (tn_it != timeouts_or_nacks_.end()) timeouts_or_nacks_.erase(tn_it); + state_->onJumpForward(production_seg); + verifier_->onJumpForward(production_seg); // the client is asking for content in the past // switch to catch up state and increase the window // this is true only if the packet is not an RTX @@ -618,11 +710,9 @@ void RTCTransportProtocol::onProbe(const ContentObject &content_object) { bool valid = state_->onProbePacketReceived(content_object); if (!valid) return; - struct nack_packet_t *probe = - (struct nack_packet_t *)content_object.getPayload()->data(); - uint32_t production_seg = probe->getProductionSegement(); + uint32_t production_seg = RTCState::getProbeParams(content_object).prod_seg; - // as for the nacks set next_segment + // As for the nacks set next_segment DLOG_IF(INFO, VLOG_IS_ON(3)) << "on probe next seg = " << indexer_verifier_->checkNextSuffix() << ", jump to " << production_seg; @@ -636,12 +726,39 @@ void RTCTransportProtocol::onContentObjectReceived( Interest &interest, ContentObject &content_object, std::error_code &ec) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received content object of size: " << content_object.payloadSize(); - uint32_t payload_size = content_object.payloadSize(); + uint32_t segment_number = content_object.getName().getSuffix(); + PayloadType payload_type = content_object.getPayloadType(); + PacketState state; + + ContentObject *content_ptr = &content_object; + ContentObject::Ptr manifest_ptr = nullptr; + + bool is_probe = + ProbeHandler::getProbeType(segment_number) != ProbeType::NOT_PROBE; + bool is_nack = !is_probe && content_object.payloadSize() == NACK_HEADER_SIZE; + bool is_fec = indexer_verifier_->isFec(segment_number); + bool is_manifest = + !is_probe && !is_nack && !is_fec && payload_type == PayloadType::MANIFEST; + bool is_data = + !is_probe && !is_nack && !is_fec && payload_type == PayloadType::DATA; + bool compute_stats = is_data || is_manifest; ec = make_error_code(protocol_error::not_reassemblable); - if (segment_number >= MIN_PROBE_SEQ) { + // A helper function to process manifests or data packets received + auto onDataPacketReceived = [this](ContentObject &content_object, + bool compute_stats) { + ldr_->onDataPacketReceived(content_object); + rc_->onDataPacketReceived(content_object, compute_stats); + updateSyncWindow(); + }; + + // First verify the packet signature and apply the corresponding policy + auth::VerificationPolicy policy = verifier_->verify(content_object, is_fec); + indexer_verifier_->applyPolicy(interest, content_object, false, policy); + + if (is_probe) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received probe " << segment_number; if (*on_content_object_input_) { (*on_content_object_input_)(*socket_->getInterface(), content_object); @@ -650,7 +767,7 @@ void RTCTransportProtocol::onContentObjectReceived( return; } - if (payload_size == NACK_HEADER_SIZE) { + if (is_nack) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received nack " << segment_number; if (*on_content_object_input_) { (*on_content_object_input_)(*socket_->getInterface(), content_object); @@ -659,57 +776,122 @@ void RTCTransportProtocol::onContentObjectReceived( return; } + // content_ptr will point either to the input data packet or to a manifest + // whose FEC header has been removed + if (is_manifest) { + manifest_ptr = removeFecHeader(content_object); + if (manifest_ptr) { + content_ptr = manifest_ptr.get(); + } + } + + // From there, the packet is either a FEC, a manifest or a data packet. DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received content " << segment_number; - bool compute_stats = true; + // Do not count timed out packets in stats auto tn_it = timeouts_or_nacks_.find(segment_number); if (tn_it != timeouts_or_nacks_.end()) { compute_stats = false; timeouts_or_nacks_.erase(tn_it); } - if (ldr_->isRtx(segment_number)) { + + // Do not count retransmissions or losses in stats + if (ldr_->isRtx(segment_number) || + ldr_->isPossibleLossWithNoRtx(segment_number)) { compute_stats = false; } - // check if the packet was already received - PacketState state = state_->isReceivedOrLost(segment_number); + // Fetch packet state + state = state_->getPacketState(segment_number); - if (state != PacketState::RECEIVED) { - // send packet to decoder - if (fec_decoder_) { - DLOG_IF(INFO, VLOG_IS_ON(4)) - << "send packet " << segment_number << " to FEC decoder"; - fec_decoder_->onDataPacket( - content_object, content_object.headerSize() + rtc::DATA_HEADER_SIZE); - } - if (!indexer_verifier_->isFec(segment_number)) { - // the packet may be alredy sent to the ap by the decoder, check again if - // it is already received - state = state_->isReceivedOrLost(segment_number); - if (state != PacketState::RECEIVED) { - DLOG_IF(INFO, VLOG_IS_ON(4)) << "Received content " << segment_number; + // Check if the packet is a retransmission + if (ldr_->isRtx(segment_number) && state != PacketState::RECEIVED) { + if (is_data || is_manifest) { + state_->onPacketRecoveredRtx(segment_number); - state_->onDataPacketReceived(content_object, compute_stats); + if (*on_content_object_input_) { + (*on_content_object_input_)(*socket_->getInterface(), content_object); + } - if (*on_content_object_input_) { - (*on_content_object_input_)(*socket_->getInterface(), content_object); - } - ec = make_error_code(protocol_error::success); + if (is_manifest) { + processManifest(interest, *content_ptr); } - } else { - DLOG_IF(INFO, VLOG_IS_ON(4)) << "Received fec " << segment_number; - state_->onFecPacketReceived(content_object); + + ec = is_manifest ? make_error_code(protocol_error::not_reassemblable) + : make_error_code(protocol_error::success); + + // The packet is considered received, return early + onDataPacketReceived(*content_ptr, compute_stats); + return; } - } else { + + if (is_fec) { + state_->onFecPacketRecoveredRtx(segment_number); + } + } + + // Fetch packet state again; it may have changed + state = state_->getPacketState(segment_number); + + // Check if the packet was already received + if (state == PacketState::RECEIVED || state == PacketState::TO_BE_RECEIVED) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received duplicated content " << segment_number << ", drop it"; ec = make_error_code(protocol_error::duplicated_content); + onDataPacketReceived(*content_ptr, compute_stats); + return; } - ldr_->onDataPacketReceived(content_object); - rc_->onDataPacketReceived(content_object); + if (!is_fec) { + state_->dataToBeReceived(segment_number); + } - updateSyncWindow(); + // Send packet to FEC decoder + if (fec_decoder_) { + DLOG_IF(INFO, VLOG_IS_ON(4)) + << "Send packet " << segment_number << " to FEC decoder"; + + uint32_t offset = is_manifest + ? content_object.headerSize() + : content_object.headerSize() + rtc::DATA_HEADER_SIZE; + uint32_t metadata = static_cast<uint32_t>(content_object.getPayloadType()); + + fec_decoder_->onDataPacket(content_object, offset, metadata); + } + + // We can return early if FEC + if (is_fec) { + DLOG_IF(INFO, VLOG_IS_ON(4)) << "Received FEC " << segment_number; + state_->onFecPacketReceived(content_object); + onDataPacketReceived(*content_ptr, compute_stats); + return; + } + + // The packet may have been already sent to the app by the decoder, check + // again if it is already received + state = state_->getPacketState( + segment_number); // state == RECEIVED or TO_BE_RECEIVED + + if (state != PacketState::RECEIVED) { + DLOG_IF(INFO, VLOG_IS_ON(4)) + << (is_manifest ? "Received manifest " : "Received data ") + << segment_number; + + if (is_manifest) { + processManifest(interest, *content_ptr); + } + + state_->onDataPacketReceived(*content_ptr, compute_stats); + + if (*on_content_object_input_) { + (*on_content_object_input_)(*socket_->getInterface(), content_object); + } + + ec = is_manifest ? make_error_code(protocol_error::not_reassemblable) + : make_error_code(protocol_error::success); + } + + onDataPacketReceived(*content_ptr, compute_stats); } void RTCTransportProtocol::sendStatsToApp( @@ -729,33 +911,149 @@ void RTCTransportProtocol::sendStatsToApp( stats_->updateReceivedNacks(received_nacks); stats_->updateReceivedFEC(received_fec); - stats_->updateAverageWindowSize(current_sync_win_); - stats_->updateLossRatio(state_->getLossRate()); - stats_->updateAverageRtt(state_->getRTT()); + stats_->updateAverageWindowSize(state_->getPendingInterestNumber()); + stats_->updateLossRatio(state_->getPerSecondLossRate()); + uint64_t rtt = state_->getAvgRTT(); + stats_->updateAverageRtt(utils::SteadyTime::Milliseconds(rtt)); + stats_->updateQueuingDelay(state_->getQueuing()); stats_->updateLostData(lost_data); stats_->updateDefinitelyLostData(definitely_lost); stats_->updateRecoveredData(recovered_losses); stats_->updateCCState((unsigned int)current_state_ ? 1 : 0); (*stats_summary_)(*socket_->getInterface(), *stats_); + bool in_congestion = rc_->inCongestionState(); + stats_->updateCongestionState(in_congestion); + double residual_losses = state_->getResidualLossRate(); + stats_->updateResidualLossRate(residual_losses); + stats_->updateQualityScore(state_->getQualityScore()); + + // set alerts + if (rtt > MAX_RTT) + stats_->setAlert(interface::TransportStatistics::statsAlerts::LATENCY); + else + stats_->clearAlert(interface::TransportStatistics::statsAlerts::LATENCY); + + if (in_congestion) + stats_->setAlert(interface::TransportStatistics::statsAlerts::CONGESTION); + else + stats_->clearAlert( + interface::TransportStatistics::statsAlerts::CONGESTION); + + if (residual_losses > MAX_RESIDUAL_LOSSES) + stats_->setAlert(interface::TransportStatistics::statsAlerts::LOSSES); + else + stats_->clearAlert(interface::TransportStatistics::statsAlerts::LOSSES); } } -void RTCTransportProtocol::onFecPackets( - std::vector<std::pair<uint32_t, fec::buffer>> &packets) { +void RTCTransportProtocol::onFecPackets(fec::BufferArray &packets) { + Packet::Format format; + socket_->getSocketOption(interface::GeneralTransportOptions::PACKET_FORMAT, + format); + + Name *name = nullptr; + socket_->getSocketOption(GeneralTransportOptions::NETWORK_NAME, &name); + for (auto &packet : packets) { - PacketState state = state_->isReceivedOrLost(packet.first); - if (state != PacketState::RECEIVED) { - state_->onPacketRecoveredFec(packet.first); - ldr_->onPacketRecoveredFec(packet.first); - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "Recovered packet " << packet.first << " through FEC."; - reassembly_->reassemble(*packet.second, packet.first); - } else { - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "Packet" << packet.first << "already received."; + uint32_t seq_number = packet.getIndex(); + uint32_t metadata = packet.getMetadata(); + fec::buffer buffer = packet.getBuffer(); + + PayloadType payload_type = static_cast<PayloadType>(metadata); + switch (payload_type) { + case PayloadType::DATA: + case PayloadType::MANIFEST: + break; + case PayloadType::UNSPECIFIED: + default: + payload_type = PayloadType::DATA; + break; } + + switch (state_->getPacketState(seq_number)) { + case PacketState::RECEIVED: + case PacketState::TO_BE_RECEIVED: { + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "Packet " << seq_number << " already received"; + break; + } + default: { + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "Recovered packet " << seq_number << " through FEC"; + + if (payload_type == PayloadType::MANIFEST) { + name->setSuffix(seq_number); + + auto interest = + core::PacketManager<>::getInstance().getPacket<Interest>(format); + interest->setName(*name); + + auto content_object = toContentObject( + *name, format, payload_type, buffer->data(), buffer->length()); + + processManifest(*interest, *content_object); + } + + state_->onPacketRecoveredFec(seq_number, buffer->length()); + ldr_->onPacketRecoveredFec(seq_number); + + if (payload_type == PayloadType::DATA) { + verifier_->onDataRecoveredFec(seq_number); + reassembly_->reassemble(*buffer, seq_number); + } + + break; + } + } + } +} + +void RTCTransportProtocol::processManifest(Interest &interest, + ContentObject &manifest) { + auth::VerificationPolicy policy = verifier_->processManifest(manifest); + indexer_verifier_->applyPolicy(interest, manifest, false, policy); +} + +ContentObject::Ptr RTCTransportProtocol::removeFecHeader( + const ContentObject &content_object) { + if (!fec_decoder_ || !fec_decoder_->getFecHeaderSize()) { + return nullptr; } + + size_t fec_header_size = fec_decoder_->getFecHeaderSize(); + const uint8_t *payload = + content_object.data() + content_object.headerSize() + fec_header_size; + size_t payload_size = content_object.payloadSize() - fec_header_size; + + ContentObject::Ptr co = + toContentObject(content_object.getName(), content_object.getFormat(), + content_object.getPayloadType(), payload, payload_size); + + return co; +} + +ContentObject::Ptr RTCTransportProtocol::toContentObject( + const Name &name, Packet::Format format, PayloadType payload_type, + const uint8_t *payload, std::size_t payload_size, + std::size_t additional_header_size) { + // Recreate ContentObject + ContentObject::Ptr co = + core::PacketManager<>::getInstance().getPacket<ContentObject>( + format, additional_header_size); + co->updateLength(payload_size); + co->append(payload_size); + co->trimStart(co->headerSize()); + + // Copy payload + std::memcpy(co->writableData(), payload, payload_size); + + // Restore network headers and some fields + co->prepend(co->headerSize()); + co->setName(name); + co->setPayloadType(payload_type); + + return co; } } // end namespace rtc diff --git a/libtransport/src/protocols/rtc/rtc.h b/libtransport/src/protocols/rtc/rtc.h index e6431264d..37706eb1c 100644 --- a/libtransport/src/protocols/rtc/rtc.h +++ b/libtransport/src/protocols/rtc/rtc.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -15,10 +15,12 @@ #pragma once -#include <protocols/datagram_reassembly.h> +#include <protocols/rtc/rtc_forwarding_strategy.h> #include <protocols/rtc/rtc_ldr.h> #include <protocols/rtc/rtc_rc.h> +#include <protocols/rtc/rtc_reassembly.h> #include <protocols/rtc/rtc_state.h> +#include <protocols/rtc/rtc_verifier.h> #include <protocols/transport_protocol.h> #include <unordered_set> @@ -44,6 +46,8 @@ class RTCTransportProtocol : public TransportProtocol { std::size_t transportHeaderLength() override; + auto shared_from_this() { return utils::shared_from(this); } + private: enum class SyncState { catch_up = 0, in_sync = 1, last }; @@ -76,6 +80,7 @@ class RTCTransportProtocol : public TransportProtocol { void onPacketDropped(Interest &interest, ContentObject &content_object, const std::error_code &reason) override {} void onReassemblyFailed(std::uint32_t missing_segment) override {} + void processManifest(Interest &interest, ContentObject &manifest); // interaction with app functions void sendStatsToApp(uint32_t retx_count, uint32_t received_bytes, @@ -84,11 +89,20 @@ class RTCTransportProtocol : public TransportProtocol { uint32_t received_nacks, uint32_t received_fec); // FEC functions - void onFecPackets(std::vector<std::pair<uint32_t, fec::buffer>> &packets); + void onFecPackets(fec::BufferArray &packets); + + // Utils + ContentObject::Ptr removeFecHeader(const ContentObject &content_object); + ContentObject::Ptr toContentObject(const Name &name, Packet::Format format, + PayloadType payload_type, + const uint8_t *payload, + std::size_t payload_size, + std::size_t additional_header_size = 0); // protocol state bool start_send_interest_; SyncState current_state_; + // cwin vars uint32_t current_sync_win_; uint32_t max_sync_win_; @@ -120,6 +134,10 @@ class RTCTransportProtocol : public TransportProtocol { std::shared_ptr<RTCState> state_; std::shared_ptr<RTCRateControl> rc_; std::shared_ptr<RTCLossDetectionAndRecovery> ldr_; + std::shared_ptr<RTCVerifier> verifier_; + + // forwarding strategy selection + RTCForwardingStrategy fwd_strategy_; uint32_t number_; }; diff --git a/libtransport/src/protocols/rtc/rtc_consts.h b/libtransport/src/protocols/rtc/rtc_consts.h index d04bc1b1f..03efd8e84 100644 --- a/libtransport/src/protocols/rtc/rtc_consts.h +++ b/libtransport/src/protocols/rtc/rtc_consts.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -45,22 +45,22 @@ const uint32_t MAX_INTERESTS_IN_BATCH = 5; // number of seq numbers per const uint32_t WAIT_FOR_INTEREST_BATCH = 20; // msec. timer that we wait to try // to aggregate interest in the // same packet -const uint32_t MAX_PACING_BATCH = 5; -// number of interest that we can send inside -// the loop before they get dropped by the -// kernel. +const uint32_t MAX_PACING_BATCH = 5; // number of interest that we can send + // inside the loop before they get dropped + // by the kernel. const uint32_t PACING_WAIT = 1000; // usec to wait betwing two pacing batch. As // for MAX_PACING_BATCH this value was // computed during tests const uint32_t MAX_RTX_IN_BATCH = 10; // max rtx to send in loop // packet const -const uint32_t HICN_HEADER_SIZE = 40 + 20; // IPv6 + TCP bytes const uint32_t RTC_INTEREST_LIFETIME = 2000; // probes sequence range const uint32_t MIN_PROBE_SEQ = 0xefffffff; -const uint32_t MIN_RTT_PROBE_SEQ = MIN_PROBE_SEQ; +const uint32_t MIN_INIT_PROBE_SEQ = MIN_PROBE_SEQ; +const uint32_t MAX_INIT_PROBE_SEQ = 0xf7ffffff - 1; +const uint32_t MIN_RTT_PROBE_SEQ = MAX_INIT_PROBE_SEQ + 1; const uint32_t MAX_RTT_PROBE_SEQ = 0xffffffff - 1; // RTT_PROBE_INTERVAL will be used during the section while // INIT_RTT_PROBE_INTERVAL is used at the beginning to @@ -78,15 +78,16 @@ const uint32_t INIT_RTT_MIN_PROBES_TO_RECV = 5; // ms const uint32_t MAX_PENDING_PROBES = 10; // congestion -const double MAX_QUEUING_DELAY = 100.0; // ms +const double MAX_QUEUING_DELAY = 50.0; // ms // data from cache const double MAX_DATA_FROM_CACHE = 0.25; // 25% // window const -const uint32_t INITIAL_WIN = 5; // pkts -const uint32_t INITIAL_WIN_MAX = 1000000; // pkts -const uint32_t WIN_MIN = 5; // pkts +const uint32_t INITIAL_WIN = 5; // pkts +const uint32_t INITIAL_WIN_MAX = 1000000; // pkts +const uint32_t WIN_MIN = 5; // pkts +const uint32_t WIN_MIN_WITH_AGGREGARED_DATA = 10; // pkts const double CATCH_UP_WIN_INCREMENT = 1.2; // used in rate control const double WIN_DECREASE_FACTOR = 0.5; @@ -105,10 +106,8 @@ const double MOVING_AVG_ALPHA = 0.8; const double MILLI_IN_A_SEC = 1000.0; const double MICRO_IN_A_SEC = 1000000.0; - -const double MAX_CACHED_PACKETS = 262144; // 2^18 - // about 50 sec of traffic at 50Mbps - // with 1200 bytes packets +const uint32_t ROUNDS_PER_SEC = (uint32_t)(MILLI_IN_A_SEC / ROUND_LEN); +const uint32_t ROUNDS_PER_MIN = (uint32_t)ROUNDS_PER_SEC * 60; const uint32_t MAX_ROUND_WHIOUT_PACKETS = (20 * MILLI_IN_A_SEC) / ROUND_LEN; // 20 sec in rounds; @@ -120,12 +119,24 @@ const uint64_t MAX_TIMER_RTX = ~0; const uint32_t SENTINEL_TIMER_INTERVAL = 100; // ms const uint32_t MAX_RTX_WITH_SENTINEL = 10; // packets const double CATCH_UP_RTT_INCREMENT = 1.2; +const double MAX_RESIDUAL_LOSS_RATE = 2.0; // % +const uint32_t WAIT_BEFORE_FEC_UPDATE = ROUNDS_PER_SEC * 5; // used by producer const uint32_t PRODUCER_STATS_INTERVAL = 200; // ms -const uint32_t MIN_PRODUCTION_RATE = 10; // pps - // min prod rate - // set running several test +const uint32_t MIN_PRODUCTION_RATE = 25; // pps, equal to min window * + // rounds in a second +const uint32_t NACK_DELAY = 1500; // ms +const uint32_t FEC_PACING_TIME = 5; // ms + +// aggregated data consts +const uint16_t MAX_RTC_PAYLOAD_SIZE = 1200; // bytes +const uint16_t MAX_AGGREGATED_PACKETS = 5; // pkt +const uint32_t AGGREGATED_PACKETS_TIMER = 2; // ms + +// alert thresholds +const uint32_t MAX_RTT = 200; // ms +const double MAX_RESIDUAL_LOSSES = 0.05; // % } // namespace rtc diff --git a/libtransport/src/protocols/rtc/rtc_data_path.cc b/libtransport/src/protocols/rtc/rtc_data_path.cc index c098088a3..b3abf5ea8 100644 --- a/libtransport/src/protocols/rtc/rtc_data_path.cc +++ b/libtransport/src/protocols/rtc/rtc_data_path.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -13,14 +13,17 @@ * limitations under the License. */ +#include <hicn/transport/utils/chrono_typedefs.h> #include <protocols/rtc/rtc_data_path.h> #include <stdlib.h> #include <algorithm> #include <cfloat> #include <chrono> +#include <cmath> #define MAX_ROUNDS_WITHOUT_PKTS 10 // 2sec +#define AVG_RTT_TIME 1000 // (ms) 1sec namespace transport { @@ -32,6 +35,8 @@ RTCDataPath::RTCDataPath(uint32_t path_id) : path_id_(path_id), min_rtt(UINT_MAX), prev_min_rtt(UINT_MAX), + max_rtt(0), + prev_max_rtt(0), min_owd(INT_MAX), // this is computed like in LEDBAT, so it is not the // real OWD, but the measured one, that depends on the // clock of sender and receiver. the only meaningful @@ -46,20 +51,46 @@ RTCDataPath::RTCDataPath(uint32_t path_id) largest_recv_seq_(0), largest_recv_seq_time_(0), avg_inter_arrival_(DBL_MAX), + rtt_sum_(0), + last_avg_rtt_compute_(0), + rtt_samples_(0), + avg_rtt_(0.0), received_nacks_(false), - received_packets_(false), + received_packets_(0), rounds_without_packets_(0), last_received_data_packet_(0), - RTT_history_(HISTORY_LEN), + min_RTT_history_(HISTORY_LEN), + max_RTT_history_(HISTORY_LEN), OWD_history_(HISTORY_LEN){}; -void RTCDataPath::insertRttSample(uint64_t rtt) { - // for the rtt we only keep track of the min one +void RTCDataPath::insertRttSample( + const utils::SteadyTime::Milliseconds& rtt_milliseconds, bool is_probe) { + // compute min rtt + uint64_t rtt = rtt_milliseconds.count(); if (rtt < min_rtt) min_rtt = rtt; - last_received_data_packet_ = - std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + + uint64_t now = utils::SteadyTime::nowMs().count(); + last_received_data_packet_ = now; + + // compute avg rtt + if (is_probe) { + // max rtt is computed only on probes to avoid to take into account the + // production time at the server + if (rtt > max_rtt) max_rtt = rtt; + + rtt_sum_ += rtt; + rtt_samples_++; + } + + if ((now - last_avg_rtt_compute_) >= AVG_RTT_TIME) { + // compute a new avg rtt + // if rtt_samples_ = 0 keep the same rtt + if (rtt_samples_ != 0) avg_rtt_ = (double)rtt_sum_ / (double)rtt_samples_; + + rtt_sum_ = 0; + rtt_samples_ = 0; + last_avg_rtt_compute_ = now; + } } void RTCDataPath::insertOwdSample(int64_t owd) { @@ -87,15 +118,13 @@ void RTCDataPath::insertOwdSample(int64_t owd) { // owd is computed only for valid data packets so we count only // this for decide if we recevie traffic or not - received_packets_ = true; + received_packets_++; } void RTCDataPath::computeInterArrivalGap(uint32_t segment_number) { // got packet in sequence, compute gap if (largest_recv_seq_ == (segment_number - 1)) { - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + uint64_t now = utils::SteadyTime::nowMs().count(); uint64_t delta = now - largest_recv_seq_time_; largest_recv_seq_ = segment_number; largest_recv_seq_time_ = now; @@ -110,10 +139,7 @@ void RTCDataPath::computeInterArrivalGap(uint32_t segment_number) { // ooo packet, update the stasts if needed if (largest_recv_seq_ <= segment_number) { largest_recv_seq_ = segment_number; - largest_recv_seq_time_ = - std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + largest_recv_seq_time_ = utils::SteadyTime::nowMs().count(); } } @@ -146,10 +172,20 @@ void RTCDataPath::roundEnd() { min_rtt = prev_min_rtt; } + // same for max_rtt + if (max_rtt != 0) { + prev_max_rtt = max_rtt; + } else { + max_rtt = prev_max_rtt; + } + if (min_rtt == 0) min_rtt = 1; + if (max_rtt == 0) max_rtt = 1; - RTT_history_.pushBack(min_rtt); + min_RTT_history_.pushBack(min_rtt); + max_RTT_history_.pushBack(max_rtt); min_rtt = UINT_MAX; + max_rtt = 0; // do the same for min owd if (min_owd != INT_MAX) { @@ -163,32 +199,47 @@ void RTCDataPath::roundEnd() { min_owd = INT_MAX; } - if (!received_packets_) + if (received_packets_ == 0) rounds_without_packets_++; else rounds_without_packets_ = 0; - received_packets_ = false; + received_packets_ = 0; } uint32_t RTCDataPath::getPathId() { return path_id_; } -double RTCDataPath::getQueuingDealy() { return queuing_delay; } +double RTCDataPath::getQueuingDealy() { + if (queuing_delay == DBL_MAX) return 0; + return queuing_delay; +} uint64_t RTCDataPath::getMinRtt() { - if (RTT_history_.size() != 0) return RTT_history_.begin(); + if (min_RTT_history_.size() != 0) return min_RTT_history_.begin(); + return 0; +} + +uint64_t RTCDataPath::getAvgRtt() { return std::round(avg_rtt_); } + +uint64_t RTCDataPath::getMaxRtt() { + if (max_RTT_history_.size() != 0) return max_RTT_history_.begin(); return 0; } int64_t RTCDataPath::getMinOwd() { if (OWD_history_.size() != 0) return OWD_history_.begin(); - return 0; + return INT_MAX; } double RTCDataPath::getJitter() { return jitter_; } uint64_t RTCDataPath::getLastPacketTS() { return last_received_data_packet_; } -void RTCDataPath::clearRtt() { RTT_history_.clear(); } +uint32_t RTCDataPath::getPacketsLastRound() { return received_packets_; } + +void RTCDataPath::clearRtt() { + min_RTT_history_.clear(); + max_RTT_history_.clear(); +} } // end namespace rtc diff --git a/libtransport/src/protocols/rtc/rtc_data_path.h b/libtransport/src/protocols/rtc/rtc_data_path.h index c5c37fc0d..5afbbb87f 100644 --- a/libtransport/src/protocols/rtc/rtc_data_path.h +++ b/libtransport/src/protocols/rtc/rtc_data_path.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -15,7 +15,9 @@ #pragma once +#include <hicn/transport/utils/chrono_typedefs.h> #include <stdint.h> +#include <utils/max_filter.h> #include <utils/min_filter.h> #include <climits> @@ -34,19 +36,23 @@ class RTCDataPath { RTCDataPath(uint32_t path_id); public: - void insertRttSample(uint64_t rtt); + void insertRttSample(const utils::SteadyTime::Milliseconds &rtt, + bool is_probe); void insertOwdSample(int64_t owd); void computeInterArrivalGap(uint32_t segment_number); void receivedNack(); uint32_t getPathId(); uint64_t getMinRtt(); + uint64_t getAvgRtt(); + uint64_t getMaxRtt(); double getQueuingDealy(); double getInterArrivalGap(); double getJitter(); bool isActive(); bool pathToProducer(); uint64_t getLastPacketTS(); + uint32_t getPacketsLastRound(); void clearRtt(); @@ -60,6 +66,9 @@ class RTCDataPath { uint64_t min_rtt; uint64_t prev_min_rtt; + uint64_t max_rtt; + uint64_t prev_max_rtt; + int64_t min_owd; int64_t prev_min_owd; @@ -74,19 +83,26 @@ class RTCDataPath { uint64_t largest_recv_seq_time_; double avg_inter_arrival_; + // compute the avg rtt over one sec + uint64_t rtt_sum_; + uint64_t last_avg_rtt_compute_; + uint32_t rtt_samples_; + double avg_rtt_; + // flags to check if a path is active // we considere a path active if it reaches a producer //(not a cache) --aka we got at least one nack on this path-- // and if we receives packets bool received_nacks_; - bool received_packets_; - uint8_t rounds_without_packets_; // if we don't get any packet + uint32_t received_packets_; + uint32_t rounds_without_packets_; // if we don't get any packet // for MAX_ROUNDS_WITHOUT_PKTS // we consider the path inactive uint64_t last_received_data_packet_; // timestamp for the last data received // on this path - utils::MinFilter<uint64_t> RTT_history_; + utils::MinFilter<uint64_t> min_RTT_history_; + utils::MaxFilter<uint64_t> max_RTT_history_; utils::MinFilter<int64_t> OWD_history_; }; diff --git a/libtransport/src/protocols/rtc/rtc_forwarding_strategy.cc b/libtransport/src/protocols/rtc/rtc_forwarding_strategy.cc new file mode 100644 index 000000000..9503eed3e --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_forwarding_strategy.cc @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <hicn/transport/interfaces/notification.h> +#include <protocols/rtc/rtc_forwarding_strategy.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +using namespace transport::interface; + +RTCForwardingStrategy::RTCForwardingStrategy() + : init_(false), + forwarder_set_(false), + selected_strategy_(NONE), + current_strategy_(NONE), + rounds_since_last_set_(0), + portal_(nullptr), + state_(nullptr) {} + +RTCForwardingStrategy::~RTCForwardingStrategy() {} + +void RTCForwardingStrategy::setCallback(interface::StrategyCallback* callback) { + callback_ = callback; +} + +void RTCForwardingStrategy::initFwdStrategy( + std::shared_ptr<core::Portal> portal, core::Prefix& prefix, RTCState* state, + strategy_t strategy) { + init_ = true; + selected_strategy_ = strategy; + if (strategy == BOTH) + current_strategy_ = BEST_PATH; + else + current_strategy_ = strategy; + rounds_since_last_set_ = 0; + prefix_ = prefix; + portal_ = portal; + state_ = state; +} + +void RTCForwardingStrategy::checkStrategy() { + if (*callback_) { + strategy_t used_strategy = selected_strategy_; + if (used_strategy == BOTH) used_strategy = current_strategy_; + assert(used_strategy == BEST_PATH || used_strategy == REPLICATION || + used_strategy == NONE); + + notification::ForwardingStrategy strategy = + notification::ForwardingStrategy::NONE; + switch (used_strategy) { + case BEST_PATH: + strategy = notification::ForwardingStrategy::BEST_PATH; + break; + case REPLICATION: + strategy = notification::ForwardingStrategy::REPLICATION; + break; + default: + break; + } + + (*callback_)(strategy); + } + + if (!init_) return; + + if (selected_strategy_ == NONE) return; + + if (selected_strategy_ == BEST_PATH) { + checkStrategyBestPath(); + return; + } + + if (selected_strategy_ == REPLICATION) { + checkStrategyReplication(); + return; + } + + checkStrategyBoth(); +} + +void RTCForwardingStrategy::checkStrategyBestPath() { + if (!forwarder_set_) { + setStrategy(BEST_PATH); + forwarder_set_ = true; + return; + } + + uint8_t qs = state_->getQualityScore(); + + if (qs >= 4 || rounds_since_last_set_ < 25) { // wait a least 5 sec + // between each switch + rounds_since_last_set_++; + return; + } + + // try to switch path + setStrategy(BEST_PATH); +} + +void RTCForwardingStrategy::checkStrategyReplication() { + if (!forwarder_set_) { + setStrategy(REPLICATION); + forwarder_set_ = true; + return; + } + + // here we have nothing to do for the moment + return; +} + +void RTCForwardingStrategy::checkStrategyBoth() { + if (!forwarder_set_) { + setStrategy(current_strategy_); + forwarder_set_ = true; + return; + } + + checkStrategyBestPath(); + + // TODO + // for the moment we use only best path. + // but later: + // 1. if both paths are bad use replication + // 2. while using replication compute the effectiveness. if the majority of + // the packets are coming from a single path, try to use bestpath +} + +void RTCForwardingStrategy::setStrategy(strategy_t strategy) { + rounds_since_last_set_ = 0; + current_strategy_ = strategy; + portal_->setForwardingStrategy(prefix_, + string_strategies_[current_strategy_]); +} + +} // namespace rtc + +} // namespace protocol + +} // namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_forwarding_strategy.h b/libtransport/src/protocols/rtc/rtc_forwarding_strategy.h new file mode 100644 index 000000000..821b28051 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_forwarding_strategy.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <core/portal.h> +#include <hicn/transport/interfaces/callbacks.h> +#include <protocols/rtc/rtc_state.h> + +#include <array> + +namespace transport { + +namespace protocol { + +namespace rtc { + +class RTCForwardingStrategy { + public: + enum strategy_t { + BEST_PATH, + REPLICATION, + BOTH, + NONE, + }; + + RTCForwardingStrategy(); + ~RTCForwardingStrategy(); + + void initFwdStrategy(std::shared_ptr<core::Portal> portal, + core::Prefix& prefix, RTCState* state, + strategy_t strategy); + + void checkStrategy(); + void setCallback(interface::StrategyCallback* callback); + + private: + void checkStrategyBestPath(); + void checkStrategyReplication(); + void checkStrategyBoth(); + + void setStrategy(strategy_t strategy); + + std::array<std::string, 4> string_strategies_ = {"bestpath", "replication", + "both", "none"}; + + bool init_; // true if all val are initializes + bool forwarder_set_; // true if the strategy is been set at least + // once + strategy_t selected_strategy_; // this is the strategy selected using socket + // options. this can also be equal to BOTH + strategy_t current_strategy_; // if both strategies can be used this + // indicates the one that is currently in use + // that can be only replication or best path + uint32_t rounds_since_last_set_; + core::Prefix prefix_; + std::shared_ptr<core::Portal> portal_; + RTCState* state_; + interface::StrategyCallback* callback_; +}; + +} // namespace rtc + +} // namespace protocol + +} // namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_indexer.h b/libtransport/src/protocols/rtc/rtc_indexer.h index 4aee242bb..cda156b22 100644 --- a/libtransport/src/protocols/rtc/rtc_indexer.h +++ b/libtransport/src/protocols/rtc/rtc_indexer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -18,8 +18,10 @@ #include <protocols/errors.h> #include <protocols/fec_utils.h> #include <protocols/indexer.h> +#include <protocols/rtc/probe_handler.h> #include <protocols/rtc/rtc_consts.h> #include <protocols/transport_protocol.h> +#include <utils/suffix_strategy.h> #include <deque> @@ -54,7 +56,7 @@ class RtcIndexer : public Indexer { n_fec_ = 0; } - uint32_t checkNextSuffix() override { return next_suffix_; } + uint32_t checkNextSuffix() const override { return next_suffix_; } uint32_t getNextSuffix() override { if (isFec(next_suffix_)) { @@ -77,7 +79,7 @@ class RtcIndexer : public Indexer { first_suffix_ = suffix % LIMIT; } - uint32_t getFirstSuffix() override { return first_suffix_; } + uint32_t getFirstSuffix() const override { return first_suffix_; } uint32_t jumpToIndex(uint32_t index) override { next_suffix_ = index % LIMIT; @@ -87,30 +89,8 @@ class RtcIndexer : public Indexer { void onContentObject(core::Interest &interest, core::ContentObject &content_object, bool reassembly) override { - setVerifier(); - auto ret = verifier_->verifyPackets(&content_object); - - switch (ret) { - case auth::VerificationPolicy::ACCEPT: { - if (reassembly) { - reassembly_->reassemble(content_object); - } - break; - } - - case auth::VerificationPolicy::UNKNOWN: - case auth::VerificationPolicy::DROP: { - transport_->onPacketDropped( - interest, content_object, - make_error_code(protocol_error::verification_failed)); - break; - } - - case auth::VerificationPolicy::ABORT: { - transport_->onContentReassembled( - make_error_code(protocol_error::session_aborted)); - break; - } + if (reassembly) { + reassembly_->reassemble(content_object); } } @@ -120,13 +100,12 @@ class RtcIndexer : public Indexer { uint32_t getNextReassemblySegment() override { throw errors::RuntimeException( "Get reassembly segment called on rtc indexer. RTC indexer does not " - "provide " - "reassembly."); + "provide reassembly."); } bool isFinalSuffixDiscovered() override { return true; } - uint32_t getFinalSuffix() override { return LIMIT; } + uint32_t getFinalSuffix() const override { return LIMIT; } void enableFec(fec::FECType fec_type) override { fec_type_ = fec_type; } @@ -137,13 +116,13 @@ class RtcIndexer : public Indexer { n_current_fec_ = n_fec_; } - uint32_t getNFec() override { return n_fec_; } + uint32_t getNFec() const override { return n_fec_; } bool isFec(uint32_t index) override { return isFec(fec_type_, index, first_suffix_); } - double getFecOverhead() override { + double getFecOverhead() const override { if (fec_type_ == fec::FECType::UNKNOWN) { return 0; } @@ -152,7 +131,7 @@ class RtcIndexer : public Indexer { return (double)n_fec_ / k; } - double getMaxFecOverhead() override { + double getMaxFecOverhead() const override { if (fec_type_ == fec::FECType::UNKNOWN) { return 0; } diff --git a/libtransport/src/protocols/rtc/rtc_ldr.cc b/libtransport/src/protocols/rtc/rtc_ldr.cc index f0de48871..1ca1cf48d 100644 --- a/libtransport/src/protocols/rtc/rtc_ldr.cc +++ b/libtransport/src/protocols/rtc/rtc_ldr.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -16,6 +16,12 @@ #include <glog/logging.h> #include <protocols/rtc/rtc_consts.h> #include <protocols/rtc/rtc_ldr.h> +#include <protocols/rtc/rtc_rs_delay.h> +#include <protocols/rtc/rtc_rs_fec_only.h> +#include <protocols/rtc/rtc_rs_low_rate.h> +#include <protocols/rtc/rtc_rs_recovery_off.h> +#include <protocols/rtc/rtc_rs_rtx_only.h> +#include <protocols/rtc/rtc_state.h> #include <algorithm> #include <unordered_set> @@ -27,146 +33,115 @@ namespace protocol { namespace rtc { RTCLossDetectionAndRecovery::RTCLossDetectionAndRecovery( - Indexer *indexer, SendRtxCallback &&callback, asio::io_service &io_service) - : rtx_on_(false), - fec_on_(false), - next_rtx_timer_(MAX_TIMER_RTX), - last_event_(0), - sentinel_timer_interval_(MAX_TIMER_RTX), - indexer_(indexer), - send_rtx_callback_(std::move(callback)) { - timer_ = std::make_unique<asio::steady_timer>(io_service); - sentinel_timer_ = std::make_unique<asio::steady_timer>(io_service); + Indexer *indexer, asio::io_service &io_service, + interface::RtcTransportRecoveryStrategies type, + RecoveryStrategy::SendRtxCallback &&callback, + interface::StrategyCallback *external_callback) { + rs_type_ = type; + if (type == interface::RtcTransportRecoveryStrategies::RECOVERY_OFF) { + rs_ = std::make_shared<RecoveryStrategyRecoveryOff>( + indexer, std::move(callback), io_service, external_callback); + } else if (type == interface::RtcTransportRecoveryStrategies::DELAY_BASED) { + rs_ = std::make_shared<RecoveryStrategyDelayBased>( + indexer, std::move(callback), io_service, external_callback); + } else if (type == interface::RtcTransportRecoveryStrategies::FEC_ONLY) { + rs_ = std::make_shared<RecoveryStrategyFecOnly>( + indexer, std::move(callback), io_service, external_callback); + } else if (type == interface::RtcTransportRecoveryStrategies::LOW_RATE || + type == interface::RtcTransportRecoveryStrategies:: + LOW_RATE_AND_BESTPATH || + type == interface::RtcTransportRecoveryStrategies:: + LOW_RATE_AND_REPLICATION || + type == interface::RtcTransportRecoveryStrategies:: + LOW_RATE_AND_ALL_FWD_STRATEGIES) { + rs_ = std::make_shared<RecoveryStrategyLowRate>( + indexer, std::move(callback), io_service, external_callback); + } else { + // default + rs_type_ = interface::RtcTransportRecoveryStrategies::RTX_ONLY; + rs_ = std::make_shared<RecoveryStrategyRtxOnly>( + indexer, std::move(callback), io_service, external_callback); + } } RTCLossDetectionAndRecovery::~RTCLossDetectionAndRecovery() {} -void RTCLossDetectionAndRecovery::turnOnRTX() { - rtx_on_ = true; - scheduleSentinelTimer(state_->getRTT() * CATCH_UP_RTT_INCREMENT); -} - -void RTCLossDetectionAndRecovery::turnOffRTX() { - rtx_on_ = false; - clear(); -} - -uint32_t RTCLossDetectionAndRecovery::computeFecPacketsToAsk(bool in_sync) { - uint32_t current_fec = indexer_->getNFec(); - double current_loss_rate = state_->getLossRate(); - double last_loss_rate = state_->getLastRoundLossRate(); - - // when in sync ask for fec only if there are losses for 2 rounds - if (in_sync && current_fec == 0 && - (current_loss_rate == 0 || last_loss_rate == 0)) - return 0; - - double loss_rate = state_->getMaxLossRate() * 1.5; - - if (!in_sync && loss_rate == 0) loss_rate = 0.05; - if (loss_rate > 0.5) loss_rate = 0.5; - - double exp_losses = (double)k_ * loss_rate; - uint32_t fec_to_ask = ceil(exp_losses / (1 - loss_rate)); - - if (fec_to_ask > (n_ - k_)) fec_to_ask = n_ - k_; - - return fec_to_ask; +void RTCLossDetectionAndRecovery::changeRecoveryStrategy( + interface::RtcTransportRecoveryStrategies type) { + if (type == rs_type_) return; + + rs_type_ = type; + if (type == interface::RtcTransportRecoveryStrategies::RECOVERY_OFF) { + rs_ = + std::make_shared<RecoveryStrategyRecoveryOff>(std::move(*(rs_.get()))); + } else if (type == interface::RtcTransportRecoveryStrategies::DELAY_BASED) { + rs_ = std::make_shared<RecoveryStrategyDelayBased>(std::move(*(rs_.get()))); + } else if (type == interface::RtcTransportRecoveryStrategies::FEC_ONLY) { + rs_ = std::make_shared<RecoveryStrategyFecOnly>(std::move(*(rs_.get()))); + } else if (type == interface::RtcTransportRecoveryStrategies::LOW_RATE || + type == interface::RtcTransportRecoveryStrategies:: + LOW_RATE_AND_BESTPATH || + type == interface::RtcTransportRecoveryStrategies:: + LOW_RATE_AND_REPLICATION || + type == interface::RtcTransportRecoveryStrategies:: + LOW_RATE_AND_ALL_FWD_STRATEGIES) { + rs_ = std::make_shared<RecoveryStrategyLowRate>(std::move(*(rs_.get()))); + } else { + // default + rs_ = std::make_shared<RecoveryStrategyRtxOnly>(std::move(*(rs_.get()))); + } } void RTCLossDetectionAndRecovery::onNewRound(bool in_sync) { - uint64_t rtt = state_->getRTT(); - if (!fec_on_ && rtt >= 100) { - // turn on fec, here we may have no info so ask for all packets - fec_on_ = true; - turnOffRTX(); - indexer_->setNFec(computeFecPacketsToAsk(in_sync)); - return; - } - - if (fec_on_ && rtt > 80) { - // keep using fec, maybe update it - indexer_->setNFec(computeFecPacketsToAsk(in_sync)); - return; - } - - if ((fec_on_ && rtt <= 80) || (!rtx_on_ && rtt <= 100)) { - // turn on rtx - fec_on_ = false; - indexer_->setNFec(0); - turnOnRTX(); - return; - } + rs_->incRoundId(); + rs_->onNewRound(in_sync); } -void RTCLossDetectionAndRecovery::onTimeout(uint32_t seq) { - // always add timeouts to the RTX list to avoid to send the same packet as if - // it was not a rtx - addToRetransmissions(seq, seq + 1); - last_event_ = getNow(); +void RTCLossDetectionAndRecovery::onTimeout(uint32_t seq, bool lost) { + if (!lost) { + detectLoss(seq, seq + 1); + } else { + rs_->onLostTimeout(seq); + } } void RTCLossDetectionAndRecovery::onPacketRecoveredFec(uint32_t seq) { - // if an RTX is scheduled for a packet recovered using FEC delete it - deleteRtx(seq); - recover_with_fec_.erase(seq); + rs_->receivedPacket(seq); } void RTCLossDetectionAndRecovery::onDataPacketReceived( const core::ContentObject &content_object) { - last_event_ = getNow(); - uint32_t seq = content_object.getName().getSuffix(); - if (deleteRtx(seq)) { - state_->onPacketRecoveredRtx(seq); - } else { - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "received data. add from " - << state_->getHighestSeqReceivedInOrder() + 1 << " to " << seq; - addToRetransmissions(state_->getHighestSeqReceivedInOrder() + 1, seq); - } + bool is_rtx = rs_->isRtx(seq); + rs_->receivedPacket(seq); + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "received data. add from " + << rs_->getState()->getHighestSeqReceivedInOrder() + 1 << " to " << seq; + if (!is_rtx) + detectLoss(rs_->getState()->getHighestSeqReceivedInOrder() + 1, seq); } void RTCLossDetectionAndRecovery::onNackPacketReceived( const core::ContentObject &nack) { - last_event_ = getNow(); - - uint32_t seq = nack.getName().getSuffix(); - struct nack_packet_t *nack_pkt = (struct nack_packet_t *)nack.getPayload()->data(); - uint32_t production_seq = nack_pkt->getProductionSegement(); + uint32_t production_seq = nack_pkt->getProductionSegment(); + uint32_t seq = nack.getName().getSuffix(); - if (production_seq > seq) { - // this is a past nack, all data before productionSeq are lost. if - // productionSeq > state_->getHighestSeqReceivedInOrder() is impossible to - // recover any packet. If this is not the case we can try to recover the - // packets between state_->getHighestSeqReceivedInOrder() and productionSeq. - // e.g.: the client receives packets 8 10 11 9 where 9 is a nack with - // productionSeq = 14. 9 is lost but we can try to recover packets 12 13 and - // 14 that are not arrived yet - deleteRtx(seq); - DLOG_IF(INFO, VLOG_IS_ON(3)) << "received past nack. add from " - << state_->getHighestSeqReceivedInOrder() + 1 - << " to " << production_seq; - addToRetransmissions(state_->getHighestSeqReceivedInOrder() + 1, - production_seq); - } else { - // future nack. here there should be a gap between the last data received - // and this packet and is it possible to recover the packets between the - // last received data and the production seq. we should not use the seq - // number of the nack since we know that is too early to ask for this seq - // number - // e.g.: // e.g.: the client receives packets 10 11 12 20 where 20 is a nack - // with productionSeq = 18. this says that all the packets between 12 and 18 - // may got lost and we should ask them - deleteRtx(seq); - DLOG_IF(INFO, VLOG_IS_ON(3)) << "received futrue nack. add from " - << state_->getHighestSeqReceivedInOrder() + 1 - << " to " << production_seq; - addToRetransmissions(state_->getHighestSeqReceivedInOrder() + 1, - production_seq); - } + // received a nack. we can try to recover all data packets between the last + // received data and the production seq in the nack. this is similar to the + // recption of a probe + // e.g.: the client receives packets 10 11 12 20 where 20 is a nack + // with productionSeq = 18. this says that all the packets between 12 and 18 + // may got lost and we should ask them + + rs_->receivedPacket(seq); + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "received nack. add from " + << rs_->getState()->getHighestSeqReceivedInOrder() + 1 << " to " + << production_seq; + detectLoss(rs_->getState()->getHighestSeqReceivedInOrder() + 1, + production_seq); } void RTCLossDetectionAndRecovery::onProbePacketReceived( @@ -174,336 +149,38 @@ void RTCLossDetectionAndRecovery::onProbePacketReceived( // we don't log the reception of a probe packet for the sentinel timer because // probes are not taken into account into the sync window. we use them as // future nacks to detect possible packets lost - struct nack_packet_t *probe_pkt = - (struct nack_packet_t *)probe.getPayload()->data(); - uint32_t production_seq = probe_pkt->getProductionSegement(); + + uint32_t production_seq = RTCState::getProbeParams(probe).prod_seg; + DLOG_IF(INFO, VLOG_IS_ON(3)) << "received probe. add from " - << state_->getHighestSeqReceivedInOrder() + 1 << " to " << production_seq; + << rs_->getState()->getHighestSeqReceivedInOrder() + 1 << " to " + << production_seq; - addToRetransmissions(state_->getHighestSeqReceivedInOrder() + 1, - production_seq); + detectLoss(rs_->getState()->getHighestSeqReceivedInOrder() + 1, + production_seq); } -void RTCLossDetectionAndRecovery::clear() { - rtx_state_.clear(); - rtx_timers_.clear(); - sentinel_timer_->cancel(); - if (next_rtx_timer_ != MAX_TIMER_RTX) { - next_rtx_timer_ = MAX_TIMER_RTX; - timer_->cancel(); - } -} +void RTCLossDetectionAndRecovery::detectLoss(uint32_t start, uint32_t stop) { + if (start >= stop) return; -void RTCLossDetectionAndRecovery::addToRetransmissions(uint32_t start, - uint32_t stop) { // skip nacked packets - if (start <= state_->getLastSeqNacked()) { - start = state_->getLastSeqNacked() + 1; + if (start <= rs_->getState()->getLastSeqNacked()) { + start = rs_->getState()->getLastSeqNacked() + 1; } // skip received or lost packets - if (start <= state_->getHighestSeqReceivedInOrder()) { - start = state_->getHighestSeqReceivedInOrder() + 1; + if (start <= rs_->getState()->getHighestSeqReceivedInOrder()) { + start = rs_->getState()->getHighestSeqReceivedInOrder() + 1; } for (uint32_t seq = start; seq < stop; seq++) { - if (state_->isReceivedOrLost(seq) == PacketState::UNKNOWN) { - if (rtx_on_) { - if (!indexer_->isFec(seq)) { - // handle it with rtx - if (!isRtx(seq)) { - state_->onLossDetected(seq); - rtxState state; - state.first_send_ = state_->getInterestSentTime(seq); - if (state.first_send_ == 0) // this interest was never sent before - state.first_send_ = getNow(); - state.next_send_ = computeNextSend(seq, true); - state.rtx_count_ = 0; - DLOG_IF(INFO, VLOG_IS_ON(4)) - << "Add " << seq << " to retransmissions. next rtx is %lu " - << state.next_send_ - getNow(); - rtx_state_.insert(std::pair<uint32_t, rtxState>(seq, state)); - rtx_timers_.insert( - std::pair<uint64_t, uint32_t>(state.next_send_, seq)); - } - } else { - // is fec, do not send it - auto it = recover_with_fec_.find(seq); - if (it == recover_with_fec_.end()) { - state_->onLossDetected(seq); - recover_with_fec_.insert(seq); - } - } - } else { - // keep track of losses but recover with FEC - auto it = recover_with_fec_.find(seq); - if (it == recover_with_fec_.end()) { - state_->onLossDetected(seq); - recover_with_fec_.insert(seq); - } - } - } - } - scheduleNextRtx(); -} - -uint64_t RTCLossDetectionAndRecovery::computeNextSend(uint32_t seq, - bool new_rtx) { - uint64_t now = getNow(); - if (new_rtx) { - // for the new rtx we wait one estimated IAT after the loss detection. this - // is bacause, assuming that packets arrive with a constant IAT, we should - // get a new packet every IAT - double prod_rate = state_->getProducerRate(); - uint32_t estimated_iat = SENTINEL_TIMER_INTERVAL; - uint32_t jitter = 0; - - if (prod_rate != 0) { - double packet_size = state_->getAveragePacketSize(); - estimated_iat = ceil(1000.0 / (prod_rate / packet_size)); - jitter = ceil(state_->getJitter()); - } - - uint32_t wait = estimated_iat + jitter; - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "first rtx for " << seq << " in " << wait - << " ms, rtt = " << state_->getRTT() << " ait = " << estimated_iat - << " jttr = " << jitter; - - return now + wait; - } else { - // wait one RTT - // however if the IAT is larger than the RTT, wait one IAT - uint32_t wait = SENTINEL_TIMER_INTERVAL; - - double prod_rate = state_->getProducerRate(); - if (prod_rate == 0) { - return now + SENTINEL_TIMER_INTERVAL; - } - - double packet_size = state_->getAveragePacketSize(); - uint32_t estimated_iat = ceil(1000.0 / (prod_rate / packet_size)); - - uint64_t rtt = state_->getRTT(); - if (rtt == 0) rtt = SENTINEL_TIMER_INTERVAL; - wait = rtt; - - if (estimated_iat > rtt) wait = estimated_iat; - - uint32_t jitter = ceil(state_->getJitter()); - wait += jitter; - - // it may happen that the channel is congested and we have some additional - // queuing delay to take into account - uint32_t queue = ceil(state_->getQueuing()); - wait += queue; - - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "next rtx for " << seq << " in " << wait - << " ms, rtt = " << state_->getRTT() << " ait = " << estimated_iat - << " jttr = " << jitter << " queue = " << queue; - - return now + wait; - } -} - -void RTCLossDetectionAndRecovery::retransmit() { - if (rtx_timers_.size() == 0) return; - - uint64_t now = getNow(); - - auto it = rtx_timers_.begin(); - std::unordered_set<uint32_t> lost_pkt; - uint32_t sent_counter = 0; - while (it != rtx_timers_.end() && it->first <= now && - sent_counter < MAX_RTX_IN_BATCH) { - uint32_t seq = it->second; - auto rtx_it = - rtx_state_.find(seq); // this should always return a valid iter - if (rtx_it->second.rtx_count_ >= RTC_MAX_RTX || - (now - rtx_it->second.first_send_) >= RTC_MAX_AGE || - seq < state_->getLastSeqNacked()) { - // max rtx reached or packet too old or packet nacked, this packet is lost - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "packet " << seq << " lost because 1) max rtx: " - << (rtx_it->second.rtx_count_ >= RTC_MAX_RTX) << " 2) max age: " - << ((now - rtx_it->second.first_send_) >= RTC_MAX_AGE) - << " 3) nacked: " << (seq < state_->getLastSeqNacked()); - lost_pkt.insert(seq); - it++; - } else { - // resend the packet - state_->onRetransmission(seq); - double prod_rate = state_->getProducerRate(); - if (prod_rate != 0) rtx_it->second.rtx_count_++; - rtx_it->second.next_send_ = computeNextSend(seq, false); - it = rtx_timers_.erase(it); - rtx_timers_.insert( - std::pair<uint64_t, uint32_t>(rtx_it->second.next_send_, seq)); - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "send rtx for sequence " << seq << ", next send in " - << (rtx_it->second.next_send_ - now); - send_rtx_callback_(seq); - sent_counter++; - } - } - - // remove packets if needed - for (auto lost_it = lost_pkt.begin(); lost_it != lost_pkt.end(); lost_it++) { - uint32_t seq = *lost_it; - state_->onPacketLost(seq); - deleteRtx(seq); - } -} - -void RTCLossDetectionAndRecovery::scheduleNextRtx() { - if (rtx_timers_.size() == 0) { - // all the rtx were removed, reset timer - next_rtx_timer_ = MAX_TIMER_RTX; - return; - } - - // check if timer is alreay set - if (next_rtx_timer_ != MAX_TIMER_RTX) { - // a new check for rtx is already scheduled - if (next_rtx_timer_ > rtx_timers_.begin()->first) { - // we need to re-schedule it - timer_->cancel(); - } else { - // wait for the next timer - return; - } - } - - // set a new timer - next_rtx_timer_ = rtx_timers_.begin()->first; - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); - uint64_t wait = 1; - if (next_rtx_timer_ != MAX_TIMER_RTX && next_rtx_timer_ > now) - wait = next_rtx_timer_ - now; - - std::weak_ptr<RTCLossDetectionAndRecovery> self(shared_from_this()); - timer_->expires_from_now(std::chrono::milliseconds(wait)); - timer_->async_wait([self](std::error_code ec) { - if (ec) return; - if (auto s = self.lock()) { - s->retransmit(); - s->next_rtx_timer_ = MAX_TIMER_RTX; - s->scheduleNextRtx(); - } - }); -} - -bool RTCLossDetectionAndRecovery::deleteRtx(uint32_t seq) { - auto it_rtx = rtx_state_.find(seq); - if (it_rtx == rtx_state_.end()) return false; // rtx not found - - uint64_t ts = it_rtx->second.next_send_; - auto it_timers = rtx_timers_.find(ts); - while (it_timers != rtx_timers_.end() && it_timers->first == ts) { - if (it_timers->second == seq) { - rtx_timers_.erase(it_timers); - break; - } - it_timers++; - } - - bool lost = it_rtx->second.rtx_count_ > 0; - rtx_state_.erase(it_rtx); - - return lost; -} - -void RTCLossDetectionAndRecovery::scheduleSentinelTimer( - uint64_t expires_from_now) { - std::weak_ptr<RTCLossDetectionAndRecovery> self(shared_from_this()); - sentinel_timer_->expires_from_now( - std::chrono::milliseconds(expires_from_now)); - sentinel_timer_->async_wait([self](std::error_code ec) { - if (ec) return; - if (auto s = self.lock()) { - s->sentinelTimer(); - } - }); -} - -void RTCLossDetectionAndRecovery::sentinelTimer() { - uint64_t now = getNow(); - - bool expired = false; - bool sent = false; - if ((now - last_event_) >= sentinel_timer_interval_) { - // at least a sentinel_timer_interval_ elapsed since last event - expired = true; - if (TRANSPORT_EXPECT_FALSE(!state_->isProducerActive())) { - // this happens at the beginning (or if the producer stops for some - // reason) we need to keep sending interest 0 until we get an answer - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "sentinel timer: the producer is not active, send packet 0"; - state_->onRetransmission(0); - send_rtx_callback_(0); - } else { - DLOG_IF(INFO, VLOG_IS_ON(3)) << "sentinel timer: the producer is active, " - "send the 10 oldest packets"; - sent = true; - uint32_t rtx = 0; - auto it = state_->getPendingInterestsMapBegin(); - auto end = state_->getPendingInterestsMapEnd(); - while (it != end && rtx < MAX_RTX_WITH_SENTINEL) { - uint32_t seq = it->first; - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "sentinel timer, add " << seq << " to the rtx list"; - addToRetransmissions(seq, seq + 1); - rtx++; - it++; - } - } - } else { - // sentinel timer did not expire because we registered at least one event - } - - uint32_t next_timer; - double prod_rate = state_->getProducerRate(); - if (TRANSPORT_EXPECT_FALSE(!state_->isProducerActive()) || prod_rate == 0) { - DLOG_IF(INFO, VLOG_IS_ON(3)) << "next timer in " << SENTINEL_TIMER_INTERVAL; - next_timer = SENTINEL_TIMER_INTERVAL; - } else { - double prod_rate = state_->getProducerRate(); - double packet_size = state_->getAveragePacketSize(); - uint32_t estimated_iat = ceil(1000.0 / (prod_rate / packet_size)); - uint32_t jitter = ceil(state_->getJitter()); - - // try to reduce the number of timers if the estimated IAT is too small - next_timer = std::max((estimated_iat + jitter) * 20, (uint32_t)1); - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "next sentinel in " << next_timer - << " ms, rate: " << ((prod_rate * 8.0) / 1000000.0) - << ", iat: " << estimated_iat << ", jitter: " << jitter; - - if (!expired) { - // discount the amout of time that is already passed - uint32_t discount = now - last_event_; - if (next_timer > discount) { - next_timer = next_timer - discount; - } else { - // in this case we trigger the timer in 1 ms - next_timer = 1; + if (rs_->getState()->getPacketState(seq) == PacketState::UNKNOWN) { + if (rs_->lossDetected(seq)) { + rs_->getState()->onLossDetected(seq); } - DLOG_IF(INFO, VLOG_IS_ON(3)) << "timer after discout: " << next_timer; - } else if (sent) { - // wait at least one producer stats interval + owd to check if the - // production rate is reducing. - uint32_t min_wait = PRODUCER_STATS_INTERVAL + ceil(state_->getQueuing()); - next_timer = std::max(next_timer, min_wait); - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "wait for updates from prod, next timer: " << next_timer; } } - - scheduleSentinelTimer(next_timer); } } // namespace rtc diff --git a/libtransport/src/protocols/rtc/rtc_ldr.h b/libtransport/src/protocols/rtc/rtc_ldr.h index 1b9f9afd6..e7f8ce5db 100644 --- a/libtransport/src/protocols/rtc/rtc_ldr.h +++ b/libtransport/src/protocols/rtc/rtc_ldr.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -15,16 +15,14 @@ #pragma once #include <hicn/transport/config.h> +#include <hicn/transport/interfaces/socket_options_keys.h> +// RtcTransportRecoveryStrategies #include <hicn/transport/core/asio_wrapper.h> #include <hicn/transport/core/content_object.h> #include <hicn/transport/core/name.h> -#include <protocols/indexer.h> -#include <protocols/rtc/rtc_consts.h> -#include <protocols/rtc/rtc_state.h> +#include <protocols/rtc/rtc_recovery_strategy.h> #include <functional> -#include <map> -#include <unordered_map> namespace transport { @@ -34,91 +32,45 @@ namespace rtc { class RTCLossDetectionAndRecovery : public std::enable_shared_from_this<RTCLossDetectionAndRecovery> { - struct rtx_state_ { - uint64_t first_send_; - uint64_t next_send_; - uint32_t rtx_count_; - }; - - using rtxState = struct rtx_state_; - using SendRtxCallback = std::function<void(uint32_t)>; - public: - RTCLossDetectionAndRecovery(Indexer *indexer, SendRtxCallback &&callback, - asio::io_service &io_service); + RTCLossDetectionAndRecovery(Indexer *indexer, asio::io_service &io_service, + interface::RtcTransportRecoveryStrategies type, + RecoveryStrategy::SendRtxCallback &&callback, + interface::StrategyCallback *external_callback); ~RTCLossDetectionAndRecovery(); - void setState(std::shared_ptr<RTCState> state) { state_ = state; } - void setFecParams(uint32_t n, uint32_t k) { - n_ = n; - k_ = k; + void setState(RTCState *state) { rs_->setState(state); } + void setRateControl(RTCRateControl *rateControl) { + rs_->setRateControl(rateControl); } - void turnOnRTX(); - void turnOffRTX(); - bool isRtxOn() { return rtx_on_; } + + void setFecParams(uint32_t n, uint32_t k) { rs_->setFecParams(n, k); } + + void turnOnRecovery() { rs_->tunrOnRecovery(); } + bool isRtxOn() { return rs_->isRtxOn(); } + + void changeRecoveryStrategy(interface::RtcTransportRecoveryStrategies type); void onNewRound(bool in_sync); - void onTimeout(uint32_t seq); + void onTimeout(uint32_t seq, bool lost); void onPacketRecoveredFec(uint32_t seq); void onDataPacketReceived(const core::ContentObject &content_object); void onNackPacketReceived(const core::ContentObject &nack); void onProbePacketReceived(const core::ContentObject &probe); - void clear(); + void clear() { rs_->clear(); } - bool isRtx(uint32_t seq) { - if (rtx_state_.find(seq) != rtx_state_.end()) return true; - return false; + bool isRtx(uint32_t seq) { return rs_->isRtx(seq); } + bool isPossibleLossWithNoRtx(uint32_t seq) { + return rs_->isPossibleLossWithNoRtx(seq); } private: - void addToRetransmissions(uint32_t start, uint32_t stop); - uint64_t computeNextSend(uint32_t seq, bool new_rtx); - void retransmit(); - void scheduleNextRtx(); - bool deleteRtx(uint32_t seq); - void scheduleSentinelTimer(uint64_t expires_from_now); - void sentinelTimer(); - uint32_t computeFecPacketsToAsk(bool in_sync); - - uint64_t getNow() { - using namespace std::chrono; - uint64_t now = - duration_cast<milliseconds>(steady_clock::now().time_since_epoch()) - .count(); - return now; - } - - // this map keeps track of the retransmitted interest, ordered from the oldest - // to the newest one. the state contains the timer of the first send of the - // interest (from pendingIntetests_), the timer of the next send (key of the - // multimap) and the number of rtx - std::map<uint32_t, rtxState> rtx_state_; - // this map stored the rtx by timer. The key is the time at which the rtx - // should be sent, and the val is the interest seq number - std::multimap<uint64_t, uint32_t> rtx_timers_; - - // lost packets that will be recovered with fec - std::unordered_set<uint32_t> recover_with_fec_; - - bool rtx_on_; - bool fec_on_; - uint64_t next_rtx_timer_; - uint64_t last_event_; - uint64_t sentinel_timer_interval_; - - // fec params - uint32_t n_; - uint32_t k_; - - std::unique_ptr<asio::steady_timer> timer_; - std::unique_ptr<asio::steady_timer> sentinel_timer_; - std::shared_ptr<RTCState> state_; - - Indexer *indexer_; + void detectLoss(uint32_t start, uint32_t stop); - SendRtxCallback send_rtx_callback_; + interface::RtcTransportRecoveryStrategies rs_type_; + std::shared_ptr<RecoveryStrategy> rs_; }; } // end namespace rtc diff --git a/libtransport/src/protocols/rtc/rtc_packet.h b/libtransport/src/protocols/rtc/rtc_packet.h index 7dc2f82c3..391aedfc6 100644 --- a/libtransport/src/protocols/rtc/rtc_packet.h +++ b/libtransport/src/protocols/rtc/rtc_packet.h @@ -24,6 +24,27 @@ * +-----------------------------------------+ */ +/* aggregated packets + * +---------------------------------+ + * |c| #pkts | len1 | len2 | .... | + * +---------------------------------- + * + * +---------------------------------+ + * |c| #pkts | resv | len 1 | + * +---------------------------------- + * + * aggregated packets header. + * header position. just after the data packet header + * + * c: 1 bit: 0 8bit encoding, 1 16bit encoding + * #pkts: 7 bits: number of application packets contained + * 8bits encoding: + * lenX: 8 bits: len in bites of packet X + * 16bits econding: + * resv: 8 bits: reserved field (unused) + * lenX: 16bits: len in bytes of packet X + */ + #pragma once #ifndef _WIN32 #include <arpa/inet.h> @@ -31,6 +52,8 @@ #include <hicn/transport/portability/win_portability.h> #endif +#include <cstring> + namespace transport { namespace protocol { @@ -82,8 +105,144 @@ struct nack_packet_t { inline uint32_t getProductionRate() const { return ntohl(prod_rate); } inline void setProductionRate(uint32_t rate) { prod_rate = htonl(rate); } - inline uint32_t getProductionSegement() const { return ntohl(prod_seg); } - inline void setProductionSegement(uint32_t seg) { prod_seg = htonl(seg); } + inline uint32_t getProductionSegment() const { return ntohl(prod_seg); } + inline void setProductionSegment(uint32_t seg) { prod_seg = htonl(seg); } +}; + +class AggrPktHeader { + public: + // XXX buf always point to the payload after the data header + AggrPktHeader(uint8_t *buf, uint16_t max_packet_len, uint16_t pkt_number) + : buf_(buf), pkt_num_(pkt_number) { + *buf_ = 0; // reset the first byte to correctly add the header + // encoding and the packet number + if (max_packet_len > 0xff) { + setAggrPktEncoding16bit(); + } else { + setAggrPktEncoding8bit(); + } + setAggrPktNUmber(pkt_number); + header_len_ = computeHeaderLen(); + memset(buf_ + 1, 0, header_len_ - 1); + } + + // XXX buf always point to the payload after the data header + AggrPktHeader(uint8_t *buf) : buf_(buf) { + encoding_ = getAggrPktEncoding(); + pkt_num_ = getAggrPktNumber(); + header_len_ = computeHeaderLen(); + } + + ~AggrPktHeader(){}; + + int addPacketToHeader(uint8_t index, uint16_t len) { + if (index > pkt_num_) return -1; + + setAggrPktLen(index, len); + return 0; + } + + int getPointerToPacket(uint8_t index, uint8_t **pkt_ptr, uint16_t *pkt_len) { + if (index > pkt_num_) return -1; + + uint16_t len = 0; + for (int i = 0; i < index; i++) + len += getAggrPktLen(i); // sum the pkts len from 0 to index - 1 + + uint16_t offset = len + header_len_; + *pkt_ptr = buf_ + offset; + *pkt_len = getAggrPktLen(index); + return 0; + } + + int getPacketOffsets(uint8_t index, uint16_t *pkt_offset, uint16_t *pkt_len) { + if (index > pkt_num_) return -1; + + uint16_t len = 0; + for (int i = 0; i < index; i++) + len += getAggrPktLen(i); // sum the pkts len from 0 to index - 1 + + uint16_t offset = len + header_len_; + *pkt_offset = offset; + *pkt_len = getAggrPktLen(index); + + return 0; + } + + uint8_t *getPayloadAppendPtr() { return buf_ + header_len_; } + + uint16_t getHeaderLen() { return header_len_; } + + uint8_t getNumberOfPackets() { return pkt_num_; } + + private: + inline uint16_t computeHeaderLen() const { + uint16_t len = 4; // min len in bytes + if (!encoding_) { + while (pkt_num_ >= len) { + len += 4; + } + } else { + while (pkt_num_ * 2 >= len) { + len += 4; + } + } + return len; + } + + inline uint8_t getAggrPktEncoding() const { + // get the first bit of the first byte + return (*buf_ >> 7); + } + + inline void setAggrPktEncoding8bit() { + // reset the first bit of the first byte + encoding_ = 0; + *buf_ &= 0x7f; + } + + inline void setAggrPktEncoding16bit() { + // set the first bit of the first byte + encoding_ = 1; + *buf_ ^= 0x80; + } + + inline uint8_t getAggrPktNumber() const { + // return the first byte with the first bit = 0 + return (*buf_ & 0x7f); + } + + inline void setAggrPktNUmber(uint8_t val) { + // set the val without modifying the first bit + *buf_ &= 0x80; // reset everithing but the first bit + val &= 0x7f; // reset the first bit + *buf_ |= val; // or the vals, done! + } + + inline uint16_t getAggrPktLen(uint8_t pkt_index) const { + pkt_index++; + if (!encoding_) { // 8 bits + return (uint16_t) * (buf_ + pkt_index); + } else { // 16 bits + uint16_t *buf_16 = (uint16_t *)buf_; + return ntohs(*(buf_16 + pkt_index)); + } + } + + inline void setAggrPktLen(uint8_t pkt_index, uint16_t len) { + pkt_index++; + if (!encoding_) { // 8 bits + *(buf_ + pkt_index) = (uint8_t)len; + } else { // 16 bits + uint16_t *buf_16 = (uint16_t *)buf_; + *(buf_16 + pkt_index) = htons(len); + } + } + + uint8_t *buf_; + uint8_t encoding_; + uint8_t pkt_num_; + uint16_t header_len_; }; } // end namespace rtc diff --git a/libtransport/src/protocols/rtc/rtc_rc.h b/libtransport/src/protocols/rtc/rtc_rc.h index 34d090092..62636ce40 100644 --- a/libtransport/src/protocols/rtc/rtc_rc.h +++ b/libtransport/src/protocols/rtc/rtc_rc.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -34,11 +34,15 @@ class RTCRateControl : public std::enable_shared_from_this<RTCRateControl> { void turnOnRateControl() { rc_on_ = true; } void setState(std::shared_ptr<RTCState> state) { protocol_state_ = state; }; - uint32_t getCongesionWindow() { return congestion_win_; }; + uint32_t getCongestionWindow() { return congestion_win_; }; + bool inCongestionState() { + if (congestion_state_ == CongestionState::Congested) return true; + return false; + } virtual void onNewRound(double round_len) = 0; - virtual void onDataPacketReceived( - const core::ContentObject &content_object) = 0; + virtual void onDataPacketReceived(const core::ContentObject &content_object, + bool compute_stats) = 0; protected: enum class CongestionState { Normal = 0, Underuse = 1, Congested = 2, Last }; diff --git a/libtransport/src/protocols/rtc/rtc_rc_congestion_detection.cc b/libtransport/src/protocols/rtc/rtc_rc_congestion_detection.cc new file mode 100644 index 000000000..6cd3094b5 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rc_congestion_detection.cc @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <protocols/rtc/rtc_consts.h> +#include <protocols/rtc/rtc_rc_congestion_detection.h> + +#include <algorithm> + +namespace transport { + +namespace protocol { + +namespace rtc { + +RTCRateControlCongestionDetection::RTCRateControlCongestionDetection() + : rounds_without_congestion_(4), last_queue_(0) {} // must be > 3 + +RTCRateControlCongestionDetection::~RTCRateControlCongestionDetection() {} + +void RTCRateControlCongestionDetection::onNewRound(double round_len) { + if (!rc_on_) return; + + double rtt = (double)protocol_state_->getMinRTT() / MILLI_IN_A_SEC; + double queue = protocol_state_->getQueuing(); + + if (rtt == 0.0) return; // no info from the producer + + if (last_queue_ == queue) { + // if last_queue == queue the consumer didn't receive any + // packet from the producer. we do not change the current congestion state. + // we just increase the counter of rounds whithout congestion if needed + // (in case of congestion the counter is already set to 0) + if (congestion_state_ == CongestionState::Normal) + rounds_without_congestion_++; + } else { + if (queue > MAX_QUEUING_DELAY) { + // here we detect congestion. + congestion_state_ = CongestionState::Congested; + rounds_without_congestion_ = 0; + } else { + // wait 3 rounds before switch back to no congestion + if (rounds_without_congestion_ > 3) { + // nothing bad is happening + congestion_state_ = CongestionState::Normal; + } + rounds_without_congestion_++; + } + last_queue_ = queue; + } +} + +void RTCRateControlCongestionDetection::onDataPacketReceived( + const core::ContentObject &content_object, bool compute_stats) { + // nothing to do + return; +} + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rc_congestion_detection.h b/libtransport/src/protocols/rtc/rtc_rc_congestion_detection.h new file mode 100644 index 000000000..9afa6c39a --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rc_congestion_detection.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include <hicn/transport/utils/shared_ptr_utils.h> +#include <protocols/rtc/rtc_rc.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +class RTCRateControlCongestionDetection : public RTCRateControl { + public: + RTCRateControlCongestionDetection(); + + ~RTCRateControlCongestionDetection(); + + void onNewRound(double round_len); + void onDataPacketReceived(const core::ContentObject &content_object, + bool compute_stats); + + auto shared_from_this() { return utils::shared_from(this); } + + private: + uint32_t rounds_without_congestion_; + double last_queue_; +}; + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rc_iat.cc b/libtransport/src/protocols/rtc/rtc_rc_iat.cc new file mode 100644 index 000000000..f06f377f3 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rc_iat.cc @@ -0,0 +1,287 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <protocols/rtc/rtc_consts.h> +#include <protocols/rtc/rtc_rc_iat.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +RTCRateControlIAT::RTCRateControlIAT() + : rounds_since_last_drop_(0), + rounds_without_congestion_(0), + rounds_with_congestion_(0), + last_queue_(0), + last_rcv_time_(0), + last_prod_time_(0), + last_seq_number_(0), + target_rate_avg_(0), + round_index_(0), + congestion_cause_(CongestionCause::UNKNOWN) {} + +RTCRateControlIAT::~RTCRateControlIAT() {} + +void RTCRateControlIAT::onNewRound(double round_len) { + if (!rc_on_) return; + + double received_rate = protocol_state_->getReceivedRate() + + protocol_state_->getRecoveredFecRate(); + + double target_rate = + protocol_state_->getProducerRate(); // * PRODUCTION_RATE_FRACTION; + double rtt = (double)protocol_state_->getMinRTT() / MILLI_IN_A_SEC; + // double packet_size = protocol_state_->getAveragePacketSize(); + double queue = protocol_state_->getQueuing(); + + if (rtt == 0.0) return; // no info from the producer + + CongestionState prev_congestion_state = congestion_state_; + + target_rate_avg_ = target_rate_avg_ * (1 - MOVING_AVG_ALPHA) + + target_rate * MOVING_AVG_ALPHA; + + if (prev_congestion_state == CongestionState::Congested) { + if (queue > MAX_QUEUING_DELAY || last_queue_ == queue) { + congestion_state_ = CongestionState::Congested; + + received_rate_.push_back(received_rate); + target_rate_.push_back(target_rate); + + // We assume the cause does not change + // Note that the first assumption about the cause could be wrong + // the cause of congestion could change + if (congestion_cause_ == CongestionCause::UNKNOWN) + if (rounds_with_congestion_ >= 1) + congestion_cause_ = apply_classification_tree( + rounds_with_congestion_ > ROUND_TO_WAIT_FORCE_DECISION); + + rounds_with_congestion_++; + } else { + congestion_state_ = CongestionState::Normal; + + // clear past history + reset_congestion_statistics(); + + // TODO maybe we can use some of these values for the stdev of the + // congestion mode + for (int i = 0; i < ROUND_HISTORY_SIZE; i++) { + iat_on_hold_[i].clear(); + } + } + } else if (queue > MAX_QUEUING_DELAY) { + if (prev_congestion_state == CongestionState::Normal) { + rounds_with_congestion_ = 0; + + if (rounds_without_congestion_ > ROUND_TO_RESET_CAUSE) + congestion_cause_ = CongestionCause::UNKNOWN; + } + congestion_state_ = CongestionState::Congested; + received_rate_.push_back(received_rate); + target_rate_.push_back(target_rate); + } else { + // nothing bad is happening + congestion_state_ = CongestionState::Normal; + reset_congestion_statistics(); + + int past_index = (round_index_ + 1) % ROUND_HISTORY_SIZE; + for (std::vector<double>::iterator it = iat_on_hold_[past_index].begin(); + it != iat_on_hold_[past_index].end(); ++it) { + congestion_free_iat_.push_back(*it); + if (congestion_free_iat_.size() > 50) { + congestion_free_iat_.erase(congestion_free_iat_.begin()); + } + } + iat_on_hold_[past_index].clear(); + round_index_ = (round_index_ + 1) % ROUND_HISTORY_SIZE; + } + + last_queue_ = queue; + + if (congestion_state_ == CongestionState::Congested) { + if (prev_congestion_state == CongestionState::Normal) { + // init the congetion window using the received rate + // disabling for the moment the congestion window setup + // congestion_win_ = (uint32_t)ceil(received_rate * rtt / packet_size); + rounds_since_last_drop_ = ROUNDS_BEFORE_TAKE_ACTION + 1; + } + + if (rounds_since_last_drop_ >= ROUNDS_BEFORE_TAKE_ACTION) { + // disabling for the moment the congestion window setup + // uint32_t win = congestion_win_ * WIN_DECREASE_FACTOR; + // congestion_win_ = std::max(win, WIN_MIN); + rounds_since_last_drop_ = 0; + return; + } + + rounds_since_last_drop_++; + } + + if (congestion_state_ == CongestionState::Normal) { + if (prev_congestion_state == CongestionState::Congested) { + rounds_without_congestion_ = 0; + } + + rounds_without_congestion_++; + if (rounds_without_congestion_ < ROUNDS_BEFORE_TAKE_ACTION) return; + + // disabling for the moment the congestion window setup + // congestion_win_ = congestion_win_ * WIN_INCREASE_FACTOR; + // congestion_win_ = std::min(congestion_win_, INITIAL_WIN_MAX); + } + + if (received_rate_.size() > 1000) + received_rate_.erase(received_rate_.begin()); + if (target_rate_.size() > 1000) target_rate_.erase(target_rate_.begin()); +} + +void RTCRateControlIAT::onDataPacketReceived( + const core::ContentObject &content_object, bool compute_stats) { + core::ParamsRTC params = RTCState::getDataParams(content_object); + + uint64_t now = utils::SteadyTime::nowMs().count(); + + uint32_t segment_number = content_object.getName().getSuffix(); + + if (segment_number == (last_seq_number_ + 1) && compute_stats) { + uint64_t iat = now - last_rcv_time_; + uint64_t ist = params.timestamp - last_prod_time_; + if (now >= last_rcv_time_ && params.timestamp > last_prod_time_) { + if (iat >= ist && ist < MIN_IST_VALUE) { + if (congestion_state_ == CongestionState::Congested) { + iat_.push_back((iat - ist)); + } else { + // no congestion, but we do not always add new values, but only when + // there is no sign of congestion + double queue = protocol_state_->getQueuing(); + if (queue <= CONGESTION_FREE_QUEUEING_DELAY) { + iat_on_hold_[round_index_].push_back((iat - ist)); + } + } + } + } + } + + last_seq_number_ = segment_number; + last_rcv_time_ = now; + last_prod_time_ = params.timestamp; + + if (iat_.size() > 1000) iat_.erase(iat_.begin()); + return; +} + +CongestionCause RTCRateControlIAT::apply_classification_tree(bool force_reply) { + if (iat_.size() <= 2 || received_rate_.size() < 2) + return CongestionCause::UNKNOWN; + + double received_ratio = 0; + double iat_ratio = 0; + double iat_stdev = compute_iat_stdev(iat_); + double iat_congestion_free_stdev = compute_iat_stdev(congestion_free_iat_); + + double iat_avg = 0.0; + + double recv_avg = 0.0; + double recv_max = 0.0; + + double target_rate_avg = 0.0; + + int counter = 0; + std::vector<double>::reverse_iterator target_it = target_rate_.rbegin(); + for (std::vector<double>::reverse_iterator it = received_rate_.rbegin(); + it != received_rate_.rend(); ++it) { + recv_avg += *it; + target_rate_avg += *target_it; + if (counter < ROUND_HISTORY_SIZE) + if (recv_max < *it) { + recv_max = *it; // we consider only the last 2 seconds + } + counter++; + target_it++; + } + recv_avg = recv_avg / received_rate_.size(); + target_rate_avg = target_rate_avg / target_rate_.size(); + + for (std::vector<double>::iterator it = iat_.begin(); it != iat_.end(); + ++it) { + iat_avg += *it; + } + iat_avg = iat_avg / iat_.size(); + + double congestion_free_iat_avg = 0.0; + for (std::vector<double>::iterator it = congestion_free_iat_.begin(); + it != congestion_free_iat_.end(); ++it) { + congestion_free_iat_avg += *it; + } + congestion_free_iat_avg = + congestion_free_iat_avg / congestion_free_iat_.size(); + + received_ratio = recv_avg / target_rate_avg; + + iat_ratio = iat_stdev / iat_congestion_free_stdev; + + CongestionCause congestion_cause = CongestionCause::UNKNOWN; + // applying classification tree model + if (received_ratio <= 0.87) + if (iat_stdev <= 6.48) + if (received_ratio <= 0.83) + congestion_cause = CongestionCause::LINK_CAPACITY; + else if (force_reply) + congestion_cause = CongestionCause::LINK_CAPACITY; + else + congestion_cause = CongestionCause::UNKNOWN; // accuracy is too low + else if (iat_ratio <= 2.46) + if (force_reply) + congestion_cause = CongestionCause::LINK_CAPACITY; + else + congestion_cause = CongestionCause::UNKNOWN; // accuracy is too low + else + congestion_cause = CongestionCause::COMPETING_CROSS_TRAFFIC; + else if (received_ratio <= 0.913 && iat_stdev <= 0.784) + congestion_cause = CongestionCause::LINK_CAPACITY; + else + congestion_cause = CongestionCause::COMPETING_CROSS_TRAFFIC; + + return congestion_cause; +} + +void RTCRateControlIAT::reset_congestion_statistics() { + iat_.clear(); + received_rate_.clear(); + target_rate_.clear(); +} + +double RTCRateControlIAT::compute_iat_stdev(std::vector<double> v) { + if (v.size() == 0) return 0; + + float sum = 0.0, mean, standard_deviation = 0.0; + for (std::vector<double>::iterator it = v.begin(); it != v.end(); it++) { + sum += *it; + } + + mean = sum / v.size(); + for (std::vector<double>::iterator it = v.begin(); it != v.end(); it++) { + standard_deviation += pow(*it - mean, 2); + } + return sqrt(standard_deviation / v.size()); +} + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rc_iat.h b/libtransport/src/protocols/rtc/rtc_rc_iat.h new file mode 100644 index 000000000..715637807 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rc_iat.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include <hicn/transport/utils/shared_ptr_utils.h> +#include <protocols/rtc/rtc_rc.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +const int ROUND_HISTORY_SIZE = 10; // equivalent to two seconds +const int ROUND_TO_WAIT_FORCE_DECISION = 5; + +// once congestion is gone, we need to wait for k rounds before changing the +// congestion cause in the case it appears again +const int ROUND_TO_RESET_CAUSE = 5; + +const int MIN_IST_VALUE = 150; // samples of ist larger than 150ms are + // discarded +const double CONGESTION_FREE_QUEUEING_DELAY = 10; + +enum class CongestionCause : uint8_t { + COMPETING_CROSS_TRAFFIC, + FRIENDLY_CROSS_TRAFFIC, + UNKNOWN_CROSS_TRAFFIC, + LINK_CAPACITY, + UNKNOWN +}; + +class RTCRateControlIAT : public RTCRateControl { + public: + RTCRateControlIAT(); + + ~RTCRateControlIAT(); + + void onNewRound(double round_len); + void onDataPacketReceived(const core::ContentObject &content_object, + bool compute_stats); + + auto shared_from_this() { return utils::shared_from(this); } + + private: + void reset_congestion_statistics(); + + double compute_iat_stdev(std::vector<double> v); + + CongestionCause apply_classification_tree(bool force_reply); + + private: + uint32_t rounds_since_last_drop_; + uint32_t rounds_without_congestion_; + uint32_t rounds_with_congestion_; + double last_queue_; + uint64_t last_rcv_time_; + uint64_t last_prod_time_; + uint32_t last_seq_number_; + double target_rate_avg_; + + // Iat values are not immediately added to the congestion free set of values + std::array<std::vector<double>, ROUND_HISTORY_SIZE> iat_on_hold_; + uint32_t round_index_; + + // with congestion statistics + std::vector<double> iat_; + std::vector<double> received_rate_; + std::vector<double> target_rate_; + + // congestion free statistics + std::vector<double> congestion_free_iat_; + + CongestionCause congestion_cause_; +}; + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rc_queue.cc b/libtransport/src/protocols/rtc/rtc_rc_queue.cc index a1c89e329..ecabc5205 100644 --- a/libtransport/src/protocols/rtc/rtc_rc_queue.cc +++ b/libtransport/src/protocols/rtc/rtc_rc_queue.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -37,7 +37,7 @@ void RTCRateControlQueue::onNewRound(double round_len) { double received_rate = protocol_state_->getReceivedRate(); double target_rate = protocol_state_->getProducerRate() * PRODUCTION_RATE_FRACTION; - double rtt = (double)protocol_state_->getRTT() / MILLI_IN_A_SEC; + double rtt = (double)protocol_state_->getMinRTT() / MILLI_IN_A_SEC; double packet_size = protocol_state_->getAveragePacketSize(); double queue = protocol_state_->getQueuing(); @@ -94,7 +94,7 @@ void RTCRateControlQueue::onNewRound(double round_len) { } void RTCRateControlQueue::onDataPacketReceived( - const core::ContentObject &content_object) { + const core::ContentObject &content_object, bool compute_stats) { // nothing to do return; } diff --git a/libtransport/src/protocols/rtc/rtc_rc_queue.h b/libtransport/src/protocols/rtc/rtc_rc_queue.h index 407354d43..cdf78fd47 100644 --- a/libtransport/src/protocols/rtc/rtc_rc_queue.h +++ b/libtransport/src/protocols/rtc/rtc_rc_queue.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -30,7 +30,8 @@ class RTCRateControlQueue : public RTCRateControl { ~RTCRateControlQueue(); void onNewRound(double round_len); - void onDataPacketReceived(const core::ContentObject &content_object); + void onDataPacketReceived(const core::ContentObject &content_object, + bool compute_stats); auto shared_from_this() { return utils::shared_from(this); } diff --git a/libtransport/src/protocols/rtc/rtc_reassembly.cc b/libtransport/src/protocols/rtc/rtc_reassembly.cc new file mode 100644 index 000000000..992bab50e --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_reassembly.cc @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <hicn/transport/interfaces/socket_consumer.h> +#include <implementation/socket_consumer.h> +#include <protocols/rtc/rtc_reassembly.h> +#include <protocols/transport_protocol.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +RtcReassembly::RtcReassembly(implementation::ConsumerSocket* icn_socket, + TransportProtocol* transport_protocol) + : DatagramReassembly(icn_socket, transport_protocol) { + is_setup_ = false; +} + +void RtcReassembly::reassemble(core::ContentObject& content_object) { + if (!is_setup_) { + is_setup_ = true; + reassembly_consumer_socket_->getSocketOption( + interface::RtcTransportOptions::AGGREGATED_DATA, data_aggregation_); + } + + auto read_buffer = content_object.getPayload(); + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Size of payload: " << read_buffer->length(); + + read_buffer->trimStart(transport_protocol_->transportHeaderLength()); + + if (data_aggregation_) { + rtc::AggrPktHeader hdr((uint8_t*)read_buffer->data()); + + for (uint8_t i = 0; i < hdr.getNumberOfPackets(); i++) { + std::unique_ptr<utils::MemBuf> segment = read_buffer->clone(); + + uint16_t pkt_start = 0; + uint16_t pkt_len = 0; + int res = hdr.getPacketOffsets(i, &pkt_start, &pkt_len); + if (res == -1) { + // this should not happen + break; + } + + segment->trimStart(pkt_start); + segment->trimEnd(segment->length() - pkt_len); + + Reassembly::read_buffer_ = std::move(segment); + Reassembly::notifyApplication(); + } + } else { + Reassembly::read_buffer_ = std::move(read_buffer); + Reassembly::notifyApplication(); + } +} + +void RtcReassembly::reassemble(utils::MemBuf& buffer, uint32_t suffix) { + if (!is_setup_) { + is_setup_ = true; + reassembly_consumer_socket_->getSocketOption( + interface::RtcTransportOptions::AGGREGATED_DATA, data_aggregation_); + } + + if (data_aggregation_) { + rtc::AggrPktHeader hdr((uint8_t*)buffer.data()); + + for (uint8_t i = 0; i < hdr.getNumberOfPackets(); i++) { + std::unique_ptr<utils::MemBuf> segment = buffer.clone(); + + uint16_t pkt_start = 0; + uint16_t pkt_len = 0; + int res = hdr.getPacketOffsets(i, &pkt_start, &pkt_len); + if (res == -1) { + // this should not happen + break; + } + + segment->trimStart(pkt_start); + segment->trimEnd(segment->length() - pkt_len); + + Reassembly::read_buffer_ = std::move(segment); + Reassembly::notifyApplication(); + } + + } else { + Reassembly::read_buffer_ = buffer.cloneOne(); + Reassembly::notifyApplication(); + } +} + +} // namespace rtc + +} // namespace protocol + +} // namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_reassembly.h b/libtransport/src/protocols/rtc/rtc_reassembly.h index 15722a6d5..132004605 100644 --- a/libtransport/src/protocols/rtc/rtc_reassembly.h +++ b/libtransport/src/protocols/rtc/rtc_reassembly.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -28,17 +28,14 @@ namespace rtc { class RtcReassembly : public DatagramReassembly { public: RtcReassembly(implementation::ConsumerSocket *icn_socket, - TransportProtocol *transport_protocol) - : DatagramReassembly(icn_socket, transport_protocol) {} - - void reassemble(core::ContentObject &content_object) override { - auto read_buffer = content_object.getPayload(); - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "Size of payload: " << read_buffer->length(); - read_buffer->trimStart(transport_protocol_->transportHeaderLength()); - Reassembly::read_buffer_ = std::move(read_buffer); - Reassembly::notifyApplication(); - } + TransportProtocol *transport_protocol); + + void reassemble(core::ContentObject &content_object) override; + void reassemble(utils::MemBuf &buffer, uint32_t suffix) override; + + private: + bool is_setup_; + bool data_aggregation_; }; } // namespace rtc diff --git a/libtransport/src/protocols/rtc/rtc_recovery_strategy.cc b/libtransport/src/protocols/rtc/rtc_recovery_strategy.cc new file mode 100644 index 000000000..888105eab --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_recovery_strategy.cc @@ -0,0 +1,418 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <glog/logging.h> +#include <hicn/transport/interfaces/notification.h> +#include <hicn/transport/interfaces/socket_options_keys.h> +#include <protocols/rtc/rtc_consts.h> +#include <protocols/rtc/rtc_recovery_strategy.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +using namespace transport::interface; + +RecoveryStrategy::RecoveryStrategy( + Indexer *indexer, SendRtxCallback &&callback, asio::io_service &io_service, + bool use_rtx, bool use_fec, interface::StrategyCallback *external_callback) + : recovery_on_(false), + next_rtx_timer_(MAX_TIMER_RTX), + send_rtx_callback_(std::move(callback)), + indexer_(indexer), + round_id_(0), + last_fec_used_(0), + callback_(external_callback) { + setRtxFec(use_rtx, use_fec); + timer_ = std::make_unique<asio::steady_timer>(io_service); +} + +RecoveryStrategy::RecoveryStrategy(RecoveryStrategy &&rs) + : rtx_state_(std::move(rs.rtx_state_)), + rtx_timers_(std::move(rs.rtx_timers_)), + recover_with_fec_(std::move(rs.recover_with_fec_)), + timer_(std::move(rs.timer_)), + next_rtx_timer_(std::move(rs.next_rtx_timer_)), + send_rtx_callback_(std::move(rs.send_rtx_callback_)), + n_(std::move(rs.n_)), + k_(std::move(rs.k_)), + indexer_(std::move(rs.indexer_)), + state_(std::move(rs.state_)), + rc_(std::move(rs.rc_)), + round_id_(std::move(rs.round_id_)), + last_fec_used_(std::move(rs.last_fec_used_)), + callback_(rs.callback_) { + setFecParams(n_, k_); +} + +RecoveryStrategy::~RecoveryStrategy() {} + +void RecoveryStrategy::setFecParams(uint32_t n, uint32_t k) { + n_ = n; + k_ = k; + + // XXX for the moment we go in steps of 5% loss rate. + // max loss rate = 95% + for (uint32_t loss_rate = 5; loss_rate < 100; loss_rate += 5) { + double dec_loss_rate = (double)loss_rate / 100.0; + double exp_losses = (double)k_ * dec_loss_rate; + uint32_t fec_to_ask = ceil(exp_losses / (1 - dec_loss_rate)); + + fec_state_ f; + f.fec_to_ask = std::min(fec_to_ask, (n_ - k_)); + f.last_update = round_id_; + f.avg_residual_losses = 0.0; + f.consecutive_use = 0; + fec_per_loss_rate_.push_back(f); + } +} + +bool RecoveryStrategy::lossDetected(uint32_t seq) { + if (isRtx(seq)) { + // this packet is already in the list of rtx + return false; + } + + auto it = recover_with_fec_.find(seq); + if (it != recover_with_fec_.end()) { + // this packet is already in list of packets to recover with fec + // this list contians also fec packets that will not be recovered with rtx + return false; + } + + // new loss detected, recover it according to the strategy + newPacketLoss(seq); + return true; +} + +void RecoveryStrategy::clear() { + rtx_state_.clear(); + rtx_timers_.clear(); + recover_with_fec_.clear(); + + if (next_rtx_timer_ != MAX_TIMER_RTX) { + next_rtx_timer_ = MAX_TIMER_RTX; + timer_->cancel(); + } +} + +// rtx functions +void RecoveryStrategy::addNewRtx(uint32_t seq, bool force) { + if (!indexer_->isFec(seq) || force) { + // this packet needs to be re-transmitted + rtxState state; + state.first_send_ = state_->getInterestSentTime(seq); + if (state.first_send_ == 0) // this interest was never sent before + state.first_send_ = getNow(); + state.next_send_ = computeNextSend(seq, true); + state.rtx_count_ = 0; + DLOG_IF(INFO, VLOG_IS_ON(4)) + << "Add " << seq << " to retransmissions. next rtx is in " + << state.next_send_ - getNow() << " ms"; + rtx_state_.insert(std::pair<uint32_t, rtxState>(seq, state)); + rtx_timers_.insert(std::pair<uint64_t, uint32_t>(state.next_send_, seq)); + + // if a new rtx is introduced, check the rtx timer + scheduleNextRtx(); + } else { + // do not re-send fec packets but keep track of them + recover_with_fec_.insert(seq); + state_->onPossibleLossWithNoRtx(seq); + } +} + +uint64_t RecoveryStrategy::computeNextSend(uint32_t seq, bool new_rtx) { + uint64_t now = getNow(); + if (new_rtx) { + // for the new rtx we wait one estimated IAT after the loss detection. this + // is bacause, assuming that packets arrive with a constant IAT, we should + // get a new packet every IAT + double prod_rate = state_->getProducerRate(); + uint32_t estimated_iat = SENTINEL_TIMER_INTERVAL; + uint32_t jitter = 0; + + if (prod_rate != 0) { + double packet_size = state_->getAveragePacketSize(); + estimated_iat = ceil(1000.0 / (prod_rate / packet_size)); + jitter = ceil(state_->getJitter()); + } + + uint32_t wait = 1; + if (estimated_iat < 18) { + // for low rate app we do not wait to send a RTX + // we consider low rate stream with less than 50pps (iat >= 20ms) + // (e.g. audio in videoconf, mobile games). + // in the check we use 18ms to accomodate for measurements errors + // for flows with higher rate wait 1 ait + jitter + wait = estimated_iat + jitter; + } + + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "first rtx for " << seq << " in " << wait + << " ms, rtt = " << state_->getMinRTT() << " ait = " << estimated_iat + << " jttr = " << jitter; + + return now + wait; + } else { + // wait one RTT + uint32_t wait = SENTINEL_TIMER_INTERVAL; + + double prod_rate = state_->getProducerRate(); + if (prod_rate == 0) { + return now + SENTINEL_TIMER_INTERVAL; + } + + double packet_size = state_->getAveragePacketSize(); + uint32_t estimated_iat = ceil(1000.0 / (prod_rate / packet_size)); + + uint64_t rtt = state_->getMinRTT(); + if (rtt == 0) rtt = SENTINEL_TIMER_INTERVAL; + wait = rtt; + + uint32_t jitter = ceil(state_->getJitter()); + wait += jitter; + + // it may happen that the channel is congested and we have some additional + // queuing delay to take into account + uint32_t queue = ceil(state_->getQueuing()); + wait += queue; + + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "next rtx for " << seq << " in " << wait + << " ms, rtt = " << state_->getMinRTT() << " ait = " << estimated_iat + << " jttr = " << jitter << " queue = " << queue; + + return now + wait; + } +} + +void RecoveryStrategy::retransmit() { + if (rtx_timers_.size() == 0) return; + + uint64_t now = getNow(); + + auto it = rtx_timers_.begin(); + std::unordered_set<uint32_t> lost_pkt; + uint32_t sent_counter = 0; + while (it != rtx_timers_.end() && it->first <= now && + sent_counter < MAX_RTX_IN_BATCH) { + uint32_t seq = it->second; + auto rtx_it = + rtx_state_.find(seq); // this should always return a valid iter + if (rtx_it->second.rtx_count_ >= RTC_MAX_RTX || + (now - rtx_it->second.first_send_) >= RTC_MAX_AGE || + seq < state_->getLastSeqNacked()) { + // max rtx reached or packet too old or packet nacked, this packet is lost + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "packet " << seq << " lost because 1) max rtx: " + << (rtx_it->second.rtx_count_ >= RTC_MAX_RTX) << " 2) max age: " + << ((now - rtx_it->second.first_send_) >= RTC_MAX_AGE) + << " 3) nacked: " << (seq < state_->getLastSeqNacked()); + lost_pkt.insert(seq); + it++; + } else { + // resend the packet + state_->onRetransmission(seq); + double prod_rate = state_->getProducerRate(); + if (prod_rate != 0) rtx_it->second.rtx_count_++; + rtx_it->second.next_send_ = computeNextSend(seq, false); + it = rtx_timers_.erase(it); + rtx_timers_.insert( + std::pair<uint64_t, uint32_t>(rtx_it->second.next_send_, seq)); + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "send rtx for sequence " << seq << ", next send in " + << (rtx_it->second.next_send_ - now); + send_rtx_callback_(seq); + sent_counter++; + } + } + + // remove packets if needed + for (auto lost_it = lost_pkt.begin(); lost_it != lost_pkt.end(); lost_it++) { + uint32_t seq = *lost_it; + state_->onPacketLost(seq); + deleteRtx(seq); + } +} + +void RecoveryStrategy::scheduleNextRtx() { + if (rtx_timers_.size() == 0) { + // all the rtx were removed, reset timer + next_rtx_timer_ = MAX_TIMER_RTX; + return; + } + + // check if timer is alreay set + if (next_rtx_timer_ != MAX_TIMER_RTX) { + // a new check for rtx is already scheduled + if (next_rtx_timer_ > rtx_timers_.begin()->first) { + // we need to re-schedule it + timer_->cancel(); + } else { + // wait for the next timer + return; + } + } + + // set a new timer + next_rtx_timer_ = rtx_timers_.begin()->first; + uint64_t now = utils::SteadyTime::nowMs().count(); + uint64_t wait = 1; + if (next_rtx_timer_ != MAX_TIMER_RTX && next_rtx_timer_ > now) + wait = next_rtx_timer_ - now; + + std::weak_ptr<RecoveryStrategy> self(shared_from_this()); + timer_->expires_from_now(std::chrono::milliseconds(wait)); + timer_->async_wait([self](const std::error_code &ec) { + if (ec) return; + if (auto s = self.lock()) { + s->retransmit(); + s->next_rtx_timer_ = MAX_TIMER_RTX; + s->scheduleNextRtx(); + } + }); +} + +void RecoveryStrategy::deleteRtx(uint32_t seq) { + auto it_rtx = rtx_state_.find(seq); + if (it_rtx == rtx_state_.end()) return; // rtx not found + + // remove the rtx from the timers list + uint64_t ts = it_rtx->second.next_send_; + auto it_timers = rtx_timers_.find(ts); + while (it_timers != rtx_timers_.end() && it_timers->first == ts) { + if (it_timers->second == seq) { + rtx_timers_.erase(it_timers); + break; + } + it_timers++; + } + // remove rtx + rtx_state_.erase(it_rtx); +} + +// fec functions +uint32_t RecoveryStrategy::computeFecPacketsToAsk(bool in_sync) { + double loss_rate = state_->getMaxLossRate() * 100; // use loss rate in % + + if (loss_rate > 95) loss_rate = 95; // max loss rate + + if (loss_rate == 0) return 0; + + // once per minute try to reduce the fec rate. it may happen that for some bin + // we ask too many fec packet. here we try to reduce this values gently + if (round_id_ % ROUNDS_PER_MIN == 0) { + reduceFec(); + } + + // keep track of the last used fec. if we use a new bin on this round reset + // consecutive use and avg loss in the prev bin + uint32_t bin = ceil(loss_rate / 5.0) - 1; + if (bin > fec_per_loss_rate_.size() - 1) bin = fec_per_loss_rate_.size() - 1; + + if (bin != last_fec_used_) { + fec_per_loss_rate_[last_fec_used_].consecutive_use = 0; + fec_per_loss_rate_[last_fec_used_].avg_residual_losses = 0.0; + } + last_fec_used_ = bin; + fec_per_loss_rate_[last_fec_used_].consecutive_use++; + + // we update the stats only once very 5 rounds (1sec) that is the rate at + // which we compute residual losses + if (round_id_ % ROUNDS_PER_SEC == 0) { + double residual_losses = state_->getResidualLossRate() * 100; + // update residual loss rate + fec_per_loss_rate_[bin].avg_residual_losses = + (fec_per_loss_rate_[bin].avg_residual_losses * MOVING_AVG_ALPHA) + + (1 - MOVING_AVG_ALPHA) * residual_losses; + + if ((fec_per_loss_rate_[bin].last_update - round_id_) < + WAIT_BEFORE_FEC_UPDATE) { + // this bin is been updated recently so don't modify it and + // return the current state + return fec_per_loss_rate_[bin].fec_to_ask; + } + + // if the residual loss rate is too high and we can ask more fec packets and + // we are using this configuration since at least 5 sec update fec + if (fec_per_loss_rate_[bin].avg_residual_losses > MAX_RESIDUAL_LOSS_RATE && + fec_per_loss_rate_[bin].fec_to_ask < (n_ - k_) && + fec_per_loss_rate_[bin].consecutive_use > WAIT_BEFORE_FEC_UPDATE) { + // so increase the number of fec packets to ask + fec_per_loss_rate_[bin].fec_to_ask++; + fec_per_loss_rate_[bin].last_update = round_id_; + fec_per_loss_rate_[bin].avg_residual_losses = 0.0; + } + } + + return fec_per_loss_rate_[bin].fec_to_ask; +} + +void RecoveryStrategy::setRtxFec(std::optional<bool> rtx_on, + std::optional<bool> fec_on) { + if (rtx_on) rtx_on_ = *rtx_on; + if (fec_on) fec_on_ = *fec_on; + + if (*callback_) { + notification::RecoveryStrategy strategy = + notification::RecoveryStrategy::RECOVERY_OFF; + + if (rtx_on_ && fec_on_) + strategy = notification::RecoveryStrategy::RTX_AND_FEC; + else if (rtx_on_) + strategy = notification::RecoveryStrategy::RTX_ONLY; + else if (fec_on_) + strategy = notification::RecoveryStrategy::FEC_ONLY; + + (*callback_)(strategy); + } +} + +// common functions +void RecoveryStrategy::onLostTimeout(uint32_t seq) { removePacketState(seq); } + +void RecoveryStrategy::removePacketState(uint32_t seq) { + auto it_fec = recover_with_fec_.find(seq); + if (it_fec != recover_with_fec_.end()) { + recover_with_fec_.erase(it_fec); + return; + } + + deleteRtx(seq); +} + +// private methods + +void RecoveryStrategy::reduceFec() { + for (uint32_t loss_rate = 5; loss_rate < 100; loss_rate += 5) { + double dec_loss_rate = (double)loss_rate / 100.0; + double exp_losses = (double)k_ * dec_loss_rate; + uint32_t fec_to_ask = ceil(exp_losses / (1 - dec_loss_rate)); + + uint32_t bin = ceil(loss_rate / 5.0) - 1; + if (fec_per_loss_rate_[bin].fec_to_ask > fec_to_ask) { + fec_per_loss_rate_[bin].fec_to_ask--; + // std::cout << "reduce fec to ask for bin " << bin << std::endl; + } + } +} + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_recovery_strategy.h b/libtransport/src/protocols/rtc/rtc_recovery_strategy.h new file mode 100644 index 000000000..9ffc69a1b --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_recovery_strategy.h @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include <hicn/transport/interfaces/callbacks.h> +#include <hicn/transport/utils/chrono_typedefs.h> +#include <protocols/indexer.h> +#include <protocols/rtc/rtc_rc.h> +#include <protocols/rtc/rtc_state.h> + +#include <map> +#include <unordered_map> + +namespace transport { + +namespace protocol { + +namespace rtc { + +class RecoveryStrategy : public std::enable_shared_from_this<RecoveryStrategy> { + protected: + struct rtx_state_ { + uint64_t first_send_; + uint64_t next_send_; + uint32_t rtx_count_; + }; + + using rtxState = struct rtx_state_; + + public: + using SendRtxCallback = std::function<void(uint32_t)>; + + RecoveryStrategy(Indexer *indexer, SendRtxCallback &&callback, + asio::io_service &io_service, bool use_rtx, bool use_fec, + interface::StrategyCallback *external_callback); + + RecoveryStrategy(RecoveryStrategy &&rs); + + virtual ~RecoveryStrategy(); + + void setRtxFec(std::optional<bool> rtx_on = {}, + std::optional<bool> fec_on = {}); + void setState(RTCState *state) { state_ = state; } + void setRateControl(RTCRateControl *rateControl) { rc_ = rateControl; } + void setFecParams(uint32_t n, uint32_t k); + + void tunrOnRecovery() { recovery_on_ = true; } + + bool isRtx(uint32_t seq) { + if (rtx_state_.find(seq) != rtx_state_.end()) return true; + return false; + } + + bool isPossibleLossWithNoRtx(uint32_t seq) { + if (recover_with_fec_.find(seq) != recover_with_fec_.end()) return true; + return false; + } + + bool isRtxOn() { return rtx_on_; } + + RTCState *getState() { return state_; } + bool lossDetected(uint32_t seq); + void clear(); + + virtual void onNewRound(bool in_sync) = 0; + virtual void newPacketLoss(uint32_t seq) = 0; + virtual void receivedPacket(uint32_t seq) = 0; + void onLostTimeout(uint32_t seq); + + void incRoundId() { round_id_++; } + + // utils + uint64_t getNow() { + uint64_t now = utils::SteadyTime::nowMs().count(); + return now; + } + + protected: + // rtx functions + void addNewRtx(uint32_t seq, bool force); + uint64_t computeNextSend(uint32_t seq, bool new_rtx); + void retransmit(); + void scheduleNextRtx(); + void deleteRtx(uint32_t seq); + + // fec functions + uint32_t computeFecPacketsToAsk(bool in_sync); + + // common functons + void removePacketState(uint32_t seq); + + bool recovery_on_; + bool rtx_on_; + bool fec_on_; + + // this map keeps track of the retransmitted interest, ordered from the oldest + // to the newest one. the state contains the timer of the first send of the + // interest (from pendingIntetests_), the timer of the next send (key of the + // multimap) and the number of rtx + std::map<uint32_t, rtxState> rtx_state_; + // this map stored the rtx by timer. The key is the time at which the rtx + // should be sent, and the val is the interest seq number + std::multimap<uint64_t, uint32_t> rtx_timers_; + + // lost packets that will be recovered with fec + std::unordered_set<uint32_t> recover_with_fec_; + + // rtx vars + std::unique_ptr<asio::steady_timer> timer_; + uint64_t next_rtx_timer_; + SendRtxCallback send_rtx_callback_; + + // fec vars + uint32_t n_; + uint32_t k_; + Indexer *indexer_; + + RTCState *state_; + RTCRateControl *rc_; + + private: + struct fec_state_ { + uint32_t fec_to_ask; + uint32_t last_update; // round id of the last update + // (wait 10 ruonds (2sec) between updates) + uint32_t consecutive_use; // consecutive ruonds where this fec was used + double avg_residual_losses; + }; + + void reduceFec(); + + uint32_t round_id_; // number of rounds + uint32_t last_fec_used_; + std::vector<fec_state_> fec_per_loss_rate_; + interface::StrategyCallback *callback_; +}; + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_delay.cc b/libtransport/src/protocols/rtc/rtc_rs_delay.cc new file mode 100644 index 000000000..e2c60ca77 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_delay.cc @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <glog/logging.h> +#include <protocols/rtc/rtc_consts.h> +#include <protocols/rtc/rtc_rs_delay.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +RecoveryStrategyDelayBased::RecoveryStrategyDelayBased( + Indexer *indexer, SendRtxCallback &&callback, asio::io_service &io_service, + interface::StrategyCallback *external_callback) + : RecoveryStrategy(indexer, std::move(callback), io_service, true, false, + external_callback), // start with rtx + congestion_state_(false), + probing_state_(false), + switch_rounds_(0) {} + +RecoveryStrategyDelayBased::RecoveryStrategyDelayBased(RecoveryStrategy &&rs) + : RecoveryStrategy(std::move(rs)) { + setRtxFec(true, false); + // we have to re-init congestion and + // probing + congestion_state_ = false; + probing_state_ = false; +} + +RecoveryStrategyDelayBased::~RecoveryStrategyDelayBased() {} + +void RecoveryStrategyDelayBased::softSwitchToFec(uint32_t fec_to_ask) { + if (fec_to_ask == 0) { + setRtxFec(true, false); + switch_rounds_ = 0; + } else { + switch_rounds_++; + if (switch_rounds_ >= 5) { + setRtxFec(false, true); + } else { + setRtxFec({}, true); + } + } +} + +void RecoveryStrategyDelayBased::onNewRound(bool in_sync) { + if (!recovery_on_) { + // disable fec so that no extra packet will be sent + // for rtx we check if recovery is on in newPacketLoss + setRtxFec(true, false); + indexer_->setNFec(0); + return; + } + + uint64_t rtt = state_->getMinRTT(); + + bool congestion = false; + // XXX at the moment we are not looking at congestion events + // congestion = rc_->inCongestionState(); + + if ((!fec_on_ && rtt >= 100) || (fec_on_ && rtt > 80) || congestion) { + // switch from rtx to fec or keep use fec. Notice that if some rtx are + // waiting to be scheduled, they will be sent normally, but no new rtx will + // be created If the loss rate is 0 keep to use RTX. + uint32_t fec_to_ask = computeFecPacketsToAsk(in_sync); + softSwitchToFec(fec_to_ask); + indexer_->setNFec(fec_to_ask); + return; + } + + if ((fec_on_ && rtt <= 80) || (!rtx_on_ && rtt <= 100)) { + // turn on rtx + softSwitchToFec(0); + indexer_->setNFec(0); + return; + } +} + +void RecoveryStrategyDelayBased::newPacketLoss(uint32_t seq) { + if (rtx_on_ && recovery_on_) { + addNewRtx(seq, false); + } else { + if (!state_->isPending(seq) && !indexer_->isFec(seq)) { + addNewRtx(seq, true); + } else { + recover_with_fec_.insert(seq); + state_->onPossibleLossWithNoRtx(seq); + } + } +} + +void RecoveryStrategyDelayBased::receivedPacket(uint32_t seq) { + removePacketState(seq); +} + +void RecoveryStrategyDelayBased::probing() { + // TODO + // for the moment ask for all fec and exit the probing phase + probing_state_ = false; + setRtxFec(false, true); + indexer_->setNFec(computeFecPacketsToAsk(true)); +} + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_delay.h b/libtransport/src/protocols/rtc/rtc_rs_delay.h new file mode 100644 index 000000000..0dd199965 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_delay.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include <protocols/rtc/rtc_recovery_strategy.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +class RecoveryStrategyDelayBased : public RecoveryStrategy { + public: + RecoveryStrategyDelayBased(Indexer *indexer, SendRtxCallback &&callback, + asio::io_service &io_service, + interface::StrategyCallback *external_callback); + + RecoveryStrategyDelayBased(RecoveryStrategy &&rs); + + ~RecoveryStrategyDelayBased(); + + void onNewRound(bool in_sync); + void newPacketLoss(uint32_t seq); + void receivedPacket(uint32_t seq); + + private: + void softSwitchToFec(uint32_t fec_to_ask); + + bool congestion_state_; + bool probing_state_; + uint32_t switch_rounds_; + + void probing(); +}; + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_fec_only.cc b/libtransport/src/protocols/rtc/rtc_rs_fec_only.cc new file mode 100644 index 000000000..36d8e39f0 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_fec_only.cc @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <glog/logging.h> +#include <protocols/rtc/rtc_consts.h> +#include <protocols/rtc/rtc_rs_fec_only.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +RecoveryStrategyFecOnly::RecoveryStrategyFecOnly( + Indexer *indexer, SendRtxCallback &&callback, asio::io_service &io_service, + interface::StrategyCallback *external_callback) + : RecoveryStrategy(indexer, std::move(callback), io_service, false, true, + external_callback), + congestion_state_(false), + probing_state_(false), + switch_rounds_(0) {} + +RecoveryStrategyFecOnly::RecoveryStrategyFecOnly(RecoveryStrategy &&rs) + : RecoveryStrategy(std::move(rs)) { + setRtxFec(false, true); + congestion_state_ = false; + probing_state_ = false; +} + +RecoveryStrategyFecOnly::~RecoveryStrategyFecOnly() {} + +void RecoveryStrategyFecOnly::onNewRound(bool in_sync) { + if (!recovery_on_) { + indexer_->setNFec(0); + return; + } + + // XXX for the moment we are considering congestion events + // if(rc_->inCongestionState()){ + // congestion_state_ = true; + // probing_state_ = false; + // indexer_->setNFec(0); + // return; + // } + + // no congestion + if (congestion_state_) { + // this is the first round after congestion + // enter probing phase + probing_state_ = true; + congestion_state_ = false; + } + + if (probing_state_) { + probing(); + } else { + uint32_t fec_to_ask = computeFecPacketsToAsk(in_sync); + // If fec_to_ask == 0 we use rtx even if in these strategy we use only fec. + // In this way the first packet loss that triggers the usage of fec can be + // recovered using rtx, otherwise it will always be lost + if (fec_to_ask == 0) { + setRtxFec(true, false); + switch_rounds_ = 0; + } else { + switch_rounds_++; + if (switch_rounds_ >= 5) { + setRtxFec(false, true); + } else { + setRtxFec({}, true); + } + } + indexer_->setNFec(fec_to_ask); + } +} + +void RecoveryStrategyFecOnly::newPacketLoss(uint32_t seq) { + if (rtx_on_ && recovery_on_) { + addNewRtx(seq, false); + } else { + if (!state_->isPending(seq) && !indexer_->isFec(seq)) { + addNewRtx(seq, true); + } else { + // if not pending add rtc + recover_with_fec_.insert(seq); + state_->onPossibleLossWithNoRtx(seq); + } + } +} + +void RecoveryStrategyFecOnly::receivedPacket(uint32_t seq) { + removePacketState(seq); +} + +void RecoveryStrategyFecOnly::probing() { + // TODO + // for the moment ask for all fec and exit the probing phase + probing_state_ = false; + uint32_t fec_to_ask = computeFecPacketsToAsk(true); + indexer_->setNFec(fec_to_ask); +} + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_fec_only.h b/libtransport/src/protocols/rtc/rtc_rs_fec_only.h new file mode 100644 index 000000000..37b505d35 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_fec_only.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include <protocols/rtc/rtc_recovery_strategy.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +class RecoveryStrategyFecOnly : public RecoveryStrategy { + public: + RecoveryStrategyFecOnly(Indexer *indexer, SendRtxCallback &&callback, + asio::io_service &io_service, + interface::StrategyCallback *external_callback); + + RecoveryStrategyFecOnly(RecoveryStrategy &&rs); + + ~RecoveryStrategyFecOnly(); + + void onNewRound(bool in_sync); + void newPacketLoss(uint32_t seq); + void receivedPacket(uint32_t seq); + + private: + bool congestion_state_; + bool probing_state_; + uint32_t switch_rounds_; + + void probing(); +}; + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_low_rate.cc b/libtransport/src/protocols/rtc/rtc_rs_low_rate.cc new file mode 100644 index 000000000..bd153d209 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_low_rate.cc @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <glog/logging.h> +#include <protocols/rtc/rtc_consts.h> +#include <protocols/rtc/rtc_rs_low_rate.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +RecoveryStrategyLowRate::RecoveryStrategyLowRate( + Indexer *indexer, SendRtxCallback &&callback, asio::io_service &io_service, + interface::StrategyCallback *external_callback) + : RecoveryStrategy(indexer, std::move(callback), io_service, false, true, + external_callback), // start with fec + fec_consecutive_rounds_((MILLI_IN_A_SEC / ROUND_LEN) * 5), // 5 sec + rtx_allowed_consecutive_rounds_(0) { + initSwitchVector(); +} + +RecoveryStrategyLowRate::RecoveryStrategyLowRate(RecoveryStrategy &&rs) + : RecoveryStrategy(std::move(rs)), + fec_consecutive_rounds_((MILLI_IN_A_SEC / ROUND_LEN) * 5), // 5 sec + rtx_allowed_consecutive_rounds_(0) { + setRtxFec(false, true); + initSwitchVector(); +} + +RecoveryStrategyLowRate::~RecoveryStrategyLowRate() {} + +void RecoveryStrategyLowRate::initSwitchVector() { + // TODO adjust thresholds here when new data are collected + // see resutls in + // https://confluence-eng-gpk1.cisco.com/conf/display/SPT/dailyreports + thresholds_t t1; + t1.rtt = 15; // 15ms + t1.loss_rtx_to_fec = 15; // 15% + t1.loss_fec_to_rtx = 10; // 10% + thresholds_t t2; + t2.rtt = 35; // 35ms + t2.loss_rtx_to_fec = 5; // 5% + t2.loss_fec_to_rtx = 1; // 1% + switch_vector.push_back(t1); + switch_vector.push_back(t2); +} + +void RecoveryStrategyLowRate::setRecoveryParameters(bool use_rtx, bool use_fec, + uint32_t fec_to_ask) { + setRtxFec(use_rtx, use_fec); + indexer_->setNFec(fec_to_ask); +} + +void RecoveryStrategyLowRate::selectRecoveryStrategy(bool in_sync) { + uint32_t fec_to_ask = computeFecPacketsToAsk(in_sync); + if (fec_to_ask == 0) { + // fec is off, turn on RTX immediatly to avoid packet losses + setRecoveryParameters(true, false, 0); + fec_consecutive_rounds_ = 0; + return; + } + + uint32_t loss_rate = std::round(state_->getPerSecondLossRate() * 100); + uint32_t rtt = state_->getAvgRTT(); + + bool use_rtx = false; + for (size_t i = 0; i < switch_vector.size(); i++) { + uint32_t max_loss_rate = 0; + if (fec_on_) + max_loss_rate = switch_vector[i].loss_fec_to_rtx; + else + max_loss_rate = switch_vector[i].loss_rtx_to_fec; + + if (rtt < switch_vector[i].rtt && loss_rate < max_loss_rate) { + use_rtx = true; + rtx_allowed_consecutive_rounds_++; + break; + } + } + + if (!use_rtx) rtx_allowed_consecutive_rounds_ = 0; + + if (use_rtx) { + if (fec_on_) { + // here we should swtich from RTX to FEC + // wait 10sec where the switch is allowed before actually switch + if (rtx_allowed_consecutive_rounds_ >= + ((MILLI_IN_A_SEC / ROUND_LEN) * 10)) { // 10 sec + // use RTX + setRecoveryParameters(true, false, 0); + fec_consecutive_rounds_ = 0; + } else { + // keep using FEC (and maybe RTX) + setRecoveryParameters(true, true, fec_to_ask); + fec_consecutive_rounds_++; + } + } else { + // keep using RTX + setRecoveryParameters(true, false, 0); + fec_consecutive_rounds_ = 0; + } + } else { + // use FEC and RTX + setRecoveryParameters(true, true, fec_to_ask); + fec_consecutive_rounds_++; + } + + // everytime that we anable FEC we keep also RTX on. in this way the first + // losses that are not covered by FEC are recovered using RTX. after 5 sec we + // disable fec + if (fec_consecutive_rounds_ >= ((MILLI_IN_A_SEC / ROUND_LEN) * 5)) { + // turn off RTX + setRtxFec(false); + } +} + +void RecoveryStrategyLowRate::onNewRound(bool in_sync) { + if (!recovery_on_) { + // disable fec so that no extra packet will be sent + // for rtx we check if recovery is on in newPacketLoss + setRtxFec(true, false); + indexer_->setNFec(0); + return; + } + + // XXX since this strategy will be used only for flow at low rate we do not + // consider congestion events like in other strategies + + selectRecoveryStrategy(in_sync); +} + +void RecoveryStrategyLowRate::newPacketLoss(uint32_t seq) { + if (rtx_on_ && recovery_on_) { + addNewRtx(seq, false); + } else { + if (!state_->isPending(seq) && !indexer_->isFec(seq)) { + addNewRtx(seq, true); + } else { + recover_with_fec_.insert(seq); + state_->onPossibleLossWithNoRtx(seq); + } + } +} + +void RecoveryStrategyLowRate::receivedPacket(uint32_t seq) { + removePacketState(seq); +} + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_low_rate.h b/libtransport/src/protocols/rtc/rtc_rs_low_rate.h new file mode 100644 index 000000000..f0c7bd0d5 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_low_rate.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include <protocols/rtc/rtc_recovery_strategy.h> + +#include <vector> + +namespace transport { + +namespace protocol { + +namespace rtc { + +struct thresholds_t { + uint32_t rtt; + uint32_t loss_rtx_to_fec; // loss rate used to move from rtx to fec + uint32_t loss_fec_to_rtx; // loss rate used to move from fec to rtx +}; + +class RecoveryStrategyLowRate : public RecoveryStrategy { + public: + RecoveryStrategyLowRate(Indexer *indexer, SendRtxCallback &&callback, + asio::io_service &io_service, + interface::StrategyCallback *external_callback); + + RecoveryStrategyLowRate(RecoveryStrategy &&rs); + + ~RecoveryStrategyLowRate(); + + void onNewRound(bool in_sync); + void newPacketLoss(uint32_t seq); + void receivedPacket(uint32_t seq); + + private: + void initSwitchVector(); + void setRecoveryParameters(bool use_rtx, bool use_fec, uint32_t fec_to_ask); + void selectRecoveryStrategy(bool in_sync); + + uint32_t fec_consecutive_rounds_; + uint32_t rtx_allowed_consecutive_rounds_; + + // this table contains the thresholds that indicates when to switch from RTX + // to FEC and viceversa. values in the vector are detected with a set of + // experiments. the vector is used in the following way: if rtt and loss rate + // are less than one of the values in the in the vector, losses are + // recovered using RTX. otherwive losses are recovered using FEC. as for FEC + // only and delay based strategy, the swith from RTX to FEC is smooth, + // meaning that FEC and RTX are used together for some rounds + std::vector<thresholds_t> switch_vector; +}; + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_recovery_off.cc b/libtransport/src/protocols/rtc/rtc_rs_recovery_off.cc new file mode 100644 index 000000000..499e978f1 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_recovery_off.cc @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <glog/logging.h> +#include <protocols/rtc/rtc_consts.h> +#include <protocols/rtc/rtc_rs_recovery_off.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +RecoveryStrategyRecoveryOff::RecoveryStrategyRecoveryOff( + Indexer *indexer, SendRtxCallback &&callback, asio::io_service &io_service, + interface::StrategyCallback *external_callback) + : RecoveryStrategy(indexer, std::move(callback), io_service, false, false, + external_callback) {} + +RecoveryStrategyRecoveryOff::RecoveryStrategyRecoveryOff(RecoveryStrategy &&rs) + : RecoveryStrategy(std::move(rs)) { + setRtxFec(false, false); +} + +RecoveryStrategyRecoveryOff::~RecoveryStrategyRecoveryOff() {} + +void RecoveryStrategyRecoveryOff::onNewRound(bool in_sync) { + // nothing to do + return; +} + +void RecoveryStrategyRecoveryOff::newPacketLoss(uint32_t seq) { + // here we only keep track of the lost packets to avoid to + // count them multple times in the counters. for this we + // use the recover_with_fec_ set + recover_with_fec_.insert(seq); + state_->onPossibleLossWithNoRtx(seq); +} + +void RecoveryStrategyRecoveryOff::receivedPacket(uint32_t seq) { + removePacketState(seq); +} + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_recovery_off.h b/libtransport/src/protocols/rtc/rtc_rs_recovery_off.h new file mode 100644 index 000000000..98cd1e6a5 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_recovery_off.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include <protocols/rtc/rtc_recovery_strategy.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +class RecoveryStrategyRecoveryOff : public RecoveryStrategy { + public: + RecoveryStrategyRecoveryOff(Indexer *indexer, SendRtxCallback &&callback, + asio::io_service &io_service, + interface::StrategyCallback *external_callback); + + RecoveryStrategyRecoveryOff(RecoveryStrategy &&rs); + + ~RecoveryStrategyRecoveryOff(); + + void onNewRound(bool in_sync); + void newPacketLoss(uint32_t seq); + void receivedPacket(uint32_t seq); +}; + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_rtx_only.cc b/libtransport/src/protocols/rtc/rtc_rs_rtx_only.cc new file mode 100644 index 000000000..c1ae9b53d --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_rtx_only.cc @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <glog/logging.h> +#include <protocols/rtc/rtc_consts.h> +#include <protocols/rtc/rtc_rs_rtx_only.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +RecoveryStrategyRtxOnly::RecoveryStrategyRtxOnly( + Indexer *indexer, SendRtxCallback &&callback, asio::io_service &io_service, + interface::StrategyCallback *external_callback) + : RecoveryStrategy(indexer, std::move(callback), io_service, true, false, + external_callback) {} + +RecoveryStrategyRtxOnly::RecoveryStrategyRtxOnly(RecoveryStrategy &&rs) + : RecoveryStrategy(std::move(rs)) { + setRtxFec(true, false); +} + +RecoveryStrategyRtxOnly::~RecoveryStrategyRtxOnly() {} + +void RecoveryStrategyRtxOnly::onNewRound(bool in_sync) { + // nothing to do + return; +} + +void RecoveryStrategyRtxOnly::newPacketLoss(uint32_t seq) { + if (!recovery_on_) { + recover_with_fec_.insert(seq); + state_->onPossibleLossWithNoRtx(seq); + return; + } + addNewRtx(seq, false); +} + +void RecoveryStrategyRtxOnly::receivedPacket(uint32_t seq) { + removePacketState(seq); +} + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/fec_base.cc b/libtransport/src/protocols/rtc/rtc_rs_rtx_only.h index 9252bc473..7ae909454 100644 --- a/libtransport/src/protocols/fec_base.cc +++ b/libtransport/src/protocols/rtc/rtc_rs_rtx_only.h @@ -13,12 +13,32 @@ * limitations under the License. */ -#include <protocols/fec/rs.h> -#include <protocols/fec_base.h> +#pragma once +#include <protocols/rtc/rtc_recovery_strategy.h> namespace transport { + namespace protocol { -namespace fec {} // namespace fec -} // namespace protocol -} // namespace transport
\ No newline at end of file +namespace rtc { + +class RecoveryStrategyRtxOnly : public RecoveryStrategy { + public: + RecoveryStrategyRtxOnly(Indexer *indexer, SendRtxCallback &&callback, + asio::io_service &io_service, + interface::StrategyCallback *external_callback); + + RecoveryStrategyRtxOnly(RecoveryStrategy &&rs); + + ~RecoveryStrategyRtxOnly(); + + void onNewRound(bool in_sync); + void newPacketLoss(uint32_t seq); + void receivedPacket(uint32_t seq); +}; + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_state.cc b/libtransport/src/protocols/rtc/rtc_state.cc index c99205a26..6a21531f8 100644 --- a/libtransport/src/protocols/rtc/rtc_state.cc +++ b/libtransport/src/protocols/rtc/rtc_state.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -24,15 +24,15 @@ namespace protocol { namespace rtc { RTCState::RTCState(Indexer *indexer, - ProbeHandler::SendProbeCallback &&rtt_probes_callback, + ProbeHandler::SendProbeCallback &&probe_callback, DiscoveredRttCallback &&discovered_rtt_callback, asio::io_service &io_service) - : indexer_(indexer), - rtt_probes_(std::make_shared<ProbeHandler>(std::move(rtt_probes_callback), - io_service)), + : loss_history_(10), // log 10sec history + indexer_(indexer), + probe_handler_(std::make_shared<ProbeHandler>(std::move(probe_callback), + io_service)), discovered_rtt_callback_(std::move(discovered_rtt_callback)) { init_rtt_timer_ = std::make_unique<asio::steady_timer>(io_service); - initParams(); } RTCState::~RTCState() {} @@ -55,9 +55,19 @@ void RTCState::initParams() { highest_seq_received_in_order_ = 0; last_seq_nacked_ = 0; loss_rate_ = 0.0; - avg_loss_rate_ = 0.0; + avg_loss_rate_ = -1.0; max_loss_rate_ = 0.0; last_round_loss_rate_ = 0.0; + + // loss rate per sec + lost_per_sec_ = 0; + total_expected_packets_ = 0; + per_sec_loss_rate_ = 0.0; + + // residual losses counters + expected_packets_ = 0; + packets_sent_to_app_ = 0; + rounds_from_last_compute_ = 0; residual_loss_rate_ = 0.0; // fec counters @@ -66,9 +76,13 @@ void RTCState::initParams() { // bw counters received_bytes_ = 0; + received_fec_bytes_ = 0; + recovered_bytes_with_fec_ = 0; + avg_packet_size_ = INIT_PACKET_SIZE; production_rate_ = 0.0; received_rate_ = 0.0; + fec_recovered_rate_ = 0.0; // nack counter nack_on_last_round_ = false; @@ -95,31 +109,30 @@ void RTCState::initParams() { path_table_.clear(); main_path_ = nullptr; - // packet received - received_or_lost_packets_.clear(); + // packet cache (not pending anymore) + packet_cache_.clear(); // pending interests pending_interests_.clear(); - // skipped interest + // used to keep track of the skipped interest last_interest_sent_ = 0; - skipped_interests_.clear(); // init rtt first_interest_sent_time_ = ~0; first_interest_sent_seq_ = 0; + // start probing the producer init_rtt_ = false; - rtt_probes_->setProbes(INIT_RTT_PROBE_INTERVAL, INIT_RTT_PROBES); - rtt_probes_->sendProbes(); + probe_handler_->setSuffixRange(MIN_INIT_PROBE_SEQ, MAX_INIT_PROBE_SEQ); + probe_handler_->setProbes(INIT_RTT_PROBE_INTERVAL, INIT_RTT_PROBES); + probe_handler_->sendProbes(); setInitRttTimer(INIT_RTT_PROBE_RESTART); } // packet events void RTCState::onSendNewInterest(const core::Name *interest_name) { - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + uint64_t now = utils::SteadyTime::nowMs().count(); uint32_t seq = interest_name->getSuffix(); pending_interests_.insert(std::pair<uint32_t, uint64_t>(seq, now)); @@ -137,11 +150,12 @@ void RTCState::onSendNewInterest(const core::Name *interest_name) { } // TODO what happen in case of jumps? - // look for skipped interests - skipped_interests_.erase(seq); // remove seq if it is there + eraseFromPacketCache( + seq); // if we send this interest we don't know its state for (uint32_t i = last_interest_sent_ + 1; i < seq; i++) { if (indexer_->isFec(i)) { - skipped_interests_.insert(i); + // only fec packets can be skipped + addToPacketCache(i, PacketState::SKIPPED); } } @@ -155,6 +169,7 @@ void RTCState::onTimeout(uint32_t seq, bool lost) { auto it = pending_interests_.find(seq); if (it != pending_interests_.end()) { pending_interests_.erase(it); + if (indexer_->isFec(seq)) pending_fec_pkt_--; } received_timeouts_++; @@ -162,11 +177,12 @@ void RTCState::onTimeout(uint32_t seq, bool lost) { } void RTCState::onLossDetected(uint32_t seq) { - if (!indexer_->isFec(seq)) { - packets_lost_++; - } else if (skipped_interests_.find(seq) == skipped_interests_.end() && - seq >= first_interest_sent_seq_) { + PacketState state = getPacketState(seq); + + // if the packet is already marked with a state, do nothing + if (state == PacketState::UNKNOWN) { packets_lost_++; + addToPacketCache(seq, PacketState::LOST); } } @@ -178,30 +194,40 @@ void RTCState::onRetransmission(uint32_t seq) { auto it = pending_interests_.find(seq); if (it != pending_interests_.end()) { pending_interests_.erase(it); -#if 0 - packets_lost_++; -#endif + if (indexer_->isFec(seq)) pending_fec_pkt_--; } sent_rtx_++; sent_rtx_last_round_++; } +void RTCState::onPossibleLossWithNoRtx(uint32_t seq) { + // if fec is on or rtx is disable we don't need to do anything to recover a + // packet. however in both cases we need to remove possible missing packets + // from the window of pendinig interest in order to free space without wating + // for the timeout. + auto it = pending_interests_.find(seq); + if (it != pending_interests_.end()) { + pending_interests_.erase(it); + if (indexer_->isFec(seq)) pending_fec_pkt_--; + } +} + void RTCState::onDataPacketReceived(const core::ContentObject &content_object, bool compute_stats) { uint32_t seq = content_object.getName().getSuffix(); + if (compute_stats) { updatePathStats(content_object, false); received_data_last_round_++; } received_data_++; + packets_sent_to_app_++; - struct data_packet_t *data_pkt = - (struct data_packet_t *)content_object.getPayload()->data(); - uint64_t production_time = data_pkt->getTimestamp(); - if (last_prod_update_ < production_time) { - last_prod_update_ = production_time; - uint32_t production_rate = data_pkt->getProductionRate(); - production_rate_ = (double)production_rate; + core::ParamsRTC params = RTCState::getDataParams(content_object); + + if (last_prod_update_ < params.timestamp) { + last_prod_update_ = params.timestamp; + production_rate_ = (double)params.prod_rate; } updatePacketSize(content_object); @@ -219,9 +245,18 @@ void RTCState::onDataPacketReceived(const core::ContentObject &content_object, void RTCState::onFecPacketReceived(const core::ContentObject &content_object) { uint32_t seq = content_object.getName().getSuffix(); - updateReceivedBytes(content_object); + // updateReceivedBytes(content_object); + received_fec_bytes_ += + (uint32_t)(content_object.headerSize() + content_object.payloadSize()); + + if (seq > highest_seq_received_) highest_seq_received_ = seq; + + PacketState state = getPacketState(seq); + if (state != PacketState::LOST) { + // increase only for not lost packets + received_fec_pkt_++; + } addRecvOrLost(seq, PacketState::RECEIVED); - received_fec_pkt_++; // the producer is responding // it is generating valid data packets so we consider it active producer_is_active_ = true; @@ -233,7 +268,7 @@ void RTCState::onNackPacketReceived(const core::ContentObject &nack, struct nack_packet_t *nack_pkt = (struct nack_packet_t *)nack.getPayload()->data(); uint64_t production_time = nack_pkt->getTimestamp(); - uint32_t production_seq = nack_pkt->getProductionSegement(); + uint32_t production_seq = nack_pkt->getProductionSegment(); uint32_t production_rate = nack_pkt->getProductionRate(); if (TRANSPORT_EXPECT_FALSE(main_path_ == nullptr) || @@ -255,6 +290,7 @@ void RTCState::onNackPacketReceived(const core::ContentObject &nack, received_nacks_++; received_nacks_last_round_++; + bool to_delete = false; if (production_seq > seq) { // old nack, seq is lost // update last nacked @@ -266,12 +302,19 @@ void RTCState::onNackPacketReceived(const core::ContentObject &nack, // future nack // remove the nack from the pending interest map // (the packet is not received/lost yet) - if (indexer_->isFec(seq)) pending_fec_pkt_--; - pending_interests_.erase(seq); + to_delete = true; } else { // this should be a quite rear event. simply remove the // packet from the pending interest list - pending_interests_.erase(seq); + to_delete = true; + } + + if (to_delete) { + auto it = pending_interests_.find(seq); + if (it != pending_interests_.end()) { + pending_interests_.erase(it); + if (indexer_->isFec(seq)) pending_fec_pkt_--; + } } // the producer is responding @@ -295,44 +338,58 @@ void RTCState::onPacketLost(uint32_t seq) { } #endif if (!indexer_->isFec(seq)) { - definitely_lost_pkt_++; - DLOG_IF(INFO, VLOG_IS_ON(4)) << "packet " << seq << " is lost"; + PacketState state = getPacketState(seq); + if (state == PacketState::LOST || state == PacketState::UNKNOWN) { + definitely_lost_pkt_++; + DLOG_IF(INFO, VLOG_IS_ON(4)) << "packet " << seq << " is lost"; + } } - addRecvOrLost(seq, PacketState::LOST); + addRecvOrLost(seq, PacketState::DEFINITELY_LOST); } void RTCState::onPacketRecoveredRtx(uint32_t seq) { + packets_sent_to_app_++; + if (seq > highest_seq_received_) highest_seq_received_ = seq; losses_recovered_++; addRecvOrLost(seq, PacketState::RECEIVED); } -void RTCState::onPacketRecoveredFec(uint32_t seq) { +void RTCState::onFecPacketRecoveredRtx(uint32_t seq) { + // This is the same as onPacketRecoveredRtx, but in this is case the + // pkt is also a FEC pkt, the addRecvOrLost will be called afterwards + if (seq > highest_seq_received_) highest_seq_received_ = seq; + losses_recovered_++; +} + +void RTCState::onPacketRecoveredFec(uint32_t seq, uint32_t size) { losses_recovered_++; + packets_sent_to_app_++; + recovered_bytes_with_fec_ += size; + + if (seq > highest_seq_received_) highest_seq_received_ = seq; + + // adding header to the count + recovered_bytes_with_fec_ += 60; // XXX get header size some where + + if (getPacketState(seq) == PacketState::UNKNOWN) + onLossDetected(seq); // the pkt was lost but didn't account for it yet + addRecvOrLost(seq, PacketState::RECEIVED); } bool RTCState::onProbePacketReceived(const core::ContentObject &probe) { uint32_t seq = probe.getName().getSuffix(); - uint64_t rtt; - - rtt = rtt_probes_->getRtt(seq); + uint64_t rtt; + rtt = probe_handler_->getRtt(seq); if (rtt == 0) return false; // this is not a valid probe - // like for data and nacks update the path stats. Here the RTT is computed - // by the probe handler. Both probes for rtt and bw are good to esimate - // info on the path + // Like for data and nacks update the path stats. Here the RTT is computed + // by the probe handler. Both probes for rtt and bw are good to estimate + // info on the path. uint32_t path_label = probe.getPathLabel(); - auto path_it = path_table_.find(path_label); - // update production rate and last_seq_nacked like in case of a nack - struct nack_packet_t *probe_pkt = - (struct nack_packet_t *)probe.getPayload()->data(); - uint64_t sender_timestamp = probe_pkt->getTimestamp(); - uint32_t production_seq = probe_pkt->getProductionSegement(); - uint32_t production_rate = probe_pkt->getProductionRate(); - if (path_it == path_table_.end()) { // found a new path std::shared_ptr<RTCDataPath> newPath = @@ -344,26 +401,26 @@ bool RTCState::onProbePacketReceived(const core::ContentObject &probe) { auto path = path_it->second; - path->insertRttSample(rtt); + path->insertRttSample(utils::SteadyTime::Milliseconds(rtt), true); path->receivedNack(); - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + uint64_t now = utils::SteadyTime::nowMs().count(); + + core::ParamsRTC params = RTCState::getProbeParams(probe); - int64_t OWD = now - sender_timestamp; + int64_t OWD = now - params.timestamp; path->insertOwdSample(OWD); - if (last_prod_update_ < sender_timestamp) { - last_production_seq_ = production_seq; - last_prod_update_ = sender_timestamp; - production_rate_ = (double)production_rate; + if (last_prod_update_ < params.timestamp) { + last_production_seq_ = params.prod_seg; + last_prod_update_ = params.timestamp; + production_rate_ = (double)params.prod_rate; } // the producer is responding // we consider it active only if the production rate is not 0 // or the production sequence numner is not 1 - if (production_rate_ != 0 || production_seq != 1) { + if (production_rate_ != 0 || params.prod_seg != 1) { producer_is_active_ = true; } @@ -375,7 +432,7 @@ bool RTCState::onProbePacketReceived(const core::ContentObject &probe) { if (!init_rtt_ && received_probes_ <= INIT_RTT_PROBES) { if (received_probes_ == 1) { // we got the first probe, wait at most INIT_RTT_PROBE_WAIT sec for the - // others + // others. main_path_ = path; setInitRttTimer(INIT_RTT_PROBE_WAIT); } @@ -393,11 +450,21 @@ bool RTCState::onProbePacketReceived(const core::ContentObject &probe) { return true; } -void RTCState::onNewRound(double round_len, bool in_sync) { - // XXX - // here we take into account only the single path case so we assume that we - // don't use two paths in parellel for this single flow +void RTCState::onJumpForward(uint32_t next_seq) { + for (uint32_t seq = highest_seq_received_in_order_ + 1; seq < next_seq; + seq++) { + auto it = pending_interests_.find(seq); + PacketState packet_state = getPacketState(seq); + if (it == pending_interests_.end() && + packet_state != PacketState::RECEIVED && + packet_state != PacketState::DEFINITELY_LOST) { + onLossDetected(seq); + onPacketLost(seq); + } + } +} +void RTCState::onNewRound(double round_len, bool in_sync) { if (path_table_.empty()) return; double bytes_per_sec = @@ -407,7 +474,24 @@ void RTCState::onNewRound(double round_len, bool in_sync) { else received_rate_ = (received_rate_ * MOVING_AVG_ALPHA) + ((1 - MOVING_AVG_ALPHA) * bytes_per_sec); + double fec_bytes_per_sec = + ((double)received_fec_bytes_ * (MILLI_IN_A_SEC / round_len)); + + if (fec_received_rate_ == 0) + fec_received_rate_ = fec_bytes_per_sec; + else + fec_received_rate_ = (fec_received_rate_ * 0.8) + (0.2 * fec_bytes_per_sec); + + double fec_recovered_bytes_per_sec = + ((double)recovered_bytes_with_fec_ * (MILLI_IN_A_SEC / round_len)); + if (fec_recovered_rate_ == 0) + fec_recovered_rate_ = fec_recovered_bytes_per_sec; + else + fec_recovered_rate_ = + (fec_recovered_rate_ * 0.8) + (0.2 * fec_recovered_bytes_per_sec); + +#if 0 // search for an active path. There should be only one active path (meaning a // path that leads to the producer socket -no cache- and from which we are // currently getting data packets) at any time. However it may happen that @@ -428,9 +512,36 @@ void RTCState::onNewRound(double round_len, bool in_sync) { } } } +#endif + + // search for an active path. Is it possible to have multiple path that are + // used at the same time. We use as reference path the one from where we gets + // more packets. This means that the path should have better lantecy or less + // channel losses + + uint32_t last_round_packets = 0; + std::shared_ptr<RTCDataPath> old_main_path = main_path_; + main_path_ = nullptr; + + for (auto it = path_table_.begin(); it != path_table_.end(); it++) { + if (it->second->isActive()) { + uint32_t pkt = it->second->getPacketsLastRound(); + if (pkt > last_round_packets) { + last_round_packets = pkt; + main_path_ = it->second; + } + } + it->second->roundEnd(); + } - // if (in_sync) updateLossRate(); - updateLossRate(); + if (main_path_ == nullptr) main_path_ = old_main_path; + + // in case we get a new main path we reset the stats of the old one. this is + // beacuse, in case we need to switch back we don't what to take decisions on + // old stats that may be outdated. + if (main_path_ != old_main_path) old_main_path->clearRtt(); + + updateLossRate(in_sync); // handle nacks if (!nack_on_last_round_ && received_bytes_ > 0) { @@ -460,6 +571,8 @@ void RTCState::onNewRound(double round_len, bool in_sync) { // reset counters received_bytes_ = 0; + received_fec_bytes_ = 0; + recovered_bytes_with_fec_ = 0; packets_lost_ = 0; definitely_lost_pkt_ = 0; losses_recovered_ = 0; @@ -516,20 +629,16 @@ void RTCState::updatePathStats(const core::ContentObject &content_object, // it means that we are processing an interest // that is not pending - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + uint64_t now = utils::SteadyTime::nowMs().count(); uint64_t RTT = now - interest_sent_time; - path->insertRttSample(RTT); + path->insertRttSample(utils::SteadyTime::Milliseconds(RTT), false); // compute OWD (the first part of the nack and data packet header are the // same, so we cast to data data packet) - struct data_packet_t *packet = - (struct data_packet_t *)content_object.getPayload()->data(); - uint64_t sender_timestamp = packet->getTimestamp(); - int64_t OWD = now - sender_timestamp; + core::ParamsRTC params = RTCState::getDataParams(content_object); + int64_t OWD = now - params.timestamp; path->insertOwdSample(OWD); // compute IAT or set path to producer @@ -543,59 +652,106 @@ void RTCState::updatePathStats(const core::ContentObject &content_object, } } -void RTCState::updateLossRate() { +void RTCState::updateLossRate(bool in_sync) { last_round_loss_rate_ = loss_rate_; loss_rate_ = 0.0; - residual_loss_rate_ = 0.0; uint32_t number_theorically_received_packets_ = highest_seq_received_ - first_seq_in_round_; - // in this case no new packet was recevied after the previuos round, avoid - // division by 0 - if (number_theorically_received_packets_ == 0) return; - // XXX this may be quite inefficient if the rate is high // maybe is better to iterate over the set? - for (uint32_t i = first_seq_in_round_; i < highest_seq_received_; i++) { - auto it = skipped_interests_.find(i); - if (it != skipped_interests_.end()) { + + uint32_t fec_packets = 0; + for (uint32_t i = (first_seq_in_round_ + 1); i < highest_seq_received_; i++) { + PacketState state = getPacketState(i); + if (state == PacketState::SKIPPED) { if (number_theorically_received_packets_ > 0) number_theorically_received_packets_--; - skipped_interests_.erase(it); } + if (indexer_->isFec(i)) fec_packets++; } + if (indexer_->isFec(highest_seq_received_)) fec_packets++; - loss_rate_ = (double)((double)(packets_lost_) / - (double)number_theorically_received_packets_); + // in this case no new packet was received after the previous round, avoid + // division by 0 + if (number_theorically_received_packets_ == 0 && packets_lost_ == 0) return; - if (rounds_ % 15 == 0) max_loss_rate_ = 0; // reset every 3 sec - if (loss_rate_ > max_loss_rate_) max_loss_rate_ = loss_rate_; + if (number_theorically_received_packets_ != 0) + loss_rate_ = (double)((double)(packets_lost_) / + (double)number_theorically_received_packets_); + else + // we didn't receive anything except NACKs that triggered losses + loss_rate_ = 1.0; - if (avg_loss_rate_ == 0) + if (avg_loss_rate_ == -1.0) avg_loss_rate_ = loss_rate_; else avg_loss_rate_ = avg_loss_rate_ * MOVING_AVG_ALPHA + loss_rate_ * (1 - MOVING_AVG_ALPHA); - residual_loss_rate_ = (double)((double)(packets_lost_ - losses_recovered_) / - (double)number_theorically_received_packets_); + // update counters for loss rate per second + total_expected_packets_ += number_theorically_received_packets_; + lost_per_sec_ += packets_lost_; + + if (in_sync) { + // update counters for residual losses + // fec packets are not sent to the app so we don't want to count them here + expected_packets_ += + ((highest_seq_received_ - first_seq_in_round_) - fec_packets); + } else { + packets_sent_to_app_ = 0; + } + + if (rounds_from_last_compute_ >= (MILLI_IN_A_SEC / ROUND_LEN)) { + // compute loss rate per second + if (lost_per_sec_ > total_expected_packets_) + lost_per_sec_ = total_expected_packets_; + + if (total_expected_packets_ == 0) + per_sec_loss_rate_ = 0; + else + per_sec_loss_rate_ = + (double)((double)(lost_per_sec_) / (double)total_expected_packets_); + + loss_history_.pushBack(per_sec_loss_rate_); + max_loss_rate_ = getMaxLoss(); + + if (in_sync && expected_packets_ != 0) { + // compute residual loss rate + if (packets_sent_to_app_ > expected_packets_) { + // this may happen if we get packet from the prev bin that get recovered + // on the current one + packets_sent_to_app_ = expected_packets_; + } + + residual_loss_rate_ = + 1.0 - ((double)packets_sent_to_app_ / (double)expected_packets_); + if (residual_loss_rate_ < 0.0) residual_loss_rate_ = 0.0; + } + + lost_per_sec_ = 0; + total_expected_packets_ = 0; + expected_packets_ = 0; + packets_sent_to_app_ = 0; + rounds_from_last_compute_ = 0; + } + + rounds_from_last_compute_++; +} - if (residual_loss_rate_ < 0) residual_loss_rate_ = 0; +void RTCState::dataToBeReceived(uint32_t seq) { + addToPacketCache(seq, PacketState::TO_BE_RECEIVED); } void RTCState::addRecvOrLost(uint32_t seq, PacketState state) { - if (indexer_->isFec(seq)) { - pending_fec_pkt_--; + auto it = pending_interests_.find(seq); + if (it != pending_interests_.end()) { + pending_interests_.erase(it); + if (indexer_->isFec(seq)) pending_fec_pkt_--; } - pending_interests_.erase(seq); - if (received_or_lost_packets_.size() >= MAX_CACHED_PACKETS) { - received_or_lost_packets_.erase(received_or_lost_packets_.begin()); - } - // notice that it may happen that a packet that we consider lost arrives after - // some time, in this case we simply overwrite the packet state. - received_or_lost_packets_[seq] = state; + addToPacketCache(seq, state); // keep track of the last packet received/lost // without holes. @@ -608,16 +764,25 @@ void RTCState::addRecvOrLost(uint32_t seq, PacketState state) { } else if (seq <= highest_seq_received_in_order_) { // here we do nothing } else if (seq > highest_seq_received_in_order_) { - // 1) there is a gap in the sequence so we do not update largest_in_seq_ - // 2) all the packets from largest_in_seq_ to seq are in - // received_or_lost_packets_ an we upate largest_in_seq_ - // or are FEC packets + // 1) there is a gap in the sequence so we do not update + // highest_seq_received_in_order_ + // 2) all the packets from highest_seq_received_in_order_ to seq are + // received or lost or are fec packetis. In this case we increase + // highest_seq_received_in_order_ until we find an hole in the sequence for (uint32_t i = highest_seq_received_in_order_ + 1; i <= seq; i++) { - if (received_or_lost_packets_.find(i) == - received_or_lost_packets_.end() && - !indexer_->isFec(i)) { - break; + PacketState state = getPacketState(i); + if ((state == PacketState::UNKNOWN || state == PacketState::LOST)) { + if (indexer_->isFec(i)) { + // this is a fec packet and we don't care to receive it + // however we may need to increse the number or lost packets + // XXX: in case we want to use rtx to recover fec packets, + // this may prevent to detect a packet loss and no rtx will be sent + onLossDetected(i); + } else { + // this is a data packet and we need to get it + break; + } } // this packet is in order so we can update the // highest_seq_received_in_order_ @@ -629,9 +794,14 @@ void RTCState::addRecvOrLost(uint32_t seq, PacketState state) { void RTCState::setInitRttTimer(uint32_t wait) { init_rtt_timer_->cancel(); init_rtt_timer_->expires_from_now(std::chrono::milliseconds(wait)); - init_rtt_timer_->async_wait([this](std::error_code ec) { + + std::weak_ptr<RTCState> self = shared_from_this(); + init_rtt_timer_->async_wait([self](const std::error_code &ec) { if (ec) return; - checkInitRttTimer(); + + if (auto ptr = self.lock()) { + ptr->checkInitRttTimer(); + } }); } @@ -639,19 +809,25 @@ void RTCState::checkInitRttTimer() { if (received_probes_ < INIT_RTT_MIN_PROBES_TO_RECV) { // we didn't received enough probes, restart received_probes_ = 0; - rtt_probes_->setProbes(INIT_RTT_PROBE_INTERVAL, INIT_RTT_PROBES); - rtt_probes_->sendProbes(); + probe_handler_->setSuffixRange(MIN_INIT_PROBE_SEQ, MAX_INIT_PROBE_SEQ); + probe_handler_->setProbes(INIT_RTT_PROBE_INTERVAL, INIT_RTT_PROBES); + probe_handler_->sendProbes(); setInitRttTimer(INIT_RTT_PROBE_RESTART); return; } + init_rtt_ = true; main_path_->roundEnd(); - rtt_probes_->setProbes(RTT_PROBE_INTERVAL, 0); - rtt_probes_->sendProbes(); + loss_history_.pushBack(probe_handler_->getProbeLossRate()); + max_loss_rate_ = getMaxLoss(); + + probe_handler_->setSuffixRange(MIN_RTT_PROBE_SEQ, MAX_RTT_PROBE_SEQ); + probe_handler_->setProbes(RTT_PROBE_INTERVAL, 0); + probe_handler_->sendProbes(); // init last_seq_nacked_. skip packets that may come from the cache double prod_rate = getProducerRate(); - double rtt = (double)getRTT() / MILLI_IN_A_SEC; + double rtt = (double)getMinRTT() / MILLI_IN_A_SEC; double packet_size = getAveragePacketSize(); uint32_t pkt_in_rtt_ = std::floor(((prod_rate / packet_size) * rtt) * 0.8); last_seq_nacked_ = last_production_seq_ + pkt_in_rtt_; @@ -659,6 +835,68 @@ void RTCState::checkInitRttTimer() { discovered_rtt_callback_(); } +double RTCState::getMaxLoss() { + if (loss_history_.size() != 0) return loss_history_.begin(); + return 0; +} + +core::ParamsRTC RTCState::getProbeParams(const core::ContentObject &probe) { + uint32_t seq = probe.getName().getSuffix(); + core::ParamsRTC params; + + switch (ProbeHandler::getProbeType(seq)) { + case ProbeType::INIT: { + core::ContentObjectManifest manifest( + const_cast<core::ContentObject &>(probe)); + manifest.decode(); + params = manifest.getParamsRTC(); + break; + } + case ProbeType::RTT: { + struct nack_packet_t *probe_pkt = + (struct nack_packet_t *)probe.getPayload()->data(); + params = core::ParamsRTC{ + .timestamp = probe_pkt->getTimestamp(), + .prod_rate = probe_pkt->getProductionRate(), + .prod_seg = probe_pkt->getProductionSegment(), + }; + break; + } + default: + break; + } + + return params; +} + +core::ParamsRTC RTCState::getDataParams(const core::ContentObject &data) { + core::ParamsRTC params; + + switch (data.getPayloadType()) { + case core::PayloadType::DATA: { + struct data_packet_t *data_pkt = + (struct data_packet_t *)data.getPayload()->data(); + params = core::ParamsRTC{ + .timestamp = data_pkt->getTimestamp(), + .prod_rate = data_pkt->getProductionRate(), + .prod_seg = data.getName().getSuffix(), + }; + break; + } + case core::PayloadType::MANIFEST: { + core::ContentObjectManifest manifest( + const_cast<core::ContentObject &>(data)); + manifest.decode(); + params = manifest.getParamsRTC(); + break; + } + default: + break; + } + + return params; +} + } // namespace rtc } // namespace protocol diff --git a/libtransport/src/protocols/rtc/rtc_state.h b/libtransport/src/protocols/rtc/rtc_state.h index 729ba7a1b..8bf48ccc2 100644 --- a/libtransport/src/protocols/rtc/rtc_state.h +++ b/libtransport/src/protocols/rtc/rtc_state.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -14,13 +14,16 @@ */ #pragma once +#include <core/facade.h> #include <hicn/transport/config.h> #include <hicn/transport/core/asio_wrapper.h> #include <hicn/transport/core/content_object.h> #include <hicn/transport/core/name.h> +#include <hicn/transport/utils/rtc_quality_score.h> #include <protocols/indexer.h> #include <protocols/rtc/probe_handler.h> #include <protocols/rtc/rtc_data_path.h> +#include <utils/max_filter.h> #include <map> #include <set> @@ -31,25 +34,50 @@ namespace protocol { namespace rtc { -enum class PacketState : uint8_t { RECEIVED, LOST, UNKNOWN }; +// packet state +// RECEIVED: the packet was already received +// LOST: the packet is marked as lost but can be recovered +// DEFINITELY_LOST: the packet is lost and cannot be recovered +// TO_BE_RECEIVED: when a packet is received is sent to the FEC decoder. the fec +// decoder may decide to send the packet directly to the app. to avoid +// duplicated the packet is marked with this state +// SKIPPED: an interest that was not sent, only for FEC packets +// UNKNOWN: unknown state +enum class PacketState : uint8_t { + RECEIVED, + TO_BE_RECEIVED, + LOST, + DEFINITELY_LOST, + SKIPPED, + UNKNOWN +}; + +class RTCState : public std::enable_shared_from_this<RTCState> { + using PendingInterestsMap = std::map<uint32_t, uint64_t>; + + private: + const double MAX_CACHED_PACKETS = 8192; // XXX this value may be too small + // for high rate apps -class RTCState : std::enable_shared_from_this<RTCState> { public: using DiscoveredRttCallback = std::function<void()>; public: - RTCState(Indexer *indexer, - ProbeHandler::SendProbeCallback &&rtt_probes_callback, + RTCState(Indexer *indexer, ProbeHandler::SendProbeCallback &&probe_callback, DiscoveredRttCallback &&discovered_rtt_callback, asio::io_service &io_service); ~RTCState(); + // initialization + void initParams(); + // packet events void onSendNewInterest(const core::Name *interest_name); void onTimeout(uint32_t seq, bool lost); void onLossDetected(uint32_t seq); void onRetransmission(uint32_t seq); + void onPossibleLossWithNoRtx(uint32_t seq); void onDataPacketReceived(const core::ContentObject &content_object, bool compute_stats); void onFecPacketReceived(const core::ContentObject &content_object); @@ -57,8 +85,10 @@ class RTCState : std::enable_shared_from_this<RTCState> { bool compute_stats); void onPacketLost(uint32_t seq); void onPacketRecoveredRtx(uint32_t seq); - void onPacketRecoveredFec(uint32_t seq); + void onFecPacketRecoveredRtx(uint32_t seq); + void onPacketRecoveredFec(uint32_t seq, uint32_t size); bool onProbePacketReceived(const core::ContentObject &probe); + void onJumpForward(uint32_t next_seq); // protocol state void onNewRound(double round_len, bool in_sync); @@ -72,10 +102,21 @@ class RTCState : std::enable_shared_from_this<RTCState> { // delay metrics bool isRttDiscovered() const { return init_rtt_; } - uint64_t getRTT() const { + uint64_t getMinRTT() const { if (mainPathIsValid()) return main_path_->getMinRtt(); return 0; } + + uint64_t getAvgRTT() const { + if (mainPathIsValid()) return main_path_->getAvgRtt(); + return 0; + } + + uint64_t getMaxRTT() const { + if (mainPathIsValid()) return main_path_->getMaxRtt(); + return 0; + } + void resetRttStats() { if (mainPathIsValid()) main_path_->clearRtt(); } @@ -98,6 +139,7 @@ class RTCState : std::enable_shared_from_this<RTCState> { uint64_t getInterestSentTime(uint32_t seq) { auto it = pending_interests_.find(seq); if (it != pending_interests_.end()) return it->second; + return 0; } @@ -110,14 +152,15 @@ class RTCState : std::enable_shared_from_this<RTCState> { return pending_interests_.size(); } - PacketState isReceivedOrLost(uint32_t seq) { - auto it = received_or_lost_packets_.find(seq); - if (it != received_or_lost_packets_.end()) return it->second; + PacketState getPacketState(uint32_t seq) { + auto it = packet_cache_.find(seq); + if (it != packet_cache_.end()) return it->second; return PacketState::UNKNOWN; } // loss rate - double getLossRate() const { return loss_rate_; } + double getPerRoundLossRate() const { return loss_rate_; } + double getPerSecondLossRate() const { return per_sec_loss_rate_; } double getAvgLossRate() const { return avg_loss_rate_; } double getMaxLossRate() const { return max_loss_rate_; } double getLastRoundLossRate() const { return last_round_loss_rate_; } @@ -134,15 +177,22 @@ class RTCState : std::enable_shared_from_this<RTCState> { return highest_seq_received_in_order_; } + double getMaxLoss(); + // fec packets uint32_t getReceivedFecPackets() const { return received_fec_pkt_; } uint32_t getPendingFecPackets() const { return pending_fec_pkt_; } // generic stats uint32_t getReceivedBytesInRound() const { return received_bytes_; } + uint32_t getReceivedFecBytesInRound() const { return received_fec_bytes_; } + uint32_t getRecoveredFecBytesInRound() const { + return recovered_bytes_with_fec_; + } uint32_t getReceivedNacksInRound() const { return received_nacks_last_round_; } + uint32_t getReceivedDataInRound() const { return received_data_last_round_; } uint32_t getSentInterestInRound() const { return sent_interests_last_round_; } uint32_t getSentRtxInRound() const { return sent_rtx_last_round_; } @@ -150,6 +200,9 @@ class RTCState : std::enable_shared_from_this<RTCState> { double getAvailableBw() const { return 0.0; }; // TODO double getProducerRate() const { return production_rate_; } double getReceivedRate() const { return received_rate_; } + double getReceivedFecRate() const { return fec_received_rate_; } + double getRecoveredFecRate() const { return fec_recovered_rate_; } + double getAveragePacketSize() const { return avg_packet_size_; } // nacks @@ -162,22 +215,46 @@ class RTCState : std::enable_shared_from_this<RTCState> { // packets from cache double getPacketFromCacheRatio() const { return data_from_cache_rate_; } - std::map<uint32_t, uint64_t>::iterator getPendingInterestsMapBegin() { + PendingInterestsMap::iterator getPendingInterestsMapBegin() { return pending_interests_.begin(); } - std::map<uint32_t, uint64_t>::iterator getPendingInterestsMapEnd() { + PendingInterestsMap::iterator getPendingInterestsMapEnd() { return pending_interests_.end(); } + // quality + uint8_t getQualityScore() { + uint8_t qs = quality_score_.getQualityScore( + getMaxRTT(), std::round(getResidualLossRate() * 100)); + return qs; + } + + // We received a data pkt that will be set to RECEIVED, but first we have to + // go through FEC. We do not want to consider this pkt as recovered, thus we + // set it as TO_BE_RECEIVED. + void dataToBeReceived(uint32_t seq); + + // Extract RTC parameters from probes (init or RTT probes) and data packets. + static core::ParamsRTC getProbeParams(const core::ContentObject &probe); + static core::ParamsRTC getDataParams(const core::ContentObject &data); + private: - void initParams(); + void addToPacketCache(uint32_t seq, PacketState state) { + // this function adds or updates the current state + if (packet_cache_.size() >= MAX_CACHED_PACKETS) { + packet_cache_.erase(packet_cache_.begin()); + } + packet_cache_[seq] = state; + } + + void eraseFromPacketCache(uint32_t seq) { packet_cache_.erase(seq); } // update stats void updateState(); void updateReceivedBytes(const core::ContentObject &content_object); void updatePacketSize(const core::ContentObject &content_object); void updatePathStats(const core::ContentObject &content_object, bool is_nack); - void updateLossRate(); + void updateLossRate(bool in_sycn); void addRecvOrLost(uint32_t seq, PacketState state); @@ -211,22 +288,39 @@ class RTCState : std::enable_shared_from_this<RTCState> { double avg_loss_rate_; double max_loss_rate_; double last_round_loss_rate_; + utils::MaxFilter<double> loss_history_; + + // per second loss rate + uint32_t lost_per_sec_; + uint32_t total_expected_packets_; + double per_sec_loss_rate_; + + // conunters for residual losses + // residual losses are computed every second and are used + // as feedback to the upper levels (e.g application) + uint32_t expected_packets_; + uint32_t packets_sent_to_app_; + uint32_t rounds_from_last_compute_; double residual_loss_rate_; // bw counters uint32_t received_bytes_; + uint32_t received_fec_bytes_; + uint32_t recovered_bytes_with_fec_; double avg_packet_size_; - double production_rate_; // rate communicated by the producer using nacks - double received_rate_; // rate recevied by the consumer + double production_rate_; // rate communicated by the producer using nacks + double received_rate_; // rate recevied by the consumer (only data) + double fec_received_rate_; // fec rate recevied by the consumer + double fec_recovered_rate_; // rate recovered using fec - // nack counter + // nack counters // the bool takes tracks only about the valid nacks (no rtx) and it is used to // switch between the states. Instead received_nacks_last_round_ logs all the // nacks for statistics bool nack_on_last_round_; uint32_t received_nacks_last_round_; - // packets counter + // packets counters uint32_t received_packets_last_round_; uint32_t received_data_last_round_; uint32_t received_data_from_cache_; @@ -234,11 +328,11 @@ class RTCState : std::enable_shared_from_this<RTCState> { uint32_t sent_interests_last_round_; uint32_t sent_rtx_last_round_; - // fec counter + // fec counters uint32_t received_fec_pkt_; uint32_t pending_fec_pkt_; - // round conunters + // round counters uint32_t rounds_; uint32_t rounds_without_nacks_; uint32_t rounds_without_packets_; @@ -261,23 +355,27 @@ class RTCState : std::enable_shared_from_this<RTCState> { // packet received // cache where to store info about the last MAX_CACHED_PACKETS - std::map<uint32_t, PacketState> received_or_lost_packets_; + // these are packets that are received or lost or definitely lost and are not + // anymore in the pending intetest list + std::map<uint32_t, PacketState> packet_cache_; // pending interests - std::map<uint32_t, uint64_t> pending_interests_; + PendingInterestsMap pending_interests_; // indexer Indexer *indexer_; - // skipped interests + // used to keep track of the skipped interests uint32_t last_interest_sent_; - std::unordered_set<uint32_t> skipped_interests_; // probes - std::shared_ptr<ProbeHandler> rtt_probes_; + std::shared_ptr<ProbeHandler> probe_handler_; bool init_rtt_; std::unique_ptr<asio::steady_timer> init_rtt_timer_; + // quality score + RTCQualityScore quality_score_; + // callbacks DiscoveredRttCallback discovered_rtt_callback_; }; diff --git a/libtransport/src/protocols/rtc/rtc_verifier.cc b/libtransport/src/protocols/rtc/rtc_verifier.cc new file mode 100644 index 000000000..29968dd02 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_verifier.cc @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <core/facade.h> +#include <protocols/rtc/rtc_packet.h> +#include <protocols/rtc/rtc_verifier.h> + +namespace transport { +namespace protocol { +namespace rtc { + +RTCVerifier::RTCVerifier(std::shared_ptr<auth::Verifier> verifier, + uint32_t max_unverified_delay) + : verifier_(verifier), max_unverified_delay_(max_unverified_delay) {} + +void RTCVerifier::setState(std::shared_ptr<RTCState> rtc_state) { + rtc_state_ = rtc_state; +} + +void RTCVerifier::setVerifier(std::shared_ptr<auth::Verifier> verifier) { + verifier_ = verifier; +} + +void RTCVerifier::setMaxUnverifiedDelay(uint32_t max_unverified_delay) { + max_unverified_delay_ = max_unverified_delay; +} + +auth::VerificationPolicy RTCVerifier::verify( + core::ContentObject &content_object, bool is_fec) { + uint32_t suffix = content_object.getName().getSuffix(); + core::PayloadType payload_type = content_object.getPayloadType(); + + bool is_probe = ProbeHandler::getProbeType(suffix) != ProbeType::NOT_PROBE; + bool is_nack = !is_probe && content_object.payloadSize() == NACK_HEADER_SIZE; + bool is_manifest = !is_probe && !is_nack && !is_fec && + payload_type == core::PayloadType::MANIFEST; + bool is_data = !is_probe && !is_nack && !is_fec && + payload_type == core::PayloadType::DATA; + + if (is_probe) return verifyProbe(content_object); + if (is_nack) return verifyNack(content_object); + if (is_fec) return verifyFec(content_object); + if (is_data) return verifyData(content_object); + if (is_manifest) return verifyManifest(content_object); + + auth::VerificationPolicy policy = auth::VerificationPolicy::ABORT; + verifier_->callVerificationFailedCallback(suffix, policy); + return policy; +} + +auth::VerificationPolicy RTCVerifier::verifyProbe( + core::ContentObject &content_object) { + switch (ProbeHandler::getProbeType(content_object.getName().getSuffix())) { + case ProbeType::INIT: { + auth::VerificationPolicy policy = verifyManifest(content_object); + if (policy != auth::VerificationPolicy::ACCEPT) { + return policy; + } + return processManifest(content_object); + } + case ProbeType::RTT: + return verifyNack(content_object); + default: + auth::VerificationPolicy policy = auth::VerificationPolicy::ABORT; + verifier_->callVerificationFailedCallback( + content_object.getName().getSuffix(), policy); + return policy; + } +} + +auth::VerificationPolicy RTCVerifier::verifyNack( + core::ContentObject &content_object) { + return verifier_->verifyPackets(&content_object); +} + +auth::VerificationPolicy RTCVerifier::verifyFec( + core::ContentObject &content_object) { + return verifier_->verifyPackets(&content_object); +} + +auth::VerificationPolicy RTCVerifier::verifyData( + core::ContentObject &content_object) { + uint32_t suffix = content_object.getName().getSuffix(); + + if (_is_ah(content_object.getFormat())) { + return verifier_->verifyPackets(&content_object); + } + + unverified_bytes_[suffix] = + content_object.headerSize() + content_object.payloadSize(); + unverified_packets_[suffix] = + content_object.computeDigest(manifest_hash_algo_); + + // An alert is raised when too much packets remain unverified + if (getTotalUnverified() > max_unverified_bytes_) { + unverified_bytes_.clear(); + unverified_packets_.clear(); + + auth::VerificationPolicy policy = auth::VerificationPolicy::ABORT; + verifier_->callVerificationFailedCallback(suffix, policy); + return policy; + } + + return auth::VerificationPolicy::ACCEPT; +} + +auth::VerificationPolicy RTCVerifier::verifyManifest( + core::ContentObject &content_object) { + return verifier_->verifyPackets(&content_object); +} + +auth::VerificationPolicy RTCVerifier::processManifest( + core::ContentObject &content_object) { + uint32_t suffix = content_object.getName().getSuffix(); + + core::ContentObjectManifest manifest(content_object); + manifest.decode(); + + // Update last manifest + if (suffix > last_manifest_) { + last_manifest_ = suffix; + } + + // Extract parameters + manifest_hash_algo_ = manifest.getHashAlgorithm(); + core::ParamsRTC params = manifest.getParamsRTC(); + + if (params.prod_rate > 0) { + max_unverified_bytes_ = static_cast<uint64_t>( + (max_unverified_delay_ / 1000.0) * params.prod_rate); + } + + if (max_unverified_bytes_ == 0 || !rtc_state_) { + auth::VerificationPolicy policy = auth::VerificationPolicy::ABORT; + verifier_->callVerificationFailedCallback(suffix, policy); + return policy; + } + + // Extract hashes + auth::Verifier::SuffixMap suffix_map = + core::ContentObjectManifest::getSuffixMap(&manifest); + + // Return early if the manifest is empty + if (suffix_map.empty()) { + return auth::VerificationPolicy::ACCEPT; + } + + // Remove lost packets from digest map + manifest_digests_.insert(suffix_map.begin(), suffix_map.end()); + for (auto it = manifest_digests_.begin(); it != manifest_digests_.end();) { + if (rtc_state_->getPacketState(it->first) == PacketState::DEFINITELY_LOST) { + unverified_packets_.erase(it->first); + unverified_bytes_.erase(it->first); + it = manifest_digests_.erase(it); + } else { + ++it; + } + } + + // Verify packets + auth::Verifier::PolicyMap policies = + verifier_->verifyHashes(unverified_packets_, manifest_digests_); + + for (const auto &policy : policies) { + switch (policy.second) { + case auth::VerificationPolicy::ACCEPT: { + manifest_digests_.erase(policy.first); + unverified_packets_.erase(policy.first); + unverified_bytes_.erase(policy.first); + break; + } + case auth::VerificationPolicy::UNKNOWN: + break; + case auth::VerificationPolicy::DROP: + case auth::VerificationPolicy::ABORT: + auth::VerificationPolicy p = policy.second; + verifier_->callVerificationFailedCallback(policy.first, p); + return p; + } + } + + return auth::VerificationPolicy::ACCEPT; +} + +void RTCVerifier::onDataRecoveredFec(uint32_t suffix) { + manifest_digests_.erase(suffix); +} + +void RTCVerifier::onJumpForward(uint32_t next_suffix) { + if (next_suffix <= last_manifest_ + 1) { + return; + } + + // When we jump forward in the suffix sequence, we remove packets that + // probably won't be verified. Those packets have a suffix in the range + // [last_manifest_ + 1, next_suffix[. + for (auto it = unverified_packets_.begin(); + it != unverified_packets_.end();) { + if (it->first > last_manifest_) { + unverified_bytes_.erase(it->first); + it = unverified_packets_.erase(it); + } else { + ++it; + } + } +} + +uint32_t RTCVerifier::getTotalUnverified() const { + uint32_t total = 0; + + for (auto bytes : unverified_bytes_) { + if (bytes.second > UINT32_MAX - total) { + total = UINT32_MAX; + break; + } + total += bytes.second; + } + + return total; +} + +uint32_t RTCVerifier::getMaxUnverified() const { return max_unverified_bytes_; } + +} // end namespace rtc +} // end namespace protocol +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_verifier.h b/libtransport/src/protocols/rtc/rtc_verifier.h new file mode 100644 index 000000000..596bd8536 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_verifier.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <core/facade.h> +#include <hicn/transport/auth/verifier.h> +#include <hicn/transport/core/content_object.h> +#include <protocols/rtc/rtc_state.h> + +namespace transport { +namespace protocol { +namespace rtc { + +class RTCVerifier { + public: + RTCVerifier(std::shared_ptr<auth::Verifier> verifier, + uint32_t max_unverified_delay); + + virtual ~RTCVerifier() = default; + + void setState(std::shared_ptr<RTCState> rtc_state); + + void setVerifier(std::shared_ptr<auth::Verifier> verifier); + + void setMaxUnverifiedDelay(uint32_t max_unverified_delay); + + void onDataRecoveredFec(uint32_t suffix); + void onJumpForward(uint32_t next_suffix); + + uint32_t getTotalUnverified() const; + uint32_t getMaxUnverified() const; + + auth::VerificationPolicy verify(core::ContentObject &content_object, + bool is_fec = false); + auth::VerificationPolicy verifyProbe(core::ContentObject &content_object); + auth::VerificationPolicy verifyNack(core::ContentObject &content_object); + auth::VerificationPolicy verifyFec(core::ContentObject &content_object); + auth::VerificationPolicy verifyData(core::ContentObject &content_object); + auth::VerificationPolicy verifyManifest(core::ContentObject &content_object); + + auth::VerificationPolicy processManifest(core::ContentObject &content_object); + + protected: + // The RTC state. + std::shared_ptr<RTCState> rtc_state_; + // The verifier instance. + std::shared_ptr<auth::Verifier> verifier_; + // Hash algorithm used by manifests. + auth::CryptoHashType manifest_hash_algo_; + // The last manifest processed. + auth::Suffix last_manifest_; + // Hold digests extracted from all manifests received. + auth::Verifier::SuffixMap manifest_digests_; + // Hold hashes of all content objects received before they are verified. + auth::Verifier::SuffixMap unverified_packets_; + // Hold number of unverified bytes. + std::unordered_map<auth::Suffix, uint32_t> unverified_bytes_; + // Maximum delay (in ms) for an unverified byte to become verifed. + uint32_t max_unverified_delay_; + // Maximum number of unverified bytes before aborting the connection. + uint64_t max_unverified_bytes_; +}; + +} // end namespace rtc +} // namespace protocol +} // namespace transport diff --git a/libtransport/src/protocols/transport_protocol.cc b/libtransport/src/protocols/transport_protocol.cc index d6954ac37..f1e49ec0b 100644 --- a/libtransport/src/protocols/transport_protocol.cc +++ b/libtransport/src/protocols/transport_protocol.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -25,7 +25,8 @@ using namespace interface; TransportProtocol::TransportProtocol(implementation::ConsumerSocket *icn_socket, Indexer *indexer, Reassembly *reassembly) - : socket_(icn_socket), + : Protocol(), + socket_(icn_socket), indexer_verifier_(indexer), reassembly_(reassembly), fec_decoder_(nullptr), @@ -36,90 +37,82 @@ TransportProtocol::TransportProtocol(implementation::ConsumerSocket *icn_socket, on_interest_satisfied_(VOID_HANDLER), on_content_object_input_(VOID_HANDLER), stats_summary_(VOID_HANDLER), + on_fwd_strategy_(VOID_HANDLER), + on_rec_strategy_(VOID_HANDLER), on_payload_(VOID_HANDLER), - fec_type_(fec::FECType::UNKNOWN), - is_running_(false) { + fec_type_(fec::FECType::UNKNOWN) { socket_->getSocketOption(GeneralTransportOptions::PORTAL, portal_); socket_->getSocketOption(OtherOptions::STATISTICS, &stats_); - // Set this transport protocol as portal's consumer callback - portal_->setConsumerCallback(this); - indexer_verifier_->setReassembly(reassembly_.get()); reassembly->setIndexer(indexer_verifier_.get()); } +TransportProtocol::~TransportProtocol() {} + int TransportProtocol::start() { // If the protocol is already running, return otherwise set as running - if (is_running_) return -1; - - // Get all callbacks references - socket_->getSocketOption(ConsumerCallbacksOptions::INTEREST_RETRANSMISSION, - &on_interest_retransmission_); - socket_->getSocketOption(ConsumerCallbacksOptions::INTEREST_OUTPUT, - &on_interest_output_); - socket_->getSocketOption(ConsumerCallbacksOptions::INTEREST_EXPIRED, - &on_interest_timeout_); - socket_->getSocketOption(ConsumerCallbacksOptions::INTEREST_SATISFIED, - &on_interest_satisfied_); - socket_->getSocketOption(ConsumerCallbacksOptions::CONTENT_OBJECT_INPUT, - &on_content_object_input_); - socket_->getSocketOption(ConsumerCallbacksOptions::STATS_SUMMARY, - &stats_summary_); - socket_->getSocketOption(ConsumerCallbacksOptions::READ_CALLBACK, - &on_payload_); - - socket_->getSocketOption(GeneralTransportOptions::ASYNC_MODE, is_async_); - - // Set it is the first time we schedule an interest - is_first_ = true; - - // Reset the protocol state machine - reset(); - - // Schedule next interests - scheduleNextInterests(); - - is_first_ = false; - - // Set the protocol as running - is_running_ = true; - - if (!is_async_) { - // Start Event loop - portal_->runEventsLoop(); - - // Not running anymore - is_running_ = false; + if (isRunning()) { + return -1; } - return 0; -} - -void TransportProtocol::stop() { - is_running_ = false; + // Start protocol on its own thread + portal_->getThread().add([this]() { + // Get all callbacks references + socket_->getSocketOption(ConsumerCallbacksOptions::INTEREST_RETRANSMISSION, + &on_interest_retransmission_); + socket_->getSocketOption(ConsumerCallbacksOptions::INTEREST_OUTPUT, + &on_interest_output_); + socket_->getSocketOption(ConsumerCallbacksOptions::INTEREST_EXPIRED, + &on_interest_timeout_); + socket_->getSocketOption(ConsumerCallbacksOptions::INTEREST_SATISFIED, + &on_interest_satisfied_); + socket_->getSocketOption(ConsumerCallbacksOptions::CONTENT_OBJECT_INPUT, + &on_content_object_input_); + socket_->getSocketOption(ConsumerCallbacksOptions::STATS_SUMMARY, + &stats_summary_); + socket_->getSocketOption(ConsumerCallbacksOptions::FWD_STRATEGY_CHANGE, + &on_fwd_strategy_); + socket_->getSocketOption(ConsumerCallbacksOptions::REC_STRATEGY_CHANGE, + &on_rec_strategy_); + socket_->getSocketOption(ConsumerCallbacksOptions::READ_CALLBACK, + &on_payload_); + + socket_->getSocketOption(GeneralTransportOptions::ASYNC_MODE, is_async_); + + std::string fec_type_str = ""; + socket_->getSocketOption(GeneralTransportOptions::FEC_TYPE, fec_type_str); + if (fec_type_str != "") { + fec_type_ = fec::FECUtils::fecTypeFromString(fec_type_str.c_str()); + } + + // Set it is the first time we schedule an interest + is_first_ = true; + + // Reset the protocol state machine + reset(); + + // Set this transport protocol as portal's consumer callback + portal_->registerTransportCallback(this); + + // Schedule next interests + scheduleNextInterests(); + + is_first_ = false; + + // Set the protocol as running + setRunning(); + }); - if (!is_async_) { - portal_->stopEventsLoop(); - } else { - portal_->clear(); - } + return 0; } void TransportProtocol::resume() { if (isRunning()) return; - is_running_ = true; + setRunning(); - scheduleNextInterests(); - - if (!is_async_) { - // Start Event loop - portal_->runEventsLoop(); - - // Not running anymore - is_running_ = false; - } + portal_->getThread().tryRunHandlerNow([this]() { scheduleNextInterests(); }); } void TransportProtocol::reset() { @@ -130,7 +123,7 @@ void TransportProtocol::reset() { } } -void TransportProtocol::onContentReassembled(std::error_code ec) { +void TransportProtocol::onContentReassembled(const std::error_code &ec) { stop(); if (!on_payload_) { @@ -153,7 +146,12 @@ void TransportProtocol::sendInterest( uint32_t len) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Sending interest for name " << interest_name; - auto interest = core::PacketManager<>::getInstance().getPacket<Interest>(); + Packet::Format format; + socket_->getSocketOption(interface::GeneralTransportOptions::PACKET_FORMAT, + format); + + auto interest = + core::PacketManager<>::getInstance().getPacket<Interest>(format); interest->setName(interest_name); for (uint32_t i = 0; i < len; i++) { @@ -176,6 +174,14 @@ void TransportProtocol::sendInterest( portal_->sendInterest(std::move(interest)); } +void TransportProtocol::onError(const std::error_code &ec) { + // error from portal: stop socket + stop(); + + // signal error to application + on_payload_->readError(ec); +} + void TransportProtocol::onTimeout(Interest::Ptr &i, const Name &n) { if (TRANSPORT_EXPECT_FALSE(!isRunning())) { return; diff --git a/libtransport/src/protocols/transport_protocol.h b/libtransport/src/protocols/transport_protocol.h index 1008a238b..ad8cf0346 100644 --- a/libtransport/src/protocols/transport_protocol.h +++ b/libtransport/src/protocols/transport_protocol.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -19,10 +19,10 @@ #include <hicn/transport/interfaces/socket_consumer.h> #include <hicn/transport/interfaces/statistics.h> #include <hicn/transport/utils/object_pool.h> -#include <implementation/socket.h> #include <protocols/data_processing_events.h> #include <protocols/fec_base.h> #include <protocols/indexer.h> +#include <protocols/protocol.h> #include <protocols/reassembly.h> #include <array> @@ -38,8 +38,10 @@ class IndexVerificationManager; using ReadCallback = interface::ConsumerSocket::ReadCallback; -class TransportProtocol : public core::Portal::ConsumerCallback, - public ContentObjectProcessingEventCallback { +class TransportProtocol + : public ContentObjectProcessingEventCallback, + public Protocol, + public std::enable_shared_from_this<TransportProtocol> { static constexpr std::size_t interest_pool_size = 4096; friend class ManifestIndexManager; @@ -48,13 +50,11 @@ class TransportProtocol : public core::Portal::ConsumerCallback, TransportProtocol(implementation::ConsumerSocket *icn_socket, Indexer *indexer, Reassembly *reassembly); - virtual ~TransportProtocol() = default; - - TRANSPORT_ALWAYS_INLINE bool isRunning() { return is_running_; } + virtual ~TransportProtocol(); virtual int start(); - virtual void stop(); + using Protocol::stop; virtual void resume(); @@ -69,7 +69,7 @@ class TransportProtocol : public core::Portal::ConsumerCallback, virtual void scheduleNextInterests() = 0; // Events generated by the indexing - virtual void onContentReassembled(std::error_code ec); + virtual void onContentReassembled(const std::error_code &ec); virtual void onPacketDropped(Interest &interest, ContentObject &content_object, const std::error_code &ec) override = 0; @@ -90,7 +90,8 @@ class TransportProtocol : public core::Portal::ConsumerCallback, AllocatorHandler &&allocator_handler) { if (!fec_decoder_) { // Try to get FEC from environment - if (const char *fec_str = std::getenv("TRANSPORT_FEC_TYPE")) { + const char *fec_str = std::getenv("TRANSPORT_FEC_TYPE"); + if (fec_str && (fec_type_ == fec::FECType::UNKNOWN)) { LOG(INFO) << "Using FEC " << fec_str; fec_type_ = fec::FECUtils::fecTypeFromString(fec_str); } @@ -114,14 +115,13 @@ class TransportProtocol : public core::Portal::ConsumerCallback, // Consumer Callback void onContentObject(Interest &i, ContentObject &c) override; void onTimeout(Interest::Ptr &i, const Name &n) override; - void onError(std::error_code ec) override {} + void onError(const std::error_code &ec) override; protected: implementation::ConsumerSocket *socket_; std::unique_ptr<Indexer> indexer_verifier_; std::unique_ptr<Reassembly> reassembly_; std::unique_ptr<fec::ConsumerFEC> fec_decoder_; - std::shared_ptr<core::Portal> portal_; // True if it si the first time we schedule an interest std::atomic<bool> is_first_; interface::TransportStatistics *stats_; @@ -134,14 +134,13 @@ class TransportProtocol : public core::Portal::ConsumerCallback, interface::ConsumerContentObjectCallback *on_content_object_input_; interface::ConsumerContentObjectCallback *on_content_object_; interface::ConsumerTimerCallback *stats_summary_; + interface::StrategyCallback *on_fwd_strategy_; + interface::StrategyCallback *on_rec_strategy_; ReadCallback *on_payload_; bool is_async_; fec::FECType fec_type_; - - private: - std::atomic<bool> is_running_; }; } // end namespace protocol |