diff options
Diffstat (limited to 'libtransport/src/protocols')
82 files changed, 6218 insertions, 2388 deletions
diff --git a/libtransport/src/protocols/CMakeLists.txt b/libtransport/src/protocols/CMakeLists.txt index b763e95e2..51879a9ed 100644 --- a/libtransport/src/protocols/CMakeLists.txt +++ b/libtransport/src/protocols/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2019 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -52,9 +52,15 @@ list(APPEND SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/errors.cc ) -set(RAAQM_CONFIG_INSTALL_PREFIX - ${CMAKE_INSTALL_FULL_SYSCONFDIR}/hicn -) +if (${CMAKE_SYSTEM_NAME} MATCHES Darwin OR ${CMAKE_SYSTEM_NAME} MATCHES Linux) + set(RAAQM_CONFIG_INSTALL_PREFIX + ${CMAKE_INSTALL_FULL_SYSCONFDIR}/hicn + ) +else() + set(RAAQM_CONFIG_INSTALL_PREFIX + ${CMAKE_INSTALL_PREFIX}/etc/hicn + ) +endif() set(raaqm_config_path ${RAAQM_CONFIG_INSTALL_PREFIX}/consumer.conf @@ -67,7 +73,7 @@ set(TRANSPORT_CONFIG install( FILES ${TRANSPORT_CONFIG} - DESTINATION ${CMAKE_INSTALL_FULL_SYSCONFDIR}/hicn + DESTINATION ${RAAQM_CONFIG_INSTALL_PREFIX} COMPONENT ${LIBTRANSPORT_COMPONENT} ) diff --git a/libtransport/src/protocols/byte_stream_reassembly.cc b/libtransport/src/protocols/byte_stream_reassembly.cc index ac36d4e61..b9eaf3bec 100644 --- a/libtransport/src/protocols/byte_stream_reassembly.cc +++ b/libtransport/src/protocols/byte_stream_reassembly.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -36,15 +36,6 @@ ByteStreamReassembly::ByteStreamReassembly( index_(Indexer::invalid_index), download_complete_(false) {} -void ByteStreamReassembly::reassemble( - std::unique_ptr<ContentObjectManifest> &&manifest) { - if (TRANSPORT_EXPECT_TRUE(manifest != nullptr) && read_buffer_->capacity()) { - received_packets_.emplace( - std::make_pair(manifest->getName().getSuffix(), nullptr)); - assembleContent(); - } -} - void ByteStreamReassembly::reassemble(ContentObject &content_object) { if (TRANSPORT_EXPECT_TRUE(read_buffer_->capacity())) { received_packets_.emplace( diff --git a/libtransport/src/protocols/byte_stream_reassembly.h b/libtransport/src/protocols/byte_stream_reassembly.h index 278740bd3..a1f965d5c 100644 --- a/libtransport/src/protocols/byte_stream_reassembly.h +++ b/libtransport/src/protocols/byte_stream_reassembly.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -29,9 +29,6 @@ class ByteStreamReassembly : public Reassembly { protected: void reassemble(core::ContentObject &content_object) override; - void reassemble( - std::unique_ptr<core::ContentObjectManifest> &&manifest) override; - void reassemble(utils::MemBuf &buffer, uint32_t suffix) override; bool copyContent(core::ContentObject &content_object); diff --git a/libtransport/src/protocols/cbr.cc b/libtransport/src/protocols/cbr.cc index 1548cc68d..e3f0f1336 100644 --- a/libtransport/src/protocols/cbr.cc +++ b/libtransport/src/protocols/cbr.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -37,11 +37,11 @@ void CbrTransportProtocol::afterDataUnsatisfied(uint64_t segment) {} void CbrTransportProtocol::afterContentReception( const Interest &interest, const ContentObject &content_object) { auto segment = content_object.getName().getSuffix(); - auto now = utils::SteadyClock::now(); - auto rtt = std::chrono::duration_cast<utils::Microseconds>( - now - interest_timepoints_[segment & mask]); + auto now = utils::SteadyTime::Clock::now(); + auto rtt = utils::SteadyTime::getDurationUs( + interest_timepoints_[segment & mask], now); // Update stats - updateStats(segment, rtt.count(), now); + updateStats(segment, rtt, now); } } // end namespace protocol diff --git a/libtransport/src/protocols/cbr.h b/libtransport/src/protocols/cbr.h index 41cdbc98c..c178dbf60 100644 --- a/libtransport/src/protocols/cbr.h +++ b/libtransport/src/protocols/cbr.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/congestion_window_protocol.h b/libtransport/src/protocols/congestion_window_protocol.h index 36ac6eb17..f9ff208cc 100644 --- a/libtransport/src/protocols/congestion_window_protocol.h +++ b/libtransport/src/protocols/congestion_window_protocol.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/data_processing_events.h b/libtransport/src/protocols/data_processing_events.h index 28732502b..182de3ed8 100644 --- a/libtransport/src/protocols/data_processing_events.h +++ b/libtransport/src/protocols/data_processing_events.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/datagram_reassembly.cc b/libtransport/src/protocols/datagram_reassembly.cc index 069873a52..a04b0eecf 100644 --- a/libtransport/src/protocols/datagram_reassembly.cc +++ b/libtransport/src/protocols/datagram_reassembly.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -29,8 +29,9 @@ void DatagramReassembly::reassemble(core::ContentObject& content_object) { auto read_buffer = content_object.getPayload(); DLOG_IF(INFO, VLOG_IS_ON(4)) << "Size of payload: " << read_buffer->length() << ". Trimming " - << transport_protocol_->transportHeaderLength(); - read_buffer->trimStart(transport_protocol_->transportHeaderLength()); + << transport_protocol_->transportHeaderLength(false); + // here we have only src data packet + read_buffer->trimStart(transport_protocol_->transportHeaderLength(false)); Reassembly::read_buffer_ = std::move(read_buffer); Reassembly::notifyApplication(); } diff --git a/libtransport/src/protocols/datagram_reassembly.h b/libtransport/src/protocols/datagram_reassembly.h index de294df06..cefdca93b 100644 --- a/libtransport/src/protocols/datagram_reassembly.h +++ b/libtransport/src/protocols/datagram_reassembly.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -29,10 +29,6 @@ class DatagramReassembly : public Reassembly { virtual void reassemble(core::ContentObject &content_object) override; void reassemble(utils::MemBuf &buffer, uint32_t suffix) override; virtual void reInitialize() override; - virtual void reassemble( - std::unique_ptr<core::ContentObjectManifest> &&manifest) override { - return; - } bool reassembleUnverified() override { return true; } }; diff --git a/libtransport/src/protocols/errors.cc b/libtransport/src/protocols/errors.cc index 183fcc574..a7dd26e16 100644 --- a/libtransport/src/protocols/errors.cc +++ b/libtransport/src/protocols/errors.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/errors.h b/libtransport/src/protocols/errors.h index 58dadae5a..33d5fbee4 100644 --- a/libtransport/src/protocols/errors.h +++ b/libtransport/src/protocols/errors.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/fec/CMakeLists.txt b/libtransport/src/protocols/fec/CMakeLists.txt index 6d61ae043..8ae0a7360 100644 --- a/libtransport/src/protocols/fec/CMakeLists.txt +++ b/libtransport/src/protocols/fec/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2019 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/fec/fec.cc b/libtransport/src/protocols/fec/fec.cc index 16a04cb98..d2105eb53 100644 --- a/libtransport/src/protocols/fec/fec.cc +++ b/libtransport/src/protocols/fec/fec.cc @@ -44,6 +44,7 @@ #include "fec.h" +#include <assert.h> #include <hicn/transport/portability/platform.h> #include <stdio.h> #include <stdlib.h> @@ -60,72 +61,6 @@ #endif /* - * compatibility stuff - */ -#ifdef MSDOS /* but also for others, e.g. sun... */ -#define NEED_BCOPY -#define bcmp(a, b, n) memcmp(a, b, n) -#endif - -#ifdef ANDROID -#define bcmp(a, b, n) memcmp(a, b, n) -#endif - -#ifdef NEED_BCOPY -#define bcopy(s, d, siz) memcpy((d), (s), (siz)) -#define bzero(d, siz) memset((d), '\0', (siz)) -#endif - -/* - * stuff used for testing purposes only - */ - -#ifdef TEST -#define DEB(x) -#define DDB(x) x -#define DEBUG 0 /* minimal debugging */ -#ifdef MSDOS -#include <time.h> -struct timeval { - unsigned long ticks; -}; -#define gettimeofday(x, dummy) \ - { (x)->ticks = clock(); } -#define DIFF_T(a, b) (1 + 1000000 * (a.ticks - b.ticks) / CLOCKS_PER_SEC) -typedef unsigned long u_long; -typedef unsigned short u_short; -#else /* typically, unix systems */ -#include <sys/time.h> -#define DIFF_T(a, b) \ - (1 + 1000000 * (a.tv_sec - b.tv_sec) + (a.tv_usec - b.tv_usec)) -#endif - -#define TICK(t) \ - { \ - struct timeval x; \ - gettimeofday(&x, NULL); \ - t = x.tv_usec + 1000000 * (x.tv_sec & 0xff); \ - } -#define TOCK(t) \ - { \ - u_long t1; \ - TICK(t1); \ - if (t1 < t) \ - t = 256000000 + t1 - t; \ - else \ - t = t1 - t; \ - if (t == 0) t = 1; \ - } - -u_long ticks[10]; /* vars for timekeeping */ -#else -#define DEB(x) -#define DDB(x) -#define TICK(x) -#define TOCK(x) -#endif /* TEST */ - -/* * You should not need to change anything beyond this point. * The first part of the file implements linear algebra in GF. * @@ -402,31 +337,17 @@ static void matmul(gf *a, gf *b, gf *c, int n, int k, int m) { } } -#ifdef DEBUGG -/* - * returns 1 if the square matrix is identiy - * (only for test) - */ -static int is_identity(gf *m, int k) { - int row, col; - for (row = 0; row < k; row++) - for (col = 0; col < k; col++) - if ((row == col && *m != 1) || (row != col && *m != 0)) - return 0; - else - m++; - return 1; -} -#endif /* debug */ - /* * invert_mat() takes a matrix and produces its inverse * k is the size of the matrix. * (Gauss-Jordan, adapted from Numerical Recipes in C) * Return non-zero if singular. */ -DEB(int pivloops = 0; int pivswaps = 0; /* diagnostic */) +int pivloops = 0; +int pivswaps = 0; /* diagnostic */ static int invert_mat(gf *src, int k) { + assert(k > 0); + gf c, *p; int irow, icol, row, col, i, ix; @@ -436,9 +357,9 @@ static int invert_mat(gf *src, int k) { int *ipiv = (int *)my_malloc(k * sizeof(int), "ipiv"); gf *id_row = NEW_GF_MATRIX(1, k); gf *temp_row = NEW_GF_MATRIX(1, k); - - bzero(id_row, k * sizeof(gf)); - DEB(pivloops = 0; pivswaps = 0; /* diagnostic */) + memset(id_row, '\0', k * sizeof(gf)); + pivloops = 0; + pivswaps = 0; /* diagnostic */ /* * ipiv marks elements already used as pivots. */ @@ -459,7 +380,7 @@ static int invert_mat(gf *src, int k) { for (row = 0; row < k; row++) { if (ipiv[row] != 1) { for (ix = 0; ix < k; ix++) { - DEB(pivloops++;) + pivloops++; if (ipiv[ix] == 0) { if (src[row * k + ix] != 0) { irow = row; @@ -497,12 +418,9 @@ static int invert_mat(gf *src, int k) { fprintf(stderr, "singular matrix 2\n"); goto fail; } - if (c != 1) { /* otherwhise this is a NOP */ - /* - * this is done often , but optimizing is not so - * fruitful, at least in the obvious ways (unrolling) - */ - DEB(pivswaps++;) + + if (c != 1) { + pivswaps++; c = inverse[c]; pivot_row[icol] = 1; for (ix = 0; ix < k; ix++) pivot_row[ix] = gf_mul(c, pivot_row[ix]); @@ -515,7 +433,7 @@ static int invert_mat(gf *src, int k) { * we can optimize the addmul). */ id_row[icol] = 1; - if (bcmp(pivot_row, id_row, k * sizeof(gf)) != 0) { + if (memcmp(pivot_row, id_row, k * sizeof(gf)) != 0) { for (p = src, ix = 0; ix < k; ix++, p += k) { if (ix != icol) { c = p[icol]; @@ -560,6 +478,8 @@ fail: */ int invert_vdm(gf *src, int k) { + assert(k > 0); + int i, j, row, col; gf *b, *c, *p; gf t, xx; @@ -614,14 +534,8 @@ int invert_vdm(gf *src, int k) { static int fec_initialized = 0; static void init_fec() { - TICK(ticks[0]); generate_gf(); - TOCK(ticks[0]); - DDB(fprintf(stderr, "generate_gf took %ldus\n", ticks[0]);) - TICK(ticks[0]); init_mul_table(); - TOCK(ticks[0]); - DDB(fprintf(stderr, "init_mul_table took %ldus\n", ticks[0]);) fec_initialized = 1; } @@ -680,19 +594,14 @@ struct fec_parms *fec_new(int k, int n) { * k*k vandermonde matrix, multiply right the bottom n-k rows * by the inverse, and construct the identity matrix at the top. */ - TICK(ticks[3]); invert_vdm(tmp_m, k); /* much faster than invert_mat */ matmul(tmp_m + k * k, tmp_m, retval->enc_matrix + k * k, n - k, k, k); /* * the upper matrix is I so do not bother with a slow multiply */ - bzero(retval->enc_matrix, k * k * sizeof(gf)); + memset(retval->enc_matrix, '\0', k * k * sizeof(gf)); for (p = retval->enc_matrix, col = 0; col < k; col++, p += k + 1) *p = 1; free(tmp_m); - TOCK(ticks[3]); - - DDB(fprintf(stderr, "--- %ld us to build encoding matrix\n", ticks[3]);) - DEB(pr_matrix(retval->enc_matrix, n, k, "encoding_matrix");) return retval; } @@ -705,13 +614,13 @@ void fec_encode(struct fec_parms *code, gf *src[], gf *fec, int index, int sz) { int i, k = code->k; gf *p; - if (GF_BITS > 8) sz /= 2; + if (GF_BITS > 8) sz /= 2; // NOSONAR if (index < k) - bcopy(src[index], fec, sz * sizeof(gf)); + memcpy(fec, src[index], sz * sizeof(gf)); else if (index < code->n) { p = &(code->enc_matrix[index * k]); - bzero(fec, sz * sizeof(gf)); + memset(fec, '\0', sz * sizeof(gf)); for (i = 0; i < k; i++) addmul(fec, src[i], p[i], sz); } else fprintf(stderr, "Invalid index %d (max %d)\n", index, code->n - 1); @@ -733,22 +642,13 @@ static int shuffle(gf *pkt[], int index[], int k) { int c = index[i]; if (index[c] == c) { - DEB(fprintf(stderr, "\nshuffle, error at %d\n", i);) + fprintf(stderr, "\nshuffle, error at %d\n", i); return 1; } SWAP(index[i], index[c], int); SWAP(pkt[i], pkt[c], gf *); } } - DEB(/* just test that it works... */ - for (i = 0; i < k; i++) { - if (index[i] < k && index[i] != i) { - fprintf(stderr, "shuffle: after\n"); - for (i = 0; i < k; i++) fprintf(stderr, "%3d ", index[i]); - fprintf(stderr, "\n"); - return 1; - } - }) return 0; } @@ -757,20 +657,16 @@ static int shuffle(gf *pkt[], int index[], int k) { * indexes. The matrix must be already allocated as * a vector of k*k elements, in row-major order */ -static gf *build_decode_matrix(struct fec_parms *code, gf *pkt[], int index[]) { +static gf *build_decode_matrix(struct fec_parms *code, int index[]) { int i, k = code->k; gf *p, *matrix = NEW_GF_MATRIX(k, k); - TICK(ticks[9]); for (i = 0, p = matrix; i < k; i++, p += k) { -#if 1 /* this is simply an optimization, not very useful indeed */ if (index[i] < k) { - bzero(p, k * sizeof(gf)); + memset(p, '\0', k * sizeof(gf)); p[i] = 1; - } else -#endif - if (index[i] < code->n) - bcopy(&(code->enc_matrix[index[i] * k]), p, k * sizeof(gf)); + } else if (index[i] < code->n) + memcpy(p, &(code->enc_matrix[index[i] * k]), k * sizeof(gf)); else { fprintf(stderr, "decode: invalid index %d (max %d)\n", index[i], code->n - 1); @@ -778,12 +674,10 @@ static gf *build_decode_matrix(struct fec_parms *code, gf *pkt[], int index[]) { return NULL; } } - TICK(ticks[9]); if (invert_mat(matrix, k)) { free(matrix); matrix = NULL; } - TOCK(ticks[9]); return matrix; } @@ -800,39 +694,29 @@ static gf *build_decode_matrix(struct fec_parms *code, gf *pkt[], int index[]) { */ int fec_decode(struct fec_parms *code, gf *pkt[], int index[], int sz) { gf *m_dec; - gf **new_pkt; + gf **new_pkt = nullptr; int row, col, k = code->k; + int i = 0; - if (GF_BITS > 8) sz /= 2; + if (GF_BITS > 8) sz /= 2; // NOSONAR if (shuffle(pkt, index, k)) /* error if true */ return 1; - m_dec = build_decode_matrix(code, pkt, index); + m_dec = build_decode_matrix(code, index); if (m_dec == NULL) return 1; /* error */ /* * do the actual decoding */ - new_pkt = (gf **)my_malloc(k * sizeof(gf *), "new pkt pointers"); + new_pkt = pkt + k; for (row = 0; row < k; row++) { if (index[row] >= k) { - new_pkt[row] = (gf *)my_malloc(sz * sizeof(gf), "new pkt buffer"); - bzero(new_pkt[row], sz * sizeof(gf)); + memset(new_pkt[i], '\0', sz * sizeof(gf)); for (col = 0; col < k; col++) - addmul(new_pkt[row], pkt[col], m_dec[row * k + col], sz); - } - } - /* - * move pkts to their final destination - */ - for (row = 0; row < k; row++) { - if (index[row] >= k) { - bcopy(new_pkt[row], pkt[row], sz * sizeof(gf)); - free(new_pkt[row]); + addmul(new_pkt[i], pkt[col], m_dec[row * k + col], sz); + i++; } } - free(new_pkt); free(m_dec); - return 0; -} +}
\ No newline at end of file diff --git a/libtransport/src/protocols/fec/rely.cc b/libtransport/src/protocols/fec/rely.cc index 7a30a62e2..9e0a06dd8 100644 --- a/libtransport/src/protocols/fec/rely.cc +++ b/libtransport/src/protocols/fec/rely.cc @@ -23,38 +23,39 @@ namespace transport { namespace protocol { namespace fec { -RelyEncoder::RelyEncoder(uint32_t k, uint32_t n, uint32_t seq_offset) +RelyEncoder::RelyEncoder(uint32_t k, uint32_t n, uint32_t /* seq_offset */) : RelyBase(k, n) { configure(kmtu, ktimeout, kmax_stream_size); set_repair_trigger(k_, n_ - k_, n_ - k_); } void RelyEncoder::onPacketProduced(core::ContentObject &content_object, - uint32_t offset) { + uint32_t offset, uint32_t metadata) { // Get pointer to payload, leaving space to insert FEC header. // TODO Check if this additional header is really needed. - auto data = content_object.writableData() + offset - sizeof(fec_header); - auto length = content_object.length() - offset + sizeof(fec_header); + auto data = content_object.writableData() + offset - sizeof(fec_metadata); + auto length = content_object.length() - offset + sizeof(fec_metadata); // Check packet length does not exceed maximum length supported by the // encoder (otherwise segmentation would take place). - assert(length < max_packet_bytes()); + DCHECK(length < max_packet_bytes()); DLOG_IF(INFO, VLOG_IS_ON(4)) - << "Encoding packet of length " << length - sizeof(fec_header); + << "Encoding packet of length " << length - sizeof(fec_metadata); - // Get the suffix. With rely we need to write it in the fec_header in order to - // be able to recognize the seq number upon recovery. + // Get the suffix. With rely we need to write it in the fec_metadata in order + // to be able to recognize the seq number upon recovery. auto suffix = content_object.getName().getSuffix(); DLOG_IF(INFO, VLOG_IS_ON(4)) << "Producing packet " << suffix << " (index == " << current_index_ << ")"; - // Consume payload. Add fec_header in front before feeding payload to encoder, - // and copy original content of packet - fec_header *h = reinterpret_cast<fec_header *>(data); - fec_header copy = *h; + // Consume payload. Add fec_metadata in front before feeding payload to + // encoder, and copy original content of packet + fec_metadata *h = reinterpret_cast<fec_metadata *>(data); + fec_metadata copy = *h; h->setSeqNumberBase(suffix); + h->setMetadataBase(metadata); auto packets = consume(data, length, getCurrentTime()); - assert(packets == 1); + DCHECK(packets == 1); // Update packet counter current_index_ += packets; @@ -62,7 +63,7 @@ void RelyEncoder::onPacketProduced(core::ContentObject &content_object, // Restore original packet content and increment data pointer to the correct // position *h = copy; - data += sizeof(fec_header); + data += sizeof(fec_metadata); // Check position of this packet inside N size block auto i = current_index_ % n_; @@ -74,24 +75,24 @@ void RelyEncoder::onPacketProduced(core::ContentObject &content_object, // TODO Optimize it by copying only the RELY header // Be sure encoder can produce - assert(can_produce()); + DCHECK(can_produce()); // Check new payload size and make sure it fits in packet buffer auto new_payload_size = produce_bytes(); - int difference = new_payload_size - length; + int difference = (int)(new_payload_size - length); - assert(difference > 0); - assert(content_object.ensureCapacity(difference)); + DCHECK(difference > 0); + DCHECK(content_object.ensureCapacity(difference)); // Update length DLOG_IF(INFO, VLOG_IS_ON(4)) << "The packet length will be incremented by " - << difference + sizeof(fec_header); - content_object.append(difference + sizeof(fec_header)); + << difference + sizeof(fec_metadata); + content_object.append(difference + sizeof(fec_metadata)); content_object.updateLength(); // Make sure we got a source packet, otherwise we would put a repair symbol // in a source packet - assert(rely::packet_is_systematic(produce_data())); + DCHECK(rely::packet_is_systematic(produce_data())); // Copy rely packet replacing old source packet. std::memcpy(data, produce_data(), new_payload_size); @@ -111,7 +112,7 @@ void RelyEncoder::onPacketProduced(core::ContentObject &content_object, while (can_produce()) { // The current index MUST be k_, because we enforce n - k repair to be // produced after k sources - assert(current_index_ == k_); + DCHECK(current_index_ == k_); buffer packet; if (!buffer_callback_) { @@ -130,7 +131,7 @@ void RelyEncoder::onPacketProduced(core::ContentObject &content_object, std::memcpy(packet->writableData(), produce_data(), produce_bytes()); // Push symbol in repair_packets - packets_.emplace_back(0, std::move(packet)); + packets_.emplace_back(0, metadata, std::move(packet)); // Advance the encoder produce_next(); @@ -143,7 +144,7 @@ void RelyEncoder::onPacketProduced(core::ContentObject &content_object, // If we have generated repair symbols, let's notify caller via the installed // callback if (packets_.size()) { - assert(packets_.size() == n_ - k_); + DCHECK(packets_.size() == n_ - k_); fec_callback_(packets_); packets_.clear(); current_index_ = 0; @@ -156,7 +157,7 @@ RelyDecoder::RelyDecoder(uint32_t k, uint32_t n, uint32_t seq_offset) } void RelyDecoder::onDataPacket(core::ContentObject &content_object, - uint32_t offset) { + uint32_t offset, uint32_t metadata) { // Adjust pointers to point to packet payload auto data = content_object.writableData() + offset; auto size = content_object.length() - offset; @@ -164,30 +165,38 @@ void RelyDecoder::onDataPacket(core::ContentObject &content_object, // Pass payload to decoder consume(data, size, getCurrentTime()); + producePackets(); +} + +void RelyDecoder::producePackets() { // Drain decoder if possible while (can_produce()) { - // Get size of decoded packet - auto size = produce_bytes(); - - // Get buffer to copy packet in - auto packet = core::PacketManager<>::getInstance().getMemBuf(); + auto fec_header_size = sizeof(fec_metadata); + auto payload_size = produce_bytes() - sizeof(fec_metadata); - // Copy buffer - packet->append(size); - std::memcpy(packet->writableData(), produce_data(), size); + buffer packet; + if (!buffer_callback_) { + packet = core::PacketManager<>::getInstance().getMemBuf(); + packet->append(payload_size); + } else { + packet = buffer_callback_(payload_size); + } // Read seq number - fec_header *h = reinterpret_cast<fec_header *>(packet->writableData()); + const fec_metadata *h = + reinterpret_cast<const fec_metadata *>(produce_data()); uint32_t index = h->getSeqNumberBase(); + uint32_t metadata = h->getMetadataBase(); DLOG_IF(INFO, VLOG_IS_ON(4)) << "The index written in the packet is " << index; - // Remove FEC header - packet->trimStart(sizeof(fec_header)); + // Copy payload + std::memcpy(packet->writableData(), produce_data() + fec_header_size, + payload_size); // Save packet in buffer - packets_.emplace_back(index, std::move(packet)); + packets_.emplace_back(index, metadata, std::move(packet)); // Advance to next packet produce_next(); @@ -198,8 +207,28 @@ void RelyDecoder::onDataPacket(core::ContentObject &content_object, fec_callback_(packets_); packets_.clear(); } + + flushOutOfOrder(); +} + +void RelyDecoder::flushOutOfOrder() { + if (flush_timer_ == nullptr) return; + flush_timer_->cancel(); + + if (has_upcoming_flush()) { + flush_timer_->expires_from_now(std::chrono::milliseconds( + std::max((int64_t)0, upcoming_flush(getCurrentTime())))); + + flush_timer_->async_wait([this](const std::error_code &ec) { + if (ec) return; + if (has_upcoming_flush()) { + flush(getCurrentTime()); + producePackets(); + } + }); + } } } // namespace fec } // namespace protocol -} // namespace transport
\ No newline at end of file +} // namespace transport diff --git a/libtransport/src/protocols/fec/rely.h b/libtransport/src/protocols/fec/rely.h index bfbdb30bc..cc81222b2 100644 --- a/libtransport/src/protocols/fec/rely.h +++ b/libtransport/src/protocols/fec/rely.h @@ -15,6 +15,7 @@ #pragma once +#include <hicn/transport/portability/endianess.h> #include <hicn/transport/utils/chrono_typedefs.h> #include <hicn/transport/utils/membuf.h> #include <protocols/fec/fec_info.h> @@ -33,8 +34,12 @@ namespace fec { * @brief Table of used codes. */ #define foreach_rely_fec_type \ + _(Rely, 1, 2) \ _(Rely, 1, 3) \ + _(Rely, 1, 4) \ _(Rely, 2, 3) \ + _(Rely, 2, 6) \ + _(Rely, 3, 9) \ _(Rely, 4, 5) \ _(Rely, 4, 6) \ _(Rely, 4, 7) \ @@ -74,11 +79,25 @@ class RelyBase : public virtual FECBase { * decoding operations. It may be removed once we know the meaning of the * fields in the rely header. */ - struct fec_header { + class fec_metadata { + public: + void setSeqNumberBase(uint32_t suffix) { + seq_number = portability::host_to_net(suffix); + } + uint32_t getSeqNumberBase() const { + return portability::net_to_host(seq_number); + } + + void setMetadataBase(uint32_t value) { + metadata = portability::host_to_net(value); + } + uint32_t getMetadataBase() const { + return portability::net_to_host(metadata); + } + + private: uint32_t seq_number; - - void setSeqNumberBase(uint32_t suffix) { seq_number = htonl(suffix); } - uint32_t getSeqNumberBase() { return ntohl(seq_number); } + uint32_t metadata; }; /** @@ -112,21 +131,20 @@ class RelyBase : public virtual FECBase { #if RELY_DEBUG return time_++; #else - auto _time = utils::SteadyClock::now().time_since_epoch(); - auto time = std::chrono::duration_cast<utils::Milliseconds>(_time).count(); - return time; + return utils::SteadyTime::nowMs().count(); #endif } - protected: uint32_t k_; uint32_t n_; + std::uint32_t seq_offset_; + /** * @brief Vector of packets to be passed to caller callbacks. For encoder it * will contain the repair packets, for decoder the recovered sources. */ - std::vector<std::pair<uint32_t, buffer>> packets_; + BufferArray packets_; /** * @brief Current index to be used for local packet count. @@ -142,50 +160,56 @@ class RelyBase : public virtual FECBase { * @brief The Rely Encoder implementation. * */ -class RelyEncoder : private RelyBase, - private rely::encoder, - public ProducerFEC { +class RelyEncoder : RelyBase, rely::encoder, public ProducerFEC { public: RelyEncoder(uint32_t k, uint32_t n, uint32_t seq_offset = 0); /** * Producers will call this function when they produce a data packet. */ - void onPacketProduced(core::ContentObject &content_object, - uint32_t offset) override; + void onPacketProduced(core::ContentObject &content_object, uint32_t offset, + uint32_t metadata = FECBase::INVALID_METADATA) override; /** * @brief Get the fec header size, if added to source packets + * there is not need to distinguish between source and FEC packets here */ - std::size_t getFecHeaderSize() override { - return header_bytes() + sizeof(fec_header) + 4; + std::size_t getFecHeaderSize(bool isFEC) override { + return header_bytes() + sizeof(fec_metadata) + 4; } - void reset() override {} + void reset() override { + // Nothing to do here + } }; -class RelyDecoder : private RelyBase, - private rely::decoder, - public ConsumerFEC { +class RelyDecoder : RelyBase, rely::decoder, public ConsumerFEC { public: RelyDecoder(uint32_t k, uint32_t n, uint32_t seq_offset = 0); /** * Consumers will call this function when they receive a data packet */ - void onDataPacket(core::ContentObject &content_object, - uint32_t offset) override; + void onDataPacket(core::ContentObject &content_object, uint32_t offset, + uint32_t metadata = FECBase::INVALID_METADATA) override; /** * @brief Get the fec header size, if added to source packets + * there is not need to distinguish between source and FEC packets here */ - std::size_t getFecHeaderSize() override { - return header_bytes() + sizeof(fec_header); + std::size_t getFecHeaderSize(bool isFEC) override { + return header_bytes() + sizeof(fec_metadata); + } + + void reset() override { + // Nothing to do here } - void reset() override {} + private: + void producePackets(); + void flushOutOfOrder(); }; } // namespace fec } // namespace protocol -} // namespace transport
\ No newline at end of file +} // namespace transport diff --git a/libtransport/src/protocols/fec/rs.cc b/libtransport/src/protocols/fec/rs.cc index 2c23d515d..d42740c32 100644 --- a/libtransport/src/protocols/fec/rs.cc +++ b/libtransport/src/protocols/fec/rs.cc @@ -46,23 +46,26 @@ bool BlockCode::addRepairSymbol(const fec::buffer &packet, uint32_t i, to_decode_ = true; DLOG_IF(INFO, VLOG_IS_ON(4)) << "Adding symbol of size " << packet->length(); return addSymbol(packet, i, offset, - packet->length() - sizeof(fec_header) - offset); + packet->length() - sizeof(fec_header) - offset, + FECBase::INVALID_METADATA); } bool BlockCode::addSourceSymbol(const fec::buffer &packet, uint32_t i, - uint32_t offset) { + uint32_t offset, uint32_t metadata) { DLOG_IF(INFO, VLOG_IS_ON(4)) << "Adding source symbol of size " << packet->length() << ", offset " << offset; - return addSymbol(packet, i, offset, packet->length() - offset); + return addSymbol(packet, i, offset, packet->length() - offset, metadata); } bool BlockCode::addSymbol(const fec::buffer &packet, uint32_t i, - uint32_t offset, std::size_t size) { + uint32_t offset, std::size_t size, + uint32_t metadata) { if (size > max_buffer_size_) { max_buffer_size_ = size; } - operator[](current_block_size_++) = std::make_tuple(i, packet, offset); + operator[](current_block_size_) = RSBufferInfo(offset, i, metadata, packet); + current_block_size_++; if (current_block_size_ >= k_) { if (to_decode_) { @@ -80,12 +83,13 @@ bool BlockCode::addSymbol(const fec::buffer &packet, uint32_t i, void BlockCode::encode() { gf *data[n_]; - uint32_t base = std::get<0>(operator[](0)); + uint32_t base = operator[](0).getIndex(); // Set packet length in first 2 bytes for (uint32_t i = 0; i < k_; i++) { - auto &packet = std::get<1>(operator[](i)); - auto offset = std::get<2>(operator[](i)); + auto &packet = operator[](i).getBuffer(); + auto offset = operator[](i).getOffset(); + auto metadata_base = operator[](i).getMetadata(); auto ret = packet->ensureCapacityAndFillUnused(max_buffer_size_ + offset, 0); @@ -98,10 +102,11 @@ void BlockCode::encode() { // Buffers should hold 2 *after* the padding, in order to be // able to set the length for the encoding operation. // packet->trimStart(offset); - uint16_t *length = reinterpret_cast<uint16_t *>(packet->writableData() + - max_buffer_size_ + offset); + fec_metadata *metadata = reinterpret_cast<fec_metadata *>( + packet->writableData() + max_buffer_size_ + offset); auto buffer_length = packet->length() - offset; - *length = htons(buffer_length); + metadata->setPacketLength(buffer_length); + metadata->setMetadataBase(metadata_base); DLOG_IF(INFO, VLOG_IS_ON(4)) << "Current buffer size: " << packet->length(); @@ -109,7 +114,7 @@ void BlockCode::encode() { } // Finish to fill source block with the buffers to hold the repair symbols - auto length = max_buffer_size_ + sizeof(fec_header) + LEN_SIZE_BYTES; + auto length = max_buffer_size_ + sizeof(fec_header) + METADATA_BYTES; for (uint32_t i = k_; i < n_; i++) { buffer packet; if (!params_.buffer_callback_) { @@ -133,19 +138,21 @@ void BlockCode::encode() { DLOG_IF(INFO, VLOG_IS_ON(4)) << "Current symbol size: " << packet->length(); data[i] = packet->writableData(); - operator[](i) = std::make_tuple(i, std::move(packet), uint32_t(0)); + operator[](i) = RSBufferInfo(uint32_t(0), i, FECBase::INVALID_METADATA, + std::move(packet)); } // Generate repair symbols and put them in corresponding buffers DLOG_IF(INFO, VLOG_IS_ON(4)) << "Calling encode with max_buffer_size_ = " << max_buffer_size_; for (uint32_t i = k_; i < n_; i++) { - fec_encode(code_, data, data[i], i, max_buffer_size_ + LEN_SIZE_BYTES); + fec_encode(code_, data, data[i], i, + (int)(max_buffer_size_ + METADATA_BYTES)); } // Re-include header in repair packets for (uint32_t i = k_; i < n_; i++) { - auto &packet = std::get<1>(operator[](i)); + auto &packet = operator[](i).getBuffer(); packet->prepend(sizeof(fec_header)); DLOG_IF(INFO, VLOG_IS_ON(4)) << "Produced repair symbol of size = " << packet->length(); @@ -153,16 +160,31 @@ void BlockCode::encode() { } void BlockCode::decode() { - gf *data[k_]; + gf *data[n_]; uint32_t index[k_]; + buffer aux_fec_packets[n_ - k_]; + // FEC packet number k0 + uint32_t k0 = 0; + + // Reorder block by index with in-place sorting + for (uint32_t i = 0; i < k_;) { + uint32_t idx = operator[](i).getIndex(); + if (idx >= k_ || idx == i) { + i++; + } else { + std::swap(operator[](i), operator[](idx)); + } + } for (uint32_t i = 0; i < k_; i++) { - auto &packet = std::get<1>(operator[](i)); - index[i] = std::get<0>(operator[](i)); - auto offset = std::get<2>(operator[](i)); + auto &packet = operator[](i).getBuffer(); + index[i] = operator[](i).getIndex(); + auto offset = operator[](i).getOffset(); + auto metadata_base = operator[](i).getMetadata(); sorted_index_[i] = index[i]; if (index[i] < k_) { + operator[](i).setReceived(); DLOG_IF(INFO, VLOG_IS_ON(4)) << "DECODE SOURCE - index " << index[i] << " - Current buffer size: " << packet->length(); @@ -173,49 +195,53 @@ void BlockCode::decode() { // able to set the length for the encoding operation packet->trimStart(offset); packet->ensureCapacityAndFillUnused(max_buffer_size_, 0); - uint16_t *length = reinterpret_cast<uint16_t *>( - packet->writableData() + max_buffer_size_ - LEN_SIZE_BYTES); - - *length = htons(packet->length()); + fec_metadata *metadata = reinterpret_cast<fec_metadata *>( + packet->writableData() + max_buffer_size_ - METADATA_BYTES); + metadata->setPacketLength(packet->length()); + metadata->setMetadataBase(metadata_base); } else { DLOG_IF(INFO, VLOG_IS_ON(4)) << "DECODE SYMBOL - index " << index[i] << " - Current buffer size: " << packet->length(); packet->trimStart(sizeof(fec_header) + offset); + aux_fec_packets[k0] = core::PacketManager<>::getInstance().getMemBuf(); + data[k_ + k0] = aux_fec_packets[k0]->writableData(); + k0++; } - data[i] = packet->writableData(); } - // We decode the source block DLOG_IF(INFO, VLOG_IS_ON(4)) << "Calling decode with max_buffer_size_ = " << max_buffer_size_; - fec_decode(code_, data, reinterpret_cast<int *>(index), max_buffer_size_); - // Find the index in the block for recovered packets - for (uint32_t i = 0; i < k_; i++) { - if (index[i] != i) { - for (uint32_t j = 0; j < k_; j++) - if (sorted_index_[j] == uint32_t(index[i])) { - sorted_index_[j] = i; - } - } - } + fec_decode(code_, data, reinterpret_cast<int *>(index), + (int)max_buffer_size_); - // Reorder block by index with in-place sorting - for (uint32_t i = 0; i < k_; i++) { - for (uint32_t j = sorted_index_[i]; j != i; j = sorted_index_[i]) { - std::swap(sorted_index_[j], sorted_index_[i]); - std::swap(operator[](j), operator[](i)); + // Find the index in the block for recovered packets + for (uint32_t i = 0, j = 0; i < k_; i++) { + if (index[i] >= k_) { + operator[](i).setBuffer(aux_fec_packets[j++]); + operator[](i).setIndex(i); } } // Adjust length according to the one written in the source packet for (uint32_t i = 0; i < k_; i++) { - auto &packet = std::get<1>(operator[](i)); - uint16_t *length = reinterpret_cast<uint16_t *>( - packet->writableData() + max_buffer_size_ - LEN_SIZE_BYTES); - packet->setLength(ntohs(*length)); + auto &packet = operator[](i).getBuffer(); + fec_metadata *metadata = reinterpret_cast<fec_metadata *>( + packet->writableData() + max_buffer_size_ - METADATA_BYTES); + DCHECK(metadata->getPacketLength() <= packet->capacity()); + // Adjust buffer length + packet->setLength(metadata->getPacketLength()); + // Adjust metadata + operator[](i).setMetadata(metadata->getMetadataBase()); + + // reset the point to the beginning of the packets for all received packets + if (operator[](i).getReceived()) { + auto &packet = operator[](i).getBuffer(); + auto offset = operator[](i).getOffset(); + packet->prepend(offset); + } } } @@ -252,12 +278,11 @@ RSEncoder::RSEncoder(uint32_t k, uint32_t n, uint32_t seq_offset) source_block_(k_, n_, seq_offset_, current_code_, *this) {} void RSEncoder::consume(const fec::buffer &packet, uint32_t index, - uint32_t offset) { - if (!source_block_.addSourceSymbol(packet, index, offset)) { - std::vector<std::pair<uint32_t, buffer>> repair_packets; + uint32_t offset, uint32_t metadata) { + if (!source_block_.addSourceSymbol(packet, index, offset, metadata)) { + fec::BufferArray repair_packets; for (uint32_t i = k_; i < n_; i++) { - repair_packets.emplace_back(std::move(std::get<0>(source_block_[i])), - std::move(std::get<1>(source_block_[i]))); + repair_packets.emplace_back(std::move(source_block_[i])); } fec_callback_(repair_packets); @@ -265,9 +290,9 @@ void RSEncoder::consume(const fec::buffer &packet, uint32_t index, } void RSEncoder::onPacketProduced(core::ContentObject &content_object, - uint32_t offset) { + uint32_t offset, uint32_t metadata) { consume(content_object.shared_from_this(), - content_object.getName().getSuffix(), offset); + content_object.getName().getSuffix(), offset, metadata); } RSDecoder::RSDecoder(uint32_t k, uint32_t n, uint32_t seq_offset) @@ -276,10 +301,15 @@ RSDecoder::RSDecoder(uint32_t k, uint32_t n, uint32_t seq_offset) void RSDecoder::recoverPackets(SourceBlocks::iterator &src_block_it) { DLOG_IF(INFO, VLOG_IS_ON(4)) << "recoverPackets for " << k_; auto &src_block = src_block_it->second; - std::vector<std::pair<uint32_t, buffer>> source_packets(k_); + auto base_index = src_block_it->first; + BufferArray source_packets(k_); + + // Iterate over packets in the block and adjust indexed accordingly. This must + // be done because indexes are from 0 to (n - k - 1), but we need indexes from + // base_index to base_index + (n - k - 1) for (uint32_t i = 0; i < src_block.getK(); i++) { - source_packets[i] = std::make_pair(src_block_it->first + i, - std::move(std::get<1>(src_block[i]))); + src_block[i].setIndex(base_index + src_block[i].getIndex()); + source_packets[i] = FECBufferInfo(std::move(src_block[i])); } setProcessed(src_block_it->first); @@ -296,9 +326,9 @@ void RSDecoder::recoverPackets(SourceBlocks::iterator &src_block_it) { } void RSDecoder::consumeSource(const fec::buffer &packet, uint32_t index, - uint32_t offset) { + uint32_t offset, uint32_t metadata) { // Normalize index - assert(index >= seq_offset_); + DCHECK(index >= seq_offset_); auto i = (index - seq_offset_) % n_; // Get base @@ -324,16 +354,19 @@ void RSDecoder::consumeSource(const fec::buffer &packet, uint32_t index, // protocol that may come in the future. auto it = src_blocks_.find(base); if (it != src_blocks_.end()) { - auto ret = it->second.addSourceSymbol(packet, i, offset); + auto ret = it->second.addSourceSymbol(packet, i, offset, metadata); if (!ret) { recoverPackets(it); } } else { DLOG_IF(INFO, VLOG_IS_ON(4)) << "Adding to parked source packets"; - auto ret = parked_packets_.emplace( - base, std::vector<std::pair<buffer, uint32_t>>()); - ret.first->second.emplace_back(packet, i); + auto ret = parked_packets_.emplace(base, BufferInfoArray()); + ret.first->second.emplace_back(offset, i, metadata, packet); + /** + * If we reached k source packets, we do not have any missing packet to + * recover via FEC. Delete the block. + */ if (ret.first->second.size() >= k_) { setProcessed(ret.first->first); parked_packets_.erase(ret.first); @@ -356,8 +389,8 @@ void RSDecoder::consumeRepair(const fec::buffer &packet, uint32_t offset) { DLOG_IF(INFO, VLOG_IS_ON(4)) << "Decoder consume called for repair symbol. BASE = " << base - << ", index = " << base + i << " and i = " << i << ". K=" << k - << ", N=" << n; + << ", index = " << base + i << " and i = " << (int)i << ". K=" << (int)k + << ", N=" << (int)n; // check if a source block already exist for this symbol auto it = src_blocks_.find(base); @@ -380,8 +413,9 @@ void RSDecoder::consumeRepair(const fec::buffer &packet, uint32_t offset) { auto it2 = parked_packets_.find(base); if (it2 != parked_packets_.end()) { for (auto &packet_index : it2->second) { - auto ret = it->second.addSourceSymbol(packet_index.first, - packet_index.second, offset); + auto ret = it->second.addSourceSymbol( + packet_index.getBuffer(), packet_index.getIndex(), + packet_index.getOffset(), packet_index.getMetadata()); if (!ret) { recoverPackets(it); // Finish to delete packets in same source block that were @@ -399,7 +433,7 @@ void RSDecoder::consumeRepair(const fec::buffer &packet, uint32_t offset) { } void RSDecoder::onDataPacket(core::ContentObject &content_object, - uint32_t offset) { + uint32_t offset, uint32_t metadata) { DLOG_IF(INFO, VLOG_IS_ON(4)) << "Calling fec for data packet " << content_object.getName() << ". Offset: " << offset; @@ -409,7 +443,7 @@ void RSDecoder::onDataPacket(core::ContentObject &content_object, if (isSymbol(suffix)) { consumeRepair(content_object.shared_from_this(), offset); } else { - consumeSource(content_object.shared_from_this(), suffix, offset); + consumeSource(content_object.shared_from_this(), suffix, offset, metadata); } } diff --git a/libtransport/src/protocols/fec/rs.h b/libtransport/src/protocols/fec/rs.h index e159ad9f7..6672eaa6b 100644 --- a/libtransport/src/protocols/fec/rs.h +++ b/libtransport/src/protocols/fec/rs.h @@ -18,6 +18,7 @@ #include <arpa/inet.h> #include <hicn/transport/portability/c_portability.h> +#include <hicn/transport/portability/endianess.h> #include <hicn/transport/utils/membuf.h> #include <protocols/fec/fec_info.h> #include <protocols/fec_base.h> @@ -34,10 +35,28 @@ namespace protocol { namespace fec { #define foreach_rs_fec_type \ + _(RS, 1, 2) \ _(RS, 1, 3) \ + _(RS, 1, 4) \ + _(RS, 2, 3) \ + _(RS, 2, 4) \ + _(RS, 2, 5) \ + _(RS, 2, 6) \ + _(RS, 3, 6) \ + _(RS, 3, 7) \ + _(RS, 3, 8) \ + _(RS, 3, 9) \ + _(RS, 3, 10) \ + _(RS, 3, 11) \ + _(RS, 3, 12) \ _(RS, 4, 5) \ _(RS, 4, 6) \ _(RS, 4, 7) \ + _(RS, 4, 8) \ + _(RS, 4, 9) \ + _(RS, 4, 10) \ + _(RS, 4, 11) \ + _(RS, 4, 12) \ _(RS, 6, 10) \ _(RS, 8, 10) \ _(RS, 8, 11) \ @@ -45,6 +64,8 @@ namespace fec { _(RS, 8, 14) \ _(RS, 8, 16) \ _(RS, 8, 32) \ + _(RS, 10, 20) \ + _(RS, 10, 25) \ _(RS, 10, 30) \ _(RS, 10, 40) \ _(RS, 10, 60) \ @@ -56,12 +77,24 @@ namespace fec { _(RS, 16, 27) \ _(RS, 17, 21) \ _(RS, 17, 34) \ + _(RS, 20, 45) \ + _(RS, 20, 50) \ + _(RS, 20, 60) \ + _(RS, 20, 70) \ + _(RS, 30, 70) \ + _(RS, 30, 75) \ + _(RS, 30, 85) \ + _(RS, 30, 95) \ _(RS, 32, 36) \ _(RS, 32, 41) \ _(RS, 32, 46) \ _(RS, 32, 54) \ _(RS, 34, 42) \ _(RS, 35, 70) \ + _(RS, 40, 95) \ + _(RS, 40, 100) \ + _(RS, 40, 110) \ + _(RS, 40, 120) \ _(RS, 52, 62) static const constexpr uint16_t MAX_SOURCE_BLOCK_SIZE = 128; @@ -73,9 +106,24 @@ static const constexpr uint16_t MAX_SOURCE_BLOCK_SIZE = 128; * std::array allows to be constructed in place, saving the allocation at the * price os knowing in advance its size. */ -using Packets = std::array<std::tuple</* index */ uint32_t, /* buffer */ buffer, - uint32_t /* offset */>, - MAX_SOURCE_BLOCK_SIZE>; +class RSBufferInfo : public FECBufferInfo { + public: + RSBufferInfo() : FECBufferInfo() {} + + RSBufferInfo(uint32_t offset, uint32_t index, uint32_t metadata, + buffer buffer) + : FECBufferInfo(index, metadata, buffer), offset_(offset) {} + + uint32_t getOffset() { return offset_; } + RSBufferInfo &setOffset(uint32_t offset) { + offset_ = offset; + return *this; + } + + private: + uint32_t offset_ = 0; +}; +using Packets = std::array<RSBufferInfo, MAX_SOURCE_BLOCK_SIZE>; /** * FEC Header, prepended to symbol packets. @@ -106,8 +154,10 @@ struct fec_header { */ uint8_t padding; - void setSeqNumberBase(uint32_t suffix) { seq_number = htonl(suffix); } - uint32_t getSeqNumberBase() { return ntohl(seq_number); } + void setSeqNumberBase(uint32_t suffix) { + seq_number = portability::host_to_net(suffix); + } + uint32_t getSeqNumberBase() { return portability::net_to_host(seq_number); } void setEncodedSymbolId(uint8_t esi) { encoded_symbol_id = esi; } uint8_t getEncodedSymbolId() { return encoded_symbol_id; } void setSourceBlockLen(uint8_t k) { source_block_len = k; } @@ -116,6 +166,8 @@ struct fec_header { uint8_t getNFecSymbols() { return n_fec_symbols; } }; +static_assert(sizeof(fec_header) <= 8, "fec_header is too large"); + class rs; /** @@ -123,10 +175,38 @@ class rs; */ class BlockCode : public Packets { /** + * @brief Metadata to include when encoding the buffers. This does not need to + * be sent over the network, but just to be included in the FEC protected + * bytes. + * + */ + class __attribute__((__packed__)) fec_metadata { + public: + void setPacketLength(uint16_t length) { + packet_length = portability::host_to_net(length); + } + uint32_t getPacketLength() { + return portability::net_to_host(packet_length); + } + + void setMetadataBase(uint32_t value) { + metadata = portability::host_to_net(value); + } + uint32_t getMetadataBase() { return portability::net_to_host(metadata); } + + private: + uint16_t packet_length; /* Used to get the real size of the packet after we + pad it */ + uint32_t + metadata; /* Caller may specify an integer for storing additional + metadata that can be used when recovering the packet. */ + }; + + /** * For variable length packet we need to prepend to the padded payload the * real length of the packet. This is *not* sent over the network. */ - static constexpr std::size_t LEN_SIZE_BYTES = 2; + static constexpr std::size_t METADATA_BYTES = sizeof(fec_metadata); public: BlockCode(uint32_t k, uint32_t n, uint32_t seq_offset, struct fec_parms *code, @@ -142,7 +222,8 @@ class BlockCode : public Packets { * Add a source symbol to the source block. */ bool addSourceSymbol(const fec::buffer &packet, uint32_t i, - uint32_t offset = 0); + uint32_t offset = 0, + uint32_t metadata = FECBase::INVALID_METADATA); /** * Get current length of source block. @@ -169,7 +250,7 @@ class BlockCode : public Packets { * Add symbol to source block **/ bool addSymbol(const fec::buffer &packet, uint32_t i, uint32_t offset, - std::size_t size); + std::size_t size, uint32_t metadata); /** * Starting from k source symbols, get the n - k repair symbols @@ -310,15 +391,19 @@ class RSEncoder : public rs, public ProducerFEC { /** * Always consume source symbols. */ - void consume(const fec::buffer &packet, uint32_t index, uint32_t offset = 0); + void consume(const fec::buffer &packet, uint32_t index, uint32_t offset = 0, + uint32_t metadata = FECBase::INVALID_METADATA); - void onPacketProduced(core::ContentObject &content_object, - uint32_t offset) override; + void onPacketProduced(core::ContentObject &content_object, uint32_t offset, + uint32_t metadata = FECBase::INVALID_METADATA) override; /** * @brief Get the fec header size, if added to source packets + * in RS the source packets do not transport any FEC header */ - std::size_t getFecHeaderSize() override { return 0; } + std::size_t getFecHeaderSize(bool isFEC) override { + return isFEC ? sizeof(fec_header) : 0; + } void clear() override { rs::clear(); @@ -348,8 +433,8 @@ class RSDecoder : public rs, public ConsumerFEC { /** * Consume source symbol */ - void consumeSource(const fec::buffer &packet, uint32_t i, - uint32_t offset = 0); + void consumeSource(const fec::buffer &packet, uint32_t i, uint32_t offset = 0, + uint32_t metadata = FECBase::INVALID_METADATA); /** * Consume repair symbol @@ -359,13 +444,16 @@ class RSDecoder : public rs, public ConsumerFEC { /** * Consumers will call this function when they receive a data packet */ - void onDataPacket(core::ContentObject &content_object, - uint32_t offset) override; + void onDataPacket(core::ContentObject &content_object, uint32_t offset, + uint32_t metadata = FECBase::INVALID_METADATA) override; /** * @brief Get the fec header size, if added to source packets + * in RS the source packets do not transport any FEC header */ - std::size_t getFecHeaderSize() override { return 0; } + std::size_t getFecHeaderSize(bool isFEC) override { + return isFEC ? sizeof(fec_header) : 0; + } /** * Clear decoder to reuse @@ -398,8 +486,8 @@ class RSDecoder : public rs, public ConsumerFEC { * not make any sense to build the source block, since we received all the * source packet of the block. */ - std::unordered_map<uint32_t, std::vector<std::pair<buffer, uint32_t>>> - parked_packets_; + using BufferInfoArray = std::vector<RSBufferInfo>; + std::unordered_map<uint32_t, BufferInfoArray> parked_packets_; }; } // namespace fec diff --git a/libtransport/src/protocols/fec_base.h b/libtransport/src/protocols/fec_base.h index a1929d85e..28f6a820a 100644 --- a/libtransport/src/protocols/fec_base.h +++ b/libtransport/src/protocols/fec_base.h @@ -15,6 +15,7 @@ #pragma once +#include <hicn/transport/core/asio_wrapper.h> #include <hicn/transport/core/content_object.h> #include <hicn/transport/errors/not_implemented_exception.h> @@ -25,12 +26,67 @@ namespace protocol { namespace fec { -using buffer = typename utils::MemBuf::Ptr; -using BufferArray = std::vector<std::pair<uint32_t, buffer>>; +using buffer = utils::MemBuf::Ptr; + +class FECBufferInfo { + public: + FECBufferInfo() + : index_(~0), metadata_(~0), received_(false), buffer_(nullptr) {} + + template <typename T> + FECBufferInfo(uint32_t index, uint32_t metadata, T &&buffer) + : index_(index), + metadata_(metadata), + received_(false), + buffer_(std::forward<T>(buffer)) {} + + // Getters + uint32_t getIndex() const { return index_; } + + uint32_t getMetadata() const { return metadata_; } + + bool getReceived() const { return received_; } + + buffer &getBuffer() { return buffer_; } + + // Setters + void setReceived() { received_ = true; } + + FECBufferInfo &setIndex(uint32_t index) { + index_ = index; + return *this; + } + + FECBufferInfo &setMetadata(uint32_t metadata) { + metadata_ = metadata; + return *this; + } + + FECBufferInfo &setBuffer(buffer &buffer) { + buffer_ = buffer; + return *this; + } + + FECBufferInfo &setBuffer(buffer &&buffer) { + buffer_ = std::move(buffer); + return *this; + } + + private: + uint32_t index_; + uint32_t metadata_; + bool received_; + buffer buffer_; +}; + +using BufferArray = typename std::vector<FECBufferInfo>; class FECBase { public: - virtual ~FECBase() = default; + static inline uint32_t INVALID_METADATA = ~0; + static inline uint32_t INVALID_INDEX = ~0; + + virtual ~FECBase() {} /** * Callback to be called after the encode or the decode operations. In the * former case it will contain the symbols, while in the latter the sources. @@ -45,8 +101,10 @@ class FECBase { /** * @brief Get size of FEC header. + * the fec header size may be different if a packet is a data packet or a FEC + * packet */ - virtual std::size_t getFecHeaderSize() = 0; + virtual std::size_t getFecHeaderSize(bool isFEC) = 0; /** * Set callback to call after packet encoding / decoding @@ -64,11 +122,21 @@ class FECBase { buffer_callback_ = buffer_callback; } + /** + * Creates the timer to flush packets. So far needed only if using Rely and + * want to avoid expired packets blocked by missing pkts to wait for a new + * packet to arrive and trigger the flush + */ + void setIOService(asio::io_service &io_service) { + flush_timer_ = std::make_unique<asio::steady_timer>(io_service); + } + virtual void reset() = 0; protected: PacketsReady fec_callback_{0}; BufferRequested buffer_callback_{0}; + std::unique_ptr<asio::steady_timer> flush_timer_; }; /** @@ -80,8 +148,9 @@ class ProducerFEC : public virtual FECBase { /** * Producers will call this function upon production of a new packet. */ - virtual void onPacketProduced(core::ContentObject &content_object, - uint32_t offset) = 0; + virtual void onPacketProduced( + core::ContentObject &content_object, uint32_t offset, + uint32_t metadata = FECBase::INVALID_METADATA) = 0; }; /** @@ -95,9 +164,10 @@ class ConsumerFEC : public virtual FECBase { * Consumers will call this function when they receive a data packet */ virtual void onDataPacket(core::ContentObject &content_object, - uint32_t offset) = 0; + uint32_t offset, + uint32_t metadata = FECBase::INVALID_METADATA) = 0; }; } // namespace fec } // namespace protocol -} // namespace transport
\ No newline at end of file +} // namespace transport diff --git a/libtransport/src/protocols/incremental_indexer_bytestream.cc b/libtransport/src/protocols/incremental_indexer_bytestream.cc index cc302a98a..b94f229e5 100644 --- a/libtransport/src/protocols/incremental_indexer_bytestream.cc +++ b/libtransport/src/protocols/incremental_indexer_bytestream.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -29,9 +29,9 @@ void IncrementalIndexer::onContentObject(core::Interest &interest, DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received content " << content_object.getName(); - assert(reassembly_); + DCHECK(reassembly_); - if (TRANSPORT_EXPECT_FALSE(content_object.testRst())) { + if (TRANSPORT_EXPECT_FALSE(content_object.isLast())) { final_suffix_ = content_object.getName().getSuffix(); } diff --git a/libtransport/src/protocols/incremental_indexer_bytestream.h b/libtransport/src/protocols/incremental_indexer_bytestream.h index c6a669629..4f9b6126f 100644 --- a/libtransport/src/protocols/incremental_indexer_bytestream.h +++ b/libtransport/src/protocols/incremental_indexer_bytestream.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -48,7 +48,7 @@ class IncrementalIndexer : public Indexer { IncrementalIndexer(const IncrementalIndexer &other) = delete; IncrementalIndexer(IncrementalIndexer &&other) - : Indexer(std::forward<Indexer>(other)), + : Indexer(other), final_suffix_(other.final_suffix_), first_suffix_(other.first_suffix_), next_download_suffix_(other.next_download_suffix_), @@ -62,7 +62,7 @@ class IncrementalIndexer : public Indexer { next_reassembly_suffix_ = first_suffix_; } - virtual uint32_t checkNextSuffix() override { + virtual uint32_t checkNextSuffix() const override { return next_download_suffix_ <= final_suffix_ ? next_download_suffix_ : Indexer::invalid_index; } @@ -76,7 +76,7 @@ class IncrementalIndexer : public Indexer { first_suffix_ = suffix; } - uint32_t getFirstSuffix() override { return first_suffix_; } + uint32_t getFirstSuffix() const override { return first_suffix_; } virtual uint32_t jumpToIndex(uint32_t index) override { next_download_suffix_ = index; @@ -95,14 +95,14 @@ class IncrementalIndexer : public Indexer { return final_suffix_ != Indexer::invalid_index; } - virtual uint32_t getFinalSuffix() override { return final_suffix_; } + virtual uint32_t getFinalSuffix() const override { return final_suffix_; } void enableFec(fec::FECType fec_type) override {} void disableFec() override {} void setNFec(uint32_t n_fec) override {} - virtual uint32_t getNFec() override { return 0; } + virtual uint32_t getNFec() const override { return 0; } virtual void onContentObject(core::Interest &interest, core::ContentObject &content_object, diff --git a/libtransport/src/protocols/index_manager_bytestream.cc b/libtransport/src/protocols/index_manager_bytestream.cc index c78dc634d..952f36e0e 100644 --- a/libtransport/src/protocols/index_manager_bytestream.cc +++ b/libtransport/src/protocols/index_manager_bytestream.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/index_manager_bytestream.h b/libtransport/src/protocols/index_manager_bytestream.h index e14c8845b..7ea31dfa5 100644 --- a/libtransport/src/protocols/index_manager_bytestream.h +++ b/libtransport/src/protocols/index_manager_bytestream.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -40,7 +40,9 @@ class IndexManager : public IncrementalIndexer { indexer_->setFirstSuffix(suffix); } - uint32_t getFirstSuffix() override { return indexer_->getFirstSuffix(); } + uint32_t getFirstSuffix() const override { + return indexer_->getFirstSuffix(); + } uint32_t getNextReassemblySegment() override { return indexer_->getNextReassemblySegment(); @@ -50,22 +52,26 @@ class IndexManager : public IncrementalIndexer { return indexer_->isFinalSuffixDiscovered(); } - uint32_t getFinalSuffix() override { return indexer_->getFinalSuffix(); } + uint32_t getFinalSuffix() const override { + return indexer_->getFinalSuffix(); + } uint32_t jumpToIndex(uint32_t index) override { return indexer_->jumpToIndex(index); } void setNFec(uint32_t n_fec) override { return indexer_->setNFec(n_fec); } - uint32_t getNFec() override { return indexer_->getNFec(); } + uint32_t getNFec() const override { return indexer_->getNFec(); } void enableFec(fec::FECType fec_type) override { return indexer_->enableFec(fec_type); } - double getFecOverhead() override { return indexer_->getFecOverhead(); } + double getFecOverhead() const override { return indexer_->getFecOverhead(); } - double getMaxFecOverhead() override { return indexer_->getMaxFecOverhead(); } + double getMaxFecOverhead() const override { + return indexer_->getMaxFecOverhead(); + } void disableFec() override { return indexer_->disableFec(); } diff --git a/libtransport/src/protocols/indexer.cc b/libtransport/src/protocols/indexer.cc index 8d4cf04d7..41465755b 100644 --- a/libtransport/src/protocols/indexer.cc +++ b/libtransport/src/protocols/indexer.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -14,10 +14,10 @@ */ #include <implementation/socket_consumer.h> +#include <protocols/errors.h> #include <protocols/indexer.h> namespace transport { - namespace protocol { using namespace interface; @@ -36,6 +36,35 @@ void Indexer::setVerifier() { } } -} // end namespace protocol +void Indexer::applyPolicy(core::Interest &interest, + core::ContentObject &content_object, bool reassembly, + auth::VerificationPolicy policy) const { + DCHECK(reassembly_ != nullptr); + + switch (policy) { + case auth::VerificationPolicy::ACCEPT: { + if (reassembly) { + reassembly_->reassemble(content_object); + } + break; + } + case auth::VerificationPolicy::UNKNOWN: + if (reassembly && reassembly_->reassembleUnverified()) { + reassembly_->reassemble(content_object); + } + break; + case auth::VerificationPolicy::DROP: + transport_->onPacketDropped( + interest, content_object, + make_error_code(protocol_error::verification_failed)); + break; + case auth::VerificationPolicy::ABORT: { + transport_->onContentReassembled( + make_error_code(protocol_error::session_aborted)); + break; + } + } +} +} // end namespace protocol } // end namespace transport diff --git a/libtransport/src/protocols/indexer.h b/libtransport/src/protocols/indexer.h index 7e3a52fb0..1bacb13aa 100644 --- a/libtransport/src/protocols/indexer.h +++ b/libtransport/src/protocols/indexer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -15,6 +15,7 @@ #pragma once +#include <hicn/transport/auth/policies.h> #include <hicn/transport/core/content_object.h> #include <hicn/transport/core/interest.h> #include <protocols/fec_utils.h> @@ -44,7 +45,7 @@ class Indexer { /** * Suffix getters */ - virtual uint32_t checkNextSuffix() = 0; + virtual uint32_t checkNextSuffix() const = 0; virtual uint32_t getNextSuffix() = 0; virtual uint32_t getNextReassemblySegment() = 0; @@ -52,24 +53,24 @@ class Indexer { * Set first suffix from where to start. */ virtual void setFirstSuffix(uint32_t suffix) = 0; - virtual uint32_t getFirstSuffix() = 0; + virtual uint32_t getFirstSuffix() const = 0; /** * Functions to set/enable/disable fec */ virtual void setNFec(uint32_t n_fec) = 0; - virtual uint32_t getNFec() = 0; + virtual uint32_t getNFec() const = 0; virtual void enableFec(fec::FECType fec_type) = 0; virtual void disableFec() = 0; virtual bool isFec(uint32_t index) { return false; } - virtual double getFecOverhead() { return 0.0; } - virtual double getMaxFecOverhead() { return 0.0; } + virtual double getFecOverhead() const { return 0.0; } + virtual double getMaxFecOverhead() const { return 0.0; } /** * Final suffix helpers. */ virtual bool isFinalSuffixDiscovered() = 0; - virtual uint32_t getFinalSuffix() = 0; + virtual uint32_t getFinalSuffix() const = 0; /** * Set reassembly protocol @@ -84,9 +85,15 @@ class Indexer { virtual void setVerifier(); /** + * Apply a verification policy + */ + virtual void applyPolicy(core::Interest &interest, + core::ContentObject &content_object, bool reassembly, + auth::VerificationPolicy policy) const; + /** * Jump to suffix. This may be useful if, for any protocol dependent - * mechanism, we need to suddenly change current suffix. This does not modify - * the way suffixes re incremented/decremented (that's part of the + * mechanism, we need to suddenly change current suffix. This does not + * modify the way suffixes re incremented/decremented (that's part of the * implementation). */ virtual uint32_t jumpToIndex(uint32_t index) = 0; @@ -108,6 +115,7 @@ class Indexer { TransportProtocol *transport_; Reassembly *reassembly_; std::shared_ptr<auth::Verifier> verifier_; + auth::CryptoHashType manifest_hash_type_; }; } // end namespace protocol diff --git a/libtransport/src/protocols/manifest_incremental_indexer_bytestream.cc b/libtransport/src/protocols/manifest_incremental_indexer_bytestream.cc index 168aa57af..0b15559a4 100644 --- a/libtransport/src/protocols/manifest_incremental_indexer_bytestream.cc +++ b/libtransport/src/protocols/manifest_incremental_indexer_bytestream.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -31,8 +31,7 @@ ManifestIncrementalIndexer::ManifestIncrementalIndexer( implementation::ConsumerSocket *icn_socket, TransportProtocol *transport) : IncrementalIndexer(icn_socket, transport), suffix_strategy_(utils::SuffixStrategyFactory::getSuffixStrategy( - NextSegmentCalculationStrategy::INCREMENTAL, next_download_suffix_, - 0)) {} + utils::NextSuffixStrategy::INCREMENTAL, next_download_suffix_)) {} void ManifestIncrementalIndexer::onContentObject( core::Interest &interest, core::ContentObject &content_object, @@ -59,12 +58,7 @@ void ManifestIncrementalIndexer::onContentObject( void ManifestIncrementalIndexer::onUntrustedManifest( core::Interest &interest, core::ContentObject &content_object, bool reassembly) { - auto manifest = - std::make_unique<ContentObjectManifest>(std::move(content_object)); - - auth::VerificationPolicy policy = verifier_->verifyPackets(manifest.get()); - - manifest->decode(); + auth::VerificationPolicy policy = verifier_->verifyPackets(&content_object); if (policy != auth::VerificationPolicy::ACCEPT) { transport_->onContentReassembled( @@ -72,35 +66,33 @@ void ManifestIncrementalIndexer::onUntrustedManifest( return; } - processTrustedManifest(interest, std::move(manifest), reassembly); + core::ContentObjectManifest manifest(content_object.shared_from_this()); + manifest.decode(); + + processTrustedManifest(interest, manifest, reassembly); } void ManifestIncrementalIndexer::processTrustedManifest( - core::Interest &interest, std::unique_ptr<ContentObjectManifest> manifest, + core::Interest &interest, core::ContentObjectManifest &manifest, bool reassembly) { - if (TRANSPORT_EXPECT_FALSE(manifest->getVersion() != - core::ManifestVersion::VERSION_1)) { - throw errors::RuntimeException("Received manifest with unknown version."); - } - - switch (manifest->getManifestType()) { + switch (manifest.getType()) { case core::ManifestType::INLINE_MANIFEST: { - suffix_strategy_->setFinalSuffix(manifest->getFinalBlockNumber()); + suffix_strategy_->setFinalSuffix( + manifest.getParamsBytestream().final_segment); // The packets to verify with the received manifest std::vector<auth::PacketPtr> packets; // Convert the received manifest to a map of packet suffixes to hashes - auth::Verifier::SuffixMap current_manifest = - core::ContentObjectManifest::getSuffixMap(manifest.get()); + auth::Verifier::SuffixMap suffix_map = manifest.getSuffixMap(); // Update 'suffix_map_' with new hashes from the received manifest and // build 'packets' - for (auto it = current_manifest.begin(); it != current_manifest.end();) { + for (auto it = suffix_map.begin(); it != suffix_map.end();) { if (unverified_segments_.find(it->first) == unverified_segments_.end()) { suffix_map_[it->first] = std::move(it->second); - current_manifest.erase(it++); + suffix_map.erase(it++); continue; } @@ -110,7 +102,7 @@ void ManifestIncrementalIndexer::processTrustedManifest( // Verify unverified segments using the received manifest auth::Verifier::PolicyMap policies = - verifier_->verifyPackets(packets, current_manifest); + verifier_->verifyPackets(packets, suffix_map); for (unsigned int i = 0; i < packets.size(); ++i) { auth::Suffix suffix = packets[i]->getName().getSuffix(); @@ -127,7 +119,9 @@ void ManifestIncrementalIndexer::processTrustedManifest( } if (reassembly) { - reassembly_->reassemble(std::move(manifest)); + auto manifest_co = + std::dynamic_pointer_cast<ContentObject>(manifest.getPacket()); + reassembly_->reassemble(*manifest_co); } break; } @@ -162,45 +156,15 @@ void ManifestIncrementalIndexer::onUntrustedContentObject( applyPolicy(interest, content_object, reassembly, policy); } -void ManifestIncrementalIndexer::applyPolicy( - core::Interest &interest, core::ContentObject &content_object, - bool reassembly, auth::VerificationPolicy policy) { - assert(reassembly_); - switch (policy) { - case auth::VerificationPolicy::ACCEPT: { - if (reassembly && !reassembly_->reassembleUnverified()) { - reassembly_->reassemble(content_object); - } - break; - } - case auth::VerificationPolicy::DROP: { - transport_->onPacketDropped( - interest, content_object, - make_error_code(protocol_error::verification_failed)); - break; - } - case auth::VerificationPolicy::ABORT: { - transport_->onContentReassembled( - make_error_code(protocol_error::session_aborted)); - break; - } - case auth::VerificationPolicy::UNKNOWN: { - if (reassembly && reassembly_->reassembleUnverified()) { - reassembly_->reassemble(content_object); - } - } - } -} - -uint32_t ManifestIncrementalIndexer::checkNextSuffix() { - return suffix_strategy_->getNextSuffix(); +uint32_t ManifestIncrementalIndexer::checkNextSuffix() const { + return suffix_strategy_->checkNextSuffix(); } uint32_t ManifestIncrementalIndexer::getNextSuffix() { auto ret = suffix_strategy_->getNextSuffix(); if (ret <= suffix_strategy_->getFinalSuffix() && - ret != utils::SuffixStrategy::INVALID_SUFFIX) { + ret != utils::SuffixStrategy::MAX_SUFFIX) { suffix_queue_.push(ret); return ret; } @@ -208,7 +172,7 @@ uint32_t ManifestIncrementalIndexer::getNextSuffix() { return Indexer::invalid_index; } -uint32_t ManifestIncrementalIndexer::getFinalSuffix() { +uint32_t ManifestIncrementalIndexer::getFinalSuffix() const { return suffix_strategy_->getFinalSuffix(); } diff --git a/libtransport/src/protocols/manifest_incremental_indexer_bytestream.h b/libtransport/src/protocols/manifest_incremental_indexer_bytestream.h index d8cf5892f..8527b55c1 100644 --- a/libtransport/src/protocols/manifest_incremental_indexer_bytestream.h +++ b/libtransport/src/protocols/manifest_incremental_indexer_bytestream.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -39,8 +39,7 @@ class ManifestIncrementalIndexer : public IncrementalIndexer { ManifestIncrementalIndexer(IncrementalIndexer &&indexer) : IncrementalIndexer(std::move(indexer)), suffix_strategy_(utils::SuffixStrategyFactory::getSuffixStrategy( - core::NextSegmentCalculationStrategy::INCREMENTAL, - next_download_suffix_, 0)) { + utils::NextSuffixStrategy::INCREMENTAL, next_download_suffix_)) { for (uint32_t i = first_suffix_; i < next_download_suffix_; i++) { suffix_queue_.push(i); } @@ -54,7 +53,7 @@ class ManifestIncrementalIndexer : public IncrementalIndexer { core::ContentObject &content_object, bool reassembly) override; - uint32_t checkNextSuffix() override; + uint32_t checkNextSuffix() const override; uint32_t getNextSuffix() override; @@ -62,7 +61,7 @@ class ManifestIncrementalIndexer : public IncrementalIndexer { bool isFinalSuffixDiscovered() override; - uint32_t getFinalSuffix() override; + uint32_t getFinalSuffix() const override; protected: std::unique_ptr<utils::SuffixStrategy> suffix_strategy_; @@ -77,14 +76,11 @@ class ManifestIncrementalIndexer : public IncrementalIndexer { core::ContentObject &content_object, bool reassembly); void processTrustedManifest(core::Interest &interest, - std::unique_ptr<ContentObjectManifest> manifest, + core::ContentObjectManifest &manifest, bool reassembly); void onUntrustedContentObject(core::Interest &interest, core::ContentObject &content_object, bool reassembly); - void applyPolicy(core::Interest &interest, - core::ContentObject &content_object, bool reassembly, - auth::VerificationPolicy policy); }; } // end namespace protocol diff --git a/libtransport/src/protocols/prod_protocol_bytestream.cc b/libtransport/src/protocols/prod_protocol_bytestream.cc index f659cb37c..7f103e12b 100644 --- a/libtransport/src/protocols/prod_protocol_bytestream.cc +++ b/libtransport/src/protocols/prod_protocol_bytestream.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -29,12 +29,7 @@ ByteStreamProductionProtocol::ByteStreamProductionProtocol( implementation::ProducerSocket *icn_socket) : ProductionProtocol(icn_socket) {} -ByteStreamProductionProtocol::~ByteStreamProductionProtocol() { - stop(); - if (listening_thread_.joinable()) { - listening_thread_.join(); - } -} +ByteStreamProductionProtocol::~ByteStreamProductionProtocol() { stop(); } uint32_t ByteStreamProductionProtocol::produceDatagram( const Name &content_name, std::unique_ptr<utils::MemBuf> &&buffer) { @@ -68,16 +63,16 @@ uint32_t ByteStreamProductionProtocol::produceStream( return 0; } - Name name(content_name); - - // Get the atomic variables to ensure they keep the same value - // during the production - // Total size of the data packet uint32_t data_packet_size; socket_->getSocketOption(GeneralTransportOptions::DATA_PACKET_SIZE, data_packet_size); + // Maximum size of a segment + uint32_t max_segment_size; + socket_->getSocketOption(GeneralTransportOptions::MAX_SEGMENT_SIZE, + max_segment_size); + // Expiry time uint32_t content_object_expiry_time; socket_->getSocketOption(GeneralTransportOptions::CONTENT_OBJECT_EXPIRY_TIME, @@ -88,106 +83,95 @@ uint32_t ByteStreamProductionProtocol::produceStream( socket_->getSocketOption(GeneralTransportOptions::HASH_ALGORITHM, hash_algo); // Suffix calculation strategy - core::NextSegmentCalculationStrategy _suffix_strategy; + std::shared_ptr<utils::SuffixStrategy> suffix_strategy; socket_->getSocketOption(GeneralTransportOptions::SUFFIX_STRATEGY, - _suffix_strategy); - auto suffix_strategy = utils::SuffixStrategyFactory::getSuffixStrategy( - _suffix_strategy, start_offset); + suffix_strategy); + suffix_strategy->reset(start_offset); + + // Default format + core::Packet::Format default_format; + socket_->getSocketOption(GeneralTransportOptions::PACKET_FORMAT, + default_format); - auto buffer_size = buffer->length(); + Name name(content_name); + size_t buffer_size = buffer->length(); + size_t signature_length = signer_->getSignatureFieldSize(); + uint32_t final_block_number = start_offset; + + // Content-related + core::Packet::Format content_format; + uint32_t content_header_size; + uint64_t content_free_space; + uint32_t nb_segments; int bytes_segmented = 0; - std::size_t header_size; - std::size_t manifest_header_size = 0; - std::size_t signature_length = 0; - std::uint32_t final_block_number = start_offset; - uint64_t free_space_for_content = 0; - core::Packet::Format format; + // Manifest-related + core::Packet::Format manifest_format; + uint32_t manifest_header_size; + uint64_t manifest_free_space; + uint32_t nb_manifests; std::shared_ptr<core::ContentObjectManifest> manifest; + uint32_t manifest_capacity = manifest_max_capacity_; bool is_last_manifest = false; - - // TODO Manifest may still be used for indexing - if (making_manifest_ && !signer_) { - LOG(FATAL) << "Making manifests without setting producer identity."; + ParamsBytestream transport_params; + + manifest_format = Packet::toAHFormat(default_format); + content_format = !manifest_max_capacity_ ? Packet::toAHFormat(default_format) + : default_format; + + content_header_size = (uint32_t)core::Packet::getHeaderSizeFromFormat( + content_format, signature_length); + manifest_header_size = (uint32_t)core::Packet::getHeaderSizeFromFormat( + manifest_format, signature_length); + content_free_space = + std::min(max_segment_size, data_packet_size - content_header_size); + manifest_free_space = + std::min(max_segment_size, data_packet_size - manifest_header_size); + + // Compute the number of segments the data will be split into + nb_segments = + uint32_t(std::ceil(double(buffer_size) / double(content_free_space))); + if (content_free_space * nb_segments < buffer_size) { + nb_segments++; } - core::Packet::Format hf_format = core::Packet::Format::HF_UNSPEC; - core::Packet::Format hf_format_ah = core::Packet::Format::HF_UNSPEC; - - if (name.getType() == HNT_CONTIGUOUS_V4 || name.getType() == HNT_IOV_V4) { - hf_format = core::Packet::Format::HF_INET_TCP; - hf_format_ah = core::Packet::Format::HF_INET_TCP_AH; - } else if (name.getType() == HNT_CONTIGUOUS_V6 || - name.getType() == HNT_IOV_V6) { - hf_format = core::Packet::Format::HF_INET6_TCP; - hf_format_ah = core::Packet::Format::HF_INET6_TCP_AH; - } else { - throw errors::RuntimeException("Unknown name format."); - } - - format = hf_format; - if (making_manifest_) { - manifest_header_size = core::Packet::getHeaderSizeFromFormat( - signer_ ? hf_format_ah : hf_format, - signer_ ? signer_->getSignatureFieldSize() : 0); - } else if (signer_) { - format = hf_format_ah; - signature_length = signer_->getSignatureFieldSize(); - } + if (manifest_max_capacity_) { + nb_manifests = static_cast<uint32_t>( + std::ceil(float(nb_segments) / manifest_capacity)); + final_block_number += nb_segments + nb_manifests - 1; + transport_params.final_segment = + is_last ? final_block_number : utils::SuffixStrategy::MAX_SUFFIX; - header_size = core::Packet::getHeaderSizeFromFormat(format, signature_length); - free_space_for_content = data_packet_size - header_size; - uint32_t number_of_segments = - uint32_t(std::ceil(double(buffer_size) / double(free_space_for_content))); - if (free_space_for_content * number_of_segments < buffer_size) { - number_of_segments++; - } - - // TODO allocate space for all the headers - if (making_manifest_) { - uint32_t segment_in_manifest = static_cast<uint32_t>( - std::floor(double(data_packet_size - manifest_header_size - - ContentObjectManifest::getManifestHeaderSize()) / - ContentObjectManifest::getManifestEntrySize()) - - 1.0); - uint32_t number_of_manifests = static_cast<uint32_t>( - std::ceil(float(number_of_segments) / segment_in_manifest)); - final_block_number += number_of_segments + number_of_manifests - 1; - - manifest.reset(ContentObjectManifest::createManifest( + manifest = ContentObjectManifest::createContentManifest( + manifest_format, name.setSuffix(suffix_strategy->getNextManifestSuffix()), - core::ManifestVersion::VERSION_1, core::ManifestType::INLINE_MANIFEST, - hash_algo, is_last_manifest, name, _suffix_strategy, - signer_ ? signer_->getSignatureFieldSize() : 0)); - manifest->setLifetime(content_object_expiry_time); - - if (is_last) { - manifest->setFinalBlockNumber(final_block_number); - } else { - manifest->setFinalBlockNumber(utils::SuffixStrategy::INVALID_SUFFIX); - } + signature_length); + manifest->setHeaders(core::ManifestType::INLINE_MANIFEST, + manifest_max_capacity_, hash_algo, is_last_manifest, + name); + manifest->setParamsBytestream(transport_params); + manifest->getPacket()->setLifetime(content_object_expiry_time); } - for (unsigned int packaged_segments = 0; - packaged_segments < number_of_segments; packaged_segments++) { - if (making_manifest_) { - if (manifest->estimateManifestSize(2) > - data_packet_size - manifest_header_size) { + auto self = shared_from_this(); + for (unsigned int packaged_segments = 0; packaged_segments < nb_segments; + packaged_segments++) { + if (manifest_max_capacity_) { + if (manifest->Encoder::manifestSize(1) > manifest_free_space) { manifest->encode(); + auto manifest_co = + std::dynamic_pointer_cast<ContentObject>(manifest->getPacket()); - // If identity set, sign manifest - if (signer_) { - signer_->signPacket(manifest.get()); - } + signer_->signPacket(manifest_co.get()); // Send the current manifest - passContentObjectToCallbacks(manifest); - - DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send manifest " << manifest->getName(); + passContentObjectToCallbacks(manifest_co, self); + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "Send manifest " << manifest_co->getName(); // Send content objects stored in the queue while (!content_queue_.empty()) { - passContentObjectToCallbacks(content_queue_.front()); + passContentObjectToCallbacks(content_queue_.front(), self); DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send content " << content_queue_.front()->getName(); content_queue_.pop(); @@ -195,86 +179,87 @@ uint32_t ByteStreamProductionProtocol::produceStream( // Create new manifest. The reference to the last manifest has been // acquired in the passContentObjectToCallbacks function, so we can - // safely release this reference - manifest.reset(ContentObjectManifest::createManifest( + // safely release this reference. + manifest = ContentObjectManifest::createContentManifest( + manifest_format, name.setSuffix(suffix_strategy->getNextManifestSuffix()), - core::ManifestVersion::VERSION_1, - core::ManifestType::INLINE_MANIFEST, hash_algo, is_last_manifest, - name, _suffix_strategy, - signer_ ? signer_->getSignatureFieldSize() : 0)); - - manifest->setLifetime(content_object_expiry_time); - manifest->setFinalBlockNumber( - is_last ? final_block_number - : utils::SuffixStrategy::INVALID_SUFFIX); + signature_length); + manifest->setHeaders(core::ManifestType::INLINE_MANIFEST, + manifest_max_capacity_, hash_algo, + is_last_manifest, name); + manifest->setParamsBytestream(transport_params); + manifest->getPacket()->setLifetime(content_object_expiry_time); } } - auto content_suffix = suffix_strategy->getNextContentSuffix(); + // Create content object + uint32_t content_suffix = suffix_strategy->getNextContentSuffix(); auto content_object = std::make_shared<ContentObject>( - name.setSuffix(content_suffix), format, - signer_ && !making_manifest_ ? signer_->getSignatureFieldSize() : 0); + name.setSuffix(content_suffix), content_format, + !manifest_max_capacity_ ? signature_length : 0); content_object->setLifetime(content_object_expiry_time); auto b = buffer->cloneOne(); - b->trimStart(free_space_for_content * packaged_segments); + b->trimStart(content_free_space * packaged_segments); b->trimEnd(b->length()); - if (TRANSPORT_EXPECT_FALSE(packaged_segments == number_of_segments - 1)) { + // Segment the input data + if (TRANSPORT_EXPECT_FALSE(packaged_segments == nb_segments - 1)) { b->append(buffer_size - bytes_segmented); bytes_segmented += (int)(buffer_size - bytes_segmented); - if (is_last && making_manifest_) { + if (is_last && manifest_max_capacity_) { is_last_manifest = true; } else if (is_last) { - content_object->setRst(); + content_object->setLast(); } } else { - b->append(free_space_for_content); - bytes_segmented += (int)(free_space_for_content); + b->append(content_free_space); + bytes_segmented += (int)(content_free_space); } + // Set the segmented data as payload content_object->appendPayload(std::move(b)); - if (making_manifest_) { - using namespace std::chrono_literals; + // Either we sign the content object or we save its hash into the current + // manifest + if (manifest_max_capacity_) { auth::CryptoHash hash = content_object->computeDigest(hash_algo); - manifest->addSuffixHash(content_suffix, hash); + manifest->addEntry(content_suffix, hash); content_queue_.push(content_object); } else { - if (signer_) { - signer_->signPacket(content_object.get()); - } - passContentObjectToCallbacks(content_object); + signer_->signPacket(content_object.get()); + passContentObjectToCallbacks(content_object, self); DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send content " << content_object->getName(); } } - if (making_manifest_) { + // We send the manifest that hasn't been fully filled yet + if (manifest_max_capacity_) { if (is_last_manifest) { - manifest->setFinalManifest(is_last_manifest); + manifest->setIsLast(is_last_manifest); } manifest->encode(); + auto manifest_co = + std::dynamic_pointer_cast<ContentObject>(manifest->getPacket()); - if (signer_) { - signer_->signPacket(manifest.get()); - } + signer_->signPacket(manifest_co.get()); - passContentObjectToCallbacks(manifest); - DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send manifest " << manifest->getName(); + passContentObjectToCallbacks(manifest_co, self); + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send manifest " << manifest_co->getName(); while (!content_queue_.empty()) { - passContentObjectToCallbacks(content_queue_.front()); + passContentObjectToCallbacks(content_queue_.front(), self); DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send content " << content_queue_.front()->getName(); content_queue_.pop(); } } - portal_->getIoService().post([this]() { + portal_->getThread().add([this, self]() { std::shared_ptr<ContentObject> co; while (object_queue_for_callbacks_.pop(co)) { if (*on_new_segment_) { @@ -296,7 +281,7 @@ uint32_t ByteStreamProductionProtocol::produceStream( } }); - portal_->getIoService().dispatch([this, buffer_size]() { + portal_->getThread().add([this, buffer_size, self]() { if (*on_content_produced_) { on_content_produced_->operator()(*socket_->getInterface(), std::make_error_code(std::errc(0)), @@ -307,9 +292,10 @@ uint32_t ByteStreamProductionProtocol::produceStream( return suffix_strategy->getTotalCount(); } -void ByteStreamProductionProtocol::scheduleSendBurst() { - portal_->getIoService().post([this]() { - std::shared_ptr<ContentObject> co; +void ByteStreamProductionProtocol::scheduleSendBurst( + const std::shared_ptr<ByteStreamProductionProtocol> &self) { + portal_->getThread().add([this, self]() { + ContentObject::Ptr co; for (uint32_t i = 0; i < burst_size; i++) { if (object_queue_for_callbacks_.pop(co)) { @@ -321,11 +307,15 @@ void ByteStreamProductionProtocol::scheduleSendBurst() { on_content_object_to_sign_->operator()(*socket_->getInterface(), *co); } + output_buffer_.insert(co); + if (*on_content_object_in_output_buffer_) { on_content_object_in_output_buffer_->operator()( *socket_->getInterface(), *co); } + portal_->sendContentObject(*co); + if (*on_content_object_output_) { on_content_object_output_->operator()(*socket_->getInterface(), *co); } @@ -337,13 +327,12 @@ void ByteStreamProductionProtocol::scheduleSendBurst() { } void ByteStreamProductionProtocol::passContentObjectToCallbacks( - const std::shared_ptr<ContentObject> &content_object) { - output_buffer_.insert(content_object); - portal_->sendContentObject(*content_object); + const std::shared_ptr<ContentObject> &content_object, + const std::shared_ptr<ByteStreamProductionProtocol> &self) { object_queue_for_callbacks_.push(std::move(content_object)); if (object_queue_for_callbacks_.size() >= burst_size) { - scheduleSendBurst(); + scheduleSendBurst(self); } } @@ -376,7 +365,5 @@ void ByteStreamProductionProtocol::onInterest(Interest &interest) { } } -void ByteStreamProductionProtocol::onError(std::error_code ec) {} - } // namespace protocol } // end namespace transport diff --git a/libtransport/src/protocols/prod_protocol_bytestream.h b/libtransport/src/protocols/prod_protocol_bytestream.h index cf36b90a5..809ad8d5c 100644 --- a/libtransport/src/protocols/prod_protocol_bytestream.h +++ b/libtransport/src/protocols/prod_protocol_bytestream.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -50,16 +50,19 @@ class ByteStreamProductionProtocol : public ProductionProtocol { uint32_t produceDatagram(const Name &content_name, const uint8_t *buffer, size_t buffer_size) override; + auto shared_from_this() { return utils::shared_from(this); } + protected: // Consumer Callback // void reset() override; void onInterest(core::Interest &i) override; - void onError(std::error_code ec) override; private: void passContentObjectToCallbacks( - const std::shared_ptr<ContentObject> &content_object); - void scheduleSendBurst(); + const std::shared_ptr<ContentObject> &content_object, + const std::shared_ptr<ByteStreamProductionProtocol> &self); + void scheduleSendBurst( + const std::shared_ptr<ByteStreamProductionProtocol> &self); private: // While manifests are being built, contents are stored in a queue diff --git a/libtransport/src/protocols/prod_protocol_rtc.cc b/libtransport/src/protocols/prod_protocol_rtc.cc index cdc882d81..83cd23ac6 100644 --- a/libtransport/src/protocols/prod_protocol_rtc.cc +++ b/libtransport/src/protocols/prod_protocol_rtc.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -13,15 +13,21 @@ * limitations under the License. */ +#include <glog/logging.h> #include <hicn/transport/core/global_object_pool.h> #include <implementation/socket_producer.h> #include <protocols/prod_protocol_rtc.h> +#include <protocols/rtc/probe_handler.h> #include <protocols/rtc/rtc_consts.h> #include <stdlib.h> #include <time.h> #include <unordered_set> +extern "C" { +#include <hicn/util/bitmap.h> +} + namespace transport { namespace protocol { @@ -31,117 +37,144 @@ RTCProductionProtocol::RTCProductionProtocol( implementation::ProducerSocket *icn_socket) : ProductionProtocol(icn_socket), current_seg_(1), + prev_produced_bytes_(0), + prev_produced_packets_(0), produced_bytes_(0), produced_packets_(0), - produced_fec_packets_(0), - max_packet_production_(1), - bytes_production_rate_(0), + max_packet_production_(UINT32_MAX), + bytes_production_rate_(UINT32_MAX), packets_production_rate_(0), - fec_packets_production_rate_(0), - last_round_(std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count()), + last_produced_data_ts_(0), + last_round_(utils::SteadyTime::nowMs().count()), allow_delayed_nacks_(false), - queue_timer_on_(false), - consumer_in_sync_(false), - on_consumer_in_sync_(nullptr) { - srand((unsigned int)time(NULL)); - prod_label_ = rand() % 256; + pending_fec_pace_(false), + max_len_(0), + queue_len_(0), + data_aggregation_(true), + data_aggregation_timer_switch_(false) { + std::uniform_int_distribution<> dis(0, 255); + prod_label_ = dis(gen_); cache_label_ = (prod_label_ + 1) % 256; - interests_queue_timer_ = - std::make_unique<asio::steady_timer>(portal_->getIoService()); - round_timer_ = std::make_unique<asio::steady_timer>(portal_->getIoService()); + round_timer_ = + std::make_unique<asio::steady_timer>(portal_->getThread().getIoService()); + fec_pacing_timer_ = + std::make_unique<asio::steady_timer>(portal_->getThread().getIoService()); + app_packets_timer_ = + std::make_unique<asio::steady_timer>(portal_->getThread().getIoService()); setOutputBufferSize(10000); - scheduleRoundTimer(); +} + +RTCProductionProtocol::~RTCProductionProtocol() {} + +void RTCProductionProtocol::setProducerParam() { + // Flow name: here we assume there is only one prefix registered in the portal + flow_name_ = portal_->getServedNamespaces().begin()->makeName(); + + // Default format + core::Packet::Format default_format; + socket_->getSocketOption(interface::GeneralTransportOptions::PACKET_FORMAT, + default_format); // FEC using namespace std::placeholders; enableFEC(std::bind(&RTCProductionProtocol::onFecPackets, this, _1), std::bind(&RTCProductionProtocol::getBuffer, this, _1)); -} -RTCProductionProtocol::~RTCProductionProtocol() {} - -void RTCProductionProtocol::registerNamespaceWithNetwork( - const Prefix &producer_namespace) { - ProductionProtocol::registerNamespaceWithNetwork(producer_namespace); - - flow_name_ = producer_namespace.getName(); - auto family = flow_name_.getAddressFamily(); - - switch (family) { - case AF_INET6: - data_header_size_ = - signer_ && !making_manifest_ - ? (uint32_t)Packet::getHeaderSizeFromFormat( - HF_INET6_TCP_AH, signer_->getSignatureFieldSize()) - : (uint32_t)Packet::getHeaderSizeFromFormat(HF_INET6_TCP); - ; - break; - case AF_INET: - data_header_size_ = - signer_ && !making_manifest_ - ? (uint32_t)Packet::getHeaderSizeFromFormat( - HF_INET_TCP_AH, signer_->getSignatureFieldSize()) - : (uint32_t)Packet::getHeaderSizeFromFormat(HF_INET_TCP); - break; - default: - throw errors::RuntimeException("Unknown name format."); - } + // Aggregated data + socket_->getSocketOption(interface::RtcTransportOptions::AGGREGATED_DATA, + data_aggregation_); + + size_t signature_size = signer_->getSignatureFieldSize(); + data_header_format_ = {!manifest_max_capacity_ + ? Packet::toAHFormat(default_format) + : default_format, + !manifest_max_capacity_ ? signature_size : 0}; + manifest_header_format_ = {Packet::toAHFormat(default_format), + signature_size}; + nack_header_format_ = {Packet::toAHFormat(default_format), signature_size}; + fec_header_format_ = {Packet::toAHFormat(default_format), signature_size}; + + // Initialize verifier for aggregated interests + std::shared_ptr<auth::Verifier> verifier; + socket_->getSocketOption(implementation::GeneralTransportOptions::VERIFIER, + verifier); + verifier_ = std::make_shared<rtc::RTCVerifier>(verifier, 0, 0); + + // Schedule round timer + scheduleRoundTimer(); } void RTCProductionProtocol::scheduleRoundTimer() { round_timer_->expires_from_now( std::chrono::milliseconds(rtc::PRODUCER_STATS_INTERVAL)); - round_timer_->async_wait([this](std::error_code ec) { + std::weak_ptr<RTCProductionProtocol> self = shared_from_this(); + round_timer_->async_wait([self](const std::error_code &ec) { if (ec) return; - updateStats(); + + auto sp = self.lock(); + if (sp && sp->isRunning()) { + sp->updateStats(true); + } }); } -void RTCProductionProtocol::updateStats() { - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); +void RTCProductionProtocol::updateStats(bool new_round) { + uint64_t now = utils::SteadyTime::nowMs().count(); uint64_t duration = now - last_round_; - if (duration == 0) duration = 1; + if (!new_round) { + duration += rtc::PRODUCER_STATS_INTERVAL; + } else { + prev_produced_bytes_ = 0; + prev_produced_packets_ = 0; + } + double per_second = rtc::MILLI_IN_A_SEC / duration; uint32_t prev_packets_production_rate = packets_production_rate_; - bytes_production_rate_ = ceil((double)produced_bytes_ * per_second); - packets_production_rate_ = ceil((double)produced_packets_ * per_second); - fec_packets_production_rate_ = - ceil((double)produced_fec_packets_ * per_second); - - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "Updating production rate: produced_bytes_ = " << produced_bytes_ - << " bps = " << bytes_production_rate_; + // bytes_production_rate_ does not take into account FEC!!! this is because + // each client requests a differen amount of FEC packet so the client itself + // increase the production rate in the right way + bytes_production_rate_ = + ceil((double)(produced_bytes_ + prev_produced_bytes_) * per_second); + packets_production_rate_ = + ceil((double)(produced_packets_ + prev_produced_packets_) * per_second); + + if (fec_encoder_ && fec_type_ != fec::FECType::UNKNOWN) { + // add fec packets looking at the fec code. we don't use directly the number + // of fec packets produced in 1 round because it may happen that different + // numbers of blocks are generated during the rounds and this creates + // inconsistencies in the estimation of the production rate + uint32_t k = fec::FECUtils::getSourceSymbols(fec_type_); + uint32_t n = fec::FECUtils::getBlockSymbols(fec_type_); + + packets_production_rate_ += + ceil((double)packets_production_rate_ / (double)k) * (n - k); + } // update the production rate as soon as it increases by 10% with respect to // the last round max_packet_production_ = - produced_packets_ + ceil((double)produced_packets_ * 0.1); + produced_packets_ + ceil((double)produced_packets_ * 0.10); if (max_packet_production_ < rtc::WIN_MIN) max_packet_production_ = rtc::WIN_MIN; - if (packets_production_rate_ != 0) { - allow_delayed_nacks_ = false; - } else if (prev_packets_production_rate == 0) { - // at least 2 rounds with production rate = 0 + if (packets_production_rate_ <= rtc::MIN_PRODUCTION_RATE || + prev_packets_production_rate <= rtc::MIN_PRODUCTION_RATE) { allow_delayed_nacks_ = true; + } else { + // at least 2 rounds with enough packets + allow_delayed_nacks_ = false; } - // check if the production rate is decreased. if yes send nacks if needed - if (prev_packets_production_rate < packets_production_rate_) { - sendNacksForPendingInterests(); + if (new_round) { + prev_produced_bytes_ = produced_bytes_; + prev_produced_packets_ = produced_packets_; + produced_bytes_ = 0; + produced_packets_ = 0; + last_round_ = now; + scheduleRoundTimer(); } - - produced_bytes_ = 0; - produced_packets_ = 0; - produced_fec_packets_ = 0; - last_round_ = now; - scheduleRoundTimer(); } uint32_t RTCProductionProtocol::produceStream( @@ -164,75 +197,275 @@ void RTCProductionProtocol::produce(ContentObject &content_object) { uint32_t RTCProductionProtocol::produceDatagram( const Name &content_name, std::unique_ptr<utils::MemBuf> &&buffer) { std::size_t buffer_size = buffer->length(); + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "Maybe Sending content object: " << content_name; + if (TRANSPORT_EXPECT_FALSE(buffer_size == 0)) return 0; + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Sending content object: " << content_name; + uint32_t data_packet_size; socket_->getSocketOption(interface::GeneralTransportOptions::DATA_PACKET_SIZE, data_packet_size); - - if (TRANSPORT_EXPECT_FALSE((buffer_size + data_header_size_ + - rtc::DATA_HEADER_SIZE) > data_packet_size)) { + // this is a source packet but we check the fec header size of FEC packet in + // order to leave room for the header when FEC packets will be generated + uint32_t fec_header = 0; + if (fec_encoder_) fec_encoder_->getFecHeaderSize(true); + uint32_t headers_size = + (uint32_t)Packet::getHeaderSizeFromFormat(data_header_format_.first, + data_header_format_.second) + + rtc::DATA_HEADER_SIZE + fec_header; + if (TRANSPORT_EXPECT_FALSE((headers_size + buffer_size) > data_packet_size)) { return 0; } + if (!data_aggregation_) { + // if data aggregation is off emptyQueue will always return doing nothing + emptyQueue(); + + sendManifest(content_name); + + // create content object + auto content_object = + core::PacketManager<>::getInstance().getPacket<ContentObject>( + data_header_format_.first, data_header_format_.second); + + // add rtc header to the payload + struct rtc::data_packet_t header; + content_object->appendPayload((const uint8_t *)&header, + rtc::DATA_HEADER_SIZE); + content_object->appendPayload(buffer->data(), buffer->length()); + + // schedule actual sending on internal thread + portal_->getThread().tryRunHandlerNow( + [this, content_object{std::move(content_object)}, + content_name]() mutable { + produceInternal(std::move(content_object), content_name); + }); + } else { + // XXX here we assume that all the packets that we push to the queue have + // the same name + auto app_pkt = utils::MemBuf::copyBuffer(buffer->data(), buffer->length()); + addPacketToQueue(std::move(app_pkt)); + } + + return 1; +} + +void RTCProductionProtocol::addPacketToQueue( + std::unique_ptr<utils::MemBuf> &&buffer) { + std::size_t buffer_size = buffer->length(); + if ((queue_len_ + buffer_size) > rtc::MAX_RTC_PAYLOAD_SIZE) { + emptyQueue(); // this should guaranty that the generated packet will never + // be larger than an MTU + } + + waiting_app_packets_.push(std::move(buffer)); + if (max_len_ < buffer_size) max_len_ = buffer_size; + queue_len_ += buffer_size; + + if (waiting_app_packets_.size() >= rtc::MAX_AGGREGATED_PACKETS) { + emptyQueue(); + } + + if (waiting_app_packets_.size() >= 1 && !data_aggregation_timer_switch_) { + data_aggregation_timer_switch_ = true; + app_packets_timer_->expires_from_now( + std::chrono::milliseconds(rtc::AGGREGATED_PACKETS_TIMER)); + std::weak_ptr<RTCProductionProtocol> self = shared_from_this(); + app_packets_timer_->async_wait([self](const std::error_code &ec) { + if (ec) return; + + auto ptr = self.lock(); + if (ptr && ptr->isRunning()) { + if (!ptr->data_aggregation_timer_switch_) return; + ptr->emptyQueue(); + } + }); + } +} + +void RTCProductionProtocol::emptyQueue() { + if (waiting_app_packets_.size() == 0) return; // queue is empty + + Name n(flow_name_); + + // cancel timer is scheduled + if (data_aggregation_timer_switch_) { + data_aggregation_timer_switch_ = false; + app_packets_timer_->cancel(); + } + + // send a manifest beforehand if the hash buffer if full + sendManifest(n); + + // create content object auto content_object = core::PacketManager<>::getInstance().getPacket<ContentObject>( - signer_ ? Format::HF_INET6_TCP_AH : Format::HF_INET6_TCP, - signer_ ? signer_->getSignatureFieldSize() : 0); + data_header_format_.first, data_header_format_.second); + // add rtc header to the payload struct rtc::data_packet_t header; content_object->appendPayload((const uint8_t *)&header, rtc::DATA_HEADER_SIZE); - content_object->appendPayload(buffer->data(), buffer->length()); - // schedule actual sending on internal thread - portal_->getIoService().dispatch([this, - content_object{std::move(content_object)}, - content_name]() mutable { - produceInternal(std::move(content_object), content_name); + // init aggregated header + rtc::AggrPktHeader hdr( + (uint8_t *)(content_object->getPayload()->data() + rtc::DATA_HEADER_SIZE), + max_len_, waiting_app_packets_.size()); + uint32_t header_size = hdr.getHeaderLen(); + content_object->append(header_size); // leave space for the aggregated header + + uint8_t index = 0; + while (waiting_app_packets_.size() != 0) { + std::unique_ptr<utils::MemBuf> pkt = + std::move(waiting_app_packets_.front()); + waiting_app_packets_.pop(); + // XXX for the moment we have a single name, so this works, otherwise we + // need to do something else + hdr.addPacketToHeader(index, pkt->length()); + // append packet + content_object->appendPayload(pkt->data(), pkt->length()); + index++; + } + + // reset queue values + max_len_ = 0; + queue_len_ = 0; + + // the packet is ready we need to send it + portal_->getThread().tryRunHandlerNow( + [this, content_object{std::move(content_object)}, n]() mutable { + produceInternal(std::move(content_object), n); + }); +} + +void RTCProductionProtocol::sendManifest(const Name &name) { + if (!manifest_max_capacity_) { + return; + } + + Name manifest_name = name; + + // If there is not enough hashes to fill a manifest, return early + if (manifest_entries_.size() < manifest_max_capacity_) { + return; + } + + // Create a new manifest + std::shared_ptr<core::ContentObjectManifest> manifest = + createManifest(manifest_name.setSuffix(current_seg_)); + auto manifest_co = + std::dynamic_pointer_cast<ContentObject>(manifest->getPacket()); + + // Fill the manifest with packet hashes that were previously saved + uint32_t nb_entries; + for (nb_entries = 0; nb_entries < manifest_max_capacity_; ++nb_entries) { + if (manifest_entries_.empty()) { + break; + } + std::pair<uint32_t, auth::CryptoHash> front = manifest_entries_.front(); + manifest->addEntry(front.first, front.second); + manifest_entries_.pop(); + } + + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "Sending manifest " << manifest_co->getName().getSuffix() + << " of size " << nb_entries; + + // Encode and send the manifest + manifest->encode(); + portal_->getThread().tryRunHandlerNow( + [this, content_object{std::move(manifest_co)}, manifest_name]() mutable { + produceInternal(std::move(content_object), manifest_name); + }); +} + +std::shared_ptr<core::ContentObjectManifest> +RTCProductionProtocol::createManifest(const Name &content_name) const { + Name name(content_name); + + auth::CryptoHashType hash_algo; + socket_->getSocketOption(interface::GeneralTransportOptions::HASH_ALGORITHM, + hash_algo); + + uint64_t now = utils::SteadyTime::nowMs().count(); + + // Create a new manifest + std::shared_ptr<core::ContentObjectManifest> manifest = + ContentObjectManifest::createContentManifest( + manifest_header_format_.first, name, manifest_header_format_.second); + manifest->setHeaders(core::ManifestType::INLINE_MANIFEST, + manifest_max_capacity_, hash_algo, false /* is_last */, + name); + + // Set connection parameters + manifest->setParamsRTC(ParamsRTC{ + .timestamp = now, + .prod_rate = bytes_production_rate_, + .prod_seg = current_seg_, + .fec_type = fec_type_, }); - return 1; + return manifest; } void RTCProductionProtocol::produceInternal( std::shared_ptr<ContentObject> &&content_object, const Name &content_name, bool fec) { + uint64_t now = utils::SteadyTime::nowMs().count(); + + if (fec && (now - last_produced_data_ts_) < rtc::FEC_PACING_TIME) { + paced_fec_packets_.push(std::pair<uint64_t, ContentObject::Ptr>( + now, std::move(content_object))); + postponeFecPacket(); + } else { + // need to check if there are FEC packets waiting to be sent + flushFecPkts(current_seg_); + producePktInternal(std::move(content_object), content_name, fec); + } +} + +void RTCProductionProtocol::producePktInternal( + std::shared_ptr<ContentObject> &&content_object, const Name &content_name, + bool fec) { + bool is_manifest = content_object->getPayloadType() == PayloadType::MANIFEST; + uint64_t now = utils::SteadyTime::nowMs().count(); + // set rtc header - struct rtc::data_packet_t *data_pkt = - (struct rtc::data_packet_t *)content_object->getPayload()->data(); - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); - data_pkt->setTimestamp(now); - data_pkt->setProductionRate(bytes_production_rate_); + if (!is_manifest) { + struct rtc::data_packet_t *data_pkt = + (struct rtc::data_packet_t *)content_object->getPayload()->data(); + data_pkt->setTimestamp(now); + data_pkt->setProductionRate(bytes_production_rate_); + } // set hicn stuff Name n(content_name); content_object->setName(n.setSuffix(current_seg_)); - content_object->setLifetime(500); // XXX this should be set by the APP - content_object->setPathLabel(prod_label_); - // sign packet - if (signer_) { - signer_->signPacket(content_object.get()); - } + uint32_t expiry_time = 0; + socket_->getSocketOption( + interface::GeneralTransportOptions::CONTENT_OBJECT_EXPIRY_TIME, + expiry_time); + if (expiry_time == interface::default_values::content_object_expiry_time) + expiry_time = 500; // the data expiration time should be set by the App. if + // the App does not specify it the default is 500ms + content_object->setLifetime(expiry_time); + content_object->setPathLabel(prod_label_); // update stats if (!fec) { produced_bytes_ += content_object->headerSize() + content_object->payloadSize(); produced_packets_++; - } else { - produced_fec_packets_++; } - if (produced_packets_ >= max_packet_production_) { + if (!data_aggregation_ && produced_packets_ >= max_packet_production_) { // in this case all the pending interests may be used to accomodate the // sudden increase in the production rate. calling the updateStats we will // notify all the clients - round_timer_->cancel(); - updateStats(); + updateStats(false); } DLOG_IF(INFO, VLOG_IS_ON(3)) @@ -240,8 +473,12 @@ void RTCProductionProtocol::produceInternal( // pass packet to FEC encoder if (fec_encoder_ && !fec) { - fec_encoder_->onPacketProduced( - *content_object, content_object->headerSize() + rtc::DATA_HEADER_SIZE); + uint32_t offset = is_manifest ? (uint32_t)content_object->headerSize() + : (uint32_t)content_object->headerSize() + + rtc::DATA_HEADER_SIZE; + uint32_t metadata = static_cast<uint32_t>(content_object->getPayloadType()); + + fec_encoder_->onPacketProduced(*content_object, offset, metadata); } output_buffer_.insert(content_object); @@ -251,18 +488,15 @@ void RTCProductionProtocol::produceInternal( *content_object); } - auto seq_it = seqs_map_.find(current_seg_); - if (seq_it != seqs_map_.end()) { - portal_->sendContentObject(*content_object); - } + // TODO we may want to send FEC only if an interest is pending in the pit in + sendContentObject(content_object, false, fec); if (*on_content_object_output_) { on_content_object_output_->operator()(*socket_->getInterface(), *content_object); } - // remove interests from the interest cache if it exists - removeFromInterestQueue(current_seg_); + if (!fec) last_produced_data_ts_ = now; // Update current segment current_seg_ = (current_seg_ + 1) % rtc::MIN_PROBE_SEQ; @@ -277,254 +511,169 @@ void RTCProductionProtocol::produceInternal( } } -void RTCProductionProtocol::onInterest(Interest &interest) { - if (*on_interest_input_) { - on_interest_input_->operator()(*socket_->getInterface(), interest); +void RTCProductionProtocol::flushFecPkts(uint32_t current_seq_num) { + // Currently we immediately send all the pending fec packets + // A pacing policy may be helpful, but we do not want to delay too much + // the packets at this moment. + while (paced_fec_packets_.size() > 0) { + producePktInternal(std::move(paced_fec_packets_.front().second), flow_name_, + true); + paced_fec_packets_.pop(); + } + fec_pacing_timer_->cancel(); + pending_fec_pace_ = false; + postponeFecPacket(); +} + +void RTCProductionProtocol::postponeFecPacket() { + if (paced_fec_packets_.size() == 0) return; + if (pending_fec_pace_) { + return; } - auto suffix = interest.firstSuffix(); - // numberOfSuffixes returns only the prefixes in the payalod - // we add + 1 to count anche the seq in the name - auto n_suffixes = interest.numberOfSuffixes() + 1; - Name name = interest.getName(); - bool prev_consumer_state = consumer_in_sync_; + uint64_t produced_time = paced_fec_packets_.front().first; + uint64_t now = utils::SteadyTime::nowMs().count(); - for (uint32_t i = 0; i < n_suffixes; i++) { - if (i > 0) { - name.setSuffix(*(suffix + (i - 1))); - } - DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received interest " << name; + uint64_t wait_time = 0; + if ((produced_time + rtc::FEC_PACING_TIME) > now) + wait_time = produced_time + rtc::FEC_PACING_TIME - now; - const std::shared_ptr<ContentObject> content_object = - output_buffer_.find(name); + fec_pacing_timer_->expires_from_now(std::chrono::milliseconds(wait_time)); + pending_fec_pace_ = true; - if (content_object) { - if (*on_interest_satisfied_output_buffer_) { - on_interest_satisfied_output_buffer_->operator()( - *socket_->getInterface(), interest); - } + std::weak_ptr<RTCProductionProtocol> self = shared_from_this(); + fec_pacing_timer_->async_wait([self](const std::error_code &ec) { + if (ec) return; - if (*on_content_object_output_) { - on_content_object_output_->operator()(*socket_->getInterface(), - *content_object); - } + auto sp = self.lock(); + if (sp && sp->isRunning()) { + if (!sp->pending_fec_pace_) return; - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "Send content %u (onInterest) " << content_object->getName(); - content_object->setPathLabel(cache_label_); - portal_->sendContentObject(*content_object); - } else { - if (*on_interest_process_) { - on_interest_process_->operator()(*socket_->getInterface(), interest); + if (sp->paced_fec_packets_.size() > 0) { + sp->producePktInternal(std::move(sp->paced_fec_packets_.front().second), + sp->flow_name_, true); + sp->paced_fec_packets_.pop(); } - processInterest(name.getSuffix(), interest.getLifetime()); + sp->pending_fec_pace_ = false; + sp->postponeFecPacket(); } - } - - if (prev_consumer_state != consumer_in_sync_ && consumer_in_sync_) - on_consumer_in_sync_(*socket_->getInterface(), interest); + }); } -void RTCProductionProtocol::processInterest(uint32_t interest_seg, - uint32_t lifetime) { - if (interest_seg == 0) { - // first packet from the consumer, reset sync state - consumer_in_sync_ = false; +void RTCProductionProtocol::onInterest(Interest &interest) { + if (*on_interest_input_) { + on_interest_input_->operator()(*socket_->getInterface(), interest); } - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + if (!interest.isValid()) throw std::runtime_error("Bad interest format"); + if (interest.hasManifest() && + verifier_->verify(interest) != auth::VerificationPolicy::ACCEPT) + throw std::runtime_error("Interset manifest verification failed"); - if (interest_seg > rtc::MIN_PROBE_SEQ) { - DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received probe " << interest_seg; - sendNack(interest_seg); - return; - } - - // if the production rate 0 use delayed nacks - if (allow_delayed_nacks_ && interest_seg >= current_seg_) { - uint64_t next_timer = ~0; - if (!timers_map_.empty()) { - next_timer = timers_map_.begin()->first; - } + uint32_t *suffix = interest.firstSuffix(); + uint32_t n_suffixes_in_manifest = interest.numberOfSuffixes(); + hicn_uword *request_bitmap = interest.getRequestBitmap(); - uint64_t expiration = now + rtc::SENTINEL_TIMER_INTERVAL; - addToInterestQueue(interest_seg, expiration); - - // here we have at least one interest in the queue, we need to start or - // update the timer - if (!queue_timer_on_) { - // set timeout - queue_timer_on_ = true; - scheduleQueueTimer(timers_map_.begin()->first - now); - } else { - // re-schedule the timer because a new interest will expires sooner - if (next_timer > timers_map_.begin()->first) { - interests_queue_timer_->cancel(); - scheduleQueueTimer(timers_map_.begin()->first - now); - } - } - return; - } + Name name = interest.getName(); + uint32_t pos = 0; // Position of current suffix in manifest - if (queue_timer_on_) { - // the producer is producing. Send nacks to packets that will expire before - // the data production and remove the timer - queue_timer_on_ = false; - interests_queue_timer_->cancel(); - sendNacksForPendingInterests(); - } + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "Received interest " << name << " (" << n_suffixes_in_manifest + << " suffixes in manifest)"; + + // Process the suffix in the interest header + // (first loop iteration), then suffixes in the manifest + do { + if (!interest.hasManifest() || + bitmap_is_set_no_check(request_bitmap, pos)) { + const std::shared_ptr<ContentObject> content_object = + output_buffer_.find(name); + + if (content_object) { + if (*on_interest_satisfied_output_buffer_) { + on_interest_satisfied_output_buffer_->operator()( + *socket_->getInterface(), interest); + } - uint32_t max_gap = (uint32_t)floor( - (double)((double)((double)lifetime * - rtc::INTEREST_LIFETIME_REDUCTION_FACTOR / - rtc::MILLI_IN_A_SEC) * - (double)(packets_production_rate_ + - fec_packets_production_rate_))); + if (*on_content_object_output_) { + on_content_object_output_->operator()(*socket_->getInterface(), + *content_object); + } - if (interest_seg < current_seg_ || interest_seg > (max_gap + current_seg_)) { - sendNack(interest_seg); - } else { - if (!consumer_in_sync_ && on_consumer_in_sync_) { - // we consider the remote consumer to be in sync as soon as it covers 70% - // of the production window with interests - uint32_t perc = ceil((double)max_gap * 0.7); - if (interest_seg > (perc + current_seg_)) { - consumer_in_sync_ = true; - // on_consumer_in_sync_(*socket_->getInterface(), interest); + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "Send content %u (onInterest) " << content_object->getName(); + content_object->setPathLabel(cache_label_); + sendContentObject(content_object); + } else { + if (*on_interest_process_) { + on_interest_process_->operator()(*socket_->getInterface(), interest); + } + processInterest(name.getSuffix(), interest.getLifetime()); } } - uint64_t expiration = - now + floor((double)lifetime * rtc::INTEREST_LIFETIME_REDUCTION_FACTOR); - addToInterestQueue(interest_seg, expiration); - } -} -void RTCProductionProtocol::onError(std::error_code ec) {} + // Retrieve next suffix in the manifest + if (interest.hasManifest()) { + uint32_t seq = *suffix; + suffix++; -void RTCProductionProtocol::scheduleQueueTimer(uint64_t wait) { - interests_queue_timer_->expires_from_now(std::chrono::milliseconds(wait)); - interests_queue_timer_->async_wait([this](std::error_code ec) { - if (ec) return; - interestQueueTimer(); - }); + name.setSuffix(seq); + interest.setName(name); + } + } while (pos++ < n_suffixes_in_manifest); } -void RTCProductionProtocol::addToInterestQueue(uint32_t interest_seg, - uint64_t expiration) { - // check if the seq number exists already - auto it_seqs = seqs_map_.find(interest_seg); - if (it_seqs != seqs_map_.end()) { - // the seq already exists - if (expiration < it_seqs->second) { - // we need to update the timer becasue we got a smaller one - // 1) remove the entry from the multimap - // 2) update this entry - auto range = timers_map_.equal_range(it_seqs->second); - for (auto it_timers = range.first; it_timers != range.second; - it_timers++) { - if (it_timers->second == it_seqs->first) { - timers_map_.erase(it_timers); - break; - } - } - timers_map_.insert( - std::pair<uint64_t, uint32_t>(expiration, interest_seg)); - it_seqs->second = expiration; - } else { - // nothing to do here +void RTCProductionProtocol::processInterest(uint32_t interest_seg, + uint32_t lifetime) { + switch (rtc::ProbeHandler::getProbeType(interest_seg)) { + case rtc::ProbeType::INIT: + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received init probe " << interest_seg; + sendManifestProbe(interest_seg); return; - } - } else { - // add the new seq - timers_map_.insert(std::pair<uint64_t, uint32_t>(expiration, interest_seg)); - seqs_map_.insert(std::pair<uint32_t, uint64_t>(interest_seg, expiration)); + case rtc::ProbeType::RTT: + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received RTT probe " << interest_seg; + sendNack(interest_seg); + return; + default: + break; } -} -void RTCProductionProtocol::sendNacksForPendingInterests() { - std::unordered_set<uint32_t> to_remove; + if (interest_seg < current_seg_) sendNack(interest_seg); +} - uint32_t packet_gap = 100000; // set it to a high value (100sec) - if (packets_production_rate_ != 0) - packet_gap = ceil(rtc::MILLI_IN_A_SEC / (double)packets_production_rate_); +void RTCProductionProtocol::sendManifestProbe(uint32_t sequence) { + Name manifest_name(flow_name_); + manifest_name.setSuffix(sequence); - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + std::shared_ptr<core::ContentObjectManifest> manifest_probe = + createManifest(manifest_name); + auto manifest_probe_co = + std::dynamic_pointer_cast<ContentObject>(manifest_probe->getPacket()); - for (auto it = seqs_map_.begin(); it != seqs_map_.end(); it++) { - if (it->first > current_seg_) { - uint64_t production_time = - ((it->first - current_seg_) * packet_gap) + now; - if (production_time >= it->second) { - sendNack(it->first); - to_remove.insert(it->first); - } - } - } + manifest_probe_co->setLifetime(0); + manifest_probe_co->setPathLabel(prod_label_); + manifest_probe->encode(); - // delete nacked interests - for (auto it = to_remove.begin(); it != to_remove.end(); it++) { - removeFromInterestQueue(*it); - } -} - -void RTCProductionProtocol::removeFromInterestQueue(uint32_t interest_seg) { - auto seq_it = seqs_map_.find(interest_seg); - if (seq_it != seqs_map_.end()) { - auto range = timers_map_.equal_range(seq_it->second); - for (auto it_timers = range.first; it_timers != range.second; it_timers++) { - if (it_timers->second == seq_it->first) { - timers_map_.erase(it_timers); - break; - } - } - seqs_map_.erase(seq_it); + if (*on_content_object_output_) { + on_content_object_output_->operator()(*socket_->getInterface(), + *manifest_probe_co); } -} -void RTCProductionProtocol::interestQueueTimer() { - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); - - for (auto it_timers = timers_map_.begin(); it_timers != timers_map_.end();) { - uint64_t expire = it_timers->first; - if (expire <= now) { - uint32_t seq = it_timers->second; - sendNack(seq); - // remove the interest from the other map - seqs_map_.erase(seq); - it_timers = timers_map_.erase(it_timers); - } else { - // stop, we are done! - break; - } - } - if (timers_map_.empty()) { - queue_timer_on_ = false; - } else { - queue_timer_on_ = true; - scheduleQueueTimer(timers_map_.begin()->first - now); - } + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send init probe " << sequence; + sendContentObject(manifest_probe_co, true, false); } void RTCProductionProtocol::sendNack(uint32_t sequence) { auto nack = core::PacketManager<>::getInstance().getPacket<ContentObject>( - signer_ ? Format::HF_INET6_TCP_AH : Format::HF_INET6_TCP, - signer_ ? signer_->getSignatureFieldSize() : 0); - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + nack_header_format_.first, nack_header_format_.second); + uint64_t now = utils::SteadyTime::nowMs().count(); uint32_t next_packet = current_seg_; uint32_t prod_rate = bytes_production_rate_; struct rtc::nack_packet_t header; header.setTimestamp(now); header.setProductionRate(prod_rate); - header.setProductionSegement(next_packet); + header.setProductionSegment(next_packet); nack->appendPayload((const uint8_t *)&header, rtc::NACK_HEADER_SIZE); Name n(flow_name_); @@ -533,33 +682,42 @@ void RTCProductionProtocol::sendNack(uint32_t sequence) { nack->setLifetime(0); nack->setPathLabel(prod_label_); - if (signer_) { - signer_->signPacket(nack.get()); + if (*on_content_object_output_) { + on_content_object_output_->operator()(*socket_->getInterface(), *nack); } - if (!consumer_in_sync_ && on_consumer_in_sync_ && - sequence < rtc::MIN_PROBE_SEQ && sequence > next_packet) { - consumer_in_sync_ = true; - auto interest = core::PacketManager<>::getInstance().getPacket<Interest>(); - interest->setName(n); - on_consumer_in_sync_(*socket_->getInterface(), *interest); + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send nack " << sequence; + sendContentObject(nack, true, false); +} + +void RTCProductionProtocol::sendContentObject( + std::shared_ptr<ContentObject> content_object, bool nack, bool fec) { + bool is_ah = HICN_PACKET_FORMAT_IS_AH(content_object->getFormat()); + + // Compute signature + if (is_ah) { + signer_->signPacket(content_object.get()); } - if (*on_content_object_output_) { - on_content_object_output_->operator()(*socket_->getInterface(), *nack); + // Compute and save data packet digest + if (manifest_max_capacity_ && !is_ah) { + auth::CryptoHashType hash_algo; + socket_->getSocketOption(interface::GeneralTransportOptions::HASH_ALGORITHM, + hash_algo); + manifest_entries_.push({content_object->getName().getSuffix(), + content_object->computeDigest(hash_algo)}); } - DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send nack " << sequence; - portal_->sendContentObject(*nack); + portal_->sendContentObject(*content_object); } -void RTCProductionProtocol::onFecPackets( - std::vector<std::pair<uint32_t, fec::buffer>> &packets) { +void RTCProductionProtocol::onFecPackets(fec::BufferArray &packets) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Produced " << packets.size() << " FEC packets"; + for (auto &packet : packets) { auto content_object = - std::static_pointer_cast<ContentObject>(packet.second); + std::static_pointer_cast<ContentObject>(packet.getBuffer()); content_object->prepend(content_object->headerSize() + rtc::DATA_HEADER_SIZE); pending_fec_packets_.push(std::move(content_object)); @@ -569,19 +727,21 @@ void RTCProductionProtocol::onFecPackets( fec::buffer RTCProductionProtocol::getBuffer(std::size_t size) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Asked buffer for FEC symbol of size " << size; + auto ret = core::PacketManager<>::getInstance().getPacket<ContentObject>( - signer_ ? Format::HF_INET6_TCP_AH : Format::HF_INET6_TCP, - signer_ ? signer_->getSignatureFieldSize() : 0); + fec_header_format_.first, fec_header_format_.second); + ret->updateLength(rtc::DATA_HEADER_SIZE + size); ret->append(rtc::DATA_HEADER_SIZE + size); ret->trimStart(ret->headerSize() + rtc::DATA_HEADER_SIZE); + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Responding with buffer of length " << ret->length(); - assert(ret->length() >= size); + DCHECK(ret->length() >= size); return ret; } } // namespace protocol -} // end namespace transport +} // namespace transport diff --git a/libtransport/src/protocols/prod_protocol_rtc.h b/libtransport/src/protocols/prod_protocol_rtc.h index 96ad5673d..285ccb646 100644 --- a/libtransport/src/protocols/prod_protocol_rtc.h +++ b/libtransport/src/protocols/prod_protocol_rtc.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -17,6 +17,7 @@ #include <hicn/transport/core/name.h> #include <protocols/production_protocol.h> +#include <protocols/rtc/rtc_verifier.h> #include <atomic> #include <map> @@ -32,6 +33,7 @@ class RTCProductionProtocol : public ProductionProtocol { using ProductionProtocol::start; using ProductionProtocol::stop; + void setProducerParam() override; void produce(ContentObject &content_object) override; uint32_t produceStream(const Name &content_name, @@ -49,56 +51,69 @@ class RTCProductionProtocol : public ProductionProtocol { buffer, buffer_size, buffer_size)); } - void registerNamespaceWithNetwork(const Prefix &producer_namespace) override; - - void setConsumerInSyncCallback( - interface::ProducerInterestCallback &&callback) { - on_consumer_in_sync_ = std::move(callback); - } + auto shared_from_this() { return utils::shared_from(this); } private: // packet handlers void onInterest(Interest &interest) override; - void onError(std::error_code ec) override; + void onError(const std::error_code &ec) override{}; void processInterest(uint32_t interest_seg, uint32_t lifetime); + void producePktInternal(std::shared_ptr<ContentObject> &&content_object, + const Name &content_name, bool fec = false); void produceInternal(std::shared_ptr<ContentObject> &&content_object, const Name &content_name, bool fec = false); void sendNack(uint32_t sequence); + void sendContentObject(std::shared_ptr<ContentObject> content_object, + bool nac = false, bool fec = false); + + // manifests + void sendManifestProbe(uint32_t sequence); + void sendManifest(const Name &content_name); + std::shared_ptr<core::ContentObjectManifest> createManifest( + const Name &name) const; // stats - void updateStats(); + void updateStats(bool new_round); void scheduleRoundTimer(); - // pending intersts functions - void addToInterestQueue(uint32_t interest_seg, uint64_t expiration); - void sendNacksForPendingInterests(); - void removeFromInterestQueue(uint32_t interest_seg); - void scheduleQueueTimer(uint64_t wait); - void interestQueueTimer(); - // FEC functions - void onFecPackets(std::vector<std::pair<uint32_t, fec::buffer>> &packets); + void onFecPackets(fec::BufferArray &packets); fec::buffer getBuffer(std::size_t size); + void postponeFecPacket(); + void dispatchFecPacket(); + void flushFecPkts(uint32_t current_seq_num); + // aggregated data functions + void emptyQueue(); + void addPacketToQueue(std::unique_ptr<utils::MemBuf> &&buffer); core::Name flow_name_; - uint32_t current_seg_; // seq id of the next packet produced - uint32_t prod_label_; // path lable of the producer - uint32_t cache_label_; // path lable for content from the producer cache - uint16_t data_header_size_; // hicn data header size + std::pair<core::Packet::Format, size_t> data_header_format_; + std::pair<core::Packet::Format, size_t> manifest_header_format_; + std::pair<core::Packet::Format, size_t> fec_header_format_; + std::pair<core::Packet::Format, size_t> nack_header_format_; + + uint32_t current_seg_; // seq id of the next packet produced + uint32_t prod_label_; // path label of the producer + uint32_t cache_label_; // path label for content from the producer cache - uint32_t produced_bytes_; // bytes produced in the last round - uint32_t produced_packets_; // packet produed in the last round - uint32_t produced_fec_packets_; // fec packets produced last round + uint32_t prev_produced_bytes_; // XXX clearly explain all these new vars + uint32_t prev_produced_packets_; + + uint32_t produced_bytes_; // bytes produced in the last round + uint32_t produced_packets_; // packet produed in the last round uint32_t max_packet_production_; // never exceed this number of packets // without update stats - uint32_t bytes_production_rate_; // bytes per sec - uint32_t packets_production_rate_; // pps - uint32_t fec_packets_production_rate_; // pps + uint32_t bytes_production_rate_; // bytes per sec + uint32_t packets_production_rate_; // pps + + uint64_t last_produced_data_ts_; // ms std::unique_ptr<asio::steady_timer> round_timer_; + std::unique_ptr<asio::steady_timer> fec_pacing_timer_; + uint64_t last_round_; // delayed nacks are used by the producer to avoid to send too @@ -108,29 +123,26 @@ class RTCProductionProtocol : public ProductionProtocol { // of the new rate. bool allow_delayed_nacks_; - // queue for the received interests - // this map maps the expiration time of an interest to - // its sequence number. the map is sorted by timeouts - // the same timeout may be used for multiple sequence numbers - // but for each sequence number we store only the smallest - // expiry time. In this way the mapping from seqs_map_ to - // timers_map_ is unique - std::multimap<uint64_t, uint32_t> timers_map_; - - // this map does the opposite, this map is not ordered - std::unordered_map<uint32_t, uint64_t> seqs_map_; - bool queue_timer_on_; - std::unique_ptr<asio::steady_timer> interests_queue_timer_; - - // this callback is called when the remote consumer is in sync with high - // probability. it is called only the first time that the switch happen. - // XXX this makes sense only in P2P mode, while in standard mode is - // impossible to know the state of the consumers so it should not be used. - bool consumer_in_sync_; - interface::ProducerInterestCallback on_consumer_in_sync_; - // Save FEC packets here before sending them std::queue<ContentObject::Ptr> pending_fec_packets_; + std::queue<std::pair<uint64_t, ContentObject::Ptr>> paced_fec_packets_; + bool pending_fec_pace_; + + // Save application packets if they are small + std::queue<std::unique_ptr<utils::MemBuf>> waiting_app_packets_; + uint16_t max_len_; // len of the largest packet + uint16_t queue_len_; // total size of all packet in the queue + bool data_aggregation_; // turns on/off data aggregation + // timer to check the queue len + std::unique_ptr<asio::steady_timer> app_packets_timer_; + bool data_aggregation_timer_switch_; // bool to check if the timer is on + + // Manifest + std::queue<std::pair<uint32_t, auth::CryptoHash>> + manifest_entries_; // map a packet suffix to a packet hash + + // Verifier for aggregated interests + std::shared_ptr<rtc::RTCVerifier> verifier_; }; } // namespace protocol diff --git a/libtransport/src/protocols/production_protocol.cc b/libtransport/src/protocols/production_protocol.cc index 6b317d47d..039a6a55a 100644 --- a/libtransport/src/protocols/production_protocol.cc +++ b/libtransport/src/protocols/production_protocol.cc @@ -24,8 +24,8 @@ using namespace interface; ProductionProtocol::ProductionProtocol( implementation::ProducerSocket *icn_socket) - : socket_(icn_socket), - is_running_(false), + : Protocol(), + socket_(icn_socket), fec_encoder_(nullptr), on_interest_input_(VOID_HANDLER), on_interest_dropped_input_buffer_(VOID_HANDLER), @@ -38,101 +38,92 @@ ProductionProtocol::ProductionProtocol( on_content_object_output_(VOID_HANDLER), on_content_object_evicted_from_output_buffer_(VOID_HANDLER), on_content_produced_(VOID_HANDLER), + producer_callback_(VOID_HANDLER), fec_type_(fec::FECType::UNKNOWN) { socket_->getSocketOption(GeneralTransportOptions::PORTAL, portal_); // TODO add statistics for producer // socket_->getSocketOption(OtherOptions::STATISTICS, &stats_); } -ProductionProtocol::~ProductionProtocol() { - if (!is_async_ && is_running_) { - stop(); - } +ProductionProtocol::~ProductionProtocol() {} - if (listening_thread_.joinable()) { - listening_thread_.join(); +int ProductionProtocol::start() { + if (isRunning()) { + return -1; } -} -int ProductionProtocol::start() { - socket_->getSocketOption(ProducerCallbacksOptions::INTEREST_INPUT, - &on_interest_input_); - socket_->getSocketOption(ProducerCallbacksOptions::INTEREST_DROP, - &on_interest_dropped_input_buffer_); - socket_->getSocketOption(ProducerCallbacksOptions::INTEREST_PASS, - &on_interest_inserted_input_buffer_); - socket_->getSocketOption(ProducerCallbacksOptions::CACHE_HIT, - &on_interest_satisfied_output_buffer_); - socket_->getSocketOption(ProducerCallbacksOptions::CACHE_MISS, - &on_interest_process_); - socket_->getSocketOption(ProducerCallbacksOptions::NEW_CONTENT_OBJECT, - &on_new_segment_); - socket_->getSocketOption(ProducerCallbacksOptions::CONTENT_OBJECT_READY, - &on_content_object_in_output_buffer_); - socket_->getSocketOption(ProducerCallbacksOptions::CONTENT_OBJECT_OUTPUT, - &on_content_object_output_); - socket_->getSocketOption(ProducerCallbacksOptions::CONTENT_OBJECT_TO_SIGN, - &on_content_object_to_sign_); - socket_->getSocketOption(ProducerCallbacksOptions::CONTENT_PRODUCED, - &on_content_produced_); - - socket_->getSocketOption(GeneralTransportOptions::ASYNC_MODE, is_async_); - socket_->getSocketOption(GeneralTransportOptions::SIGNER, signer_); - socket_->getSocketOption(GeneralTransportOptions::MAKE_MANIFEST, - making_manifest_); - - bool first = true; - - for (core::Prefix &producer_namespace : served_namespaces_) { - if (first) { - core::BindConfig bind_config(producer_namespace, 1000); - portal_->bind(bind_config); - portal_->setProducerCallback(this); - first = !first; - } else { - portal_->registerRoute(producer_namespace); + portal_->getThread().addAndWaitForExecution([this]() { + socket_->getSocketOption(ProducerCallbacksOptions::INTEREST_INPUT, + &on_interest_input_); + socket_->getSocketOption(ProducerCallbacksOptions::INTEREST_DROP, + &on_interest_dropped_input_buffer_); + socket_->getSocketOption(ProducerCallbacksOptions::INTEREST_PASS, + &on_interest_inserted_input_buffer_); + socket_->getSocketOption(ProducerCallbacksOptions::CACHE_HIT, + &on_interest_satisfied_output_buffer_); + socket_->getSocketOption(ProducerCallbacksOptions::CACHE_MISS, + &on_interest_process_); + socket_->getSocketOption(ProducerCallbacksOptions::NEW_CONTENT_OBJECT, + &on_new_segment_); + socket_->getSocketOption(ProducerCallbacksOptions::CONTENT_OBJECT_READY, + &on_content_object_in_output_buffer_); + socket_->getSocketOption(ProducerCallbacksOptions::CONTENT_OBJECT_OUTPUT, + &on_content_object_output_); + socket_->getSocketOption(ProducerCallbacksOptions::CONTENT_OBJECT_TO_SIGN, + &on_content_object_to_sign_); + socket_->getSocketOption(ProducerCallbacksOptions::CONTENT_PRODUCED, + &on_content_produced_); + socket_->getSocketOption(ProducerCallbacksOptions::PRODUCER_CALLBACK, + &producer_callback_); + + socket_->getSocketOption(GeneralTransportOptions::ASYNC_MODE, is_async_); + socket_->getSocketOption(GeneralTransportOptions::SIGNER, signer_); + socket_->getSocketOption(GeneralTransportOptions::MANIFEST_MAX_CAPACITY, + manifest_max_capacity_); + + std::string fec_type_str = ""; + socket_->getSocketOption(GeneralTransportOptions::FEC_TYPE, fec_type_str); + if (fec_type_str != "") { + fec_type_ = fec::FECUtils::fecTypeFromString(fec_type_str.c_str()); } - } - is_running_ = true; + portal_->registerTransportCallback(this); + setProducerParam(); - if (!is_async_) { - listening_thread_ = std::thread([this]() { portal_->runEventsLoop(); }); - } + setRunning(); + }); return 0; } -void ProductionProtocol::stop() { - is_running_ = false; - - if (!is_async_) { - portal_->stopEventsLoop(); - } else { - portal_->clear(); - } -} - void ProductionProtocol::produce(ContentObject &content_object) { - if (*on_content_object_in_output_buffer_) { - on_content_object_in_output_buffer_->operator()(*socket_->getInterface(), - content_object); - } + auto content_object_ptr = content_object.shared_from_this(); + portal_->getThread().add([this, co = std::move(content_object_ptr)]() { + if (*on_content_object_in_output_buffer_) { + on_content_object_in_output_buffer_->operator()(*socket_->getInterface(), + *co); + } - output_buffer_.insert(std::static_pointer_cast<ContentObject>( - content_object.shared_from_this())); + output_buffer_.insert(co); - if (*on_content_object_output_) { - on_content_object_output_->operator()(*socket_->getInterface(), - content_object); - } + if (*on_content_object_output_) { + on_content_object_output_->operator()(*socket_->getInterface(), *co); + } - portal_->sendContentObject(content_object); + portal_->sendContentObject(*co); + }); } -void ProductionProtocol::registerNamespaceWithNetwork( - const Prefix &producer_namespace) { - served_namespaces_.push_back(producer_namespace); +void ProductionProtocol::sendMapme() { portal_->sendMapme(); } + +void ProductionProtocol::onError(const std::error_code &ec) { + // Stop production protocol + stop(); + + // Call error callback + if (producer_callback_) { + producer_callback_->produceError(ec); + } } } // namespace protocol diff --git a/libtransport/src/protocols/production_protocol.h b/libtransport/src/protocols/production_protocol.h index 7366311eb..09718631f 100644 --- a/libtransport/src/protocols/production_protocol.h +++ b/libtransport/src/protocols/production_protocol.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -22,6 +22,7 @@ #include <implementation/socket.h> #include <protocols/fec_base.h> #include <protocols/fec_utils.h> +#include <protocols/protocol.h> #include <utils/content_store.h> #include <atomic> @@ -33,17 +34,20 @@ namespace protocol { using namespace core; -class ProductionProtocol : public Portal::ProducerCallback { +class ProductionProtocol + : public Protocol, + public std::enable_shared_from_this<ProductionProtocol> { public: ProductionProtocol(implementation::ProducerSocket *icn_socket); virtual ~ProductionProtocol(); - bool isRunning() { return is_running_; } - virtual int start(); - virtual void stop(); + using Protocol::stop; + + virtual void setProducerParam(){}; virtual void produce(ContentObject &content_object); + virtual void sendMapme(); virtual uint32_t produceStream(const Name &content_name, std::unique_ptr<utils::MemBuf> &&buffer, bool is_last = true, @@ -61,22 +65,21 @@ class ProductionProtocol : public Portal::ProducerCallback { void setOutputBufferSize(std::size_t size) { output_buffer_.setLimit(size); } std::size_t getOutputBufferSize() { return output_buffer_.getLimit(); } - virtual void registerNamespaceWithNetwork(const Prefix &producer_namespace); - const std::list<Prefix> &getNamespaces() const { return served_namespaces_; } - protected: // Producer callback virtual void onInterest(core::Interest &i) override = 0; - virtual void onError(std::error_code ec) override{}; + virtual void onError(const std::error_code &ec) override; template <typename FECHandler, typename AllocatorHandler> void enableFEC(FECHandler &&fec_handler, AllocatorHandler &&allocator_handler) { if (!fec_encoder_) { // Try to get FEC from environment - if (const char *fec_str = std::getenv("TRANSPORT_FEC_TYPE")) { + const char *fec_str = std::getenv("TRANSPORT_FEC_TYPE"); + if (fec_str && (fec_type_ == fec::FECType::UNKNOWN)) { LOG(INFO) << "Using FEC " << fec_str; fec_type_ = fec::FECUtils::fecTypeFromString(fec_str); + CHECK(fec_type_ != fec::FECType::UNKNOWN); } if (fec_type_ == fec::FECType::UNKNOWN) { @@ -95,11 +98,6 @@ class ProductionProtocol : public Portal::ProducerCallback { // Thread pool responsible for IO operations (send data / receive interests) std::vector<utils::EventThread> io_threads_; - - // TODO remove this thread - std::thread listening_thread_; - std::shared_ptr<Portal> portal_; - std::atomic<bool> is_running_; interface::ProductionStatistics *stats_; std::unique_ptr<fec::ProducerFEC> fec_encoder_; @@ -119,15 +117,14 @@ class ProductionProtocol : public Portal::ProducerCallback { interface::ProducerContentCallback *on_content_produced_; + interface::ProducerSocket::Callback *producer_callback_; + // Output buffer utils::ContentStore output_buffer_; - // List ot routes served by current producer protocol - std::list<Prefix> served_namespaces_; - // Signature and manifest std::shared_ptr<auth::Signer> signer_; - bool making_manifest_; + uint32_t manifest_max_capacity_; bool is_async_; fec::FECType fec_type_; diff --git a/libtransport/src/protocols/protocol.h b/libtransport/src/protocols/protocol.h new file mode 100644 index 000000000..a9f929db9 --- /dev/null +++ b/libtransport/src/protocols/protocol.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <core/portal.h> +#include <hicn/transport/errors/runtime_exception.h> +#include <hicn/transport/utils/noncopyable.h> + +#include <random> + +namespace transport { + +namespace protocol { + +class Protocol : public core::Portal::TransportCallback, utils::NonCopyable { + public: + virtual void stop() { + portal_->getThread().addAndWaitForExecution([this]() { + unSetRunning(); + portal_->unregisterTransportCallback(); + portal_->clear(); + }); + } + + virtual void onInterest(core::Interest &i) { + throw errors::RuntimeException("Not implemented"); + } + + virtual void onContentObject(core::Interest &i, core::ContentObject &c) { + throw errors::RuntimeException("Not implemented"); + } + + virtual void onTimeout(core::Interest::Ptr &i, const core::Name &n) { + throw errors::RuntimeException("Not implemented"); + } + + virtual void onError(const std::error_code &ec) { + throw errors::RuntimeException("Not implemented"); + } + + bool isRunning() { return is_running_; } + void setRunning() { is_running_ = true; } + void unSetRunning() { is_running_ = false; } + + protected: + Protocol() : portal_(nullptr), is_running_(false), gen_(rd_()) {} + virtual ~Protocol() {} + + protected: + std::shared_ptr<core::Portal> portal_; + std::atomic_bool is_running_; + + // Random engine + std::random_device rd_; + std::mt19937 gen_; +}; + +} // namespace protocol + +} // namespace transport
\ No newline at end of file diff --git a/libtransport/src/protocols/raaqm.cc b/libtransport/src/protocols/raaqm.cc index 1247af400..bcbc15aef 100644 --- a/libtransport/src/protocols/raaqm.cc +++ b/libtransport/src/protocols/raaqm.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -36,8 +36,9 @@ RaaqmTransportProtocol::RaaqmTransportProtocol( current_window_size_(1), interests_in_flight_(0), cur_path_(nullptr), - t0_(utils::SteadyClock::now()), + t0_(utils::SteadyTime::Clock::now()), rate_estimator_(nullptr), + dis_(0, 1.0), schedule_interests_(true) { init(); } @@ -60,7 +61,7 @@ void RaaqmTransportProtocol::reset() { // Reset protocol variables interests_in_flight_ = 0; - t0_ = utils::SteadyClock::now(); + t0_ = utils::SteadyTime::Clock::now(); // Optionally reset congestion window bool reset_window; @@ -370,7 +371,7 @@ void RaaqmTransportProtocol::onPacketDropped(Interest &interest, } interest_retransmissions_[segment & mask]++; - interest_to_retransmit_.push(segment); + interest_to_retransmit_.push((unsigned int)segment); } else { LOG(ERROR) << "Stop: received not trusted packet " << interest_retransmissions_[segment & mask] << " times"; @@ -389,6 +390,8 @@ void RaaqmTransportProtocol::sendInterest( uint32_t len) { interests_in_flight_++; interest_retransmissions_[interest_name.getSuffix() & mask]++; + interest_timepoints_[interest_name.getSuffix() & mask] = + utils::SteadyTime::Clock::now(); TransportProtocol::sendInterest(interest_name, additional_suffixes, len); } @@ -426,7 +429,7 @@ void RaaqmTransportProtocol::onInterestTimeout(Interest::Ptr &interest, return; } - interest_to_retransmit_.push(segment); + interest_to_retransmit_.push((unsigned int)segment); scheduleNextInterests(); } else { LOG(ERROR) << "Stop: reached max retx limit."; @@ -477,7 +480,7 @@ void RaaqmTransportProtocol::scheduleNextInterests() { } } -void RaaqmTransportProtocol::onContentReassembled(std::error_code ec) { +void RaaqmTransportProtocol::onContentReassembled(const std::error_code &ec) { rate_estimator_->onDownloadFinished(); TransportProtocol::onContentReassembled(ec); schedule_interests_ = false; @@ -487,18 +490,18 @@ void RaaqmTransportProtocol::updateRtt(uint64_t segment) { if (TRANSPORT_EXPECT_FALSE(!cur_path_)) { throw std::runtime_error("RAAQM ERROR: no current path found, exit"); } else { - auto now = utils::SteadyClock::now(); - utils::Microseconds rtt = std::chrono::duration_cast<utils::Microseconds>( - now - interest_timepoints_[segment & mask]); + auto now = utils::SteadyTime::Clock::now(); + auto rtt = utils::SteadyTime::getDurationUs( + interest_timepoints_[segment & mask], now); // Update stats - updateStats((uint32_t)segment, rtt.count(), now); + updateStats((uint32_t)segment, rtt, now); if (rate_estimator_) { - rate_estimator_->onRttUpdate((double)rtt.count()); + rate_estimator_->onRttUpdate(rtt); } - cur_path_->insertNewRtt(rtt.count(), now); + cur_path_->insertNewRtt(rtt, now); cur_path_->smoothTimer(); if (cur_path_->newPropagationDelayAvailable()) { @@ -510,27 +513,27 @@ void RaaqmTransportProtocol::updateRtt(uint64_t segment) { void RaaqmTransportProtocol::RAAQM() { if (!cur_path_) { throw errors::RuntimeException("ERROR: no current path found, exit"); - exit(EXIT_FAILURE); } else { // Change drop probability according to RTT statistics cur_path_->updateDropProb(); - double coin = ((double)rand() / (RAND_MAX)); + double coin = dis_(gen_); if (coin <= cur_path_->getDropProb()) { decreaseWindow(); } } } -void RaaqmTransportProtocol::updateStats(uint32_t suffix, uint64_t rtt, - utils::TimePoint &now) { +void RaaqmTransportProtocol::updateStats( + uint32_t suffix, const utils::SteadyTime::Microseconds &rtt, + utils::SteadyTime::TimePoint &now) { // Update RTT statistics stats_->updateAverageRtt(rtt); stats_->updateAverageWindowSize(current_window_size_); // Call statistics callback if (*stats_summary_) { - auto dt = std::chrono::duration_cast<utils::Milliseconds>(now - t0_); + auto dt = utils::SteadyTime::getDurationMs(t0_, now); uint32_t timer_interval_milliseconds = 0; socket_->getSocketOption(GeneralTransportOptions::STATS_INTERVAL, diff --git a/libtransport/src/protocols/raaqm.h b/libtransport/src/protocols/raaqm.h index ffbb30d3a..a7ef23b68 100644 --- a/libtransport/src/protocols/raaqm.h +++ b/libtransport/src/protocols/raaqm.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -23,6 +23,7 @@ #include <protocols/transport_protocol.h> #include <queue> +#include <random> #include <vector> namespace transport { @@ -55,8 +56,9 @@ class RaaqmTransportProtocol : public TransportProtocol, const ContentObject &content_object); virtual void afterDataUnsatisfied(uint64_t segment); - virtual void updateStats(uint32_t suffix, uint64_t rtt, - utils::TimePoint &now); + virtual void updateStats(uint32_t suffix, + const utils::SteadyTime::Microseconds &rtt, + utils::SteadyTime::TimePoint &now); private: void init(); @@ -73,7 +75,7 @@ class RaaqmTransportProtocol : public TransportProtocol, *additional_suffixes = nullptr, uint32_t len = 0) override; - void onContentReassembled(std::error_code ec) override; + void onContentReassembled(const std::error_code &ec) override; void updateRtt(uint64_t segment); void RAAQM(); void updatePathTable(const ContentObject &content_object); @@ -81,13 +83,15 @@ class RaaqmTransportProtocol : public TransportProtocol, void checkForStalePaths(); void printRtt(); + auto shared_from_this() { return utils::shared_from(this); } + protected: // Congestion window management double current_window_size_; // Protocol management uint64_t interests_in_flight_; std::array<std::uint32_t, buffer_size> interest_retransmissions_; - std::array<utils::TimePoint, buffer_size> interest_timepoints_; + std::array<utils::SteadyTime::TimePoint, buffer_size> interest_timepoints_; std::queue<uint32_t> interest_to_retransmit_; private: @@ -102,13 +106,16 @@ class RaaqmTransportProtocol : public TransportProtocol, PathTable path_table_; // TimePoints for statistic - utils::TimePoint t0_; + utils::SteadyTime::TimePoint t0_; bool set_interest_filter_; // for rate-estimation at packet level IcnRateEstimator *rate_estimator_; + // Real distribution + std::uniform_real_distribution<> dis_; + // params for autotuning bool raaqm_autotune_; double default_beta_; diff --git a/libtransport/src/protocols/raaqm_data_path.cc b/libtransport/src/protocols/raaqm_data_path.cc index f2c21b9ef..b8e6e6285 100644 --- a/libtransport/src/protocols/raaqm_data_path.cc +++ b/libtransport/src/protocols/raaqm_data_path.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -23,16 +23,18 @@ namespace protocol { RaaqmDataPath::RaaqmDataPath(double drop_factor, double minimum_drop_probability, unsigned new_timer, unsigned int samples, - uint64_t new_rtt, uint64_t new_rtt_min, - uint64_t new_rtt_max, unsigned new_pd) + const utils::SteadyTime::Milliseconds &new_rtt, + const utils::SteadyTime::Milliseconds &new_rtt_min, + const utils::SteadyTime::Milliseconds &new_rtt_max, + unsigned new_pd) : drop_factor_(drop_factor), minimum_drop_probability_(minimum_drop_probability), timer_(new_timer), samples_(samples), - rtt_(new_rtt), - rtt_min_(new_rtt_min), - rtt_max_(new_rtt_max), + rtt_(new_rtt.count()), + rtt_min_(new_rtt_min.count()), + rtt_max_(new_rtt_max.count()), prop_delay_(new_pd), new_prop_delay_(false), drop_prob_(0), @@ -43,14 +45,15 @@ RaaqmDataPath::RaaqmDataPath(double drop_factor, raw_data_bytes_received_(0), last_raw_data_bytes_received_(0), rtt_samples_(samples_), - last_received_pkt_(utils::SteadyClock::now()), + last_received_pkt_(utils::SteadyTime::Clock::now()), average_rtt_(0), alpha_(ALPHA) {} -RaaqmDataPath &RaaqmDataPath::insertNewRtt(uint64_t new_rtt, - const utils::TimePoint &now) { - rtt_ = new_rtt; - rtt_samples_.pushBack(new_rtt); +RaaqmDataPath &RaaqmDataPath::insertNewRtt( + const utils::SteadyTime::Microseconds &new_rtt, + const utils::SteadyTime::TimePoint &now) { + rtt_ = new_rtt.count() / 1000; + rtt_samples_.pushBack(rtt_); rtt_max_ = rtt_samples_.rBegin(); rtt_min_ = rtt_samples_.begin(); @@ -143,10 +146,8 @@ unsigned int RaaqmDataPath::getPropagationDelay() { } bool RaaqmDataPath::isStale() { - utils::TimePoint now = utils::SteadyClock::now(); - auto time = - std::chrono::duration_cast<utils::Microseconds>(now - last_received_pkt_) - .count(); + utils::SteadyTime::TimePoint now = utils::SteadyTime::Clock::now(); + auto time = utils::SteadyTime::getDurationUs(last_received_pkt_, now).count(); if (time > 2000000) { return true; } diff --git a/libtransport/src/protocols/raaqm_data_path.h b/libtransport/src/protocols/raaqm_data_path.h index c0b53a690..dd24dad51 100644 --- a/libtransport/src/protocols/raaqm_data_path.h +++ b/libtransport/src/protocols/raaqm_data_path.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -34,8 +34,13 @@ class RaaqmDataPath { public: RaaqmDataPath(double drop_factor, double minimum_drop_probability, unsigned new_timer, unsigned int samples, - uint64_t new_rtt = 1000, uint64_t new_rtt_min = 1000, - uint64_t new_rtt_max = 1000, unsigned new_pd = UINT_MAX); + const utils::SteadyTime::Milliseconds &new_rtt = + utils::SteadyTime::Milliseconds(1000), + const utils::SteadyTime::Milliseconds &new_rtt_min = + utils::SteadyTime::Milliseconds(1000), + const utils::SteadyTime::Milliseconds &new_rtt_max = + utils::SteadyTime::Milliseconds(1000), + unsigned new_pd = UINT_MAX); public: /* @@ -44,7 +49,8 @@ class RaaqmDataPath { * max of RTT. * @param new_rtt is the value of the new RTT */ - RaaqmDataPath &insertNewRtt(uint64_t new_rtt, const utils::TimePoint &now); + RaaqmDataPath &insertNewRtt(const utils::SteadyTime::Microseconds &new_rtt, + const utils::SteadyTime::TimePoint &now); /** * @brief Update the path statistics @@ -214,7 +220,7 @@ class RaaqmDataPath { /** * Time of the last call to the path reporter method */ - utils::TimePoint last_received_pkt_; + utils::SteadyTime::TimePoint last_received_pkt_; double average_rtt_; double alpha_; diff --git a/libtransport/src/protocols/rate_estimation.cc b/libtransport/src/protocols/rate_estimation.cc index 2337e18be..01c18c6cb 100644 --- a/libtransport/src/protocols/rate_estimation.cc +++ b/libtransport/src/protocols/rate_estimation.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -14,6 +14,7 @@ */ #include <glog/logging.h> +#include <hicn/transport/errors/runtime_exception.h> #include <hicn/transport/interfaces/socket_options_default_values.h> #include <protocols/rate_estimation.h> @@ -92,8 +93,8 @@ InterRttEstimator::InterRttEstimator(double alpha_arg) { this->win_current_ = 1.0; pthread_mutex_init(&(this->mutex_), NULL); - this->start_time_ = std::chrono::steady_clock::now(); - this->begin_batch_ = std::chrono::steady_clock::now(); + this->start_time_ = utils::SteadyTime::now(); + this->begin_batch_ = utils::SteadyTime::now(); } InterRttEstimator::~InterRttEstimator() { @@ -105,33 +106,35 @@ InterRttEstimator::~InterRttEstimator() { pthread_mutex_destroy(&(this->mutex_)); } -void InterRttEstimator::onRttUpdate(double rtt) { +void InterRttEstimator::onRttUpdate( + const utils::SteadyTime::Microseconds &rtt) { pthread_mutex_lock(&(this->mutex_)); - this->rtt_ = rtt; + this->rtt_ = rtt.count() / 1000.0; this->number_of_packets_++; - this->avg_rtt_ += rtt; + this->avg_rtt_ += this->rtt_; pthread_mutex_unlock(&(this->mutex_)); if (!thread_is_running_) { my_th_ = (pthread_t *)malloc(sizeof(pthread_t)); if (!my_th_) { - LOG(ERROR) << "Error allocating thread."; - my_th_ = NULL; + throw errors::RuntimeException("Error allocating thread."); } - if (/*int err = */ pthread_create(my_th_, NULL, transport::protocol::Timer, - (void *)this)) { - LOG(ERROR) << "Error creating the thread"; + + if (pthread_create(my_th_, NULL, transport::protocol::Timer, + (void *)this)) { + free(my_th_); my_th_ = NULL; + throw errors::RuntimeException("Error allocating thread."); } + thread_is_running_ = true; } } void InterRttEstimator::onWindowIncrease(double win_current) { - TimePoint end = std::chrono::steady_clock::now(); + auto end = utils::SteadyTime::now(); auto delay = - std::chrono::duration_cast<Microseconds>(end - this->begin_batch_) - .count(); + utils::SteadyTime::getDurationUs(this->begin_batch_, end).count(); pthread_mutex_lock(&(this->mutex_)); this->avg_win_ += this->win_current_ * delay; @@ -139,14 +142,13 @@ void InterRttEstimator::onWindowIncrease(double win_current) { this->win_change_ += delay; pthread_mutex_unlock(&(this->mutex_)); - this->begin_batch_ = std::chrono::steady_clock::now(); + this->begin_batch_ = utils::SteadyTime::now(); } void InterRttEstimator::onWindowDecrease(double win_current) { - TimePoint end = std::chrono::steady_clock::now(); + auto end = utils::SteadyTime::now(); auto delay = - std::chrono::duration_cast<Microseconds>(end - this->begin_batch_) - .count(); + utils::SteadyTime::getDurationUs(this->begin_batch_, end).count(); pthread_mutex_lock(&(this->mutex_)); this->avg_win_ += this->win_current_ * delay; @@ -154,25 +156,24 @@ void InterRttEstimator::onWindowDecrease(double win_current) { this->win_change_ += delay; pthread_mutex_unlock(&(this->mutex_)); - this->begin_batch_ = std::chrono::steady_clock::now(); + this->begin_batch_ = utils::SteadyTime::now(); } ALaTcpEstimator::ALaTcpEstimator() { this->estimation_ = 0.0; this->observer_ = NULL; - this->start_time_ = std::chrono::steady_clock::now(); + this->start_time_ = utils::SteadyTime::now(); this->totalSize_ = 0.0; } void ALaTcpEstimator::onStart() { this->totalSize_ = 0.0; - this->start_time_ = std::chrono::steady_clock::now(); + this->start_time_ = utils::SteadyTime::now(); } void ALaTcpEstimator::onDownloadFinished() { - TimePoint end = std::chrono::steady_clock::now(); - auto delay = - std::chrono::duration_cast<Microseconds>(end - this->start_time_).count(); + auto end = utils::SteadyTime::now(); + auto delay = utils::SteadyTime::getDurationUs(this->start_time_, end).count(); this->estimation_ = this->totalSize_ * 8 * 1000000 / delay; if (observer_) { observer_->notifyStats(this->estimation_); @@ -192,22 +193,21 @@ SimpleEstimator::SimpleEstimator(double alphaArg, int batching_param) { this->number_of_packets_ = 0; this->base_alpha_ = alphaArg; this->alpha_ = alphaArg; - this->start_time_ = std::chrono::steady_clock::now(); - this->begin_batch_ = std::chrono::steady_clock::now(); + this->start_time_ = utils::SteadyTime::now(); + this->begin_batch_ = utils::SteadyTime::now(); } void SimpleEstimator::onStart() { this->estimated_ = false; this->number_of_packets_ = 0; this->total_size_ = 0.0; - this->start_time_ = std::chrono::steady_clock::now(); - this->begin_batch_ = std::chrono::steady_clock::now(); + this->start_time_ = utils::SteadyTime::now(); + this->begin_batch_ = utils::SteadyTime::now(); } void SimpleEstimator::onDownloadFinished() { - TimePoint end = std::chrono::steady_clock::now(); - auto delay = - std::chrono::duration_cast<Microseconds>(end - this->start_time_).count(); + auto end = utils::SteadyTime::now(); + auto delay = utils::SteadyTime::getDurationUs(this->start_time_, end).count(); if (observer_) { observer_->notifyDownloadTime((double)delay); } @@ -229,8 +229,7 @@ void SimpleEstimator::onDownloadFinished() { } else { if (this->number_of_packets_ >= (int)(75.0 * (double)this->batching_param_ / 100.0)) { - delay = std::chrono::duration_cast<Microseconds>(end - this->begin_batch_) - .count(); + delay = utils::SteadyTime::getDurationUs(this->begin_batch_, end).count(); // Assuming all packets carry max_packet_size_ bytes of data // (8*max_packet_size_ bits); 1000000 factor to convert us to seconds if (this->estimation_) { @@ -249,22 +248,21 @@ void SimpleEstimator::onDownloadFinished() { } this->number_of_packets_ = 0; this->total_size_ = 0.0; - this->start_time_ = std::chrono::steady_clock::now(); - this->begin_batch_ = std::chrono::steady_clock::now(); + this->start_time_ = utils::SteadyTime::now(); + this->begin_batch_ = utils::SteadyTime::now(); } void SimpleEstimator::onDataReceived(int packet_size) { this->total_size_ += packet_size; } -void SimpleEstimator::onRttUpdate(double rtt) { +void SimpleEstimator::onRttUpdate(const utils::SteadyTime::Microseconds &rtt) { this->number_of_packets_++; if (this->number_of_packets_ == this->batching_param_) { - TimePoint end = std::chrono::steady_clock::now(); + auto end = utils::SteadyTime::now(); auto delay = - std::chrono::duration_cast<Microseconds>(end - this->begin_batch_) - .count(); + utils::SteadyTime::getDurationUs(this->begin_batch_, end).count(); // Assuming all packets carry max_packet_size_ bytes of data // (8*max_packet_size_ bits); 1000000 factor to convert us to seconds if (this->estimation_) { @@ -280,7 +278,7 @@ void SimpleEstimator::onRttUpdate(double rtt) { this->alpha_ = this->base_alpha_; this->number_of_packets_ = 0; this->total_size_ = 0.0; - this->begin_batch_ = std::chrono::steady_clock::now(); + this->begin_batch_ = utils::SteadyTime::now(); } } @@ -297,13 +295,14 @@ BatchingPacketsEstimator::BatchingPacketsEstimator(double alpha_arg, this->max_packet_size_ = 0; this->estimation_ = 0.0; this->win_current_ = 1.0; - this->begin_batch_ = std::chrono::steady_clock::now(); - this->start_time_ = std::chrono::steady_clock::now(); + this->begin_batch_ = utils::SteadyTime::now(); + this->start_time_ = utils::SteadyTime::now(); } -void BatchingPacketsEstimator::onRttUpdate(double rtt) { +void BatchingPacketsEstimator::onRttUpdate( + const utils::SteadyTime::Microseconds &rtt) { this->number_of_packets_++; - this->avg_rtt_ += rtt; + this->avg_rtt_ += rtt.count() / 1000.0; if (number_of_packets_ == this->batching_param_) { if (estimation_ == 0) { @@ -329,25 +328,23 @@ void BatchingPacketsEstimator::onRttUpdate(double rtt) { } void BatchingPacketsEstimator::onWindowIncrease(double win_current) { - TimePoint end = std::chrono::steady_clock::now(); + auto end = utils::SteadyTime::now(); auto delay = - std::chrono::duration_cast<Microseconds>(end - this->begin_batch_) - .count(); + utils::SteadyTime::getDurationUs(this->begin_batch_, end).count(); this->avg_win_ += this->win_current_ * delay; this->win_current_ = win_current; this->win_change_ += delay; - this->begin_batch_ = std::chrono::steady_clock::now(); + this->begin_batch_ = utils::SteadyTime::now(); } void BatchingPacketsEstimator::onWindowDecrease(double win_current) { - TimePoint end = std::chrono::steady_clock::now(); + auto end = utils::SteadyTime::now(); auto delay = - std::chrono::duration_cast<Microseconds>(end - this->begin_batch_) - .count(); + utils::SteadyTime::getDurationUs(this->begin_batch_, end).count(); this->avg_win_ += this->win_current_ * delay; this->win_current_ = win_current; this->win_change_ += delay; - this->begin_batch_ = std::chrono::steady_clock::now(); + this->begin_batch_ = utils::SteadyTime::now(); } } // end namespace protocol diff --git a/libtransport/src/protocols/rate_estimation.h b/libtransport/src/protocols/rate_estimation.h index 42ae74194..d809b2b7c 100644 --- a/libtransport/src/protocols/rate_estimation.h +++ b/libtransport/src/protocols/rate_estimation.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -16,6 +16,7 @@ #pragma once #include <hicn/transport/interfaces/statistics.h> +#include <hicn/transport/utils/noncopyable.h> #include <protocols/raaqm_data_path.h> #include <chrono> @@ -24,16 +25,13 @@ namespace transport { namespace protocol { -class IcnRateEstimator { +class IcnRateEstimator : utils::NonCopyable { public: - using TimePoint = std::chrono::steady_clock::time_point; - using Microseconds = std::chrono::microseconds; - IcnRateEstimator(){}; virtual ~IcnRateEstimator(){}; - virtual void onRttUpdate(double rtt){}; + virtual void onRttUpdate(const utils::SteadyTime::Microseconds &rtt){}; virtual void onDataReceived(int packetSize){}; @@ -48,9 +46,10 @@ class IcnRateEstimator { virtual void setObserver(interface::IcnObserver *observer) { this->observer_ = observer; }; + interface::IcnObserver *observer_; - TimePoint start_time_; - TimePoint begin_batch_; + utils::SteadyTime::TimePoint start_time_; + utils::SteadyTime::TimePoint begin_batch_; double base_alpha_; double alpha_; double estimation_; @@ -67,7 +66,7 @@ class InterRttEstimator : public IcnRateEstimator { ~InterRttEstimator(); - void onRttUpdate(double rtt); + void onRttUpdate(const utils::SteadyTime::Microseconds &rtt); void onDataReceived(int packet_size) { if (packet_size > this->max_packet_size_) { @@ -102,7 +101,7 @@ class BatchingPacketsEstimator : public IcnRateEstimator { public: BatchingPacketsEstimator(double alpha_arg, int batchingParam); - void onRttUpdate(double rtt); + void onRttUpdate(const utils::SteadyTime::Microseconds &rtt); void onDataReceived(int packet_size) { if (packet_size > this->max_packet_size_) { @@ -149,7 +148,7 @@ class SimpleEstimator : public IcnRateEstimator { public: SimpleEstimator(double alpha, int batching_param); - void onRttUpdate(double rtt); + void onRttUpdate(const utils::SteadyTime::Microseconds &rtt); void onDataReceived(int packet_size); diff --git a/libtransport/src/protocols/reassembly.cc b/libtransport/src/protocols/reassembly.cc index ce24fce1b..065458f7e 100644 --- a/libtransport/src/protocols/reassembly.cc +++ b/libtransport/src/protocols/reassembly.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: diff --git a/libtransport/src/protocols/reassembly.h b/libtransport/src/protocols/reassembly.h index e072ad123..c0c4de3d8 100644 --- a/libtransport/src/protocols/reassembly.h +++ b/libtransport/src/protocols/reassembly.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -36,7 +36,7 @@ class Reassembly { public: class ContentReassembledCallback { public: - virtual void onContentReassembled(std::error_code ec) = 0; + virtual void onContentReassembled(const std::error_code &ec) = 0; }; Reassembly(implementation::ConsumerSocket *icn_socket, @@ -47,22 +47,16 @@ class Reassembly { virtual ~Reassembly() = default; /** - * Hanle reassembly of content object. + * Handle reassembly of content object. */ virtual void reassemble(core::ContentObject &content_object) = 0; /** - * Hanle reassembly of content object. + * Handle reassembly of content object. */ virtual void reassemble(utils::MemBuf &buffer, uint32_t suffix) = 0; /** - * Handle reassembly of manifest - */ - virtual void reassemble( - std::unique_ptr<core::ContentObjectManifest> &&manifest) = 0; - - /** * Reset reassembler for new round */ virtual void reInitialize() = 0; diff --git a/libtransport/src/protocols/rtc/CMakeLists.txt b/libtransport/src/protocols/rtc/CMakeLists.txt index 873b345d0..be8e0189c 100644 --- a/libtransport/src/protocols/rtc/CMakeLists.txt +++ b/libtransport/src/protocols/rtc/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2019 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -16,22 +16,43 @@ list(APPEND HEADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/rtc.h ${CMAKE_CURRENT_SOURCE_DIR}/rtc_consts.h ${CMAKE_CURRENT_SOURCE_DIR}/rtc_data_path.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_forwarding_strategy.h ${CMAKE_CURRENT_SOURCE_DIR}/rtc_indexer.h ${CMAKE_CURRENT_SOURCE_DIR}/rtc_ldr.h ${CMAKE_CURRENT_SOURCE_DIR}/rtc_packet.h ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rc.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rc_congestion_detection.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rc_iat.h ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rc_queue.h ${CMAKE_CURRENT_SOURCE_DIR}/rtc_reassembly.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_recovery_strategy.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_delay.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_fec_only.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_low_rate.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_recovery_off.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_rtx_only.h ${CMAKE_CURRENT_SOURCE_DIR}/rtc_state.h + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_verifier.h ) list(APPEND SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/probe_handler.cc ${CMAKE_CURRENT_SOURCE_DIR}/rtc.cc ${CMAKE_CURRENT_SOURCE_DIR}/rtc_data_path.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_forwarding_strategy.cc ${CMAKE_CURRENT_SOURCE_DIR}/rtc_ldr.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rc_congestion_detection.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rc_iat.cc ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rc_queue.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_reassembly.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_recovery_strategy.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_delay.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_fec_only.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_low_rate.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_recovery_off.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_rs_rtx_only.cc ${CMAKE_CURRENT_SOURCE_DIR}/rtc_state.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rtc_verifier.cc ) set(SOURCE_FILES ${SOURCE_FILES} PARENT_SCOPE) diff --git a/libtransport/src/protocols/rtc/probe_handler.cc b/libtransport/src/protocols/rtc/probe_handler.cc index abaca6ad9..60eceeb19 100644 --- a/libtransport/src/protocols/rtc/probe_handler.cc +++ b/libtransport/src/protocols/rtc/probe_handler.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -13,6 +13,8 @@ * limitations under the License. */ +#include <glog/logging.h> +#include <hicn/transport/utils/chrono_typedefs.h> #include <protocols/rtc/probe_handler.h> #include <protocols/rtc/rtc_consts.h> @@ -27,6 +29,7 @@ ProbeHandler::ProbeHandler(SendProbeCallback &&send_callback, : probe_interval_(0), max_probes_(0), sent_probes_(0), + recv_probes_(0), probe_timer_(std::make_unique<asio::steady_timer>(io_service)), rand_eng_((std::random_device())()), distr_(MIN_RTT_PROBE_SEQ, MAX_RTT_PROBE_SEQ), @@ -34,22 +37,38 @@ ProbeHandler::ProbeHandler(SendProbeCallback &&send_callback, ProbeHandler::~ProbeHandler() {} -uint64_t ProbeHandler::getRtt(uint32_t seq) { +uint64_t ProbeHandler::getRtt(uint32_t seq, bool is_valid) { auto it = pending_probes_.find(seq); if (it == pending_probes_.end()) return 0; - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + if (!is_valid) { + // delete the probe anyway + pending_probes_.erase(it); + valid_batch_ = false; + return 0; + } + + uint64_t now = utils::SteadyTime::nowMs().count(); uint64_t rtt = now - it->second; if (rtt < 1) rtt = 1; pending_probes_.erase(it); + recv_probes_++; return rtt; } +double ProbeHandler::getProbeLossRate() { + if (!valid_batch_) return 1.0; + return 1.0 - ((double)recv_probes_ / (double)sent_probes_); +} + +void ProbeHandler::setSuffixRange(uint32_t min, uint32_t max) { + DCHECK(min <= max && min >= MIN_PROBE_SEQ); + distr_ = std::uniform_int_distribution<uint32_t>(min, max); +} + void ProbeHandler::setProbes(uint32_t probe_interval, uint32_t max_probes) { stopProbes(); probe_interval_ = probe_interval; @@ -60,16 +79,30 @@ void ProbeHandler::stopProbes() { probe_interval_ = 0; max_probes_ = 0; sent_probes_ = 0; + recv_probes_ = 0; + valid_batch_ = true; probe_timer_->cancel(); } void ProbeHandler::sendProbes() { if (probe_interval_ == 0) return; + + std::weak_ptr<ProbeHandler> self(shared_from_this()); + probe_timer_->expires_from_now(std::chrono::microseconds(probe_interval_)); + probe_timer_->async_wait([self](const std::error_code &ec) { + if (ec) return; + auto s = self.lock(); + if (s) { + s->generateProbe(); + } + }); +} + +void ProbeHandler::generateProbe() { + if (probe_interval_ == 0) return; if (max_probes_ != 0 && sent_probes_ >= max_probes_) return; - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + uint64_t now = utils::SteadyTime::nowMs().count(); uint32_t seq = distr_(rand_eng_); pending_probes_.insert(std::pair<uint32_t, uint64_t>(seq, now)); @@ -88,16 +121,17 @@ void ProbeHandler::sendProbes() { } } - if (probe_interval_ == 0) return; + sendProbes(); +} - std::weak_ptr<ProbeHandler> self(shared_from_this()); - probe_timer_->expires_from_now(std::chrono::microseconds(probe_interval_)); - probe_timer_->async_wait([self](std::error_code ec) { - if (ec) return; - if (auto s = self.lock()) { - s->sendProbes(); - } - }); +ProbeType ProbeHandler::getProbeType(uint32_t seq) { + if (MIN_INIT_PROBE_SEQ <= seq && seq <= MAX_INIT_PROBE_SEQ) { + return ProbeType::INIT; + } + if (MIN_RTT_PROBE_SEQ <= seq && seq <= MAX_RTT_PROBE_SEQ) { + return ProbeType::RTT; + } + return ProbeType::NOT_PROBE; } } // namespace rtc diff --git a/libtransport/src/protocols/rtc/probe_handler.h b/libtransport/src/protocols/rtc/probe_handler.h index e34b23df0..d989194d4 100644 --- a/libtransport/src/protocols/rtc/probe_handler.h +++ b/libtransport/src/protocols/rtc/probe_handler.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -26,6 +26,12 @@ namespace protocol { namespace rtc { +enum class ProbeType { + NOT_PROBE, + INIT, + RTT, +}; + class ProbeHandler : public std::enable_shared_from_this<ProbeHandler> { public: using SendProbeCallback = std::function<void(uint32_t)>; @@ -35,31 +41,45 @@ class ProbeHandler : public std::enable_shared_from_this<ProbeHandler> { ~ProbeHandler(); - // if the function returns 0 the probe is not valaid - uint64_t getRtt(uint32_t seq); + // If the function returns 0 the probe is not valid. + uint64_t getRtt(uint32_t seq, bool is_valid); + + // this function may return a residual loss rate higher than the real one if + // we don't wait enough time for the probes to come back + double getProbeLossRate(); + + // Set the probe suffix range [min, max] + void setSuffixRange(uint32_t min, uint32_t max); - // reset the probes parameters. it stop the current probing. - // to restar call sendProbes. - // probe_interval = 0 means that no event will be scheduled - // max_probe = 0 means no limit to the number of probe to send + // Reset the probes parameters and stops the current probing. + // probe_interval = 0 means that no event will be scheduled. + // max_probe = 0 means no limit to the number of probe to send. void setProbes(uint32_t probe_interval, uint32_t max_probes); - // stop to schedule probes void stopProbes(); void sendProbes(); + static ProbeType getProbeType(uint32_t seq); + private: + void generateProbe(); + uint32_t probe_interval_; // us uint32_t max_probes_; // packets uint32_t sent_probes_; // packets + uint32_t recv_probes_; // packets + + bool valid_batch_; // if at least one probe in a batch is considered not + // valid (e.g. prod rate == ~0) the full batch is invalid. + // the bool is set to true when sendProbe is called std::unique_ptr<asio::steady_timer> probe_timer_; - // map from seqnumber to timestamp + // Map from packet suffixes to timestamp std::unordered_map<uint32_t, uint64_t> pending_probes_; - // random generator + // Random generator std::default_random_engine rand_eng_; std::uniform_int_distribution<uint32_t> distr_; diff --git a/libtransport/src/protocols/rtc/rtc.cc b/libtransport/src/protocols/rtc/rtc.cc index 0cb4cda1d..9a56269f3 100644 --- a/libtransport/src/protocols/rtc/rtc.cc +++ b/libtransport/src/protocols/rtc/rtc.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -22,7 +22,7 @@ #include <protocols/rtc/rtc.h> #include <protocols/rtc/rtc_consts.h> #include <protocols/rtc/rtc_indexer.h> -#include <protocols/rtc/rtc_rc_queue.h> +#include <protocols/rtc/rtc_rc_congestion_detection.h> #include <algorithm> @@ -37,13 +37,16 @@ using namespace interface; RTCTransportProtocol::RTCTransportProtocol( implementation::ConsumerSocket *icn_socket) : TransportProtocol(icn_socket, new RtcIndexer<>(icn_socket, this), - new DatagramReassembly(icn_socket, this)), + new RtcReassembly(icn_socket, this)), + max_aggregated_interest_(1), number_(0) { icn_socket->getSocketOption(PORTAL, portal_); - round_timer_ = std::make_unique<asio::steady_timer>(portal_->getIoService()); + round_timer_ = + std::make_unique<asio::steady_timer>(portal_->getThread().getIoService()); scheduler_timer_ = - std::make_unique<asio::steady_timer>(portal_->getIoService()); - pacing_timer_ = std::make_unique<asio::steady_timer>(portal_->getIoService()); + std::make_unique<asio::steady_timer>(portal_->getThread().getIoService()); + pacing_timer_ = + std::make_unique<asio::steady_timer>(portal_->getThread().getIoService()); } RTCTransportProtocol::~RTCTransportProtocol() {} @@ -53,33 +56,75 @@ void RTCTransportProtocol::resume() { TransportProtocol::resume(); } -std::size_t RTCTransportProtocol::transportHeaderLength() { +std::size_t RTCTransportProtocol::transportHeaderLength(bool isFEC) { return DATA_HEADER_SIZE + - (fec_decoder_ != nullptr ? fec_decoder_->getFecHeaderSize() : 0); + (fec_decoder_ != nullptr ? fec_decoder_->getFecHeaderSize(isFEC) : 0); } // private void RTCTransportProtocol::initParams() { TransportProtocol::reset(); + std::weak_ptr<RTCTransportProtocol> self = shared_from_this(); - rc_ = std::make_shared<RTCRateControlQueue>(); + fwd_strategy_.setCallback([self](notification::Strategy strategy) { + auto ptr = self.lock(); + if (ptr && ptr->isRunning()) { + if (*ptr->on_fwd_strategy_) (*ptr->on_fwd_strategy_)(strategy); + } + }); + + std::shared_ptr<auth::Verifier> verifier; + socket_->getSocketOption(GeneralTransportOptions::VERIFIER, verifier); + + uint32_t factor_relevant; + socket_->getSocketOption(GeneralTransportOptions::MANIFEST_FACTOR_RELEVANT, + factor_relevant); + + uint32_t factor_alert; + socket_->getSocketOption(GeneralTransportOptions::MANIFEST_FACTOR_ALERT, + factor_alert); + + rc_ = std::make_shared<RTCRateControlCongestionDetection>(); ldr_ = std::make_shared<RTCLossDetectionAndRecovery>( - indexer_verifier_.get(), - std::bind(&RTCTransportProtocol::sendRtxInterest, this, - std::placeholders::_1), - portal_->getIoService()); + indexer_verifier_.get(), portal_->getThread().getIoService(), + interface::RtcTransportRecoveryStrategies::RTX_ONLY, + [self](uint32_t seq) { + auto ptr = self.lock(); + if (ptr && ptr->isRunning()) { + ptr->sendRtxInterest(seq); + } + }, + [self](notification::Strategy strategy) { + auto ptr = self.lock(); + if (ptr && ptr->isRunning()) { + if (*ptr->on_rec_strategy_) (*ptr->on_rec_strategy_)(strategy); + } + }); + + verifier_ = + std::make_shared<RTCVerifier>(verifier, factor_relevant, factor_alert); state_ = std::make_shared<RTCState>( indexer_verifier_.get(), - std::bind(&RTCTransportProtocol::sendProbeInterest, this, - std::placeholders::_1), - std::bind(&RTCTransportProtocol::discoveredRtt, this), - portal_->getIoService()); + [self](uint32_t seq) { + auto ptr = self.lock(); + if (ptr && ptr->isRunning()) { + ptr->sendProbeInterest(seq); + } + }, + [self]() { + auto ptr = self.lock(); + if (ptr && ptr->isRunning()) { + ptr->discoveredRtt(); + } + }, + portal_->getThread().getIoService()); rc_->setState(state_); - // TODO: for the moment we keep the congestion control disabled - // rc_->tunrOnRateControl(); - ldr_->setState(state_); + rc_->turnOnRateControl(); + ldr_->setState(state_.get()); + ldr_->setRateControl(rc_.get()); + verifier_->setState(state_); // protocol state start_send_interest_ = false; @@ -94,15 +139,20 @@ void RTCTransportProtocol::initParams() { last_interest_sent_time_ = 0; last_interest_sent_seq_ = 0; -#if 0 - if(portal_->isConnectedToFwd()){ - max_aggregated_interest_ = 1; - }else{ - max_aggregated_interest_ = MAX_INTERESTS_IN_BATCH; + // Aggregated interests setup + bool aggregated_interests_on; + socket_->getSocketOption(RtcTransportOptions::AGGREGATED_INTERESTS, + aggregated_interests_on); + if (aggregated_interests_on) { + if (const char *max_aggr = std::getenv("MAX_AGGREGATED_INTERESTS")) + max_aggregated_interest_ = (uint32_t)std::stoul(std::string(max_aggr)); + else + max_aggregated_interest_ = MAX_INTERESTS_IN_BATCH; + + max_aggregated_interest_ = std::min<uint32_t>(max_aggregated_interest_, + 1 + MAX_SUFFIXES_IN_MANIFEST); } -#else - max_aggregated_interest_ = 1; -#endif + LOG(INFO) << "Max Aggregated: " << max_aggregated_interest_; max_sent_int_ = std::ceil((double)MAX_PACING_BATCH / (double)max_aggregated_interest_); @@ -120,20 +170,8 @@ void RTCTransportProtocol::initParams() { socket_->setSocketOption(GeneralTransportOptions::INTEREST_LIFETIME, RTC_INTEREST_LIFETIME); - // FEC - using namespace std::placeholders; - enableFEC(std::bind(&RTCTransportProtocol::onFecPackets, this, _1), - /* We leave the buffer allocation to the fec decoder */ - fec::FECBase::BufferRequested(0)); - - if (fec_decoder_) { - indexer_verifier_->enableFec(fec_type_); - indexer_verifier_->setNFec(0); - ldr_->setFecParams(fec::FECUtils::getBlockSymbols(fec_type_), - fec::FECUtils::getSourceSymbols(fec_type_)); - } else { - indexer_verifier_->disableFec(); - } + // init state params + state_->initParams(); } // private @@ -162,61 +200,86 @@ void RTCTransportProtocol::inactiveProducer() { void RTCTransportProtocol::newRound() { round_timer_->expires_from_now(std::chrono::milliseconds(ROUND_LEN)); - // TODO pass weak_ptr here - round_timer_->async_wait([this, n{number_}](std::error_code ec) { - if (ec) return; - if (n != number_) { + std::weak_ptr<RTCTransportProtocol> self = shared_from_this(); + round_timer_->async_wait([self](const std::error_code &ec) { + if (ec) { + return; + } + + auto ptr = self.lock(); + + if (!ptr || !ptr->isRunning()) { return; } + auto &state = ptr->state_; + // saving counters that will be reset on new round - uint32_t sent_retx = state_->getSentRtxInRound(); - uint32_t received_bytes = state_->getReceivedBytesInRound(); - uint32_t sent_interest = state_->getSentInterestInRound(); - uint32_t lost_data = state_->getLostData(); - uint32_t definitely_lost = state_->getDefinitelyLostPackets(); - uint32_t recovered_losses = state_->getRecoveredLosses(); - uint32_t received_nacks = state_->getReceivedNacksInRound(); - uint32_t received_fec = state_->getReceivedFecPackets(); - - bool in_sync = (current_state_ == SyncState::in_sync); - ldr_->onNewRound(in_sync); - state_->onNewRound((double)ROUND_LEN, in_sync); - rc_->onNewRound((double)ROUND_LEN); + uint32_t sent_retx = state->getSentRtxInRound(); + uint32_t received_bytes = + (state->getReceivedBytesInRound() + // data packets received + state->getReceivedFecBytesInRound()); // fec packets received + uint32_t sent_interest = state->getSentInterestInRound(); + uint32_t lost_data = state->getLostData(); + uint32_t definitely_lost = state->getDefinitelyLostPackets(); + uint32_t recovered_losses = state->getRecoveredLosses(); + uint32_t received_nacks = state->getReceivedNacksInRound(); + uint32_t received_fec = state->getReceivedFecPackets(); // update sync state if needed - if (current_state_ == SyncState::in_sync) { - double cache_rate = state_->getPacketFromCacheRatio(); + double cache_rate = state->getPacketFromCacheRatio(); + uint32_t round_without_nacks = state->getRoundsWithoutNacks(); + + if (ptr->current_state_ == SyncState::in_sync) { if (cache_rate > MAX_DATA_FROM_CACHE) { - current_state_ = SyncState::catch_up; + ptr->current_state_ = SyncState::catch_up; } } else { - double target_rate = state_->getProducerRate() * PRODUCTION_RATE_FRACTION; - double received_rate = state_->getReceivedRate(); - uint32_t round_without_nacks = state_->getRoundsWithoutNacks(); - double cache_ratio = state_->getPacketFromCacheRatio(); if (round_without_nacks >= ROUNDS_IN_SYNC_BEFORE_SWITCH && - received_rate >= target_rate && cache_ratio < MAX_DATA_FROM_CACHE) { - current_state_ = SyncState::in_sync; + cache_rate < MAX_DATA_FROM_CACHE) { + ptr->current_state_ = SyncState::in_sync; } } + bool in_sync = (ptr->current_state_ == SyncState::in_sync); + ptr->ldr_->onNewRound(in_sync); + ptr->state_->onNewRound((double)ROUND_LEN, in_sync); + ptr->rc_->onNewRound((double)ROUND_LEN); + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Calling updateSyncWindow in newRound function"; - updateSyncWindow(); + ptr->updateSyncWindow(); - sendStatsToApp(sent_retx, received_bytes, sent_interest, lost_data, - definitely_lost, recovered_losses, received_nacks, - received_fec); - newRound(); + ptr->sendStatsToApp(sent_retx, received_bytes, sent_interest, lost_data, + definitely_lost, recovered_losses, received_nacks, + received_fec); + ptr->fwd_strategy_.checkStrategy(); + ptr->newRound(); }); } void RTCTransportProtocol::discoveredRtt() { start_send_interest_ = true; - ldr_->turnOnRTX(); + uint32_t strategy; + socket_->getSocketOption(RtcTransportOptions::RECOVERY_STRATEGY, strategy); + ldr_->changeRecoveryStrategy( + (interface::RtcTransportRecoveryStrategies)strategy); + + bool content_sharing_mode; + socket_->getSocketOption(RtcTransportOptions::CONTENT_SHARING_MODE, + content_sharing_mode); + if (content_sharing_mode) ldr_->setContentSharingMode(); + ldr_->turnOnRecovery(); ldr_->onNewRound(false); + + // set forwarding strategy switch if selected + Name *name = nullptr; + socket_->getSocketOption(GeneralTransportOptions::NETWORK_NAME, &name); + Prefix prefix(*name, 128); + fwd_strategy_.initFwdStrategy( + portal_, prefix, state_.get(), + (interface::RtcTransportRecoveryStrategies)strategy); updateSyncWindow(); } @@ -233,6 +296,12 @@ void RTCTransportProtocol::computeMaxSyncWindow() { return; } + bool content_sharing_mode; + socket_->getSocketOption(RtcTransportOptions::CONTENT_SHARING_MODE, + content_sharing_mode); + if (content_sharing_mode && (production_rate < MIN_PROD_RATE_SHARING_MODE)) + production_rate = MIN_PROD_RATE_SHARING_MODE; + production_rate += (production_rate * indexer_verifier_->getMaxFecOverhead()); uint32_t lifetime = default_values::interest_lifetime; @@ -244,7 +313,7 @@ void RTCTransportProtocol::computeMaxSyncWindow() { (production_rate * lifetime_ms * INTEREST_LIFETIME_REDUCTION_FACTOR) / packet_size); - max_sync_win_ = std::min(max_sync_win_, rc_->getCongesionWindow()); + max_sync_win_ = std::min(max_sync_win_, rc_->getCongestionWindow()); } void RTCTransportProtocol::updateSyncWindow() { @@ -259,25 +328,19 @@ void RTCTransportProtocol::updateSyncWindow() { } double prod_rate = state_->getProducerRate(); - double rtt = (double)state_->getRTT() / MILLI_IN_A_SEC; + double rtt = (double)state_->getMinRTT() / MILLI_IN_A_SEC; double packet_size = state_->getAveragePacketSize(); + bool content_sharing_mode; + socket_->getSocketOption(RtcTransportOptions::CONTENT_SHARING_MODE, + content_sharing_mode); + if (content_sharing_mode && (prod_rate < MIN_PROD_RATE_SHARING_MODE)) + prod_rate = MIN_PROD_RATE_SHARING_MODE; // if some of the info are not available do not update the current win if (prod_rate != 0.0 && rtt != 0.0 && packet_size != 0.0) { - double fec_interest_overhead = (double)state_->getPendingFecPackets() / - (double)(state_->getPendingInterestNumber() - - state_->getPendingFecPackets()); - - double fec_overhead = - std::max(indexer_verifier_->getFecOverhead(), fec_interest_overhead); - - prod_rate += (prod_rate * fec_overhead); - current_sync_win_ = (uint32_t)ceil(prod_rate * rtt / packet_size); - uint32_t buffer = PRODUCER_BUFFER_MS; - if (rtt > 150) - buffer = buffer * 2; // if the RTT is too large we increase the - // the size of the buffer + uint32_t buffer = PRODUCER_BUFFER_MS + ((double)state_->getMinRTT() / 2.0); + current_sync_win_ += ceil(prod_rate * (buffer / MILLI_IN_A_SEC) / packet_size); @@ -285,22 +348,22 @@ void RTCTransportProtocol::updateSyncWindow() { current_sync_win_ = current_sync_win_ * CATCH_UP_WIN_INCREMENT; } + uint32_t min_win = WIN_MIN; + bool aggregated_data_on; + socket_->getSocketOption(RtcTransportOptions::AGGREGATED_DATA, + aggregated_data_on); + if (aggregated_data_on) { + min_win = WIN_MIN_WITH_AGGREGARED_DATA; + min_win += (min_win * (1 - (std::max(0.3, rtt) - rtt) / 0.3)); + } + current_sync_win_ = std::min(current_sync_win_, max_sync_win_); - current_sync_win_ = std::max(current_sync_win_, WIN_MIN); + current_sync_win_ = std::max(current_sync_win_, min_win); } scheduleNextInterests(); } -void RTCTransportProtocol::decreaseSyncWindow() { - // called on future nack - // we have a new sample of the production rate, so update max win first - computeMaxSyncWindow(); - current_sync_win_--; - current_sync_win_ = std::max(current_sync_win_, WIN_MIN); - scheduleNextInterests(); -} - void RTCTransportProtocol::sendRtxInterest(uint32_t seq) { if (!isRunning() && !is_first_) return; @@ -322,21 +385,39 @@ void RTCTransportProtocol::sendProbeInterest(uint32_t seq) { socket_->getSocketOption(GeneralTransportOptions::NETWORK_NAME, &interest_name); - DLOG_IF(INFO, VLOG_IS_ON(3)) << "send probe " << seq; + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send probe " << seq; interest_name->setSuffix(seq); sendInterest(*interest_name); } +void RTCTransportProtocol::sendInterestForTimeout(uint32_t seq) { + if (!isRunning() && !is_first_) return; + + Name *interest_name = nullptr; + socket_->getSocketOption(GeneralTransportOptions::NETWORK_NAME, + &interest_name); + + // we got a timeout for this packet so it is not pending anymore + interest_name->setSuffix(seq); + state_->onSendNewInterest(interest_name); + sendInterest(*interest_name); +} + void RTCTransportProtocol::scheduleNextInterests() { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Schedule next interests"; - if (!isRunning() && !is_first_) return; + if (!isRunning() && !is_first_) { + return; + } - if (pacing_timer_on_) return; // wait pacing timer for the next send + if (pacing_timer_on_) { + return; // wait pacing timer for the next send + } - if (!start_send_interest_) + if (!start_send_interest_) { return; // RTT discovering phase is not finished so // do not start to send interests + } if (TRANSPORT_EXPECT_FALSE(!state_->isProducerActive())) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Inactive producer."; @@ -352,7 +433,7 @@ void RTCTransportProtocol::scheduleNextInterests() { &interest_name); uint32_t next_seg = 0; - DLOG_IF(INFO, VLOG_IS_ON(3)) << "send interest " << next_seg; + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Send interest " << next_seg; interest_name->setSuffix(next_seg); if (portal_->interestIsPending(*interest_name)) { @@ -370,28 +451,36 @@ void RTCTransportProtocol::scheduleNextInterests() { << " -- current_sync_win_: " << current_sync_win_; uint32_t pending = state_->getPendingInterestNumber(); - if (pending >= current_sync_win_) return; // no space in the window + uint32_t pending_fec = state_->getPendingFecPackets(); - if ((current_sync_win_ - pending) < max_aggregated_interest_) { + if ((pending - pending_fec) >= current_sync_win_) + return; // no space in the window + + // XXX double check if aggregated interests are still working here + if ((current_sync_win_ - (pending - pending_fec)) < + max_aggregated_interest_) { if (scheduler_timer_on_) return; // timer already scheduled - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + uint64_t now = utils::SteadyTime::nowMs().count(); uint64_t time = now - last_interest_sent_time_; if (time < WAIT_FOR_INTEREST_BATCH) { uint64_t next = WAIT_FOR_INTEREST_BATCH - time; scheduler_timer_on_ = true; scheduler_timer_->expires_from_now(std::chrono::milliseconds(next)); - scheduler_timer_->async_wait([this](std::error_code ec) { + + std::weak_ptr<RTCTransportProtocol> self = shared_from_this(); + scheduler_timer_->async_wait([self](const std::error_code &ec) { if (ec) return; - if (!scheduler_timer_on_) return; - scheduler_timer_on_ = false; - scheduleNextInterests(); + auto ptr = self.lock(); + if (ptr && ptr->isRunning()) { + if (!ptr->scheduler_timer_on_) return; + ptr->scheduler_timer_on_ = false; + ptr->scheduleNextInterests(); + } }); - return; // whait for the timer + return; // wait for the timer } } @@ -403,10 +492,10 @@ void RTCTransportProtocol::scheduleNextInterests() { indexer_verifier_->jumpToIndex(state_->getLastSeqNacked() + 1); } - // skipe received packets - if (indexer_verifier_->checkNextSuffix() <= - state_->getHighestSeqReceivedInOrder()) { - indexer_verifier_->jumpToIndex(state_->getHighestSeqReceivedInOrder() + 1); + // skip received packets + uint32_t max_received = state_->getHighestSeqReceivedInOrder(); + if (indexer_verifier_->checkNextSuffix() <= max_received) { + indexer_verifier_->jumpToIndex(max_received + 1); } uint32_t sent_interests = 0; @@ -417,30 +506,31 @@ void RTCTransportProtocol::scheduleNextInterests() { socket_->getSocketOption(GeneralTransportOptions::NETWORK_NAME, &name); std::array<uint32_t, MAX_AGGREGATED_INTEREST> additional_suffixes; - while ((state_->getPendingInterestNumber() < current_sync_win_) && + while (((state_->getPendingInterestNumber() - + state_->getPendingFecPackets()) < current_sync_win_) && (sent_interests < max_sent_int_)) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "In while loop. Window size: " << current_sync_win_; uint32_t next_seg = indexer_verifier_->getNextSuffix(); - name->setSuffix(next_seg); // send the packet only if: // 1) it is not pending yet (not true for rtx) - // 2) the packet is not received or lost + // 2) the packet is not received or def lost // 3) is not in the rtx list // 4) is fec and is not in order (!= last sent + 1) + PacketState packet_state = state_->getPacketState(next_seg); if (portal_->interestIsPending(*name) || - state_->isReceivedOrLost(next_seg) != PacketState::UNKNOWN || - ldr_->isRtx(next_seg) || + packet_state == PacketState::RECEIVED || + packet_state == PacketState::DEFINITELY_LOST || ldr_->isRtx(next_seg) || (indexer_verifier_->isFec(next_seg) && next_seg != last_interest_sent_seq_ + 1)) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "skip interest " << next_seg << " because: pending " - << portal_->interestIsPending(*name) << ", recv " - << (state_->isReceivedOrLost(next_seg) != PacketState::UNKNOWN) - << ", rtx " << (ldr_->isRtx(next_seg)) << ", is old fec " + << portal_->interestIsPending(*name) << ", recv or lost" + << (int)packet_state << ", rtx " << (ldr_->isRtx(next_seg)) + << ", is old fec " << ((indexer_verifier_->isFec(next_seg) && next_seg != last_interest_sent_seq_ + 1)); continue; @@ -462,10 +552,7 @@ void RTCTransportProtocol::scheduleNextInterests() { sent_packets++; sent_interests++; sendInterest(interest_name, &additional_suffixes, aggregated_counter - 1); - last_interest_sent_time_ = - std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + last_interest_sent_time_ = utils::SteadyTime::nowMs().count(); aggregated_counter = 0; } } @@ -473,25 +560,29 @@ void RTCTransportProtocol::scheduleNextInterests() { // exiting the while we may have some pending interest to send if (aggregated_counter != 0) { sent_packets++; - last_interest_sent_time_ = - std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + last_interest_sent_time_ = utils::SteadyTime::nowMs().count(); sendInterest(interest_name, &additional_suffixes, aggregated_counter - 1); } - if (state_->getPendingInterestNumber() < current_sync_win_) { + if ((state_->getPendingInterestNumber() - state_->getPendingFecPackets()) < + current_sync_win_) { // we still have space in the window but we already sent too many packets // wait PACING_WAIT to avoid drops in the kernel pacing_timer_on_ = true; pacing_timer_->expires_from_now(std::chrono::microseconds(PACING_WAIT)); - scheduler_timer_->async_wait([this](std::error_code ec) { + + std::weak_ptr<RTCTransportProtocol> self = shared_from_this(); + scheduler_timer_->async_wait([self](const std::error_code &ec) { if (ec) return; - if (!pacing_timer_on_) return; - pacing_timer_on_ = false; - scheduleNextInterests(); + auto ptr = self.lock(); + if (ptr && ptr->isRunning()) { + if (!ptr->pacing_timer_on_) return; + + ptr->pacing_timer_on_ = false; + ptr->scheduleNextInterests(); + } }); } } @@ -500,19 +591,18 @@ void RTCTransportProtocol::onInterestTimeout(Interest::Ptr &interest, const Name &name) { uint32_t segment_number = name.getSuffix(); - if (segment_number >= MIN_PROBE_SEQ) { + if (ProbeHandler::getProbeType(segment_number) != ProbeType::NOT_PROBE) { // this is a timeout on a probe, do nothing return; } - PacketState state = state_->isReceivedOrLost(segment_number); - if (state != PacketState::UNKNOWN) { + PacketState state = state_->getPacketState(segment_number); + if (state == PacketState::RECEIVED || state == PacketState::DEFINITELY_LOST) { // we may recover a packets using fec, ignore this timer return; } timeouts_or_nacks_.insert(segment_number); - if (TRANSPORT_EXPECT_TRUE(state_->isProducerActive()) && segment_number <= state_->getHighestSeqReceived()) { // we retransmit packets only if the producer is active, otherwise we @@ -524,13 +614,18 @@ void RTCTransportProtocol::onInterestTimeout(Interest::Ptr &interest, DLOG_IF(INFO, VLOG_IS_ON(3)) << "handle timeout for packet " << segment_number << " using rtx"; if (ldr_->isRtxOn()) { - ldr_->onTimeout(segment_number); - if (indexer_verifier_->isFec(segment_number)) + if (indexer_verifier_->isFec(segment_number)) { + // if this is a fec packet we do not recover it with rtx so we consider + // the packet to be lost + ldr_->onTimeout(segment_number, true); state_->onTimeout(segment_number, true); - else + } else { + ldr_->onTimeout(segment_number, false); state_->onTimeout(segment_number, false); + } } else { // in this case we wil never recover the timeout + ldr_->onTimeout(segment_number, true); state_->onTimeout(segment_number, true); } scheduleNextInterests(); @@ -548,18 +643,18 @@ void RTCTransportProtocol::onInterestTimeout(Interest::Ptr &interest, << "On timeout next seg = " << indexer_verifier_->checkNextSuffix() << ", jump to " << segment_number; // add an extra space in the window - current_sync_win_++; indexer_verifier_->jumpToIndex(segment_number); } state_->onTimeout(segment_number, false); + sendInterestForTimeout(segment_number); scheduleNextInterests(); } void RTCTransportProtocol::onNack(const ContentObject &content_object) { struct nack_packet_t *nack = (struct nack_packet_t *)content_object.getPayload()->data(); - uint32_t production_seg = nack->getProductionSegement(); + uint32_t production_seg = nack->getProductionSegment(); uint32_t nack_segment = content_object.getName().getSuffix(); bool is_rtx = ldr_->isRtx(nack_segment); @@ -592,12 +687,11 @@ void RTCTransportProtocol::onNack(const ContentObject &content_object) { // remove the nack is it exists if (tn_it != timeouts_or_nacks_.end()) timeouts_or_nacks_.erase(tn_it); + state_->onJumpForward(production_seg); // the client is asking for content in the past // switch to catch up state and increase the window // this is true only if the packet is not an RTX if (!is_rtx) current_state_ = SyncState::catch_up; - - updateSyncWindow(); } else { // if production_seg == nack_segment we consider this a future nack, since // production_seg is not yet created. this may happen in case of low @@ -610,25 +704,50 @@ void RTCTransportProtocol::onNack(const ContentObject &content_object) { // the client is asking for content in the future // switch to in sync state and decrease the window current_state_ = SyncState::in_sync; - decreaseSyncWindow(); } + updateSyncWindow(); } void RTCTransportProtocol::onProbe(const ContentObject &content_object) { - bool valid = state_->onProbePacketReceived(content_object); - if (!valid) return; + uint32_t suffix = content_object.getName().getSuffix(); + ParamsRTC params = RTCState::getProbeParams(content_object); + + if (ProbeHandler::getProbeType(suffix) == ProbeType::INIT) { + fec::FECType fec_type = params.fec_type; + + if (fec_type != fec::FECType::UNKNOWN && !fec_decoder_) { + // Update FEC type + fec_type_ = fec_type; + + // Enable FEC + enableFEC(std::bind(&RTCTransportProtocol::onFecPackets, this, + std::placeholders::_1), + fec::FECBase::BufferRequested(0)); + + // Update FEC parameters + indexer_verifier_->enableFec(fec_type); + indexer_verifier_->setNFec(0); + ldr_->setFecParams(fec::FECUtils::getBlockSymbols(fec_type), + fec::FECUtils::getSourceSymbols(fec_type)); + fec_decoder_->setIOService(portal_->getThread().getIoService()); + } else if (fec_type == fec::FECType::UNKNOWN) { + indexer_verifier_->disableFec(); + } + } - struct nack_packet_t *probe = - (struct nack_packet_t *)content_object.getPayload()->data(); - uint32_t production_seg = probe->getProductionSegement(); + if (!state_->onProbePacketReceived(content_object)) return; - // as for the nacks set next_segment + // As for NACKs, set next_segment DLOG_IF(INFO, VLOG_IS_ON(3)) << "on probe next seg = " << indexer_verifier_->checkNextSuffix() - << ", jump to " << production_seg; - indexer_verifier_->jumpToIndex(production_seg); - - ldr_->onProbePacketReceived(content_object); + << ", jump to " << params.prod_seg; + indexer_verifier_->jumpToIndex(params.prod_seg); + + bool loss_detected = ldr_->onProbePacketReceived(content_object); + // we are not out of sync here but we are starting to download content from + // the cache, maybe beacuse the production rate increased suddenly. for this + // reason we put the state to catch up to increase the window + if (loss_detected) current_state_ = SyncState::catch_up; updateSyncWindow(); } @@ -636,12 +755,39 @@ void RTCTransportProtocol::onContentObjectReceived( Interest &interest, ContentObject &content_object, std::error_code &ec) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received content object of size: " << content_object.payloadSize(); - uint32_t payload_size = content_object.payloadSize(); + uint32_t segment_number = content_object.getName().getSuffix(); + PayloadType payload_type = content_object.getPayloadType(); + PacketState state; + + ContentObject *content_ptr = &content_object; + ContentObject::Ptr manifest_ptr = nullptr; + + bool is_probe = + ProbeHandler::getProbeType(segment_number) != ProbeType::NOT_PROBE; + bool is_nack = !is_probe && content_object.payloadSize() == NACK_HEADER_SIZE; + bool is_fec = indexer_verifier_->isFec(segment_number); + bool is_manifest = + !is_probe && !is_nack && !is_fec && payload_type == PayloadType::MANIFEST; + bool is_data = + !is_probe && !is_nack && !is_fec && payload_type == PayloadType::DATA; + bool compute_stats = is_data || is_manifest; ec = make_error_code(protocol_error::not_reassemblable); - if (segment_number >= MIN_PROBE_SEQ) { + // A helper function to process manifests or data packets received + auto onDataPacketReceived = [this](ContentObject &content_object, + bool compute_stats) { + ldr_->onDataPacketReceived(content_object); + rc_->onDataPacketReceived(content_object, compute_stats); + updateSyncWindow(); + }; + + // First verify the packet signature and apply the corresponding policy + auth::VerificationPolicy policy = verifier_->verify(content_object, is_fec); + indexer_verifier_->applyPolicy(interest, content_object, false, policy); + + if (is_probe) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received probe " << segment_number; if (*on_content_object_input_) { (*on_content_object_input_)(*socket_->getInterface(), content_object); @@ -650,7 +796,7 @@ void RTCTransportProtocol::onContentObjectReceived( return; } - if (payload_size == NACK_HEADER_SIZE) { + if (is_nack) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received nack " << segment_number; if (*on_content_object_input_) { (*on_content_object_input_)(*socket_->getInterface(), content_object); @@ -659,57 +805,115 @@ void RTCTransportProtocol::onContentObjectReceived( return; } + // content_ptr will point either to the input data packet or to a manifest + // whose FEC header has been removed + if (is_manifest) { + manifest_ptr = removeFecHeader(content_object); + if (manifest_ptr) { + content_ptr = manifest_ptr.get(); + } + } + + // From there, the packet is either a FEC, a manifest or a data packet. DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received content " << segment_number; - bool compute_stats = true; + // Do not count timed out packets in stats auto tn_it = timeouts_or_nacks_.find(segment_number); if (tn_it != timeouts_or_nacks_.end()) { compute_stats = false; timeouts_or_nacks_.erase(tn_it); } - if (ldr_->isRtx(segment_number)) { + + // Do not count retransmissions or losses in stats + if (ldr_->isRtx(segment_number) || + ldr_->isPossibleLossWithNoRtx(segment_number)) { compute_stats = false; } - // check if the packet was already received - PacketState state = state_->isReceivedOrLost(segment_number); + // Fetch packet state + state = state_->getPacketState(segment_number); - if (state != PacketState::RECEIVED) { - // send packet to decoder - if (fec_decoder_) { - DLOG_IF(INFO, VLOG_IS_ON(4)) - << "send packet " << segment_number << " to FEC decoder"; - fec_decoder_->onDataPacket( - content_object, content_object.headerSize() + rtc::DATA_HEADER_SIZE); - } - if (!indexer_verifier_->isFec(segment_number)) { - // the packet may be alredy sent to the ap by the decoder, check again if - // it is already received - state = state_->isReceivedOrLost(segment_number); - if (state != PacketState::RECEIVED) { - DLOG_IF(INFO, VLOG_IS_ON(4)) << "Received content " << segment_number; + // Check if the packet is a retransmission + if (ldr_->isRtx(segment_number) && state != PacketState::RECEIVED) { + if (is_data || is_manifest) { + uint64_t rtt = ldr_->getRtxRtt(segment_number); + state_->onPacketRecoveredRtx(content_object, rtt); - state_->onDataPacketReceived(content_object, compute_stats); + if (*on_content_object_input_) { + (*on_content_object_input_)(*socket_->getInterface(), content_object); + } - if (*on_content_object_input_) { - (*on_content_object_input_)(*socket_->getInterface(), content_object); - } - ec = make_error_code(protocol_error::success); + if (is_manifest) { + processManifest(interest, *content_ptr); } - } else { - DLOG_IF(INFO, VLOG_IS_ON(4)) << "Received fec " << segment_number; - state_->onFecPacketReceived(content_object); + + ec = is_manifest ? make_error_code(protocol_error::not_reassemblable) + : make_error_code(protocol_error::success); + + // The packet is considered received, return early + onDataPacketReceived(*content_ptr, compute_stats); + // this is a rtx but we may need to feed it in the decoder + decodePacket(content_object, is_manifest); + return; } - } else { + + if (is_fec) { + state_->onFecPacketRecoveredRtx(content_object); + } + } + + // Fetch packet state again; it may have changed + state = state_->getPacketState(segment_number); + + // Check if the packet was already received + if (state == PacketState::RECEIVED || state == PacketState::TO_BE_RECEIVED) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Received duplicated content " << segment_number << ", drop it"; ec = make_error_code(protocol_error::duplicated_content); + onDataPacketReceived(*content_ptr, compute_stats); + return; } - ldr_->onDataPacketReceived(content_object); - rc_->onDataPacketReceived(content_object); + if (!is_fec) { + state_->dataToBeReceived(segment_number); + } - updateSyncWindow(); + // send packet to the decoder + decodePacket(content_object, is_manifest); + + // We can return early if FEC + if (is_fec) { + DLOG_IF(INFO, VLOG_IS_ON(4)) << "Received FEC " << segment_number; + state_->onFecPacketReceived(content_object); + onDataPacketReceived(*content_ptr, compute_stats); + return; + } + + // The packet may have been already sent to the app by the decoder, check + // again if it is already received + state = state_->getPacketState( + segment_number); // state == RECEIVED or TO_BE_RECEIVED + + if (state != PacketState::RECEIVED) { + DLOG_IF(INFO, VLOG_IS_ON(4)) + << (is_manifest ? "Received manifest " : "Received data ") + << segment_number; + + if (is_manifest) { + processManifest(interest, *content_ptr); + } + + state_->onDataPacketReceived(*content_ptr, compute_stats); + + if (*on_content_object_input_) { + (*on_content_object_input_)(*socket_->getInterface(), content_object); + } + + ec = is_manifest ? make_error_code(protocol_error::not_reassemblable) + : make_error_code(protocol_error::success); + } + + onDataPacketReceived(*content_ptr, compute_stats); } void RTCTransportProtocol::sendStatsToApp( @@ -729,33 +933,165 @@ void RTCTransportProtocol::sendStatsToApp( stats_->updateReceivedNacks(received_nacks); stats_->updateReceivedFEC(received_fec); - stats_->updateAverageWindowSize(current_sync_win_); - stats_->updateLossRatio(state_->getLossRate()); - stats_->updateAverageRtt(state_->getRTT()); + stats_->updateAverageWindowSize(state_->getPendingInterestNumber()); + stats_->updateLossRatio(state_->getPerSecondLossRate()); + uint64_t rtt = state_->getAvgRTT(); + stats_->updateAverageRtt(utils::SteadyTime::Microseconds(rtt * 1000)); + stats_->updateQueuingDelay(state_->getQueuing()); stats_->updateLostData(lost_data); stats_->updateDefinitelyLostData(definitely_lost); stats_->updateRecoveredData(recovered_losses); stats_->updateCCState((unsigned int)current_state_ ? 1 : 0); (*stats_summary_)(*socket_->getInterface(), *stats_); + bool in_congestion = rc_->inCongestionState(); + stats_->updateCongestionState(in_congestion); + double residual_losses = state_->getResidualLossRate(); + stats_->updateResidualLossRate(residual_losses); + stats_->updateQualityScore(state_->getQualityScore()); + + // set alerts + if (rtt > MAX_RTT) + stats_->setAlert(interface::TransportStatistics::statsAlerts::LATENCY); + else + stats_->clearAlert(interface::TransportStatistics::statsAlerts::LATENCY); + + if (in_congestion) + stats_->setAlert(interface::TransportStatistics::statsAlerts::CONGESTION); + else + stats_->clearAlert( + interface::TransportStatistics::statsAlerts::CONGESTION); + + if (residual_losses > MAX_RESIDUAL_LOSSES) + stats_->setAlert(interface::TransportStatistics::statsAlerts::LOSSES); + else + stats_->clearAlert(interface::TransportStatistics::statsAlerts::LOSSES); } } -void RTCTransportProtocol::onFecPackets( - std::vector<std::pair<uint32_t, fec::buffer>> &packets) { +void RTCTransportProtocol::decodePacket(ContentObject &content_object, + bool is_manifest) { + if (!fec_decoder_) return; + + DLOG_IF(INFO, VLOG_IS_ON(4)) + << "Send packet " << content_object.getName() << " to FEC decoder"; + + uint32_t offset = + is_manifest + ? (uint32_t)content_object.headerSize() + : (uint32_t)(content_object.headerSize() + rtc::DATA_HEADER_SIZE); + uint32_t metadata = static_cast<uint32_t>(content_object.getPayloadType()); + + fec_decoder_->onDataPacket(content_object, offset, metadata); +} + +void RTCTransportProtocol::onFecPackets(fec::BufferArray &packets) { + Packet::Format format; + socket_->getSocketOption(interface::GeneralTransportOptions::PACKET_FORMAT, + format); + + Name *name = nullptr; + socket_->getSocketOption(GeneralTransportOptions::NETWORK_NAME, &name); + for (auto &packet : packets) { - PacketState state = state_->isReceivedOrLost(packet.first); - if (state != PacketState::RECEIVED) { - state_->onPacketRecoveredFec(packet.first); - ldr_->onPacketRecoveredFec(packet.first); - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "Recovered packet " << packet.first << " through FEC."; - reassembly_->reassemble(*packet.second, packet.first); - } else { - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "Packet" << packet.first << "already received."; + uint32_t seq_number = packet.getIndex(); + uint32_t metadata = packet.getMetadata(); + fec::buffer buffer = packet.getBuffer(); + + PayloadType payload_type = static_cast<PayloadType>(metadata); + switch (payload_type) { + case PayloadType::DATA: + case PayloadType::MANIFEST: + break; + case PayloadType::UNSPECIFIED: + default: + payload_type = PayloadType::DATA; + break; } + + switch (state_->getPacketState(seq_number)) { + case PacketState::RECEIVED: + case PacketState::TO_BE_RECEIVED: { + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "Packet " << seq_number << " already received"; + break; + } + default: { + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "Recovered packet " << seq_number << " through FEC"; + + if (payload_type == PayloadType::MANIFEST) { + name->setSuffix(seq_number); + + auto interest = + core::PacketManager<>::getInstance().getPacket<Interest>(format); + interest->setName(*name); + + auto content_object = toContentObject( + *name, format, payload_type, buffer->data(), buffer->length()); + + processManifest(*interest, *content_object); + } + + state_->onPacketRecoveredFec(seq_number, (uint32_t)buffer->length()); + ldr_->onPacketRecoveredFec(seq_number); + + if (payload_type == PayloadType::DATA) { + verifier_->onDataRecoveredFec(seq_number); + reassembly_->reassemble(*buffer, seq_number); + } + + break; + } + } + } +} + +void RTCTransportProtocol::processManifest(Interest &interest, + ContentObject &manifest) { + auth::VerificationPolicy policy = verifier_->processManifest(manifest); + indexer_verifier_->applyPolicy(interest, manifest, false, policy); +} + +ContentObject::Ptr RTCTransportProtocol::removeFecHeader( + const ContentObject &content_object) { + if (!fec_decoder_ || !fec_decoder_->getFecHeaderSize(false)) { + return nullptr; } + + size_t fec_header_size = fec_decoder_->getFecHeaderSize(false); + const uint8_t *payload = + content_object.data() + content_object.headerSize() + fec_header_size; + size_t payload_size = content_object.payloadSize() - fec_header_size; + + ContentObject::Ptr co = + toContentObject(content_object.getName(), content_object.getFormat(), + content_object.getPayloadType(), payload, payload_size); + + return co; +} + +ContentObject::Ptr RTCTransportProtocol::toContentObject( + const Name &name, Packet::Format format, PayloadType payload_type, + const uint8_t *payload, std::size_t payload_size, + std::size_t additional_header_size) { + // Recreate ContentObject + ContentObject::Ptr co = + core::PacketManager<>::getInstance().getPacket<ContentObject>( + format, additional_header_size); + co->updateLength(payload_size); + co->append(payload_size); + co->trimStart(co->headerSize()); + + // Copy payload + std::memcpy(co->writableData(), payload, payload_size); + + // Restore network headers and some fields + co->prepend(co->headerSize()); + co->setName(name); + co->setPayloadType(payload_type); + + return co; } } // end namespace rtc diff --git a/libtransport/src/protocols/rtc/rtc.h b/libtransport/src/protocols/rtc/rtc.h index e6431264d..a8a474216 100644 --- a/libtransport/src/protocols/rtc/rtc.h +++ b/libtransport/src/protocols/rtc/rtc.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -15,10 +15,12 @@ #pragma once -#include <protocols/datagram_reassembly.h> +#include <protocols/rtc/rtc_forwarding_strategy.h> #include <protocols/rtc/rtc_ldr.h> #include <protocols/rtc/rtc_rc.h> +#include <protocols/rtc/rtc_reassembly.h> #include <protocols/rtc/rtc_state.h> +#include <protocols/rtc/rtc_verifier.h> #include <protocols/transport_protocol.h> #include <unordered_set> @@ -42,7 +44,9 @@ class RTCTransportProtocol : public TransportProtocol { void resume() override; - std::size_t transportHeaderLength() override; + std::size_t transportHeaderLength(bool isFEC) override; + + auto shared_from_this() { return utils::shared_from(this); } private: enum class SyncState { catch_up = 0, in_sync = 1, last }; @@ -61,11 +65,11 @@ class RTCTransportProtocol : public TransportProtocol { // window functions void computeMaxSyncWindow(); void updateSyncWindow(); - void decreaseSyncWindow(); // packet functions void sendRtxInterest(uint32_t seq); void sendProbeInterest(uint32_t seq); + void sendInterestForTimeout(uint32_t seq); void scheduleNextInterests() override; void onInterestTimeout(Interest::Ptr &interest, const Name &name) override; void onNack(const ContentObject &content_object); @@ -76,6 +80,7 @@ class RTCTransportProtocol : public TransportProtocol { void onPacketDropped(Interest &interest, ContentObject &content_object, const std::error_code &reason) override {} void onReassemblyFailed(std::uint32_t missing_segment) override {} + void processManifest(Interest &interest, ContentObject &manifest); // interaction with app functions void sendStatsToApp(uint32_t retx_count, uint32_t received_bytes, @@ -84,11 +89,22 @@ class RTCTransportProtocol : public TransportProtocol { uint32_t received_nacks, uint32_t received_fec); // FEC functions - void onFecPackets(std::vector<std::pair<uint32_t, fec::buffer>> &packets); + // send the received content object to the decoder + void decodePacket(ContentObject &content_object, bool is_manifest); + void onFecPackets(fec::BufferArray &packets); + + // Utils + ContentObject::Ptr removeFecHeader(const ContentObject &content_object); + ContentObject::Ptr toContentObject(const Name &name, Packet::Format format, + PayloadType payload_type, + const uint8_t *payload, + std::size_t payload_size, + std::size_t additional_header_size = 0); // protocol state bool start_send_interest_; SyncState current_state_; + // cwin vars uint32_t current_sync_win_; uint32_t max_sync_win_; @@ -120,6 +136,10 @@ class RTCTransportProtocol : public TransportProtocol { std::shared_ptr<RTCState> state_; std::shared_ptr<RTCRateControl> rc_; std::shared_ptr<RTCLossDetectionAndRecovery> ldr_; + std::shared_ptr<RTCVerifier> verifier_; + + // forwarding strategy selection + RTCForwardingStrategy fwd_strategy_; uint32_t number_; }; diff --git a/libtransport/src/protocols/rtc/rtc_consts.h b/libtransport/src/protocols/rtc/rtc_consts.h index d04bc1b1f..29b5a3a12 100644 --- a/libtransport/src/protocols/rtc/rtc_consts.h +++ b/libtransport/src/protocols/rtc/rtc_consts.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -34,7 +34,7 @@ const double INTEREST_LIFETIME_REDUCTION_FACTOR = 0.8; // increasing this number we increase the time that an // interest will wait for the data packet to be produced // at the producer socket -const uint32_t PRODUCER_BUFFER_MS = 200; // ms +const uint32_t PRODUCER_BUFFER_MS = 300; // ms // interest scheduler // const uint32_t MAX_INTERESTS_IN_BATCH = 5; @@ -45,22 +45,22 @@ const uint32_t MAX_INTERESTS_IN_BATCH = 5; // number of seq numbers per const uint32_t WAIT_FOR_INTEREST_BATCH = 20; // msec. timer that we wait to try // to aggregate interest in the // same packet -const uint32_t MAX_PACING_BATCH = 5; -// number of interest that we can send inside -// the loop before they get dropped by the -// kernel. +const uint32_t MAX_PACING_BATCH = 5; // number of interest that we can send + // inside the loop before they get dropped + // by the kernel. const uint32_t PACING_WAIT = 1000; // usec to wait betwing two pacing batch. As // for MAX_PACING_BATCH this value was // computed during tests const uint32_t MAX_RTX_IN_BATCH = 10; // max rtx to send in loop // packet const -const uint32_t HICN_HEADER_SIZE = 40 + 20; // IPv6 + TCP bytes -const uint32_t RTC_INTEREST_LIFETIME = 2000; +const uint32_t RTC_INTEREST_LIFETIME = 4000; // probes sequence range const uint32_t MIN_PROBE_SEQ = 0xefffffff; -const uint32_t MIN_RTT_PROBE_SEQ = MIN_PROBE_SEQ; +const uint32_t MIN_INIT_PROBE_SEQ = MIN_PROBE_SEQ; +const uint32_t MAX_INIT_PROBE_SEQ = 0xf7ffffff - 1; +const uint32_t MIN_RTT_PROBE_SEQ = MAX_INIT_PROBE_SEQ + 1; const uint32_t MAX_RTT_PROBE_SEQ = 0xffffffff - 1; // RTT_PROBE_INTERVAL will be used during the section while // INIT_RTT_PROBE_INTERVAL is used at the beginning to @@ -72,25 +72,28 @@ const uint32_t INIT_RTT_PROBES = 40; // number of probes to init RTT // to get an answer. we wait 100ms between each try const uint32_t INIT_RTT_PROBE_RESTART = 100; // ms // once we get the first probe we wait at most 60ms for the others -const uint32_t INIT_RTT_PROBE_WAIT = 30; // ms +const uint32_t INIT_RTT_PROBE_WAIT = + ((INIT_RTT_PROBES * INIT_RTT_PROBE_INTERVAL) / 1000) * 2; // ms // we reuires at least 5 probes to be recevied const uint32_t INIT_RTT_MIN_PROBES_TO_RECV = 5; // ms const uint32_t MAX_PENDING_PROBES = 10; // congestion -const double MAX_QUEUING_DELAY = 100.0; // ms +const double MAX_QUEUING_DELAY = 50.0; // ms // data from cache -const double MAX_DATA_FROM_CACHE = 0.25; // 25% +const double MAX_DATA_FROM_CACHE = 0.10; // 10% // window const -const uint32_t INITIAL_WIN = 5; // pkts -const uint32_t INITIAL_WIN_MAX = 1000000; // pkts -const uint32_t WIN_MIN = 5; // pkts +const uint32_t INITIAL_WIN = 5; // pkts +const uint32_t INITIAL_WIN_MAX = 1000000; // pkts +const uint32_t WIN_MIN = 5; // pkts +const uint32_t WIN_MIN_WITH_AGGREGARED_DATA = 10; // pkts const double CATCH_UP_WIN_INCREMENT = 1.2; // used in rate control const double WIN_DECREASE_FACTOR = 0.5; const double WIN_INCREASE_FACTOR = 1.5; +const uint32_t MIN_PROD_RATE_SHARING_MODE = 125000; // 1Mbps in bytes // round in congestion const double ROUNDS_BEFORE_TAKE_ACTION = 5; @@ -105,10 +108,8 @@ const double MOVING_AVG_ALPHA = 0.8; const double MILLI_IN_A_SEC = 1000.0; const double MICRO_IN_A_SEC = 1000000.0; - -const double MAX_CACHED_PACKETS = 262144; // 2^18 - // about 50 sec of traffic at 50Mbps - // with 1200 bytes packets +const uint32_t ROUNDS_PER_SEC = (uint32_t)(MILLI_IN_A_SEC / ROUND_LEN); +const uint32_t ROUNDS_PER_MIN = (uint32_t)ROUNDS_PER_SEC * 60; const uint32_t MAX_ROUND_WHIOUT_PACKETS = (20 * MILLI_IN_A_SEC) / ROUND_LEN; // 20 sec in rounds; @@ -120,12 +121,91 @@ const uint64_t MAX_TIMER_RTX = ~0; const uint32_t SENTINEL_TIMER_INTERVAL = 100; // ms const uint32_t MAX_RTX_WITH_SENTINEL = 10; // packets const double CATCH_UP_RTT_INCREMENT = 1.2; +const double MAX_RESIDUAL_LOSS_RATE = 1.0; // % +const uint32_t WAIT_BEFORE_FEC_UPDATE = ROUNDS_PER_SEC; +const uint32_t MAX_RTT_BEFORE_FEC = 60; // ms // used by producer const uint32_t PRODUCER_STATS_INTERVAL = 200; // ms -const uint32_t MIN_PRODUCTION_RATE = 10; // pps - // min prod rate - // set running several test +const uint32_t MIN_PRODUCTION_RATE = 25; // pps, equal to min window * + // rounds in a second +const uint32_t FEC_PACING_TIME = 5; // ms + +// aggregated data consts +const uint16_t MAX_RTC_PAYLOAD_SIZE = 1200; // bytes +const uint16_t MAX_AGGREGATED_PACKETS = 5; // pkt +const uint32_t AGGREGATED_PACKETS_TIMER = 2; // ms + +// alert thresholds +const uint32_t MAX_RTT = 200; // ms +const double MAX_RESIDUAL_LOSSES = 0.05; // % + +const uint8_t FEC_MATRIX[64][10] = { + {1, 2, 2, 2, 3, 3, 4, 5, 5, 6}, // k = 1 + {1, 2, 3, 3, 4, 5, 5, 6, 7, 9}, + {2, 2, 3, 4, 5, 6, 7, 8, 9, 11}, + {2, 3, 4, 5, 5, 7, 8, 9, 11, 13}, + {2, 3, 4, 5, 6, 7, 9, 10, 12, 14}, // k = 5 + {2, 3, 4, 6, 7, 8, 10, 12, 14, 16}, + {2, 4, 5, 6, 8, 9, 11, 13, 15, 18}, + {3, 4, 5, 7, 8, 10, 12, 14, 16, 19}, + {3, 4, 6, 7, 9, 11, 13, 15, 18, 21}, + {3, 4, 6, 8, 9, 11, 14, 16, 19, 23}, // k = 10 + {3, 5, 6, 8, 10, 12, 14, 17, 20, 24}, + {3, 5, 7, 8, 10, 13, 15, 18, 21, 26}, + {3, 5, 7, 9, 11, 13, 16, 19, 23, 27}, + {3, 5, 7, 9, 12, 14, 17, 20, 24, 28}, + {4, 6, 8, 10, 12, 15, 18, 21, 25, 30}, // k = 15 + {4, 6, 8, 10, 13, 15, 19, 22, 26, 31}, + {4, 6, 8, 11, 13, 16, 19, 23, 27, 33}, + {4, 6, 9, 11, 14, 17, 20, 24, 29, 34}, + {4, 6, 9, 11, 14, 17, 21, 25, 30, 35}, + {4, 7, 9, 12, 15, 18, 22, 26, 31, 37}, // k = 20 + {4, 7, 9, 12, 15, 19, 22, 27, 32, 38}, + {4, 7, 10, 13, 16, 19, 23, 28, 33, 40}, + {5, 7, 10, 13, 16, 20, 24, 29, 34, 41}, + {5, 7, 10, 13, 17, 20, 25, 30, 35, 42}, + {5, 8, 11, 14, 17, 21, 26, 31, 37, 44}, // k = 25 + {5, 8, 11, 14, 18, 22, 26, 31, 38, 45}, + {5, 8, 11, 15, 18, 22, 27, 32, 39, 46}, + {5, 8, 11, 15, 19, 23, 28, 33, 40, 48}, + {5, 8, 12, 15, 19, 24, 28, 34, 41, 49}, + {5, 9, 12, 16, 20, 24, 29, 35, 42, 50}, // k = 30 + {5, 9, 12, 16, 20, 25, 30, 36, 43, 51}, + {5, 9, 13, 16, 21, 25, 31, 37, 44, 53}, + {6, 9, 13, 17, 21, 26, 31, 38, 45, 54}, + {6, 9, 13, 17, 22, 26, 32, 39, 46, 55}, + {6, 10, 13, 17, 22, 27, 33, 40, 47, 57}, // k = 35 + {6, 10, 14, 18, 22, 28, 34, 40, 48, 58}, + {6, 10, 14, 18, 23, 28, 34, 41, 49, 59}, + {6, 10, 14, 19, 23, 29, 35, 42, 50, 60}, + {6, 10, 14, 19, 24, 29, 36, 43, 52, 62}, + {6, 10, 15, 19, 24, 30, 36, 44, 53, 63}, // k = 40 + {6, 11, 15, 20, 25, 31, 37, 45, 54, 64}, + {6, 11, 15, 20, 25, 31, 38, 46, 55, 65}, + {7, 11, 15, 20, 26, 32, 39, 46, 56, 67}, + {7, 11, 16, 21, 26, 32, 39, 47, 57, 68}, + {7, 11, 16, 21, 27, 33, 40, 48, 58, 69}, // k = 45 + {7, 11, 16, 21, 27, 33, 41, 49, 59, 70}, + {7, 12, 16, 22, 27, 34, 41, 50, 60, 72}, + {7, 12, 17, 22, 28, 34, 42, 51, 61, 73}, + {7, 12, 17, 22, 28, 35, 43, 52, 62, 74}, + {7, 12, 17, 23, 29, 36, 43, 52, 63, 75}, // k = 50 + {7, 12, 17, 23, 29, 36, 44, 53, 64, 77}, + {7, 12, 18, 23, 30, 37, 45, 54, 65, 78}, + {7, 13, 18, 24, 30, 37, 45, 55, 66, 79}, + {8, 13, 18, 24, 31, 38, 46, 56, 67, 80}, + {8, 13, 18, 24, 31, 38, 47, 57, 68, 82}, // k = 55 + {8, 13, 19, 25, 31, 39, 47, 57, 69, 83}, + {8, 13, 19, 25, 32, 39, 48, 58, 70, 84}, + {8, 13, 19, 25, 32, 40, 49, 59, 71, 85}, + {8, 14, 19, 26, 33, 41, 50, 60, 72, 86}, + {8, 14, 20, 26, 33, 41, 50, 61, 73, 88}, // k = 60 + {8, 14, 20, 26, 34, 42, 51, 61, 74, 89}, + {8, 14, 20, 27, 34, 42, 52, 62, 75, 90}, + {8, 14, 20, 27, 34, 43, 52, 63, 76, 91}, + {8, 14, 21, 27, 35, 43, 53, 64, 77, 92}, // k = 64 +}; } // namespace rtc diff --git a/libtransport/src/protocols/rtc/rtc_data_path.cc b/libtransport/src/protocols/rtc/rtc_data_path.cc index c098088a3..a421396b1 100644 --- a/libtransport/src/protocols/rtc/rtc_data_path.cc +++ b/libtransport/src/protocols/rtc/rtc_data_path.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -13,14 +13,17 @@ * limitations under the License. */ +#include <hicn/transport/utils/chrono_typedefs.h> #include <protocols/rtc/rtc_data_path.h> #include <stdlib.h> #include <algorithm> #include <cfloat> #include <chrono> +#include <cmath> #define MAX_ROUNDS_WITHOUT_PKTS 10 // 2sec +#define AVG_RTT_TIME 1000 // (ms) 1sec namespace transport { @@ -32,6 +35,8 @@ RTCDataPath::RTCDataPath(uint32_t path_id) : path_id_(path_id), min_rtt(UINT_MAX), prev_min_rtt(UINT_MAX), + max_rtt(0), + prev_max_rtt(0), min_owd(INT_MAX), // this is computed like in LEDBAT, so it is not the // real OWD, but the measured one, that depends on the // clock of sender and receiver. the only meaningful @@ -46,20 +51,48 @@ RTCDataPath::RTCDataPath(uint32_t path_id) largest_recv_seq_(0), largest_recv_seq_time_(0), avg_inter_arrival_(DBL_MAX), + rtt_sum_(0), + last_avg_rtt_compute_(0), + rtt_samples_(0), + avg_rtt_(0.0), received_nacks_(false), - received_packets_(false), + received_packets_(0), rounds_without_packets_(0), last_received_data_packet_(0), - RTT_history_(HISTORY_LEN), + min_RTT_history_(HISTORY_LEN), + max_RTT_history_(HISTORY_LEN), OWD_history_(HISTORY_LEN){}; -void RTCDataPath::insertRttSample(uint64_t rtt) { - // for the rtt we only keep track of the min one +void RTCDataPath::insertRttSample( + const utils::SteadyTime::Milliseconds& rtt_milliseconds, bool is_probe) { + // compute min rtt + uint64_t rtt = rtt_milliseconds.count(); if (rtt < min_rtt) min_rtt = rtt; - last_received_data_packet_ = - std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + + uint64_t now = utils::SteadyTime::nowMs().count(); + last_received_data_packet_ = now; + + // compute avg rtt + if (is_probe) { + // max rtt is computed only on probes to avoid to take into account the + // production time at the server + if (rtt > max_rtt) max_rtt = rtt; + + rtt_sum_ += rtt; + rtt_samples_++; + } + + if ((now - last_avg_rtt_compute_) >= AVG_RTT_TIME) { + // compute a new avg rtt + // if rtt_samples_ = 0 keep the same rtt + if (rtt_samples_ != 0) avg_rtt_ = (double)rtt_sum_ / (double)rtt_samples_; + + rtt_sum_ = 0; + rtt_samples_ = 0; + last_avg_rtt_compute_ = now; + } + + received_packets_++; } void RTCDataPath::insertOwdSample(int64_t owd) { @@ -84,18 +117,12 @@ void RTCDataPath::insertOwdSample(int64_t owd) { int64_t diff = std::abs(owd - last_owd_); last_owd_ = owd; jitter_ += (1.0 / 16.0) * ((double)diff - jitter_); - - // owd is computed only for valid data packets so we count only - // this for decide if we recevie traffic or not - received_packets_ = true; } void RTCDataPath::computeInterArrivalGap(uint32_t segment_number) { // got packet in sequence, compute gap if (largest_recv_seq_ == (segment_number - 1)) { - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + uint64_t now = utils::SteadyTime::nowMs().count(); uint64_t delta = now - largest_recv_seq_time_; largest_recv_seq_ = segment_number; largest_recv_seq_time_ = now; @@ -110,10 +137,7 @@ void RTCDataPath::computeInterArrivalGap(uint32_t segment_number) { // ooo packet, update the stasts if needed if (largest_recv_seq_ <= segment_number) { largest_recv_seq_ = segment_number; - largest_recv_seq_time_ = - std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + largest_recv_seq_time_ = utils::SteadyTime::nowMs().count(); } } @@ -124,12 +148,17 @@ double RTCDataPath::getInterArrivalGap() { return avg_inter_arrival_; } -bool RTCDataPath::isActive() { +bool RTCDataPath::isValidProducer() { if (received_nacks_ && rounds_without_packets_ < MAX_ROUNDS_WITHOUT_PKTS) return true; return false; } +bool RTCDataPath::isActive() { + if (rounds_without_packets_ < MAX_ROUNDS_WITHOUT_PKTS) return true; + return false; +} + bool RTCDataPath::pathToProducer() { if (received_nacks_) return true; return false; @@ -146,10 +175,20 @@ void RTCDataPath::roundEnd() { min_rtt = prev_min_rtt; } + // same for max_rtt + if (max_rtt != 0) { + prev_max_rtt = max_rtt; + } else { + max_rtt = prev_max_rtt; + } + if (min_rtt == 0) min_rtt = 1; + if (max_rtt == 0) max_rtt = 1; - RTT_history_.pushBack(min_rtt); + min_RTT_history_.pushBack(min_rtt); + max_RTT_history_.pushBack(max_rtt); min_rtt = UINT_MAX; + max_rtt = 0; // do the same for min owd if (min_owd != INT_MAX) { @@ -163,32 +202,47 @@ void RTCDataPath::roundEnd() { min_owd = INT_MAX; } - if (!received_packets_) + if (received_packets_ == 0) rounds_without_packets_++; else rounds_without_packets_ = 0; - received_packets_ = false; + received_packets_ = 0; } uint32_t RTCDataPath::getPathId() { return path_id_; } -double RTCDataPath::getQueuingDealy() { return queuing_delay; } +double RTCDataPath::getQueuingDealy() { + if (queuing_delay == DBL_MAX) return 0; + return queuing_delay; +} uint64_t RTCDataPath::getMinRtt() { - if (RTT_history_.size() != 0) return RTT_history_.begin(); + if (min_RTT_history_.size() != 0) return min_RTT_history_.begin(); + return 0; +} + +uint64_t RTCDataPath::getAvgRtt() { return std::round(avg_rtt_); } + +uint64_t RTCDataPath::getMaxRtt() { + if (max_RTT_history_.size() != 0) return max_RTT_history_.begin(); return 0; } int64_t RTCDataPath::getMinOwd() { if (OWD_history_.size() != 0) return OWD_history_.begin(); - return 0; + return INT_MAX; } double RTCDataPath::getJitter() { return jitter_; } uint64_t RTCDataPath::getLastPacketTS() { return last_received_data_packet_; } -void RTCDataPath::clearRtt() { RTT_history_.clear(); } +uint32_t RTCDataPath::getPacketsLastRound() { return received_packets_; } + +void RTCDataPath::clearRtt() { + min_RTT_history_.clear(); + max_RTT_history_.clear(); +} } // end namespace rtc diff --git a/libtransport/src/protocols/rtc/rtc_data_path.h b/libtransport/src/protocols/rtc/rtc_data_path.h index c5c37fc0d..ba5201fe8 100644 --- a/libtransport/src/protocols/rtc/rtc_data_path.h +++ b/libtransport/src/protocols/rtc/rtc_data_path.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -15,7 +15,9 @@ #pragma once +#include <hicn/transport/utils/chrono_typedefs.h> #include <stdint.h> +#include <utils/max_filter.h> #include <utils/min_filter.h> #include <climits> @@ -34,19 +36,24 @@ class RTCDataPath { RTCDataPath(uint32_t path_id); public: - void insertRttSample(uint64_t rtt); + void insertRttSample(const utils::SteadyTime::Milliseconds &rtt, + bool is_probe); void insertOwdSample(int64_t owd); void computeInterArrivalGap(uint32_t segment_number); void receivedNack(); uint32_t getPathId(); uint64_t getMinRtt(); + uint64_t getAvgRtt(); + uint64_t getMaxRtt(); double getQueuingDealy(); double getInterArrivalGap(); double getJitter(); - bool isActive(); - bool pathToProducer(); + bool isActive(); // pakets recevied from this path in the last rounds + bool pathToProducer(); // path from a producer + bool isValidProducer(); // path from a producer that is also active uint64_t getLastPacketTS(); + uint32_t getPacketsLastRound(); void clearRtt(); @@ -60,6 +67,9 @@ class RTCDataPath { uint64_t min_rtt; uint64_t prev_min_rtt; + uint64_t max_rtt; + uint64_t prev_max_rtt; + int64_t min_owd; int64_t prev_min_owd; @@ -74,19 +84,26 @@ class RTCDataPath { uint64_t largest_recv_seq_time_; double avg_inter_arrival_; + // compute the avg rtt over one sec + uint64_t rtt_sum_; + uint64_t last_avg_rtt_compute_; + uint32_t rtt_samples_; + double avg_rtt_; + // flags to check if a path is active // we considere a path active if it reaches a producer //(not a cache) --aka we got at least one nack on this path-- // and if we receives packets bool received_nacks_; - bool received_packets_; - uint8_t rounds_without_packets_; // if we don't get any packet + uint32_t received_packets_; + uint32_t rounds_without_packets_; // if we don't get any packet // for MAX_ROUNDS_WITHOUT_PKTS // we consider the path inactive uint64_t last_received_data_packet_; // timestamp for the last data received // on this path - utils::MinFilter<uint64_t> RTT_history_; + utils::MinFilter<uint64_t> min_RTT_history_; + utils::MaxFilter<uint64_t> max_RTT_history_; utils::MinFilter<int64_t> OWD_history_; }; diff --git a/libtransport/src/protocols/rtc/rtc_forwarding_strategy.cc b/libtransport/src/protocols/rtc/rtc_forwarding_strategy.cc new file mode 100644 index 000000000..4bbd7eac0 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_forwarding_strategy.cc @@ -0,0 +1,217 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <hicn/transport/interfaces/notification.h> +#include <protocols/rtc/rtc_consts.h> +#include <protocols/rtc/rtc_forwarding_strategy.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +using namespace transport::interface; + +const double FWD_MAX_QUEUE = 30.0; // ms +const double FWD_MAX_RTT = MAX_RTT_BEFORE_FEC; // ms +const double FWD_MAX_LOSS_RATE = 0.1; + +RTCForwardingStrategy::RTCForwardingStrategy() + : low_rate_app_(false), + init_(false), + forwarder_set_(false), + selected_strategy_(NONE), + current_strategy_(NONE), + rounds_since_last_set_(0), + portal_(nullptr), + state_(nullptr) {} + +RTCForwardingStrategy::~RTCForwardingStrategy() {} + +void RTCForwardingStrategy::setCallback( + interface::StrategyCallback&& callback) { + callback_ = std::move(callback); +} + +void RTCForwardingStrategy::initFwdStrategy( + std::shared_ptr<core::Portal> portal, core::Prefix& prefix, RTCState* state, + interface::RtcTransportRecoveryStrategies strategy) { + switch (strategy) { + case interface::RtcTransportRecoveryStrategies::LOW_RATE_AND_BESTPATH: + init_ = true; + low_rate_app_ = true; + selected_strategy_ = BEST_PATH; + current_strategy_ = BEST_PATH; + break; + case interface::RtcTransportRecoveryStrategies::LOW_RATE_AND_REPLICATION: + init_ = true; + low_rate_app_ = true; + selected_strategy_ = REPLICATION; + current_strategy_ = REPLICATION; + break; + case interface::RtcTransportRecoveryStrategies:: + LOW_RATE_AND_ALL_FWD_STRATEGIES: + init_ = true; + low_rate_app_ = true; + selected_strategy_ = BEST_PATH; + current_strategy_ = BEST_PATH; + break; + case interface::RtcTransportRecoveryStrategies::DELAY_AND_BESTPATH: + init_ = true; + low_rate_app_ = false; + selected_strategy_ = BEST_PATH; + current_strategy_ = BEST_PATH; + break; + case interface::RtcTransportRecoveryStrategies::DELAY_AND_REPLICATION: + init_ = true; + low_rate_app_ = false; + selected_strategy_ = REPLICATION; + current_strategy_ = REPLICATION; + break; + case interface::RtcTransportRecoveryStrategies::RECOVERY_OFF: + case interface::RtcTransportRecoveryStrategies::RTX_ONLY: + case interface::RtcTransportRecoveryStrategies::FEC_ONLY: + case interface::RtcTransportRecoveryStrategies::DELAY_BASED: + case interface::RtcTransportRecoveryStrategies::LOW_RATE: + case interface::RtcTransportRecoveryStrategies::FEC_ONLY_LOW_RES_LOSSES: + default: + // fwd strategies are not used + init_ = false; + } + + if (init_) { + rounds_since_last_set_ = 0; + prefix_ = prefix; + portal_ = portal; + state_ = state; + } +} + +void RTCForwardingStrategy::checkStrategy() { + strategy_t used_strategy = selected_strategy_; + if (used_strategy == BOTH) used_strategy = current_strategy_; + assert(used_strategy == BEST_PATH || used_strategy == REPLICATION || + used_strategy == NONE); + + notification::ForwardingStrategy strategy = + notification::ForwardingStrategy::NONE; + switch (used_strategy) { + case BEST_PATH: + strategy = notification::ForwardingStrategy::BEST_PATH; + break; + case REPLICATION: + strategy = notification::ForwardingStrategy::REPLICATION; + break; + default: + break; + } + callback_(strategy); + + if (!init_) return; + + if (selected_strategy_ == NONE) return; + + if (selected_strategy_ == BEST_PATH) { + checkStrategyBestPath(); + return; + } + + if (selected_strategy_ == REPLICATION) { + checkStrategyReplication(); + return; + } + + checkStrategyBoth(); +} + +void RTCForwardingStrategy::checkStrategyBestPath() { + if (!forwarder_set_) { + setStrategy(BEST_PATH); + forwarder_set_ = true; + return; + } + + if (low_rate_app_) { + // this is used for gaming + uint8_t qs = state_->getQualityScore(); + + if (qs >= 4 || rounds_since_last_set_ < 25) { // wait a least 5 sec + // between each switch + rounds_since_last_set_++; + return; + } + + // try to switch path + setStrategy(BEST_PATH); + } else { + if (rounds_since_last_set_ < 25) { // wait a least 5 sec + // between each switch + rounds_since_last_set_++; + return; + } + + double queue = state_->getQueuing(); + double rtt = state_->getAvgRTT(); + double loss_rate = state_->getPerSecondLossRate(); + + if (queue >= FWD_MAX_QUEUE || rtt >= FWD_MAX_RTT || + loss_rate > FWD_MAX_LOSS_RATE) { + // try to switch path + setStrategy(BEST_PATH); + } + } +} + +void RTCForwardingStrategy::checkStrategyReplication() { + if (!forwarder_set_) { + setStrategy(REPLICATION); + forwarder_set_ = true; + return; + } + + // here we have nothing to do for the moment + return; +} + +void RTCForwardingStrategy::checkStrategyBoth() { + if (!forwarder_set_) { + setStrategy(current_strategy_); + forwarder_set_ = true; + return; + } + + checkStrategyBestPath(); + + // TODO + // for the moment we use only best path. + // for later: + // 1. if both paths are bad use replication + // 2. while using replication compute the effectiveness. if the majority of + // the packets are coming from a single path, try to use bestpath +} + +void RTCForwardingStrategy::setStrategy(strategy_t strategy) { + rounds_since_last_set_ = 0; + current_strategy_ = strategy; + portal_->setForwardingStrategy(prefix_, + string_strategies_[current_strategy_]); +} + +} // namespace rtc + +} // namespace protocol + +} // namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_forwarding_strategy.h b/libtransport/src/protocols/rtc/rtc_forwarding_strategy.h new file mode 100644 index 000000000..c2227e09f --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_forwarding_strategy.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <core/portal.h> +#include <hicn/transport/interfaces/callbacks.h> +#include <protocols/rtc/rtc_state.h> + +#include <array> + +namespace transport { + +namespace protocol { + +namespace rtc { + +class RTCForwardingStrategy { + public: + enum strategy_t { + BEST_PATH, + REPLICATION, + BOTH, + NONE, + }; + + RTCForwardingStrategy(); + ~RTCForwardingStrategy(); + + void initFwdStrategy(std::shared_ptr<core::Portal> portal, + core::Prefix& prefix, RTCState* state, + interface::RtcTransportRecoveryStrategies strategy); + + void checkStrategy(); + void setCallback(interface::StrategyCallback&& callback); + + private: + void checkStrategyBestPath(); + void checkStrategyReplication(); + void checkStrategyBoth(); + + void setStrategy(strategy_t strategy); + + std::array<std::string, 4> string_strategies_ = {"bestpath", "replication", + "both", "none"}; + + bool low_rate_app_; // if set to true the best path strategy will + // trigger a path switch based on the quality + // score, otherwise it will use the RTT, + // queuing delay and loss rate + bool init_; // true if all val are initializes + bool forwarder_set_; // true if the strategy is been set at least + // once + strategy_t selected_strategy_; // this is the strategy selected using socket + // options. this can also be equal to BOTH + strategy_t current_strategy_; // if both strategies can be used this + // indicates the one that is currently in use + // that can be only replication or best path + uint32_t rounds_since_last_set_; + core::Prefix prefix_; + std::shared_ptr<core::Portal> portal_; + RTCState* state_; + interface::StrategyCallback callback_; +}; + +} // namespace rtc + +} // namespace protocol + +} // namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_indexer.h b/libtransport/src/protocols/rtc/rtc_indexer.h index 4aee242bb..f87fcaaa2 100644 --- a/libtransport/src/protocols/rtc/rtc_indexer.h +++ b/libtransport/src/protocols/rtc/rtc_indexer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -18,8 +18,10 @@ #include <protocols/errors.h> #include <protocols/fec_utils.h> #include <protocols/indexer.h> +#include <protocols/rtc/probe_handler.h> #include <protocols/rtc/rtc_consts.h> #include <protocols/transport_protocol.h> +#include <utils/suffix_strategy.h> #include <deque> @@ -45,7 +47,7 @@ class RtcIndexer : public Indexer { n_fec_(0), n_current_fec_(n_fec_) {} - RtcIndexer(RtcIndexer &&other) : Indexer(std::forward<Indexer>(other)) {} + RtcIndexer(RtcIndexer &&other) : Indexer(other) {} ~RtcIndexer() {} @@ -54,7 +56,7 @@ class RtcIndexer : public Indexer { n_fec_ = 0; } - uint32_t checkNextSuffix() override { return next_suffix_; } + uint32_t checkNextSuffix() const override { return next_suffix_; } uint32_t getNextSuffix() override { if (isFec(next_suffix_)) { @@ -77,7 +79,7 @@ class RtcIndexer : public Indexer { first_suffix_ = suffix % LIMIT; } - uint32_t getFirstSuffix() override { return first_suffix_; } + uint32_t getFirstSuffix() const override { return first_suffix_; } uint32_t jumpToIndex(uint32_t index) override { next_suffix_ = index % LIMIT; @@ -87,30 +89,8 @@ class RtcIndexer : public Indexer { void onContentObject(core::Interest &interest, core::ContentObject &content_object, bool reassembly) override { - setVerifier(); - auto ret = verifier_->verifyPackets(&content_object); - - switch (ret) { - case auth::VerificationPolicy::ACCEPT: { - if (reassembly) { - reassembly_->reassemble(content_object); - } - break; - } - - case auth::VerificationPolicy::UNKNOWN: - case auth::VerificationPolicy::DROP: { - transport_->onPacketDropped( - interest, content_object, - make_error_code(protocol_error::verification_failed)); - break; - } - - case auth::VerificationPolicy::ABORT: { - transport_->onContentReassembled( - make_error_code(protocol_error::session_aborted)); - break; - } + if (reassembly) { + reassembly_->reassemble(content_object); } } @@ -120,13 +100,12 @@ class RtcIndexer : public Indexer { uint32_t getNextReassemblySegment() override { throw errors::RuntimeException( "Get reassembly segment called on rtc indexer. RTC indexer does not " - "provide " - "reassembly."); + "provide reassembly."); } bool isFinalSuffixDiscovered() override { return true; } - uint32_t getFinalSuffix() override { return LIMIT; } + uint32_t getFinalSuffix() const override { return LIMIT; } void enableFec(fec::FECType fec_type) override { fec_type_ = fec_type; } @@ -137,13 +116,13 @@ class RtcIndexer : public Indexer { n_current_fec_ = n_fec_; } - uint32_t getNFec() override { return n_fec_; } + uint32_t getNFec() const override { return n_fec_; } bool isFec(uint32_t index) override { return isFec(fec_type_, index, first_suffix_); } - double getFecOverhead() override { + double getFecOverhead() const override { if (fec_type_ == fec::FECType::UNKNOWN) { return 0; } @@ -152,7 +131,7 @@ class RtcIndexer : public Indexer { return (double)n_fec_ / k; } - double getMaxFecOverhead() override { + double getMaxFecOverhead() const override { if (fec_type_ == fec::FECType::UNKNOWN) { return 0; } diff --git a/libtransport/src/protocols/rtc/rtc_ldr.cc b/libtransport/src/protocols/rtc/rtc_ldr.cc index f0de48871..6e88a8636 100644 --- a/libtransport/src/protocols/rtc/rtc_ldr.cc +++ b/libtransport/src/protocols/rtc/rtc_ldr.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -16,6 +16,12 @@ #include <glog/logging.h> #include <protocols/rtc/rtc_consts.h> #include <protocols/rtc/rtc_ldr.h> +#include <protocols/rtc/rtc_rs_delay.h> +#include <protocols/rtc/rtc_rs_fec_only.h> +#include <protocols/rtc/rtc_rs_low_rate.h> +#include <protocols/rtc/rtc_rs_recovery_off.h> +#include <protocols/rtc/rtc_rs_rtx_only.h> +#include <protocols/rtc/rtc_state.h> #include <algorithm> #include <unordered_set> @@ -27,483 +33,200 @@ namespace protocol { namespace rtc { RTCLossDetectionAndRecovery::RTCLossDetectionAndRecovery( - Indexer *indexer, SendRtxCallback &&callback, asio::io_service &io_service) - : rtx_on_(false), - fec_on_(false), - next_rtx_timer_(MAX_TIMER_RTX), - last_event_(0), - sentinel_timer_interval_(MAX_TIMER_RTX), - indexer_(indexer), - send_rtx_callback_(std::move(callback)) { - timer_ = std::make_unique<asio::steady_timer>(io_service); - sentinel_timer_ = std::make_unique<asio::steady_timer>(io_service); + Indexer *indexer, asio::io_service &io_service, + interface::RtcTransportRecoveryStrategies type, + RecoveryStrategy::SendRtxCallback &&callback, + interface::StrategyCallback &&external_callback) { + if (type == interface::RtcTransportRecoveryStrategies::RECOVERY_OFF) { + rs_ = std::make_shared<RecoveryStrategyRecoveryOff>( + indexer, std::move(callback), io_service, type, + std::move(external_callback)); + } else if (type == interface::RtcTransportRecoveryStrategies::DELAY_BASED || + type == interface::RtcTransportRecoveryStrategies:: + DELAY_AND_BESTPATH || + type == interface::RtcTransportRecoveryStrategies:: + DELAY_AND_REPLICATION) { + rs_ = std::make_shared<RecoveryStrategyDelayBased>( + indexer, std::move(callback), io_service, type, + std::move(external_callback)); + } else if (type == interface::RtcTransportRecoveryStrategies::FEC_ONLY || + type == interface::RtcTransportRecoveryStrategies:: + FEC_ONLY_LOW_RES_LOSSES) { + rs_ = std::make_shared<RecoveryStrategyFecOnly>( + indexer, std::move(callback), io_service, type, + std::move(external_callback)); + } else if (type == interface::RtcTransportRecoveryStrategies::LOW_RATE || + type == interface::RtcTransportRecoveryStrategies:: + LOW_RATE_AND_BESTPATH || + type == interface::RtcTransportRecoveryStrategies:: + LOW_RATE_AND_REPLICATION || + type == interface::RtcTransportRecoveryStrategies:: + LOW_RATE_AND_ALL_FWD_STRATEGIES) { + rs_ = std::make_shared<RecoveryStrategyLowRate>( + indexer, std::move(callback), io_service, type, + std::move(external_callback)); + } else { + // default + type = interface::RtcTransportRecoveryStrategies::RTX_ONLY; + rs_ = std::make_shared<RecoveryStrategyRtxOnly>( + indexer, std::move(callback), io_service, type, + std::move(external_callback)); + } } RTCLossDetectionAndRecovery::~RTCLossDetectionAndRecovery() {} -void RTCLossDetectionAndRecovery::turnOnRTX() { - rtx_on_ = true; - scheduleSentinelTimer(state_->getRTT() * CATCH_UP_RTT_INCREMENT); -} - -void RTCLossDetectionAndRecovery::turnOffRTX() { - rtx_on_ = false; - clear(); -} - -uint32_t RTCLossDetectionAndRecovery::computeFecPacketsToAsk(bool in_sync) { - uint32_t current_fec = indexer_->getNFec(); - double current_loss_rate = state_->getLossRate(); - double last_loss_rate = state_->getLastRoundLossRate(); - - // when in sync ask for fec only if there are losses for 2 rounds - if (in_sync && current_fec == 0 && - (current_loss_rate == 0 || last_loss_rate == 0)) - return 0; - - double loss_rate = state_->getMaxLossRate() * 1.5; - - if (!in_sync && loss_rate == 0) loss_rate = 0.05; - if (loss_rate > 0.5) loss_rate = 0.5; - - double exp_losses = (double)k_ * loss_rate; - uint32_t fec_to_ask = ceil(exp_losses / (1 - loss_rate)); - - if (fec_to_ask > (n_ - k_)) fec_to_ask = n_ - k_; - - return fec_to_ask; +void RTCLossDetectionAndRecovery::changeRecoveryStrategy( + interface::RtcTransportRecoveryStrategies type) { + if (type == rs_->getType()) return; + + rs_->updateType(type); + if (type == interface::RtcTransportRecoveryStrategies::RECOVERY_OFF) { + rs_ = + std::make_shared<RecoveryStrategyRecoveryOff>(std::move(*(rs_.get()))); + } else if (type == interface::RtcTransportRecoveryStrategies::DELAY_BASED || + type == interface::RtcTransportRecoveryStrategies:: + DELAY_AND_BESTPATH || + type == interface::RtcTransportRecoveryStrategies:: + DELAY_AND_REPLICATION) { + rs_ = std::make_shared<RecoveryStrategyDelayBased>(std::move(*(rs_.get()))); + } else if (type == interface::RtcTransportRecoveryStrategies::FEC_ONLY || + type == interface::RtcTransportRecoveryStrategies:: + FEC_ONLY_LOW_RES_LOSSES) { + rs_ = std::make_shared<RecoveryStrategyFecOnly>(std::move(*(rs_.get()))); + } else if (type == interface::RtcTransportRecoveryStrategies::LOW_RATE || + type == interface::RtcTransportRecoveryStrategies:: + LOW_RATE_AND_BESTPATH || + type == interface::RtcTransportRecoveryStrategies:: + LOW_RATE_AND_REPLICATION || + type == interface::RtcTransportRecoveryStrategies:: + LOW_RATE_AND_ALL_FWD_STRATEGIES) { + rs_ = std::make_shared<RecoveryStrategyLowRate>(std::move(*(rs_.get()))); + } else { + // default + rs_ = std::make_shared<RecoveryStrategyRtxOnly>(std::move(*(rs_.get()))); + } } void RTCLossDetectionAndRecovery::onNewRound(bool in_sync) { - uint64_t rtt = state_->getRTT(); - if (!fec_on_ && rtt >= 100) { - // turn on fec, here we may have no info so ask for all packets - fec_on_ = true; - turnOffRTX(); - indexer_->setNFec(computeFecPacketsToAsk(in_sync)); - return; - } - - if (fec_on_ && rtt > 80) { - // keep using fec, maybe update it - indexer_->setNFec(computeFecPacketsToAsk(in_sync)); - return; - } - - if ((fec_on_ && rtt <= 80) || (!rtx_on_ && rtt <= 100)) { - // turn on rtx - fec_on_ = false; - indexer_->setNFec(0); - turnOnRTX(); - return; - } + rs_->incRoundId(); + rs_->onNewRound(in_sync); } -void RTCLossDetectionAndRecovery::onTimeout(uint32_t seq) { - // always add timeouts to the RTX list to avoid to send the same packet as if - // it was not a rtx - addToRetransmissions(seq, seq + 1); - last_event_ = getNow(); +bool RTCLossDetectionAndRecovery::onTimeout(uint32_t seq, bool lost) { + if (!lost) { + return detectLoss(seq, seq + 1, false); + } else { + rs_->onLostTimeout(seq); + } + return false; } -void RTCLossDetectionAndRecovery::onPacketRecoveredFec(uint32_t seq) { - // if an RTX is scheduled for a packet recovered using FEC delete it - deleteRtx(seq); - recover_with_fec_.erase(seq); +bool RTCLossDetectionAndRecovery::onPacketRecoveredFec(uint32_t seq) { + rs_->receivedPacket(seq); + return false; } -void RTCLossDetectionAndRecovery::onDataPacketReceived( +bool RTCLossDetectionAndRecovery::onDataPacketReceived( const core::ContentObject &content_object) { - last_event_ = getNow(); - uint32_t seq = content_object.getName().getSuffix(); - if (deleteRtx(seq)) { - state_->onPacketRecoveredRtx(seq); - } else { - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "received data. add from " - << state_->getHighestSeqReceivedInOrder() + 1 << " to " << seq; - addToRetransmissions(state_->getHighestSeqReceivedInOrder() + 1, seq); - } + bool is_rtx = rs_->isRtx(seq); + rs_->receivedPacket(seq); + bool ret = false; + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "received data. add from " + << rs_->getState()->getHighestSeqReceived() + 1 << " to " << seq; + if (!is_rtx) + ret = detectLoss(rs_->getState()->getHighestSeqReceived() + 1, seq, false); + + rs_->getState()->updateHighestSeqReceived(seq); + return ret; } -void RTCLossDetectionAndRecovery::onNackPacketReceived( +bool RTCLossDetectionAndRecovery::onNackPacketReceived( const core::ContentObject &nack) { - last_event_ = getNow(); - - uint32_t seq = nack.getName().getSuffix(); - struct nack_packet_t *nack_pkt = (struct nack_packet_t *)nack.getPayload()->data(); - uint32_t production_seq = nack_pkt->getProductionSegement(); + uint32_t production_seq = nack_pkt->getProductionSegment(); + uint32_t seq = nack.getName().getSuffix(); - if (production_seq > seq) { - // this is a past nack, all data before productionSeq are lost. if - // productionSeq > state_->getHighestSeqReceivedInOrder() is impossible to - // recover any packet. If this is not the case we can try to recover the - // packets between state_->getHighestSeqReceivedInOrder() and productionSeq. - // e.g.: the client receives packets 8 10 11 9 where 9 is a nack with - // productionSeq = 14. 9 is lost but we can try to recover packets 12 13 and - // 14 that are not arrived yet - deleteRtx(seq); - DLOG_IF(INFO, VLOG_IS_ON(3)) << "received past nack. add from " - << state_->getHighestSeqReceivedInOrder() + 1 - << " to " << production_seq; - addToRetransmissions(state_->getHighestSeqReceivedInOrder() + 1, - production_seq); - } else { - // future nack. here there should be a gap between the last data received - // and this packet and is it possible to recover the packets between the - // last received data and the production seq. we should not use the seq - // number of the nack since we know that is too early to ask for this seq - // number - // e.g.: // e.g.: the client receives packets 10 11 12 20 where 20 is a nack - // with productionSeq = 18. this says that all the packets between 12 and 18 - // may got lost and we should ask them - deleteRtx(seq); - DLOG_IF(INFO, VLOG_IS_ON(3)) << "received futrue nack. add from " - << state_->getHighestSeqReceivedInOrder() + 1 - << " to " << production_seq; - addToRetransmissions(state_->getHighestSeqReceivedInOrder() + 1, - production_seq); - } + // received a nack. we can try to recover all data packets between the last + // received data and the production seq in the nack. this is similar to the + // recption of a probe + // e.g.: the client receives packets 10 11 12 20 where 20 is a nack + // with productionSeq = 18. this says that all the packets between 12 and 18 + // may got lost and we should ask them + + rs_->receivedPacket(seq); + DLOG_IF(INFO, VLOG_IS_ON(3)) << "received nack. add from " + << rs_->getState()->getHighestSeqReceived() + 1 + << " to " << production_seq; + + // if it is a future nack store it in the list set of nacked seq + if (production_seq <= seq) rs_->receivedFutureNack(seq); + + // call the detectLoss function using the probe flag = true. in fact the + // losses detected using nacks are the same as the one detected using probes, + // we should not increase the loss counter + return detectLoss(rs_->getState()->getHighestSeqReceived() + 1, + production_seq, true); } -void RTCLossDetectionAndRecovery::onProbePacketReceived( +bool RTCLossDetectionAndRecovery::onProbePacketReceived( const core::ContentObject &probe) { // we don't log the reception of a probe packet for the sentinel timer because // probes are not taken into account into the sync window. we use them as // future nacks to detect possible packets lost - struct nack_packet_t *probe_pkt = - (struct nack_packet_t *)probe.getPayload()->data(); - uint32_t production_seq = probe_pkt->getProductionSegement(); - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "received probe. add from " - << state_->getHighestSeqReceivedInOrder() + 1 << " to " << production_seq; - addToRetransmissions(state_->getHighestSeqReceivedInOrder() + 1, - production_seq); -} + uint32_t production_seq = RTCState::getProbeParams(probe).prod_seg; -void RTCLossDetectionAndRecovery::clear() { - rtx_state_.clear(); - rtx_timers_.clear(); - sentinel_timer_->cancel(); - if (next_rtx_timer_ != MAX_TIMER_RTX) { - next_rtx_timer_ = MAX_TIMER_RTX; - timer_->cancel(); - } + DLOG_IF(INFO, VLOG_IS_ON(3)) << "received probe. add from " + << rs_->getState()->getHighestSeqReceived() + 1 + << " to " << production_seq; + + return detectLoss(rs_->getState()->getHighestSeqReceived() + 1, + production_seq, true); } -void RTCLossDetectionAndRecovery::addToRetransmissions(uint32_t start, - uint32_t stop) { +bool RTCLossDetectionAndRecovery::detectLoss(uint32_t start, uint32_t stop, + bool recv_probe) { + if (start >= stop) return false; + // skip nacked packets - if (start <= state_->getLastSeqNacked()) { - start = state_->getLastSeqNacked() + 1; + if (start <= rs_->getState()->getLastSeqNacked()) { + start = rs_->getState()->getLastSeqNacked() + 1; } // skip received or lost packets - if (start <= state_->getHighestSeqReceivedInOrder()) { - start = state_->getHighestSeqReceivedInOrder() + 1; + if (start <= rs_->getState()->getHighestSeqReceived()) { + start = rs_->getState()->getHighestSeqReceived() + 1; } + bool loss_detected = false; for (uint32_t seq = start; seq < stop; seq++) { - if (state_->isReceivedOrLost(seq) == PacketState::UNKNOWN) { - if (rtx_on_) { - if (!indexer_->isFec(seq)) { - // handle it with rtx - if (!isRtx(seq)) { - state_->onLossDetected(seq); - rtxState state; - state.first_send_ = state_->getInterestSentTime(seq); - if (state.first_send_ == 0) // this interest was never sent before - state.first_send_ = getNow(); - state.next_send_ = computeNextSend(seq, true); - state.rtx_count_ = 0; - DLOG_IF(INFO, VLOG_IS_ON(4)) - << "Add " << seq << " to retransmissions. next rtx is %lu " - << state.next_send_ - getNow(); - rtx_state_.insert(std::pair<uint32_t, rtxState>(seq, state)); - rtx_timers_.insert( - std::pair<uint64_t, uint32_t>(state.next_send_, seq)); - } + if (rs_->getState()->getPacketState(seq) == PacketState::UNKNOWN) { + if (rs_->lossDetected(seq)) { + loss_detected = true; + if ((recv_probe || rs_->wasNacked(seq)) && !rs_->isFecOn()) { + // these losses were detected using a probe and fec is off. + // in this case most likelly the procotol is about to go out of sync + // and the packets are not really lost (e.g. increase in prod rate). + // for this reason we do not + // count the losses in the stats. Instead we do the following + // 1. send RTX for the packets in case they were really lost + // 2. return to the RTC protocol that a loss was detected using a + // probe. the protocol will switch to catch_up mode to increase the + // size of the window + rs_->requestPossibleLostPacket(seq); } else { - // is fec, do not send it - auto it = recover_with_fec_.find(seq); - if (it == recover_with_fec_.end()) { - state_->onLossDetected(seq); - recover_with_fec_.insert(seq); - } - } - } else { - // keep track of losses but recover with FEC - auto it = recover_with_fec_.find(seq); - if (it == recover_with_fec_.end()) { - state_->onLossDetected(seq); - recover_with_fec_.insert(seq); + // if fec is on we don't need to mask pontetial losses, so increase + // the loss rate + rs_->notifyNewLossDetedcted(seq); } } } } - scheduleNextRtx(); -} - -uint64_t RTCLossDetectionAndRecovery::computeNextSend(uint32_t seq, - bool new_rtx) { - uint64_t now = getNow(); - if (new_rtx) { - // for the new rtx we wait one estimated IAT after the loss detection. this - // is bacause, assuming that packets arrive with a constant IAT, we should - // get a new packet every IAT - double prod_rate = state_->getProducerRate(); - uint32_t estimated_iat = SENTINEL_TIMER_INTERVAL; - uint32_t jitter = 0; - - if (prod_rate != 0) { - double packet_size = state_->getAveragePacketSize(); - estimated_iat = ceil(1000.0 / (prod_rate / packet_size)); - jitter = ceil(state_->getJitter()); - } - - uint32_t wait = estimated_iat + jitter; - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "first rtx for " << seq << " in " << wait - << " ms, rtt = " << state_->getRTT() << " ait = " << estimated_iat - << " jttr = " << jitter; - - return now + wait; - } else { - // wait one RTT - // however if the IAT is larger than the RTT, wait one IAT - uint32_t wait = SENTINEL_TIMER_INTERVAL; - - double prod_rate = state_->getProducerRate(); - if (prod_rate == 0) { - return now + SENTINEL_TIMER_INTERVAL; - } - - double packet_size = state_->getAveragePacketSize(); - uint32_t estimated_iat = ceil(1000.0 / (prod_rate / packet_size)); - - uint64_t rtt = state_->getRTT(); - if (rtt == 0) rtt = SENTINEL_TIMER_INTERVAL; - wait = rtt; - - if (estimated_iat > rtt) wait = estimated_iat; - - uint32_t jitter = ceil(state_->getJitter()); - wait += jitter; - - // it may happen that the channel is congested and we have some additional - // queuing delay to take into account - uint32_t queue = ceil(state_->getQueuing()); - wait += queue; - - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "next rtx for " << seq << " in " << wait - << " ms, rtt = " << state_->getRTT() << " ait = " << estimated_iat - << " jttr = " << jitter << " queue = " << queue; - - return now + wait; - } -} - -void RTCLossDetectionAndRecovery::retransmit() { - if (rtx_timers_.size() == 0) return; - - uint64_t now = getNow(); - - auto it = rtx_timers_.begin(); - std::unordered_set<uint32_t> lost_pkt; - uint32_t sent_counter = 0; - while (it != rtx_timers_.end() && it->first <= now && - sent_counter < MAX_RTX_IN_BATCH) { - uint32_t seq = it->second; - auto rtx_it = - rtx_state_.find(seq); // this should always return a valid iter - if (rtx_it->second.rtx_count_ >= RTC_MAX_RTX || - (now - rtx_it->second.first_send_) >= RTC_MAX_AGE || - seq < state_->getLastSeqNacked()) { - // max rtx reached or packet too old or packet nacked, this packet is lost - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "packet " << seq << " lost because 1) max rtx: " - << (rtx_it->second.rtx_count_ >= RTC_MAX_RTX) << " 2) max age: " - << ((now - rtx_it->second.first_send_) >= RTC_MAX_AGE) - << " 3) nacked: " << (seq < state_->getLastSeqNacked()); - lost_pkt.insert(seq); - it++; - } else { - // resend the packet - state_->onRetransmission(seq); - double prod_rate = state_->getProducerRate(); - if (prod_rate != 0) rtx_it->second.rtx_count_++; - rtx_it->second.next_send_ = computeNextSend(seq, false); - it = rtx_timers_.erase(it); - rtx_timers_.insert( - std::pair<uint64_t, uint32_t>(rtx_it->second.next_send_, seq)); - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "send rtx for sequence " << seq << ", next send in " - << (rtx_it->second.next_send_ - now); - send_rtx_callback_(seq); - sent_counter++; - } - } - - // remove packets if needed - for (auto lost_it = lost_pkt.begin(); lost_it != lost_pkt.end(); lost_it++) { - uint32_t seq = *lost_it; - state_->onPacketLost(seq); - deleteRtx(seq); - } -} - -void RTCLossDetectionAndRecovery::scheduleNextRtx() { - if (rtx_timers_.size() == 0) { - // all the rtx were removed, reset timer - next_rtx_timer_ = MAX_TIMER_RTX; - return; - } - - // check if timer is alreay set - if (next_rtx_timer_ != MAX_TIMER_RTX) { - // a new check for rtx is already scheduled - if (next_rtx_timer_ > rtx_timers_.begin()->first) { - // we need to re-schedule it - timer_->cancel(); - } else { - // wait for the next timer - return; - } - } - - // set a new timer - next_rtx_timer_ = rtx_timers_.begin()->first; - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); - uint64_t wait = 1; - if (next_rtx_timer_ != MAX_TIMER_RTX && next_rtx_timer_ > now) - wait = next_rtx_timer_ - now; - - std::weak_ptr<RTCLossDetectionAndRecovery> self(shared_from_this()); - timer_->expires_from_now(std::chrono::milliseconds(wait)); - timer_->async_wait([self](std::error_code ec) { - if (ec) return; - if (auto s = self.lock()) { - s->retransmit(); - s->next_rtx_timer_ = MAX_TIMER_RTX; - s->scheduleNextRtx(); - } - }); -} - -bool RTCLossDetectionAndRecovery::deleteRtx(uint32_t seq) { - auto it_rtx = rtx_state_.find(seq); - if (it_rtx == rtx_state_.end()) return false; // rtx not found - - uint64_t ts = it_rtx->second.next_send_; - auto it_timers = rtx_timers_.find(ts); - while (it_timers != rtx_timers_.end() && it_timers->first == ts) { - if (it_timers->second == seq) { - rtx_timers_.erase(it_timers); - break; - } - it_timers++; - } - - bool lost = it_rtx->second.rtx_count_ > 0; - rtx_state_.erase(it_rtx); - - return lost; -} - -void RTCLossDetectionAndRecovery::scheduleSentinelTimer( - uint64_t expires_from_now) { - std::weak_ptr<RTCLossDetectionAndRecovery> self(shared_from_this()); - sentinel_timer_->expires_from_now( - std::chrono::milliseconds(expires_from_now)); - sentinel_timer_->async_wait([self](std::error_code ec) { - if (ec) return; - if (auto s = self.lock()) { - s->sentinelTimer(); - } - }); -} - -void RTCLossDetectionAndRecovery::sentinelTimer() { - uint64_t now = getNow(); - - bool expired = false; - bool sent = false; - if ((now - last_event_) >= sentinel_timer_interval_) { - // at least a sentinel_timer_interval_ elapsed since last event - expired = true; - if (TRANSPORT_EXPECT_FALSE(!state_->isProducerActive())) { - // this happens at the beginning (or if the producer stops for some - // reason) we need to keep sending interest 0 until we get an answer - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "sentinel timer: the producer is not active, send packet 0"; - state_->onRetransmission(0); - send_rtx_callback_(0); - } else { - DLOG_IF(INFO, VLOG_IS_ON(3)) << "sentinel timer: the producer is active, " - "send the 10 oldest packets"; - sent = true; - uint32_t rtx = 0; - auto it = state_->getPendingInterestsMapBegin(); - auto end = state_->getPendingInterestsMapEnd(); - while (it != end && rtx < MAX_RTX_WITH_SENTINEL) { - uint32_t seq = it->first; - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "sentinel timer, add " << seq << " to the rtx list"; - addToRetransmissions(seq, seq + 1); - rtx++; - it++; - } - } - } else { - // sentinel timer did not expire because we registered at least one event - } - - uint32_t next_timer; - double prod_rate = state_->getProducerRate(); - if (TRANSPORT_EXPECT_FALSE(!state_->isProducerActive()) || prod_rate == 0) { - DLOG_IF(INFO, VLOG_IS_ON(3)) << "next timer in " << SENTINEL_TIMER_INTERVAL; - next_timer = SENTINEL_TIMER_INTERVAL; - } else { - double prod_rate = state_->getProducerRate(); - double packet_size = state_->getAveragePacketSize(); - uint32_t estimated_iat = ceil(1000.0 / (prod_rate / packet_size)); - uint32_t jitter = ceil(state_->getJitter()); - - // try to reduce the number of timers if the estimated IAT is too small - next_timer = std::max((estimated_iat + jitter) * 20, (uint32_t)1); - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "next sentinel in " << next_timer - << " ms, rate: " << ((prod_rate * 8.0) / 1000000.0) - << ", iat: " << estimated_iat << ", jitter: " << jitter; - - if (!expired) { - // discount the amout of time that is already passed - uint32_t discount = now - last_event_; - if (next_timer > discount) { - next_timer = next_timer - discount; - } else { - // in this case we trigger the timer in 1 ms - next_timer = 1; - } - DLOG_IF(INFO, VLOG_IS_ON(3)) << "timer after discout: " << next_timer; - } else if (sent) { - // wait at least one producer stats interval + owd to check if the - // production rate is reducing. - uint32_t min_wait = PRODUCER_STATS_INTERVAL + ceil(state_->getQueuing()); - next_timer = std::max(next_timer, min_wait); - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "wait for updates from prod, next timer: " << next_timer; - } - } - - scheduleSentinelTimer(next_timer); + return loss_detected; } } // namespace rtc diff --git a/libtransport/src/protocols/rtc/rtc_ldr.h b/libtransport/src/protocols/rtc/rtc_ldr.h index 1b9f9afd6..24f22ffed 100644 --- a/libtransport/src/protocols/rtc/rtc_ldr.h +++ b/libtransport/src/protocols/rtc/rtc_ldr.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -15,16 +15,14 @@ #pragma once #include <hicn/transport/config.h> +#include <hicn/transport/interfaces/socket_options_keys.h> +// RtcTransportRecoveryStrategies #include <hicn/transport/core/asio_wrapper.h> #include <hicn/transport/core/content_object.h> #include <hicn/transport/core/name.h> -#include <protocols/indexer.h> -#include <protocols/rtc/rtc_consts.h> -#include <protocols/rtc/rtc_state.h> +#include <protocols/rtc/rtc_recovery_strategy.h> #include <functional> -#include <map> -#include <unordered_map> namespace transport { @@ -34,91 +32,50 @@ namespace rtc { class RTCLossDetectionAndRecovery : public std::enable_shared_from_this<RTCLossDetectionAndRecovery> { - struct rtx_state_ { - uint64_t first_send_; - uint64_t next_send_; - uint32_t rtx_count_; - }; - - using rtxState = struct rtx_state_; - using SendRtxCallback = std::function<void(uint32_t)>; - public: - RTCLossDetectionAndRecovery(Indexer *indexer, SendRtxCallback &&callback, - asio::io_service &io_service); + RTCLossDetectionAndRecovery(Indexer *indexer, asio::io_service &io_service, + interface::RtcTransportRecoveryStrategies type, + RecoveryStrategy::SendRtxCallback &&callback, + interface::StrategyCallback &&external_callback); ~RTCLossDetectionAndRecovery(); - void setState(std::shared_ptr<RTCState> state) { state_ = state; } - void setFecParams(uint32_t n, uint32_t k) { - n_ = n; - k_ = k; + void setState(RTCState *state) { rs_->setState(state); } + void setRateControl(RTCRateControl *rateControl) { + rs_->setRateControl(rateControl); } - void turnOnRTX(); - void turnOffRTX(); - bool isRtxOn() { return rtx_on_; } - void onNewRound(bool in_sync); - void onTimeout(uint32_t seq); - void onPacketRecoveredFec(uint32_t seq); - void onDataPacketReceived(const core::ContentObject &content_object); - void onNackPacketReceived(const core::ContentObject &nack); - void onProbePacketReceived(const core::ContentObject &probe); + void setFecParams(uint32_t n, uint32_t k) { rs_->setFecParams(n, k); } - void clear(); - - bool isRtx(uint32_t seq) { - if (rtx_state_.find(seq) != rtx_state_.end()) return true; - return false; - } + void setContentSharingMode() { rs_->setContentSharingMode(); } + void turnOnRecovery() { rs_->turnOnRecovery(); } + bool isRtxOn() { return rs_->isRtxOn(); } - private: - void addToRetransmissions(uint32_t start, uint32_t stop); - uint64_t computeNextSend(uint32_t seq, bool new_rtx); - void retransmit(); - void scheduleNextRtx(); - bool deleteRtx(uint32_t seq); - void scheduleSentinelTimer(uint64_t expires_from_now); - void sentinelTimer(); - uint32_t computeFecPacketsToAsk(bool in_sync); - - uint64_t getNow() { - using namespace std::chrono; - uint64_t now = - duration_cast<milliseconds>(steady_clock::now().time_since_epoch()) - .count(); - return now; - } + void changeRecoveryStrategy(interface::RtcTransportRecoveryStrategies type); - // this map keeps track of the retransmitted interest, ordered from the oldest - // to the newest one. the state contains the timer of the first send of the - // interest (from pendingIntetests_), the timer of the next send (key of the - // multimap) and the number of rtx - std::map<uint32_t, rtxState> rtx_state_; - // this map stored the rtx by timer. The key is the time at which the rtx - // should be sent, and the val is the interest seq number - std::multimap<uint64_t, uint32_t> rtx_timers_; + void onNewRound(bool in_sync); - // lost packets that will be recovered with fec - std::unordered_set<uint32_t> recover_with_fec_; + // the following functions return true if a loss is detected, false otherwise + bool onTimeout(uint32_t seq, bool lost); + bool onPacketRecoveredFec(uint32_t seq); + bool onDataPacketReceived(const core::ContentObject &content_object); + bool onNackPacketReceived(const core::ContentObject &nack); + bool onProbePacketReceived(const core::ContentObject &probe); - bool rtx_on_; - bool fec_on_; - uint64_t next_rtx_timer_; - uint64_t last_event_; - uint64_t sentinel_timer_interval_; + void clear() { rs_->clear(); } - // fec params - uint32_t n_; - uint32_t k_; + bool isRtx(uint32_t seq) { return rs_->isRtx(seq); } + bool isPossibleLossWithNoRtx(uint32_t seq) { + return rs_->isPossibleLossWithNoRtx(seq); + } - std::unique_ptr<asio::steady_timer> timer_; - std::unique_ptr<asio::steady_timer> sentinel_timer_; - std::shared_ptr<RTCState> state_; + uint64_t getRtxRtt(uint32_t seq) { return rs_->getRtxRtt(seq); } - Indexer *indexer_; + private: + // returns true if a loss is detected, false otherwise + bool detectLoss(uint32_t start, uint32_t stop, bool recv_probe); - SendRtxCallback send_rtx_callback_; + std::shared_ptr<RecoveryStrategy> rs_; }; } // end namespace rtc diff --git a/libtransport/src/protocols/rtc/rtc_packet.h b/libtransport/src/protocols/rtc/rtc_packet.h index 7dc2f82c3..ffbbd78fd 100644 --- a/libtransport/src/protocols/rtc/rtc_packet.h +++ b/libtransport/src/protocols/rtc/rtc_packet.h @@ -24,6 +24,27 @@ * +-----------------------------------------+ */ +/* aggregated packets + * +---------------------------------+ + * |c| #pkts | len1 | len2 | .... | + * +---------------------------------- + * + * +---------------------------------+ + * |c| #pkts | resv | len 1 | + * +---------------------------------- + * + * aggregated packets header. + * header position. just after the data packet header + * + * c: 1 bit: 0 8bit encoding, 1 16bit encoding + * #pkts: 7 bits: number of application packets contained + * 8bits encoding: + * lenX: 8 bits: len in bites of packet X + * 16bits econding: + * resv: 8 bits: reserved field (unused) + * lenX: 16bits: len in bytes of packet X + */ + #pragma once #ifndef _WIN32 #include <arpa/inet.h> @@ -31,30 +52,16 @@ #include <hicn/transport/portability/win_portability.h> #endif +#include <hicn/transport/portability/endianess.h> + +#include <cstring> + namespace transport { namespace protocol { namespace rtc { -inline uint64_t _ntohll(const uint64_t *input) { - uint64_t return_val; - uint8_t *tmp = (uint8_t *)&return_val; - - tmp[0] = (uint8_t)(*input >> 56); - tmp[1] = (uint8_t)(*input >> 48); - tmp[2] = (uint8_t)(*input >> 40); - tmp[3] = (uint8_t)(*input >> 32); - tmp[4] = (uint8_t)(*input >> 24); - tmp[5] = (uint8_t)(*input >> 16); - tmp[6] = (uint8_t)(*input >> 8); - tmp[7] = (uint8_t)(*input >> 0); - - return return_val; -} - -inline uint64_t _htonll(const uint64_t *input) { return (_ntohll(input)); } - const uint32_t DATA_HEADER_SIZE = 12; // bytes // XXX: sizeof(data_packet_t) is 16 // beacuse of padding @@ -64,11 +71,19 @@ struct data_packet_t { uint64_t timestamp; uint32_t prod_rate; - inline uint64_t getTimestamp() const { return _ntohll(×tamp); } - inline void setTimestamp(uint64_t time) { timestamp = _htonll(&time); } + inline uint64_t getTimestamp() const { + return portability::net_to_host(timestamp); + } + inline void setTimestamp(uint64_t time) { + timestamp = portability::host_to_net(time); + } - inline uint32_t getProductionRate() const { return ntohl(prod_rate); } - inline void setProductionRate(uint32_t rate) { prod_rate = htonl(rate); } + inline uint32_t getProductionRate() const { + return portability::net_to_host(prod_rate); + } + inline void setProductionRate(uint32_t rate) { + prod_rate = portability::host_to_net(rate); + } }; struct nack_packet_t { @@ -76,14 +91,162 @@ struct nack_packet_t { uint32_t prod_rate; uint32_t prod_seg; - inline uint64_t getTimestamp() const { return _ntohll(×tamp); } - inline void setTimestamp(uint64_t time) { timestamp = _htonll(&time); } + inline uint64_t getTimestamp() const { + return portability::net_to_host(timestamp); + } + inline void setTimestamp(uint64_t time) { + timestamp = portability::host_to_net(time); + } + + inline uint32_t getProductionRate() const { + return portability::net_to_host(prod_rate); + } + inline void setProductionRate(uint32_t rate) { + prod_rate = portability::host_to_net(rate); + } + + inline uint32_t getProductionSegment() const { + return portability::net_to_host(prod_seg); + } + inline void setProductionSegment(uint32_t seg) { + prod_seg = portability::host_to_net(seg); + } +}; + +class AggrPktHeader { + public: + // XXX buf always point to the payload after the data header + AggrPktHeader(uint8_t *buf, uint16_t max_packet_len, uint16_t pkt_number) + : buf_(buf), pkt_num_(pkt_number) { + *buf_ = 0; // reset the first byte to correctly add the header + // encoding and the packet number + if (max_packet_len > 0xff) { + setAggrPktEncoding16bit(); + } else { + setAggrPktEncoding8bit(); + } + setAggrPktNUmber(pkt_number); + header_len_ = computeHeaderLen(); + memset(buf_ + 1, 0, header_len_ - 1); + } + + // XXX buf always point to the payload after the data header + AggrPktHeader(uint8_t *buf) : buf_(buf) { + encoding_ = getAggrPktEncoding(); + pkt_num_ = getAggrPktNumber(); + header_len_ = computeHeaderLen(); + } + + ~AggrPktHeader(){}; + + int addPacketToHeader(uint8_t index, uint16_t len) { + if (index > pkt_num_) return -1; + + setAggrPktLen(index, len); + return 0; + } + + int getPointerToPacket(uint8_t index, uint8_t **pkt_ptr, uint16_t *pkt_len) { + if (index > pkt_num_) return -1; + + uint16_t len = 0; + for (int i = 0; i < index; i++) + len += getAggrPktLen(i); // sum the pkts len from 0 to index - 1 + + uint16_t offset = len + header_len_; + *pkt_ptr = buf_ + offset; + *pkt_len = getAggrPktLen(index); + return 0; + } + + int getPacketOffsets(uint8_t index, uint16_t *pkt_offset, uint16_t *pkt_len) { + if (index > pkt_num_) return -1; + + uint16_t len = 0; + for (int i = 0; i < index; i++) + len += getAggrPktLen(i); // sum the pkts len from 0 to index - 1 + + uint16_t offset = len + header_len_; + *pkt_offset = offset; + *pkt_len = getAggrPktLen(index); + + return 0; + } + + uint8_t *getPayloadAppendPtr() { return buf_ + header_len_; } + + uint16_t getHeaderLen() { return header_len_; } + + uint8_t getNumberOfPackets() { return pkt_num_; } + + private: + inline uint16_t computeHeaderLen() const { + uint16_t len = 4; // min len in bytes + if (!encoding_) { + while (pkt_num_ >= len) { + len += 4; + } + } else { + while (pkt_num_ * 2 >= len) { + len += 4; + } + } + return len; + } + + inline uint8_t getAggrPktEncoding() const { + // get the first bit of the first byte + return (*buf_ >> 7); + } + + inline void setAggrPktEncoding8bit() { + // reset the first bit of the first byte + encoding_ = 0; + *buf_ &= 0x7f; + } + + inline void setAggrPktEncoding16bit() { + // set the first bit of the first byte + encoding_ = 1; + *buf_ ^= 0x80; + } + + inline uint8_t getAggrPktNumber() const { + // return the first byte with the first bit = 0 + return (*buf_ & 0x7f); + } + + inline void setAggrPktNUmber(uint8_t val) { + // set the val without modifying the first bit + *buf_ &= 0x80; // reset everithing but the first bit + val &= 0x7f; // reset the first bit + *buf_ |= val; // or the vals, done! + } + + inline uint16_t getAggrPktLen(uint8_t pkt_index) const { + pkt_index++; + if (!encoding_) { // 8 bits + return (uint16_t) * (buf_ + pkt_index); + } else { // 16 bits + uint16_t *buf_16 = (uint16_t *)buf_; + return portability::net_to_host(*(buf_16 + pkt_index)); + } + } - inline uint32_t getProductionRate() const { return ntohl(prod_rate); } - inline void setProductionRate(uint32_t rate) { prod_rate = htonl(rate); } + inline void setAggrPktLen(uint8_t pkt_index, uint16_t len) { + pkt_index++; + if (!encoding_) { // 8 bits + *(buf_ + pkt_index) = (uint8_t)len; + } else { // 16 bits + uint16_t *buf_16 = (uint16_t *)buf_; + *(buf_16 + pkt_index) = portability::host_to_net(len); + } + } - inline uint32_t getProductionSegement() const { return ntohl(prod_seg); } - inline void setProductionSegement(uint32_t seg) { prod_seg = htonl(seg); } + uint8_t *buf_; + uint8_t encoding_; + uint8_t pkt_num_; + uint16_t header_len_; }; } // end namespace rtc diff --git a/libtransport/src/protocols/rtc/rtc_rc.h b/libtransport/src/protocols/rtc/rtc_rc.h index 34d090092..62636ce40 100644 --- a/libtransport/src/protocols/rtc/rtc_rc.h +++ b/libtransport/src/protocols/rtc/rtc_rc.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -34,11 +34,15 @@ class RTCRateControl : public std::enable_shared_from_this<RTCRateControl> { void turnOnRateControl() { rc_on_ = true; } void setState(std::shared_ptr<RTCState> state) { protocol_state_ = state; }; - uint32_t getCongesionWindow() { return congestion_win_; }; + uint32_t getCongestionWindow() { return congestion_win_; }; + bool inCongestionState() { + if (congestion_state_ == CongestionState::Congested) return true; + return false; + } virtual void onNewRound(double round_len) = 0; - virtual void onDataPacketReceived( - const core::ContentObject &content_object) = 0; + virtual void onDataPacketReceived(const core::ContentObject &content_object, + bool compute_stats) = 0; protected: enum class CongestionState { Normal = 0, Underuse = 1, Congested = 2, Last }; diff --git a/libtransport/src/protocols/rtc/rtc_rc_congestion_detection.cc b/libtransport/src/protocols/rtc/rtc_rc_congestion_detection.cc new file mode 100644 index 000000000..6cd3094b5 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rc_congestion_detection.cc @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <protocols/rtc/rtc_consts.h> +#include <protocols/rtc/rtc_rc_congestion_detection.h> + +#include <algorithm> + +namespace transport { + +namespace protocol { + +namespace rtc { + +RTCRateControlCongestionDetection::RTCRateControlCongestionDetection() + : rounds_without_congestion_(4), last_queue_(0) {} // must be > 3 + +RTCRateControlCongestionDetection::~RTCRateControlCongestionDetection() {} + +void RTCRateControlCongestionDetection::onNewRound(double round_len) { + if (!rc_on_) return; + + double rtt = (double)protocol_state_->getMinRTT() / MILLI_IN_A_SEC; + double queue = protocol_state_->getQueuing(); + + if (rtt == 0.0) return; // no info from the producer + + if (last_queue_ == queue) { + // if last_queue == queue the consumer didn't receive any + // packet from the producer. we do not change the current congestion state. + // we just increase the counter of rounds whithout congestion if needed + // (in case of congestion the counter is already set to 0) + if (congestion_state_ == CongestionState::Normal) + rounds_without_congestion_++; + } else { + if (queue > MAX_QUEUING_DELAY) { + // here we detect congestion. + congestion_state_ = CongestionState::Congested; + rounds_without_congestion_ = 0; + } else { + // wait 3 rounds before switch back to no congestion + if (rounds_without_congestion_ > 3) { + // nothing bad is happening + congestion_state_ = CongestionState::Normal; + } + rounds_without_congestion_++; + } + last_queue_ = queue; + } +} + +void RTCRateControlCongestionDetection::onDataPacketReceived( + const core::ContentObject &content_object, bool compute_stats) { + // nothing to do + return; +} + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/fec_base.cc b/libtransport/src/protocols/rtc/rtc_rc_congestion_detection.h index 9252bc473..9afa6c39a 100644 --- a/libtransport/src/protocols/fec_base.cc +++ b/libtransport/src/protocols/rtc/rtc_rc_congestion_detection.h @@ -13,12 +13,35 @@ * limitations under the License. */ -#include <protocols/fec/rs.h> -#include <protocols/fec_base.h> +#pragma once +#include <hicn/transport/utils/shared_ptr_utils.h> +#include <protocols/rtc/rtc_rc.h> namespace transport { + namespace protocol { -namespace fec {} // namespace fec -} // namespace protocol -} // namespace transport
\ No newline at end of file +namespace rtc { + +class RTCRateControlCongestionDetection : public RTCRateControl { + public: + RTCRateControlCongestionDetection(); + + ~RTCRateControlCongestionDetection(); + + void onNewRound(double round_len); + void onDataPacketReceived(const core::ContentObject &content_object, + bool compute_stats); + + auto shared_from_this() { return utils::shared_from(this); } + + private: + uint32_t rounds_without_congestion_; + double last_queue_; +}; + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rc_iat.cc b/libtransport/src/protocols/rtc/rtc_rc_iat.cc new file mode 100644 index 000000000..f06f377f3 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rc_iat.cc @@ -0,0 +1,287 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <protocols/rtc/rtc_consts.h> +#include <protocols/rtc/rtc_rc_iat.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +RTCRateControlIAT::RTCRateControlIAT() + : rounds_since_last_drop_(0), + rounds_without_congestion_(0), + rounds_with_congestion_(0), + last_queue_(0), + last_rcv_time_(0), + last_prod_time_(0), + last_seq_number_(0), + target_rate_avg_(0), + round_index_(0), + congestion_cause_(CongestionCause::UNKNOWN) {} + +RTCRateControlIAT::~RTCRateControlIAT() {} + +void RTCRateControlIAT::onNewRound(double round_len) { + if (!rc_on_) return; + + double received_rate = protocol_state_->getReceivedRate() + + protocol_state_->getRecoveredFecRate(); + + double target_rate = + protocol_state_->getProducerRate(); // * PRODUCTION_RATE_FRACTION; + double rtt = (double)protocol_state_->getMinRTT() / MILLI_IN_A_SEC; + // double packet_size = protocol_state_->getAveragePacketSize(); + double queue = protocol_state_->getQueuing(); + + if (rtt == 0.0) return; // no info from the producer + + CongestionState prev_congestion_state = congestion_state_; + + target_rate_avg_ = target_rate_avg_ * (1 - MOVING_AVG_ALPHA) + + target_rate * MOVING_AVG_ALPHA; + + if (prev_congestion_state == CongestionState::Congested) { + if (queue > MAX_QUEUING_DELAY || last_queue_ == queue) { + congestion_state_ = CongestionState::Congested; + + received_rate_.push_back(received_rate); + target_rate_.push_back(target_rate); + + // We assume the cause does not change + // Note that the first assumption about the cause could be wrong + // the cause of congestion could change + if (congestion_cause_ == CongestionCause::UNKNOWN) + if (rounds_with_congestion_ >= 1) + congestion_cause_ = apply_classification_tree( + rounds_with_congestion_ > ROUND_TO_WAIT_FORCE_DECISION); + + rounds_with_congestion_++; + } else { + congestion_state_ = CongestionState::Normal; + + // clear past history + reset_congestion_statistics(); + + // TODO maybe we can use some of these values for the stdev of the + // congestion mode + for (int i = 0; i < ROUND_HISTORY_SIZE; i++) { + iat_on_hold_[i].clear(); + } + } + } else if (queue > MAX_QUEUING_DELAY) { + if (prev_congestion_state == CongestionState::Normal) { + rounds_with_congestion_ = 0; + + if (rounds_without_congestion_ > ROUND_TO_RESET_CAUSE) + congestion_cause_ = CongestionCause::UNKNOWN; + } + congestion_state_ = CongestionState::Congested; + received_rate_.push_back(received_rate); + target_rate_.push_back(target_rate); + } else { + // nothing bad is happening + congestion_state_ = CongestionState::Normal; + reset_congestion_statistics(); + + int past_index = (round_index_ + 1) % ROUND_HISTORY_SIZE; + for (std::vector<double>::iterator it = iat_on_hold_[past_index].begin(); + it != iat_on_hold_[past_index].end(); ++it) { + congestion_free_iat_.push_back(*it); + if (congestion_free_iat_.size() > 50) { + congestion_free_iat_.erase(congestion_free_iat_.begin()); + } + } + iat_on_hold_[past_index].clear(); + round_index_ = (round_index_ + 1) % ROUND_HISTORY_SIZE; + } + + last_queue_ = queue; + + if (congestion_state_ == CongestionState::Congested) { + if (prev_congestion_state == CongestionState::Normal) { + // init the congetion window using the received rate + // disabling for the moment the congestion window setup + // congestion_win_ = (uint32_t)ceil(received_rate * rtt / packet_size); + rounds_since_last_drop_ = ROUNDS_BEFORE_TAKE_ACTION + 1; + } + + if (rounds_since_last_drop_ >= ROUNDS_BEFORE_TAKE_ACTION) { + // disabling for the moment the congestion window setup + // uint32_t win = congestion_win_ * WIN_DECREASE_FACTOR; + // congestion_win_ = std::max(win, WIN_MIN); + rounds_since_last_drop_ = 0; + return; + } + + rounds_since_last_drop_++; + } + + if (congestion_state_ == CongestionState::Normal) { + if (prev_congestion_state == CongestionState::Congested) { + rounds_without_congestion_ = 0; + } + + rounds_without_congestion_++; + if (rounds_without_congestion_ < ROUNDS_BEFORE_TAKE_ACTION) return; + + // disabling for the moment the congestion window setup + // congestion_win_ = congestion_win_ * WIN_INCREASE_FACTOR; + // congestion_win_ = std::min(congestion_win_, INITIAL_WIN_MAX); + } + + if (received_rate_.size() > 1000) + received_rate_.erase(received_rate_.begin()); + if (target_rate_.size() > 1000) target_rate_.erase(target_rate_.begin()); +} + +void RTCRateControlIAT::onDataPacketReceived( + const core::ContentObject &content_object, bool compute_stats) { + core::ParamsRTC params = RTCState::getDataParams(content_object); + + uint64_t now = utils::SteadyTime::nowMs().count(); + + uint32_t segment_number = content_object.getName().getSuffix(); + + if (segment_number == (last_seq_number_ + 1) && compute_stats) { + uint64_t iat = now - last_rcv_time_; + uint64_t ist = params.timestamp - last_prod_time_; + if (now >= last_rcv_time_ && params.timestamp > last_prod_time_) { + if (iat >= ist && ist < MIN_IST_VALUE) { + if (congestion_state_ == CongestionState::Congested) { + iat_.push_back((iat - ist)); + } else { + // no congestion, but we do not always add new values, but only when + // there is no sign of congestion + double queue = protocol_state_->getQueuing(); + if (queue <= CONGESTION_FREE_QUEUEING_DELAY) { + iat_on_hold_[round_index_].push_back((iat - ist)); + } + } + } + } + } + + last_seq_number_ = segment_number; + last_rcv_time_ = now; + last_prod_time_ = params.timestamp; + + if (iat_.size() > 1000) iat_.erase(iat_.begin()); + return; +} + +CongestionCause RTCRateControlIAT::apply_classification_tree(bool force_reply) { + if (iat_.size() <= 2 || received_rate_.size() < 2) + return CongestionCause::UNKNOWN; + + double received_ratio = 0; + double iat_ratio = 0; + double iat_stdev = compute_iat_stdev(iat_); + double iat_congestion_free_stdev = compute_iat_stdev(congestion_free_iat_); + + double iat_avg = 0.0; + + double recv_avg = 0.0; + double recv_max = 0.0; + + double target_rate_avg = 0.0; + + int counter = 0; + std::vector<double>::reverse_iterator target_it = target_rate_.rbegin(); + for (std::vector<double>::reverse_iterator it = received_rate_.rbegin(); + it != received_rate_.rend(); ++it) { + recv_avg += *it; + target_rate_avg += *target_it; + if (counter < ROUND_HISTORY_SIZE) + if (recv_max < *it) { + recv_max = *it; // we consider only the last 2 seconds + } + counter++; + target_it++; + } + recv_avg = recv_avg / received_rate_.size(); + target_rate_avg = target_rate_avg / target_rate_.size(); + + for (std::vector<double>::iterator it = iat_.begin(); it != iat_.end(); + ++it) { + iat_avg += *it; + } + iat_avg = iat_avg / iat_.size(); + + double congestion_free_iat_avg = 0.0; + for (std::vector<double>::iterator it = congestion_free_iat_.begin(); + it != congestion_free_iat_.end(); ++it) { + congestion_free_iat_avg += *it; + } + congestion_free_iat_avg = + congestion_free_iat_avg / congestion_free_iat_.size(); + + received_ratio = recv_avg / target_rate_avg; + + iat_ratio = iat_stdev / iat_congestion_free_stdev; + + CongestionCause congestion_cause = CongestionCause::UNKNOWN; + // applying classification tree model + if (received_ratio <= 0.87) + if (iat_stdev <= 6.48) + if (received_ratio <= 0.83) + congestion_cause = CongestionCause::LINK_CAPACITY; + else if (force_reply) + congestion_cause = CongestionCause::LINK_CAPACITY; + else + congestion_cause = CongestionCause::UNKNOWN; // accuracy is too low + else if (iat_ratio <= 2.46) + if (force_reply) + congestion_cause = CongestionCause::LINK_CAPACITY; + else + congestion_cause = CongestionCause::UNKNOWN; // accuracy is too low + else + congestion_cause = CongestionCause::COMPETING_CROSS_TRAFFIC; + else if (received_ratio <= 0.913 && iat_stdev <= 0.784) + congestion_cause = CongestionCause::LINK_CAPACITY; + else + congestion_cause = CongestionCause::COMPETING_CROSS_TRAFFIC; + + return congestion_cause; +} + +void RTCRateControlIAT::reset_congestion_statistics() { + iat_.clear(); + received_rate_.clear(); + target_rate_.clear(); +} + +double RTCRateControlIAT::compute_iat_stdev(std::vector<double> v) { + if (v.size() == 0) return 0; + + float sum = 0.0, mean, standard_deviation = 0.0; + for (std::vector<double>::iterator it = v.begin(); it != v.end(); it++) { + sum += *it; + } + + mean = sum / v.size(); + for (std::vector<double>::iterator it = v.begin(); it != v.end(); it++) { + standard_deviation += pow(*it - mean, 2); + } + return sqrt(standard_deviation / v.size()); +} + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rc_iat.h b/libtransport/src/protocols/rtc/rtc_rc_iat.h new file mode 100644 index 000000000..715637807 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rc_iat.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include <hicn/transport/utils/shared_ptr_utils.h> +#include <protocols/rtc/rtc_rc.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +const int ROUND_HISTORY_SIZE = 10; // equivalent to two seconds +const int ROUND_TO_WAIT_FORCE_DECISION = 5; + +// once congestion is gone, we need to wait for k rounds before changing the +// congestion cause in the case it appears again +const int ROUND_TO_RESET_CAUSE = 5; + +const int MIN_IST_VALUE = 150; // samples of ist larger than 150ms are + // discarded +const double CONGESTION_FREE_QUEUEING_DELAY = 10; + +enum class CongestionCause : uint8_t { + COMPETING_CROSS_TRAFFIC, + FRIENDLY_CROSS_TRAFFIC, + UNKNOWN_CROSS_TRAFFIC, + LINK_CAPACITY, + UNKNOWN +}; + +class RTCRateControlIAT : public RTCRateControl { + public: + RTCRateControlIAT(); + + ~RTCRateControlIAT(); + + void onNewRound(double round_len); + void onDataPacketReceived(const core::ContentObject &content_object, + bool compute_stats); + + auto shared_from_this() { return utils::shared_from(this); } + + private: + void reset_congestion_statistics(); + + double compute_iat_stdev(std::vector<double> v); + + CongestionCause apply_classification_tree(bool force_reply); + + private: + uint32_t rounds_since_last_drop_; + uint32_t rounds_without_congestion_; + uint32_t rounds_with_congestion_; + double last_queue_; + uint64_t last_rcv_time_; + uint64_t last_prod_time_; + uint32_t last_seq_number_; + double target_rate_avg_; + + // Iat values are not immediately added to the congestion free set of values + std::array<std::vector<double>, ROUND_HISTORY_SIZE> iat_on_hold_; + uint32_t round_index_; + + // with congestion statistics + std::vector<double> iat_; + std::vector<double> received_rate_; + std::vector<double> target_rate_; + + // congestion free statistics + std::vector<double> congestion_free_iat_; + + CongestionCause congestion_cause_; +}; + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rc_queue.cc b/libtransport/src/protocols/rtc/rtc_rc_queue.cc index a1c89e329..ecabc5205 100644 --- a/libtransport/src/protocols/rtc/rtc_rc_queue.cc +++ b/libtransport/src/protocols/rtc/rtc_rc_queue.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -37,7 +37,7 @@ void RTCRateControlQueue::onNewRound(double round_len) { double received_rate = protocol_state_->getReceivedRate(); double target_rate = protocol_state_->getProducerRate() * PRODUCTION_RATE_FRACTION; - double rtt = (double)protocol_state_->getRTT() / MILLI_IN_A_SEC; + double rtt = (double)protocol_state_->getMinRTT() / MILLI_IN_A_SEC; double packet_size = protocol_state_->getAveragePacketSize(); double queue = protocol_state_->getQueuing(); @@ -94,7 +94,7 @@ void RTCRateControlQueue::onNewRound(double round_len) { } void RTCRateControlQueue::onDataPacketReceived( - const core::ContentObject &content_object) { + const core::ContentObject &content_object, bool compute_stats) { // nothing to do return; } diff --git a/libtransport/src/protocols/rtc/rtc_rc_queue.h b/libtransport/src/protocols/rtc/rtc_rc_queue.h index 407354d43..cdf78fd47 100644 --- a/libtransport/src/protocols/rtc/rtc_rc_queue.h +++ b/libtransport/src/protocols/rtc/rtc_rc_queue.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -30,7 +30,8 @@ class RTCRateControlQueue : public RTCRateControl { ~RTCRateControlQueue(); void onNewRound(double round_len); - void onDataPacketReceived(const core::ContentObject &content_object); + void onDataPacketReceived(const core::ContentObject &content_object, + bool compute_stats); auto shared_from_this() { return utils::shared_from(this); } diff --git a/libtransport/src/protocols/rtc/rtc_reassembly.cc b/libtransport/src/protocols/rtc/rtc_reassembly.cc new file mode 100644 index 000000000..b1b0fcaba --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_reassembly.cc @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <hicn/transport/interfaces/socket_consumer.h> +#include <implementation/socket_consumer.h> +#include <protocols/rtc/rtc_reassembly.h> +#include <protocols/transport_protocol.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +RtcReassembly::RtcReassembly(implementation::ConsumerSocket* icn_socket, + TransportProtocol* transport_protocol) + : DatagramReassembly(icn_socket, transport_protocol) { + is_setup_ = false; +} + +void RtcReassembly::reassemble(core::ContentObject& content_object) { + if (!is_setup_) { + is_setup_ = true; + reassembly_consumer_socket_->getSocketOption( + interface::RtcTransportOptions::AGGREGATED_DATA, data_aggregation_); + } + + auto read_buffer = content_object.getPayload(); + DLOG_IF(INFO, VLOG_IS_ON(3)) << "Size of payload: " << read_buffer->length(); + + read_buffer->trimStart(transport_protocol_->transportHeaderLength(false)); + + if (data_aggregation_) { + rtc::AggrPktHeader hdr((uint8_t*)read_buffer->data()); + + for (uint8_t i = 0; i < hdr.getNumberOfPackets(); i++) { + std::unique_ptr<utils::MemBuf> segment = read_buffer->clone(); + + uint16_t pkt_start = 0; + uint16_t pkt_len = 0; + int res = hdr.getPacketOffsets(i, &pkt_start, &pkt_len); + if (res == -1) { + // this should not happen + break; + } + + segment->trimStart(pkt_start); + segment->trimEnd(segment->length() - pkt_len); + + Reassembly::read_buffer_ = std::move(segment); + Reassembly::notifyApplication(); + } + } else { + Reassembly::read_buffer_ = std::move(read_buffer); + Reassembly::notifyApplication(); + } +} + +void RtcReassembly::reassemble(utils::MemBuf& buffer, uint32_t suffix) { + if (!is_setup_) { + is_setup_ = true; + reassembly_consumer_socket_->getSocketOption( + interface::RtcTransportOptions::AGGREGATED_DATA, data_aggregation_); + } + + if (data_aggregation_) { + rtc::AggrPktHeader hdr((uint8_t*)buffer.data()); + + for (uint8_t i = 0; i < hdr.getNumberOfPackets(); i++) { + std::unique_ptr<utils::MemBuf> segment = buffer.clone(); + + uint16_t pkt_start = 0; + uint16_t pkt_len = 0; + int res = hdr.getPacketOffsets(i, &pkt_start, &pkt_len); + if (res == -1) { + // this should not happen + break; + } + + segment->trimStart(pkt_start); + segment->trimEnd(segment->length() - pkt_len); + + Reassembly::read_buffer_ = std::move(segment); + Reassembly::notifyApplication(); + } + + } else { + Reassembly::read_buffer_ = buffer.cloneOne(); + Reassembly::notifyApplication(); + } +} + +} // namespace rtc + +} // namespace protocol + +} // namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_reassembly.h b/libtransport/src/protocols/rtc/rtc_reassembly.h index 15722a6d5..132004605 100644 --- a/libtransport/src/protocols/rtc/rtc_reassembly.h +++ b/libtransport/src/protocols/rtc/rtc_reassembly.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -28,17 +28,14 @@ namespace rtc { class RtcReassembly : public DatagramReassembly { public: RtcReassembly(implementation::ConsumerSocket *icn_socket, - TransportProtocol *transport_protocol) - : DatagramReassembly(icn_socket, transport_protocol) {} - - void reassemble(core::ContentObject &content_object) override { - auto read_buffer = content_object.getPayload(); - DLOG_IF(INFO, VLOG_IS_ON(3)) - << "Size of payload: " << read_buffer->length(); - read_buffer->trimStart(transport_protocol_->transportHeaderLength()); - Reassembly::read_buffer_ = std::move(read_buffer); - Reassembly::notifyApplication(); - } + TransportProtocol *transport_protocol); + + void reassemble(core::ContentObject &content_object) override; + void reassemble(utils::MemBuf &buffer, uint32_t suffix) override; + + private: + bool is_setup_; + bool data_aggregation_; }; } // namespace rtc diff --git a/libtransport/src/protocols/rtc/rtc_recovery_strategy.cc b/libtransport/src/protocols/rtc/rtc_recovery_strategy.cc new file mode 100644 index 000000000..257fdd09b --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_recovery_strategy.cc @@ -0,0 +1,420 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <glog/logging.h> +#include <hicn/transport/interfaces/notification.h> +#include <hicn/transport/interfaces/socket_options_keys.h> +#include <protocols/rtc/rtc_consts.h> +#include <protocols/rtc/rtc_recovery_strategy.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +using namespace transport::interface; + +RecoveryStrategy::RecoveryStrategy( + Indexer *indexer, SendRtxCallback &&callback, asio::io_service &io_service, + bool use_rtx, bool use_fec, + interface::RtcTransportRecoveryStrategies rs_type, + interface::StrategyCallback &&external_callback) + : rs_type_(rs_type), + recovery_on_(false), + content_sharing_mode_(false), + rtx_during_fec_(0), + next_rtx_timer_(MAX_TIMER_RTX), + send_rtx_callback_(std::move(callback)), + indexer_(indexer), + round_id_(0), + last_fec_used_(0), + callback_(std::move(external_callback)) { + setRtxFec(use_rtx, use_fec); + timer_ = std::make_unique<asio::steady_timer>(io_service); +} + +RecoveryStrategy::RecoveryStrategy(RecoveryStrategy &&rs) + : rs_type_(rs.rs_type_), + content_sharing_mode_(rs.content_sharing_mode_), + rtx_during_fec_(0), + rtx_state_(std::move(rs.rtx_state_)), + rtx_timers_(std::move(rs.rtx_timers_)), + recover_with_fec_(std::move(rs.recover_with_fec_)), + timer_(std::move(rs.timer_)), + next_rtx_timer_(std::move(rs.next_rtx_timer_)), + send_rtx_callback_(std::move(rs.send_rtx_callback_)), + n_(std::move(rs.n_)), + k_(std::move(rs.k_)), + indexer_(std::move(rs.indexer_)), + state_(std::move(rs.state_)), + rc_(std::move(rs.rc_)), + round_id_(std::move(rs.round_id_)), + last_fec_used_(std::move(rs.last_fec_used_)), + callback_(std::move(rs.callback_)) { + setFecParams(n_, k_); +} + +RecoveryStrategy::~RecoveryStrategy() {} + +void RecoveryStrategy::setFecParams(uint32_t n, uint32_t k) { + // if rs_type == FEC_ONLY_LOW_RES_LOSSES max k == 64 + n_ = n; + k_ = k; + + // XXX for the moment we go in steps of 5% loss rate. + uint32_t i = 0; + for (uint32_t loss_rate = 5; loss_rate < 100; loss_rate += 5) { + uint32_t fec_to_ask = 0; + if (n_ != 0 && k_ != 0) { + if (rs_type_ == + interface::RtcTransportRecoveryStrategies::FEC_ONLY_LOW_RES_LOSSES) { + // the max loss rate in the matrix is 50% + uint32_t index = i; + if (i > 9) index = 9; + fec_to_ask = FEC_MATRIX[k_ - 1][index]; + } else { + double dec_loss_rate = (double)(loss_rate + 5); + if (dec_loss_rate == 100.0) dec_loss_rate = 95.0; + dec_loss_rate = dec_loss_rate / 100.0; + double exp_losses = ceil((double)k_ * dec_loss_rate); + fec_to_ask = ceil((exp_losses / (1 - dec_loss_rate)) * 1.25); + } + } + fec_to_ask = std::min(fec_to_ask, (n_ - k_)); + fec_per_loss_rate_.push_back(fec_to_ask); + + i++; + } +} + +uint64_t RecoveryStrategy::getRtxRtt(uint32_t seq) { + auto it = rtx_state_.find(seq); + + if (it == rtx_state_.end()) return 0; + + // we can compute the RTT of an RTX only if it was send once. Infact if the + // RTX was sent twice or more the data may be alredy in flight and the RTT + // will be underestimated. This may happen also for packets that we + // retransmitted too soon. in that case the RTT will be filtered out by + // checking the path label + if (it->second.rtx_count_ != 1) return 0; + + // this a potentialy valid packet, compute the RTT + return (utils::SteadyTime::nowMs().count() - it->second.last_send_); +} + +bool RecoveryStrategy::lossDetected(uint32_t seq) { + if (isRtx(seq)) { + // this packet is already in the list of rtx + return false; + } + + auto it_fec = recover_with_fec_.find(seq); + if (it_fec != recover_with_fec_.end()) { + // this packet is already in list of packets to recover with fec + // this list contians also fec packets that will not be recovered with rtx + return false; + } + + auto it_nack = nacked_seq_.find(seq); + if (it_nack != nacked_seq_.end()) { + // this packet was nacked so we do not use it to determine the loss rate + return false; + } + + return true; +} + +void RecoveryStrategy::notifyNewLossDetedcted(uint32_t seq) { + // new loss detected + // first record the loss. second do what is needed to recover it + state_->onLossDetected(seq); + newPacketLoss(seq); +} + +void RecoveryStrategy::requestPossibleLostPacket(uint32_t seq) { + // these are packets for which we send a RTX but we do not increase the loss + // counter beacuse we don't know if they are lost or not + addNewRtx(seq, false); +} + +void RecoveryStrategy::receivedFutureNack(uint32_t seq) { + nacked_seq_.insert(seq); +} + +void RecoveryStrategy::clear() { + rtx_state_.clear(); + rtx_timers_.clear(); + recover_with_fec_.clear(); + + if (next_rtx_timer_ != MAX_TIMER_RTX) { + next_rtx_timer_ = MAX_TIMER_RTX; + timer_->cancel(); + } +} + +// rtx functions +void RecoveryStrategy::addNewRtx(uint32_t seq, bool force) { + if (!indexer_->isFec(seq) || force) { + // this packet needs to be re-transmitted + rtxState state; + state.first_send_ = state_->getInterestSentTime(seq); + if (state.first_send_ == 0) // this interest was never sent before + state.first_send_ = getNow(); + state.last_send_ = state.first_send_; // we didn't send an RTX for this + // packet yet + state.rtx_count_ = 0; + state.next_send_ = computeNextSend(seq, state.rtx_count_); + DLOG_IF(INFO, VLOG_IS_ON(4)) + << "Add " << seq << " to retransmissions. next rtx is in " + << state.next_send_ - getNow() << " ms"; + rtx_state_.insert(std::pair<uint32_t, rtxState>(seq, state)); + rtx_timers_.insert(std::pair<uint64_t, uint32_t>(state.next_send_, seq)); + + // if a new rtx is introduced, check the rtx timer + scheduleNextRtx(); + } else { + // do not re-send fec packets but keep track of them + recover_with_fec_.insert(seq); + state_->onPossibleLossWithNoRtx(seq); + } +} + +uint64_t RecoveryStrategy::computeNextSend(uint32_t seq, uint32_t rtx_counter) { + uint64_t now = getNow(); + if (rtx_counter == 0) { + uint32_t wait = 1; + if (content_sharing_mode_) return now + wait; + + uint32_t jitter = SENTINEL_TIMER_INTERVAL; + double prod_rate = state_->getProducerRate(); + if (prod_rate != 0) jitter = ceil(state_->getJitter()); + + wait += jitter; + + DLOG_IF(INFO, VLOG_IS_ON(3)) << "first rtx for " << seq << " in " << wait + << " ms, jitter = " << jitter; + + return now + wait; + } else { + // wait one RTT. if an edge is known use the edge RTT for the first 5 rtx + double prod_rate = state_->getProducerRate(); + if (prod_rate == 0) { + return now + SENTINEL_TIMER_INTERVAL; + } + + uint64_t rtt = 0; + // if the transport detects an edge we try first to get the RTX from the + // edge. if no interest get a reply we move to the full RTT + if (rtx_counter < 5 && (state_->getEdgeRtt() != 0)) { + rtt = state_->getEdgeRtt(); + } else { + rtt = state_->getAvgRTT(); + } + + if (rtt == 0) rtt = SENTINEL_TIMER_INTERVAL; + + if (content_sharing_mode_) return now + rtt; + + uint32_t wait = (uint32_t)rtt; + + uint32_t jitter = ceil(state_->getJitter()); + wait += jitter; + + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "next rtx for " << seq << " in " << wait << " ms, rtt = " << rtt + << " jtter = " << jitter; + + return now + wait; + } +} + +void RecoveryStrategy::retransmit() { + if (rtx_timers_.size() == 0) return; + + uint64_t now = getNow(); + + auto it = rtx_timers_.begin(); + std::unordered_set<uint32_t> lost_pkt; + uint32_t sent_counter = 0; + while (it != rtx_timers_.end() && it->first <= now && + sent_counter < MAX_RTX_IN_BATCH) { + uint32_t seq = it->second; + auto rtx_it = + rtx_state_.find(seq); // this should always return a valid iter + if (rtx_it->second.rtx_count_ >= RTC_MAX_RTX || + (now - rtx_it->second.first_send_) >= RTC_MAX_AGE || + seq < state_->getLastSeqNacked()) { + // max rtx reached or packet too old or packet nacked, this packet is lost + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "packet " << seq << " lost because 1) max rtx: " + << (rtx_it->second.rtx_count_ >= RTC_MAX_RTX) << " 2) max age: " + << ((now - rtx_it->second.first_send_) >= RTC_MAX_AGE) + << " 3) nacked: " << (seq < state_->getLastSeqNacked()); + lost_pkt.insert(seq); + it++; + } else { + // resend the packet + state_->onRetransmission(seq); + double prod_rate = state_->getProducerRate(); + if (prod_rate != 0) rtx_it->second.rtx_count_++; + rtx_it->second.last_send_ = now; + rtx_it->second.next_send_ = + computeNextSend(seq, rtx_it->second.rtx_count_); + it = rtx_timers_.erase(it); + rtx_timers_.insert( + std::pair<uint64_t, uint32_t>(rtx_it->second.next_send_, seq)); + DLOG_IF(INFO, VLOG_IS_ON(3)) + << "send rtx for sequence " << seq << ", next send in " + << (rtx_it->second.next_send_ - now); + + // if fec is on increase the number of RTX send during fec + if (fec_on_) rtx_during_fec_++; + send_rtx_callback_(seq); + sent_counter++; + } + } + + // remove packets if needed + for (auto lost_it = lost_pkt.begin(); lost_it != lost_pkt.end(); lost_it++) { + uint32_t seq = *lost_it; + state_->onPacketLost(seq); + deleteRtx(seq); + } +} + +void RecoveryStrategy::scheduleNextRtx() { + if (rtx_timers_.size() == 0) { + // all the rtx were removed, reset timer + next_rtx_timer_ = MAX_TIMER_RTX; + return; + } + + // check if timer is alreay set + if (next_rtx_timer_ != MAX_TIMER_RTX) { + // a new check for rtx is already scheduled + if (next_rtx_timer_ > rtx_timers_.begin()->first) { + // we need to re-schedule it + timer_->cancel(); + } else { + // wait for the next timer + return; + } + } + + // set a new timer + next_rtx_timer_ = rtx_timers_.begin()->first; + uint64_t now = utils::SteadyTime::nowMs().count(); + uint64_t wait = 1; + if (next_rtx_timer_ != MAX_TIMER_RTX && next_rtx_timer_ > now) + wait = next_rtx_timer_ - now; + + std::weak_ptr<RecoveryStrategy> self(shared_from_this()); + timer_->expires_from_now(std::chrono::milliseconds(wait)); + timer_->async_wait([self](const std::error_code &ec) { + if (ec) return; + if (auto s = self.lock()) { + s->retransmit(); + s->next_rtx_timer_ = MAX_TIMER_RTX; + s->scheduleNextRtx(); + } + }); +} + +void RecoveryStrategy::deleteRtx(uint32_t seq) { + auto it_rtx = rtx_state_.find(seq); + if (it_rtx == rtx_state_.end()) return; // rtx not found + + // remove the rtx from the timers list + uint64_t ts = it_rtx->second.next_send_; + auto it_timers = rtx_timers_.find(ts); + while (it_timers != rtx_timers_.end() && it_timers->first == ts) { + if (it_timers->second == seq) { + rtx_timers_.erase(it_timers); + break; + } + it_timers++; + } + + // remove rtx + rtx_state_.erase(it_rtx); +} + +// fec functions +uint32_t RecoveryStrategy::computeFecPacketsToAsk() { + double loss_rate = state_->getMaxLossRate() * 100; // use loss rate in % + + if (loss_rate > 95) loss_rate = 95; // max loss rate + + if (loss_rate == 0) return 0; + + // keep track of the last used fec. if we use a new bin on this round reset + // consecutive use and avg loss in the prev bin + uint32_t bin = ceil(loss_rate / 5.0) - 1; + if (bin > fec_per_loss_rate_.size() - 1) + bin = (uint32_t)fec_per_loss_rate_.size() - 1; + + return fec_per_loss_rate_[bin]; +} + +void RecoveryStrategy::setRtxFec(std::optional<bool> rtx_on, + std::optional<bool> fec_on) { + if (rtx_on) rtx_on_ = *rtx_on; + if (fec_on) { + if (fec_on_ == false && (*fec_on) == true) { // turn on fec + // reset the number of RTX sent during fec + rtx_during_fec_ = 0; + } + fec_on_ = *fec_on; + } + + notification::RecoveryStrategy strategy = + notification::RecoveryStrategy::RECOVERY_OFF; + + if (rtx_on_ && fec_on_) + strategy = notification::RecoveryStrategy::RTX_AND_FEC; + else if (rtx_on_) + strategy = notification::RecoveryStrategy::RTX_ONLY; + else if (fec_on_) + strategy = notification::RecoveryStrategy::FEC_ONLY; + + callback_(strategy); +} + +// common functions +void RecoveryStrategy::onLostTimeout(uint32_t seq) { removePacketState(seq); } + +void RecoveryStrategy::removePacketState(uint32_t seq) { + auto it_fec = recover_with_fec_.find(seq); + if (it_fec != recover_with_fec_.end()) { + recover_with_fec_.erase(it_fec); + return; + } + + auto it_nack = nacked_seq_.find(seq); + if (it_nack != nacked_seq_.end()) { + nacked_seq_.erase(it_nack); + return; + } + + deleteRtx(seq); +} + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_recovery_strategy.h b/libtransport/src/protocols/rtc/rtc_recovery_strategy.h new file mode 100644 index 000000000..405e1ebba --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_recovery_strategy.h @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include <hicn/transport/interfaces/callbacks.h> +#include <hicn/transport/utils/chrono_typedefs.h> +#include <protocols/indexer.h> +#include <protocols/rtc/rtc_rc.h> +#include <protocols/rtc/rtc_state.h> + +#include <map> +#include <optional> +#include <unordered_map> + +namespace transport { + +namespace protocol { + +namespace rtc { + +class RecoveryStrategy : public std::enable_shared_from_this<RecoveryStrategy> { + protected: + struct rtx_state_ { + uint64_t first_send_; // first time this interest was sent + uint64_t last_send_; // last time this rtx was sent + uint64_t next_send_; // next retransmission time + uint32_t rtx_count_; // number or rtx + }; + + using rtxState = struct rtx_state_; + + public: + using SendRtxCallback = std::function<void(uint32_t)>; + + RecoveryStrategy(Indexer *indexer, SendRtxCallback &&callback, + asio::io_service &io_service, bool use_rtx, bool use_fec, + interface::RtcTransportRecoveryStrategies rs_type, + interface::StrategyCallback &&external_callback); + + RecoveryStrategy(RecoveryStrategy &&rs); + + virtual ~RecoveryStrategy(); + + void setRtxFec(std::optional<bool> rtx_on = {}, + std::optional<bool> fec_on = {}); + void setState(RTCState *state) { state_ = state; } + void setRateControl(RTCRateControl *rateControl) { rc_ = rateControl; } + void setFecParams(uint32_t n, uint32_t k); + void setContentSharingMode() { content_sharing_mode_ = true; } + + bool isRtx(uint32_t seq) { + if (rtx_state_.find(seq) != rtx_state_.end()) return true; + return false; + } + + bool isPossibleLossWithNoRtx(uint32_t seq) { + if (recover_with_fec_.find(seq) != recover_with_fec_.end()) return true; + return false; + } + + bool wasNacked(uint32_t seq) { + if (nacked_seq_.find(seq) != nacked_seq_.end()) return true; + return false; + } + + interface::RtcTransportRecoveryStrategies getType() { + return rs_type_; + } + void updateType(interface::RtcTransportRecoveryStrategies type) { + rs_type_ = type; + } + bool isRtxOn() { return rtx_on_; } + bool isFecOn() { return fec_on_; } + + RTCState *getState() { return state_; } + + // if the function returns 0 it means that the packet is not an RTX or it is + // not a valid packet to safely compute the RTT + uint64_t getRtxRtt(uint32_t seq); + bool lossDetected(uint32_t seq); + void notifyNewLossDetedcted(uint32_t seq); + void requestPossibleLostPacket(uint32_t seq); + void receivedFutureNack(uint32_t seq); + void clear(); + + virtual void turnOnRecovery() = 0; + virtual void onNewRound(bool in_sync) = 0; + virtual void newPacketLoss(uint32_t seq) = 0; + virtual void receivedPacket(uint32_t seq) = 0; + void onLostTimeout(uint32_t seq); + + void incRoundId() { round_id_++; } + + // utils + uint64_t getNow() { + uint64_t now = utils::SteadyTime::nowMs().count(); + return now; + } + + protected: + // rtx functions + void addNewRtx(uint32_t seq, bool force); + uint64_t computeNextSend(uint32_t seq, uint32_t rtx_counter); + void retransmit(); + void scheduleNextRtx(); + void deleteRtx(uint32_t seq); + + // fec functions + uint32_t computeFecPacketsToAsk(); + + // common functons + void removePacketState(uint32_t seq); + + interface::RtcTransportRecoveryStrategies rs_type_; + bool recovery_on_; + bool rtx_on_; + bool fec_on_; + bool content_sharing_mode_; + + // number of RTX sent after fec turned on + // this is used to take into account jitter and out of order packets + // if we detect losses but we do not sent any RTX it means that the holes in + // the sequence are caused by the jitter + uint32_t rtx_during_fec_; + + // this map keeps track of the retransmitted interest, ordered from the oldest + // to the newest one. the state contains the timer of the first send of the + // interest (from pendingIntetests_), the timer of the next send (key of the + // multimap) and the number of rtx + std::map<uint32_t, rtxState> rtx_state_; + // this map stored the rtx by timer. The key is the time at which the rtx + // should be sent, and the val is the interest seq number + std::multimap<uint64_t, uint32_t> rtx_timers_; + + // lost packets that will be recovered with fec + std::unordered_set<uint32_t> recover_with_fec_; + + // packet for which we recived a future nack + // in case we detect a loss for a nacked packet we send an RTX but we do not + // increase the loss counter. this is done because it may happen that the + // producer rate checkes over time and in flight interest may be satified by + // data packet after the reception of nacks + std::unordered_set<uint32_t> nacked_seq_; + + // rtx vars + std::unique_ptr<asio::steady_timer> timer_; + uint64_t next_rtx_timer_; + SendRtxCallback send_rtx_callback_; + + // fec vars + uint32_t n_; + uint32_t k_; + Indexer *indexer_; + + RTCState *state_; + RTCRateControl *rc_; + + private: + uint32_t round_id_; // number of rounds + uint32_t last_fec_used_; + std::vector<uint32_t> fec_per_loss_rate_; + interface::StrategyCallback callback_; +}; + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_delay.cc b/libtransport/src/protocols/rtc/rtc_rs_delay.cc new file mode 100644 index 000000000..7d7a01133 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_delay.cc @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <glog/logging.h> +#include <protocols/rtc/rtc_consts.h> +#include <protocols/rtc/rtc_rs_delay.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +RecoveryStrategyDelayBased::RecoveryStrategyDelayBased( + Indexer *indexer, SendRtxCallback &&callback, asio::io_service &io_service, + interface::RtcTransportRecoveryStrategies rs_type, + interface::StrategyCallback &&external_callback) + : RecoveryStrategy(indexer, std::move(callback), io_service, true, false, + rs_type, + std::move(external_callback)), // start with rtx + congestion_state_(false), + probing_state_(false), + switch_rounds_(0) {} + +RecoveryStrategyDelayBased::RecoveryStrategyDelayBased(RecoveryStrategy &&rs) + : RecoveryStrategy(std::move(rs)) { + setRtxFec(true, false); + // we have to re-init congestion and + // probing + switch_rounds_ = 0; + congestion_state_ = false; + probing_state_ = false; +} + +RecoveryStrategyDelayBased::~RecoveryStrategyDelayBased() {} + +void RecoveryStrategyDelayBased::turnOnRecovery() { + recovery_on_ = true; + uint64_t rtt = state_->getMinRTT(); + uint32_t fec_to_ask = computeFecPacketsToAsk(); + if (rtt > MAX_RTT_BEFORE_FEC && fec_to_ask > 0) { + // we need to start FEC (see fec only strategy for more details) + setRtxFec(true, true); + rtx_during_fec_ = 1; // avoid to stop fec + indexer_->setNFec(fec_to_ask); + } else { + // use RTX + setRtxFec(true, false); + switch_rounds_ = 0; + } +} + +void RecoveryStrategyDelayBased::softSwitchToFec(uint32_t fec_to_ask) { + if (fec_to_ask == 0) { + setRtxFec(true, false); + switch_rounds_ = 0; + } else { + switch_rounds_++; + if (switch_rounds_ >= ((RTC_INTEREST_LIFETIME / ROUND_LEN) * 2) && + rtx_during_fec_ != 0) { // go to fec only if it is needed (RTX are on) + setRtxFec(false, true); + } else { + setRtxFec(true, true); + } + } +} + +void RecoveryStrategyDelayBased::onNewRound(bool in_sync) { + if (!recovery_on_) { + // disable fec so that no extra packet will be sent + // for rtx we check if recovery is on in newPacketLoss + setRtxFec(true, false); + indexer_->setNFec(0); + return; + } + + uint64_t rtt = state_->getAvgRTT(); + + // XXX at the moment we are not looking at congestion events + // bool congestion = rc_->inCongestionState(); + + if ((!fec_on_ && rtt >= MAX_RTT_BEFORE_FEC) || + (fec_on_ && rtt > (MAX_RTT_BEFORE_FEC - 10))) { + // switch from rtx to fec or keep use fec. Notice that if some rtx are + // waiting to be scheduled, they will be sent normally, but no new rtx will + // be created if the loss rate is 0 keep to use RTX. + uint32_t fec_to_ask = computeFecPacketsToAsk(); + softSwitchToFec(fec_to_ask); + if (rtx_during_fec_ == 0) // if we do not send any RTX the losses + // registered may be due to jitter + indexer_->setNFec(0); + else + indexer_->setNFec(fec_to_ask); + return; + } + + if ((fec_on_ && rtt <= (MAX_RTT_BEFORE_FEC - 10)) || + (!rtx_on_ && rtt <= MAX_RTT_BEFORE_FEC)) { + // turn on rtx + softSwitchToFec(0); + indexer_->setNFec(0); + return; + } +} + +void RecoveryStrategyDelayBased::newPacketLoss(uint32_t seq) { + if (rtx_on_ && recovery_on_) { + addNewRtx(seq, false); + } else { + if (!state_->isPending(seq) && !indexer_->isFec(seq)) { + addNewRtx(seq, true); + } else { + recover_with_fec_.insert(seq); + state_->onPossibleLossWithNoRtx(seq); + } + } +} + +void RecoveryStrategyDelayBased::receivedPacket(uint32_t seq) { + removePacketState(seq); +} + +void RecoveryStrategyDelayBased::probing() { + // TODO + // for the moment ask for all fec and exit the probing phase + probing_state_ = false; + setRtxFec(false, true); + indexer_->setNFec(computeFecPacketsToAsk()); +} + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_delay.h b/libtransport/src/protocols/rtc/rtc_rs_delay.h new file mode 100644 index 000000000..9e1c41388 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_delay.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include <protocols/rtc/rtc_recovery_strategy.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +class RecoveryStrategyDelayBased : public RecoveryStrategy { + public: + RecoveryStrategyDelayBased(Indexer *indexer, SendRtxCallback &&callback, + asio::io_service &io_service, + interface::RtcTransportRecoveryStrategies rs_type, + interface::StrategyCallback &&external_callback); + + RecoveryStrategyDelayBased(RecoveryStrategy &&rs); + + ~RecoveryStrategyDelayBased(); + + void turnOnRecovery(); + void onNewRound(bool in_sync); + void newPacketLoss(uint32_t seq); + void receivedPacket(uint32_t seq); + + private: + void softSwitchToFec(uint32_t fec_to_ask); + + bool congestion_state_; + bool probing_state_; + uint32_t switch_rounds_; + + void probing(); +}; + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_fec_only.cc b/libtransport/src/protocols/rtc/rtc_rs_fec_only.cc new file mode 100644 index 000000000..5b10823ec --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_fec_only.cc @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <glog/logging.h> +#include <protocols/rtc/rtc_consts.h> +#include <protocols/rtc/rtc_rs_fec_only.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +RecoveryStrategyFecOnly::RecoveryStrategyFecOnly( + Indexer *indexer, SendRtxCallback &&callback, asio::io_service &io_service, + interface::RtcTransportRecoveryStrategies rs_type, + interface::StrategyCallback &&external_callback) + : RecoveryStrategy(indexer, std::move(callback), io_service, true, false, + rs_type, std::move(external_callback)), + congestion_state_(false), + probing_state_(false), + switch_rounds_(0) {} + +RecoveryStrategyFecOnly::RecoveryStrategyFecOnly(RecoveryStrategy &&rs) + : RecoveryStrategy(std::move(rs)) { + setRtxFec(true, false); + switch_rounds_ = 0; + congestion_state_ = false; + probing_state_ = false; +} + +RecoveryStrategyFecOnly::~RecoveryStrategyFecOnly() {} + +void RecoveryStrategyFecOnly::turnOnRecovery() { + recovery_on_ = true; + // init strategy + uint32_t fec_to_ask = computeFecPacketsToAsk(); + if (fec_to_ask > 0) { + // the probing phase detected a lossy link. we immedialty start the fec and + // we disable the check to prevent to send fec packets before RTX. in fact + // here we know that we have losses and it is not a problem of OOO packets + setRtxFec(true, true); + rtx_during_fec_ = 1; // avoid to stop fec + indexer_->setNFec(fec_to_ask); + } else { + // keep only RTX on + setRtxFec(true, true); + } +} + +void RecoveryStrategyFecOnly::onNewRound(bool in_sync) { + if (!recovery_on_) { + indexer_->setNFec(0); + return; + } + + // XXX for the moment we are considering congestion events + // if(rc_->inCongestionState()){ + // congestion_state_ = true; + // probing_state_ = false; + // indexer_->setNFec(0); + // return; + // } + + // no congestion + if (congestion_state_) { + // this is the first round after congestion + // enter probing phase + probing_state_ = true; + congestion_state_ = false; + } + + if (probing_state_) { + probing(); + } else { + uint32_t fec_to_ask = computeFecPacketsToAsk(); + // If fec_to_ask == 0 we use rtx even if in these strategy we use only fec. + // In this way the first packet loss that triggers the usage of fec can be + // recovered using rtx, otherwise it will always be lost + if (fec_to_ask == 0) { + setRtxFec(true, false); + switch_rounds_ = 0; + } else { + switch_rounds_++; + if (switch_rounds_ >= ((RTC_INTEREST_LIFETIME / ROUND_LEN) * 2) && + rtx_during_fec_ != + 0) { // go to fec only if it is needed (RTX are on) + setRtxFec(false, true); + } else { + setRtxFec(true, true); // keep both + } + } + if (rtx_during_fec_ == 0) // if we do not send any RTX the losses + // registered may be due to jitter + indexer_->setNFec(0); + else + indexer_->setNFec(fec_to_ask); + } +} + +void RecoveryStrategyFecOnly::newPacketLoss(uint32_t seq) { + if (rtx_on_ && recovery_on_) { + addNewRtx(seq, false); + } else { + if (!state_->isPending(seq) && !indexer_->isFec(seq)) { + addNewRtx(seq, true); + } else { + // if not pending add to list to recover with fec + recover_with_fec_.insert(seq); + state_->onPossibleLossWithNoRtx(seq); + } + } +} + +void RecoveryStrategyFecOnly::receivedPacket(uint32_t seq) { + removePacketState(seq); +} + +void RecoveryStrategyFecOnly::probing() { + // TODO + // for the moment ask for all fec and exit the probing phase + probing_state_ = false; + uint32_t fec_to_ask = computeFecPacketsToAsk(); + indexer_->setNFec(fec_to_ask); +} + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_fec_only.h b/libtransport/src/protocols/rtc/rtc_rs_fec_only.h new file mode 100644 index 000000000..42df25bd9 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_fec_only.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include <protocols/rtc/rtc_recovery_strategy.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +class RecoveryStrategyFecOnly : public RecoveryStrategy { + public: + RecoveryStrategyFecOnly(Indexer *indexer, SendRtxCallback &&callback, + asio::io_service &io_service, + interface::RtcTransportRecoveryStrategies rs_type, + interface::StrategyCallback &&external_callback); + + RecoveryStrategyFecOnly(RecoveryStrategy &&rs); + + ~RecoveryStrategyFecOnly(); + + void turnOnRecovery(); + void onNewRound(bool in_sync); + void newPacketLoss(uint32_t seq); + void receivedPacket(uint32_t seq); + + private: + bool congestion_state_; + bool probing_state_; + uint32_t switch_rounds_; + + void probing(); +}; + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_low_rate.cc b/libtransport/src/protocols/rtc/rtc_rs_low_rate.cc new file mode 100644 index 000000000..dbad563cd --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_low_rate.cc @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <glog/logging.h> +#include <protocols/rtc/rtc_consts.h> +#include <protocols/rtc/rtc_rs_low_rate.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +RecoveryStrategyLowRate::RecoveryStrategyLowRate( + Indexer *indexer, SendRtxCallback &&callback, asio::io_service &io_service, + interface::RtcTransportRecoveryStrategies rs_type, + interface::StrategyCallback &&external_callback) + : RecoveryStrategy(indexer, std::move(callback), io_service, false, true, + rs_type, + std::move(external_callback)), // start with fec + fec_consecutive_rounds_((MILLI_IN_A_SEC / ROUND_LEN) * 5), // 5 sec + rtx_allowed_consecutive_rounds_(0) { + initSwitchVector(); +} + +RecoveryStrategyLowRate::RecoveryStrategyLowRate(RecoveryStrategy &&rs) + : RecoveryStrategy(std::move(rs)), + fec_consecutive_rounds_((MILLI_IN_A_SEC / ROUND_LEN) * 5), // 5 sec + rtx_allowed_consecutive_rounds_(0) { + setRtxFec(false, true); + initSwitchVector(); +} + +RecoveryStrategyLowRate::~RecoveryStrategyLowRate() {} + +void RecoveryStrategyLowRate::initSwitchVector() { + // TODO adjust thresholds here when new data are collected + // see resutls in + // https://confluence-eng-gpk1.cisco.com/conf/display/SPT/dailyreports + thresholds_t t1; + t1.rtt = 15; // 15ms + t1.loss_rtx_to_fec = 15; // 15% + t1.loss_fec_to_rtx = 10; // 10% + thresholds_t t2; + t2.rtt = 35; // 35ms + t2.loss_rtx_to_fec = 5; // 5% + t2.loss_fec_to_rtx = 1; // 1% + switch_vector.push_back(t1); + switch_vector.push_back(t2); +} + +void RecoveryStrategyLowRate::setRecoveryParameters(bool use_rtx, bool use_fec, + uint32_t fec_to_ask) { + setRtxFec(use_rtx, use_fec); + indexer_->setNFec(fec_to_ask); +} + +void RecoveryStrategyLowRate::selectRecoveryStrategy(bool in_sync) { + uint32_t fec_to_ask = computeFecPacketsToAsk(); + if (fec_to_ask == 0) { + // fec is off, turn on RTX immediatly to avoid packet losses + setRecoveryParameters(true, false, 0); + fec_consecutive_rounds_ = 0; + return; + } + + uint32_t loss_rate = std::round(state_->getPerSecondLossRate() * 100); + uint32_t rtt = (uint32_t)state_->getAvgRTT(); + + bool use_rtx = false; + for (size_t i = 0; i < switch_vector.size(); i++) { + uint32_t max_loss_rate = 0; + if (fec_on_) + max_loss_rate = switch_vector[i].loss_fec_to_rtx; + else + max_loss_rate = switch_vector[i].loss_rtx_to_fec; + + if (rtt < switch_vector[i].rtt && loss_rate < max_loss_rate) { + use_rtx = true; + rtx_allowed_consecutive_rounds_++; + break; + } + } + + if (!use_rtx) rtx_allowed_consecutive_rounds_ = 0; + + if (use_rtx) { + if (fec_on_) { + // here we should swtich from RTX to FEC + // wait 10sec where the switch is allowed before actually switch + if (rtx_allowed_consecutive_rounds_ >= + ((MILLI_IN_A_SEC / ROUND_LEN) * 10)) { // 10 sec + // use RTX + setRecoveryParameters(true, false, 0); + fec_consecutive_rounds_ = 0; + } else { + // keep using FEC (and maybe RTX) + setRecoveryParameters(true, true, fec_to_ask); + fec_consecutive_rounds_++; + } + } else { + // keep using RTX + setRecoveryParameters(true, false, 0); + fec_consecutive_rounds_ = 0; + } + } else { + // use FEC and RTX + setRecoveryParameters(true, true, fec_to_ask); + fec_consecutive_rounds_++; + } + + // everytime that we anable FEC we keep also RTX on. in this way the first + // losses that are not covered by FEC are recovered using RTX. after 5 sec we + // disable fec + if (fec_consecutive_rounds_ >= ((MILLI_IN_A_SEC / ROUND_LEN) * 5)) { + // turn off RTX + setRtxFec(false); + } +} + +void RecoveryStrategyLowRate::turnOnRecovery() { + recovery_on_ = 1; + // the stategy will be init in the new round function +} + +void RecoveryStrategyLowRate::onNewRound(bool in_sync) { + if (!recovery_on_) { + // disable fec so that no extra packet will be sent + // for rtx we check if recovery is on in newPacketLoss + setRtxFec(true, false); + indexer_->setNFec(0); + return; + } + + // XXX since this strategy will be used only for flow at low rate we do not + // consider congestion events like in other strategies + + selectRecoveryStrategy(in_sync); +} + +void RecoveryStrategyLowRate::newPacketLoss(uint32_t seq) { + if (rtx_on_ && recovery_on_) { + addNewRtx(seq, false); + } else { + if (!state_->isPending(seq) && !indexer_->isFec(seq)) { + addNewRtx(seq, true); + } else { + recover_with_fec_.insert(seq); + state_->onPossibleLossWithNoRtx(seq); + } + } +} + +void RecoveryStrategyLowRate::receivedPacket(uint32_t seq) { + removePacketState(seq); +} + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_low_rate.h b/libtransport/src/protocols/rtc/rtc_rs_low_rate.h new file mode 100644 index 000000000..0e76efaca --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_low_rate.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include <protocols/rtc/rtc_recovery_strategy.h> + +#include <vector> + +namespace transport { + +namespace protocol { + +namespace rtc { + +struct thresholds_t { + uint32_t rtt; + uint32_t loss_rtx_to_fec; // loss rate used to move from rtx to fec + uint32_t loss_fec_to_rtx; // loss rate used to move from fec to rtx +}; + +class RecoveryStrategyLowRate : public RecoveryStrategy { + public: + RecoveryStrategyLowRate(Indexer *indexer, SendRtxCallback &&callback, + asio::io_service &io_service, + interface::RtcTransportRecoveryStrategies rs_type, + interface::StrategyCallback &&external_callback); + + RecoveryStrategyLowRate(RecoveryStrategy &&rs); + + ~RecoveryStrategyLowRate(); + + void turnOnRecovery(); + void onNewRound(bool in_sync); + void newPacketLoss(uint32_t seq); + void receivedPacket(uint32_t seq); + + private: + void initSwitchVector(); + void setRecoveryParameters(bool use_rtx, bool use_fec, uint32_t fec_to_ask); + void selectRecoveryStrategy(bool in_sync); + + uint32_t fec_consecutive_rounds_; + uint32_t rtx_allowed_consecutive_rounds_; + + // this table contains the thresholds that indicates when to switch from RTX + // to FEC and viceversa. values in the vector are detected with a set of + // experiments. the vector is used in the following way: if rtt and loss rate + // are less than one of the values in the in the vector, losses are + // recovered using RTX. otherwive losses are recovered using FEC. as for FEC + // only and delay based strategy, the swith from RTX to FEC is smooth, + // meaning that FEC and RTX are used together for some rounds + std::vector<thresholds_t> switch_vector; +}; + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_recovery_off.cc b/libtransport/src/protocols/rtc/rtc_rs_recovery_off.cc new file mode 100644 index 000000000..00c6a0504 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_recovery_off.cc @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <glog/logging.h> +#include <protocols/rtc/rtc_consts.h> +#include <protocols/rtc/rtc_rs_recovery_off.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +RecoveryStrategyRecoveryOff::RecoveryStrategyRecoveryOff( + Indexer *indexer, SendRtxCallback &&callback, asio::io_service &io_service, + interface::RtcTransportRecoveryStrategies rs_type, + interface::StrategyCallback &&external_callback) + : RecoveryStrategy(indexer, std::move(callback), io_service, false, false, + rs_type, std::move(external_callback)) {} + +RecoveryStrategyRecoveryOff::RecoveryStrategyRecoveryOff(RecoveryStrategy &&rs) + : RecoveryStrategy(std::move(rs)) { + setRtxFec(false, false); +} + +RecoveryStrategyRecoveryOff::~RecoveryStrategyRecoveryOff() {} + +void RecoveryStrategyRecoveryOff::turnOnRecovery() { + // nothing to do + return; +} +void RecoveryStrategyRecoveryOff::onNewRound(bool in_sync) { + // nothing to do + return; +} + +void RecoveryStrategyRecoveryOff::newPacketLoss(uint32_t seq) { + // here we only keep track of the lost packets to avoid to + // count them multple times in the counters. for this we + // use the recover_with_fec_ set + recover_with_fec_.insert(seq); + state_->onPossibleLossWithNoRtx(seq); +} + +void RecoveryStrategyRecoveryOff::receivedPacket(uint32_t seq) { + removePacketState(seq); +} + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_recovery_off.h b/libtransport/src/protocols/rtc/rtc_rs_recovery_off.h new file mode 100644 index 000000000..3d59cc473 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_recovery_off.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include <protocols/rtc/rtc_recovery_strategy.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +class RecoveryStrategyRecoveryOff : public RecoveryStrategy { + public: + RecoveryStrategyRecoveryOff(Indexer *indexer, SendRtxCallback &&callback, + asio::io_service &io_service, + interface::RtcTransportRecoveryStrategies rs_type, + interface::StrategyCallback &&external_callback); + + RecoveryStrategyRecoveryOff(RecoveryStrategy &&rs); + + ~RecoveryStrategyRecoveryOff(); + + void turnOnRecovery(); + void onNewRound(bool in_sync); + void newPacketLoss(uint32_t seq); + void receivedPacket(uint32_t seq); +}; + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_rtx_only.cc b/libtransport/src/protocols/rtc/rtc_rs_rtx_only.cc new file mode 100644 index 000000000..4d7cf7a82 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_rtx_only.cc @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <glog/logging.h> +#include <protocols/rtc/rtc_consts.h> +#include <protocols/rtc/rtc_rs_rtx_only.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +RecoveryStrategyRtxOnly::RecoveryStrategyRtxOnly( + Indexer *indexer, SendRtxCallback &&callback, asio::io_service &io_service, + interface::RtcTransportRecoveryStrategies rs_type, + interface::StrategyCallback &&external_callback) + : RecoveryStrategy(indexer, std::move(callback), io_service, true, false, + rs_type, std::move(external_callback)) {} + +RecoveryStrategyRtxOnly::RecoveryStrategyRtxOnly(RecoveryStrategy &&rs) + : RecoveryStrategy(std::move(rs)) { + setRtxFec(true, false); +} + +RecoveryStrategyRtxOnly::~RecoveryStrategyRtxOnly() {} + +void RecoveryStrategyRtxOnly::turnOnRecovery() { + recovery_on_ = true; + setRtxFec(true, false); +} + +void RecoveryStrategyRtxOnly::onNewRound(bool in_sync) { + // nothing to do + return; +} + +void RecoveryStrategyRtxOnly::newPacketLoss(uint32_t seq) { + if (!recovery_on_) { + recover_with_fec_.insert(seq); + state_->onPossibleLossWithNoRtx(seq); + return; + } + addNewRtx(seq, false); +} + +void RecoveryStrategyRtxOnly::receivedPacket(uint32_t seq) { + removePacketState(seq); +} + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_rs_rtx_only.h b/libtransport/src/protocols/rtc/rtc_rs_rtx_only.h new file mode 100644 index 000000000..03dbed1c7 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_rs_rtx_only.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include <protocols/rtc/rtc_recovery_strategy.h> + +namespace transport { + +namespace protocol { + +namespace rtc { + +class RecoveryStrategyRtxOnly : public RecoveryStrategy { + public: + RecoveryStrategyRtxOnly(Indexer *indexer, SendRtxCallback &&callback, + asio::io_service &io_service, + interface::RtcTransportRecoveryStrategies rs_type, + interface::StrategyCallback &&external_callback); + + RecoveryStrategyRtxOnly(RecoveryStrategy &&rs); + + ~RecoveryStrategyRtxOnly(); + + void turnOnRecovery(); + void onNewRound(bool in_sync); + void newPacketLoss(uint32_t seq); + void receivedPacket(uint32_t seq); +}; + +} // end namespace rtc + +} // end namespace protocol + +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_state.cc b/libtransport/src/protocols/rtc/rtc_state.cc index c99205a26..82ac0b9c1 100644 --- a/libtransport/src/protocols/rtc/rtc_state.cc +++ b/libtransport/src/protocols/rtc/rtc_state.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -24,15 +24,15 @@ namespace protocol { namespace rtc { RTCState::RTCState(Indexer *indexer, - ProbeHandler::SendProbeCallback &&rtt_probes_callback, + ProbeHandler::SendProbeCallback &&probe_callback, DiscoveredRttCallback &&discovered_rtt_callback, asio::io_service &io_service) - : indexer_(indexer), - rtt_probes_(std::make_shared<ProbeHandler>(std::move(rtt_probes_callback), - io_service)), + : loss_history_(10), // log 10sec history + indexer_(indexer), + probe_handler_(std::make_shared<ProbeHandler>(std::move(probe_callback), + io_service)), discovered_rtt_callback_(std::move(discovered_rtt_callback)) { init_rtt_timer_ = std::make_unique<asio::steady_timer>(io_service); - initParams(); } RTCState::~RTCState() {} @@ -55,9 +55,18 @@ void RTCState::initParams() { highest_seq_received_in_order_ = 0; last_seq_nacked_ = 0; loss_rate_ = 0.0; - avg_loss_rate_ = 0.0; - max_loss_rate_ = 0.0; + avg_loss_rate_ = -1.0; last_round_loss_rate_ = 0.0; + + // loss rate per sec + lost_per_sec_ = 0; + total_expected_packets_ = 0; + per_sec_loss_rate_ = 0.0; + + // residual losses counters + expected_packets_ = 0; + packets_sent_to_app_ = 0; + rounds_from_last_compute_ = 0; residual_loss_rate_ = 0.0; // fec counters @@ -66,19 +75,22 @@ void RTCState::initParams() { // bw counters received_bytes_ = 0; + received_fec_bytes_ = 0; + recovered_bytes_with_fec_ = 0; + avg_packet_size_ = INIT_PACKET_SIZE; production_rate_ = 0.0; received_rate_ = 0.0; + fec_recovered_rate_ = 0.0; // nack counter - nack_on_last_round_ = false; + past_nack_on_last_round_ = false; received_nacks_last_round_ = 0; // packets counter received_packets_last_round_ = 0; received_data_last_round_ = 0; received_data_from_cache_ = 0; - data_from_cache_rate_ = 0; sent_interests_last_round_ = 0; sent_rtx_last_round_ = 0; @@ -89,37 +101,37 @@ void RTCState::initParams() { last_production_seq_ = 0; producer_is_active_ = false; - last_prod_update_ = 0; + last_prod_update_seq_ = 0; // paths stats path_table_.clear(); main_path_ = nullptr; + edge_path_ = nullptr; - // packet received - received_or_lost_packets_.clear(); + // packet cache (not pending anymore) + packet_cache_.clear(); // pending interests pending_interests_.clear(); - // skipped interest + // used to keep track of the skipped interest last_interest_sent_ = 0; - skipped_interests_.clear(); // init rtt first_interest_sent_time_ = ~0; first_interest_sent_seq_ = 0; + // start probing the producer init_rtt_ = false; - rtt_probes_->setProbes(INIT_RTT_PROBE_INTERVAL, INIT_RTT_PROBES); - rtt_probes_->sendProbes(); + probe_handler_->setSuffixRange(MIN_INIT_PROBE_SEQ, MAX_INIT_PROBE_SEQ); + probe_handler_->setProbes(INIT_RTT_PROBE_INTERVAL, INIT_RTT_PROBES); + probe_handler_->sendProbes(); setInitRttTimer(INIT_RTT_PROBE_RESTART); } // packet events void RTCState::onSendNewInterest(const core::Name *interest_name) { - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + uint64_t now = utils::SteadyTime::nowMs().count(); uint32_t seq = interest_name->getSuffix(); pending_interests_.insert(std::pair<uint32_t, uint64_t>(seq, now)); @@ -137,11 +149,12 @@ void RTCState::onSendNewInterest(const core::Name *interest_name) { } // TODO what happen in case of jumps? - // look for skipped interests - skipped_interests_.erase(seq); // remove seq if it is there + eraseFromPacketCache( + seq); // if we send this interest we don't know its state for (uint32_t i = last_interest_sent_ + 1; i < seq; i++) { if (indexer_->isFec(i)) { - skipped_interests_.insert(i); + // only fec packets can be skipped + addToPacketCache(i, PacketState::SKIPPED); } } @@ -155,6 +168,7 @@ void RTCState::onTimeout(uint32_t seq, bool lost) { auto it = pending_interests_.find(seq); if (it != pending_interests_.end()) { pending_interests_.erase(it); + if (indexer_->isFec(seq)) pending_fec_pkt_--; } received_timeouts_++; @@ -162,11 +176,14 @@ void RTCState::onTimeout(uint32_t seq, bool lost) { } void RTCState::onLossDetected(uint32_t seq) { - if (!indexer_->isFec(seq)) { - packets_lost_++; - } else if (skipped_interests_.find(seq) == skipped_interests_.end() && - seq >= first_interest_sent_seq_) { + PacketState state = getPacketState(seq); + + // if the packet is already marked with a state, do nothing + // to be considered lost the packet must be pending + if (state == PacketState::UNKNOWN && + pending_interests_.find(seq) != pending_interests_.end()) { packets_lost_++; + addToPacketCache(seq, PacketState::LOST); } } @@ -178,38 +195,46 @@ void RTCState::onRetransmission(uint32_t seq) { auto it = pending_interests_.find(seq); if (it != pending_interests_.end()) { pending_interests_.erase(it); -#if 0 - packets_lost_++; -#endif + if (indexer_->isFec(seq)) pending_fec_pkt_--; } sent_rtx_++; sent_rtx_last_round_++; } +void RTCState::onPossibleLossWithNoRtx(uint32_t seq) { + // if fec is on or rtx is disable we don't need to do anything to recover a + // packet. however in both cases we need to remove possible missing packets + // from the window of pendinig interest in order to free space without wating + // for the timeout. + auto it = pending_interests_.find(seq); + if (it != pending_interests_.end()) { + pending_interests_.erase(it); + if (indexer_->isFec(seq)) pending_fec_pkt_--; + } +} + void RTCState::onDataPacketReceived(const core::ContentObject &content_object, bool compute_stats) { uint32_t seq = content_object.getName().getSuffix(); + if (compute_stats) { updatePathStats(content_object, false); received_data_last_round_++; } received_data_++; + packets_sent_to_app_++; - struct data_packet_t *data_pkt = - (struct data_packet_t *)content_object.getPayload()->data(); - uint64_t production_time = data_pkt->getTimestamp(); - if (last_prod_update_ < production_time) { - last_prod_update_ = production_time; - uint32_t production_rate = data_pkt->getProductionRate(); - production_rate_ = (double)production_rate; + core::ParamsRTC params = RTCState::getDataParams(content_object); + + if (last_prod_update_seq_ < seq) { + last_prod_update_seq_ = seq; + production_rate_ = (double)params.prod_rate; } updatePacketSize(content_object); - updateReceivedBytes(content_object); + updateReceivedBytes(content_object, false); addRecvOrLost(seq, PacketState::RECEIVED); - if (seq > highest_seq_received_) highest_seq_received_ = seq; - // the producer is responding // it is generating valid data packets so we consider it active producer_is_active_ = true; @@ -219,9 +244,14 @@ void RTCState::onDataPacketReceived(const core::ContentObject &content_object, void RTCState::onFecPacketReceived(const core::ContentObject &content_object) { uint32_t seq = content_object.getName().getSuffix(); - updateReceivedBytes(content_object); + updateReceivedBytes(content_object, true); + + PacketState state = getPacketState(seq); + if (state != PacketState::LOST) { + // increase only for not lost packets + received_fec_pkt_++; + } addRecvOrLost(seq, PacketState::RECEIVED); - received_fec_pkt_++; // the producer is responding // it is generating valid data packets so we consider it active producer_is_active_ = true; @@ -232,14 +262,12 @@ void RTCState::onNackPacketReceived(const core::ContentObject &nack, uint32_t seq = nack.getName().getSuffix(); struct nack_packet_t *nack_pkt = (struct nack_packet_t *)nack.getPayload()->data(); - uint64_t production_time = nack_pkt->getTimestamp(); - uint32_t production_seq = nack_pkt->getProductionSegement(); + uint32_t production_seq = nack_pkt->getProductionSegment(); uint32_t production_rate = nack_pkt->getProductionRate(); if (TRANSPORT_EXPECT_FALSE(main_path_ == nullptr) || - last_prod_update_ < production_time) { + last_prod_update_seq_ < production_seq) { // update production rate - last_prod_update_ = production_time; last_production_seq_ = production_seq; production_rate_ = (double)production_rate; } @@ -247,7 +275,6 @@ void RTCState::onNackPacketReceived(const core::ContentObject &nack, if (compute_stats) { // this is not an RTX updatePathStats(nack, true); - nack_on_last_round_ = true; } // for statistics pourpose we log all nacks, also the one received for @@ -255,83 +282,132 @@ void RTCState::onNackPacketReceived(const core::ContentObject &nack, received_nacks_++; received_nacks_last_round_++; + bool to_delete = false; if (production_seq > seq) { // old nack, seq is lost // update last nacked if (last_seq_nacked_ < seq) last_seq_nacked_ = seq; DLOG_IF(INFO, VLOG_IS_ON(3)) << "lost packet " << seq << " beacuse of a past nack"; + if (compute_stats) past_nack_on_last_round_ = true; onPacketLost(seq); } else if (seq > production_seq) { // future nack // remove the nack from the pending interest map // (the packet is not received/lost yet) - if (indexer_->isFec(seq)) pending_fec_pkt_--; - pending_interests_.erase(seq); + to_delete = true; } else { // this should be a quite rear event. simply remove the // packet from the pending interest list - pending_interests_.erase(seq); + to_delete = true; } - // the producer is responding - // we consider it active only if the production rate is not 0 - // or the production sequence number is not 1 - if (production_rate_ != 0 || production_seq != 1) { - producer_is_active_ = true; + if (to_delete) { + auto it = pending_interests_.find(seq); + if (it != pending_interests_.end()) { + pending_interests_.erase(it); + if (indexer_->isFec(seq)) pending_fec_pkt_--; + } } received_packets_last_round_++; } void RTCState::onPacketLost(uint32_t seq) { -#if 0 - DLOG_IF(INFO, VLOG_IS_ON(3)) << "packet " << seq << " is lost"; - auto it = pending_interests_.find(seq); - if (it != pending_interests_.end()) { - // this packet was never retransmitted so it does - // not appear in the loss count - packets_lost_++; - } -#endif if (!indexer_->isFec(seq)) { - definitely_lost_pkt_++; - DLOG_IF(INFO, VLOG_IS_ON(4)) << "packet " << seq << " is lost"; + PacketState state = getPacketState(seq); + if (state == PacketState::LOST || + (state == PacketState::UNKNOWN && + pending_interests_.find(seq) != pending_interests_.end())) { + definitely_lost_pkt_++; + DLOG_IF(INFO, VLOG_IS_ON(4)) << "packet " << seq << " is lost"; + } } - addRecvOrLost(seq, PacketState::LOST); + + addRecvOrLost(seq, PacketState::DEFINITELY_LOST); } -void RTCState::onPacketRecoveredRtx(uint32_t seq) { - losses_recovered_++; +void RTCState::onPacketRecoveredRtx(const core::ContentObject &content_object, + uint64_t rtt) { + uint32_t seq = content_object.getName().getSuffix(); + packets_sent_to_app_++; + + // increase the recovered packet counter only if the packet was marked as LOST + // before. + PacketState state = getPacketState(seq); + if (state == PacketState::LOST) losses_recovered_++; + addRecvOrLost(seq, PacketState::RECEIVED); + updateReceivedBytes(content_object, false); + + if (rtt == 0) return; // nothing to do + + uint32_t path_label = content_object.getPathLabel(); + auto path_it = path_table_.find(path_label); + if (path_it == path_table_.end()) { + // this is a new path and it must be a cache + std::shared_ptr<RTCDataPath> newPath = + std::make_shared<RTCDataPath>(path_label); + auto ret = path_table_.insert( + std::pair<uint32_t, std::shared_ptr<RTCDataPath>>(path_label, newPath)); + path_it = ret.first; + } + + auto path = path_it->second; + if (path->pathToProducer()) + return; // this packet is coming from a producer + // even if we sent an RTX. this may happen + // for RTX that are sent too fast or in + // case of multipath + + path->insertRttSample(utils::SteadyTime::Milliseconds(rtt), true); } -void RTCState::onPacketRecoveredFec(uint32_t seq) { +void RTCState::onFecPacketRecoveredRtx( + const core::ContentObject &content_object) { + // This is the same as onPacketRecoveredRtx, but in this is case the + // pkt is also a FEC pkt, the addRecvOrLost will be called afterwards losses_recovered_++; + updateReceivedBytes(content_object, true); +} + +void RTCState::onPacketRecoveredFec(uint32_t seq, uint32_t size) { + losses_recovered_++; + packets_sent_to_app_++; + recovered_bytes_with_fec_ += size; + + // adding header to the count + recovered_bytes_with_fec_ += 60; // XXX get header size some where + + // the packet could be not marked as lost yet. onLossDetected checks if add in + // the packet in the lost count or not + onLossDetected(seq); + addRecvOrLost(seq, PacketState::RECEIVED); } bool RTCState::onProbePacketReceived(const core::ContentObject &probe) { uint32_t seq = probe.getName().getSuffix(); - uint64_t rtt; + core::ParamsRTC params = RTCState::getProbeParams(probe); - rtt = rtt_probes_->getRtt(seq); + bool is_valid = true; + uint32_t max = UINT32_MAX; + if (params.prod_rate == max) is_valid = false; + uint64_t rtt; + rtt = probe_handler_->getRtt(seq, is_valid); if (rtt == 0) return false; // this is not a valid probe - // like for data and nacks update the path stats. Here the RTT is computed - // by the probe handler. Both probes for rtt and bw are good to esimate - // info on the path - uint32_t path_label = probe.getPathLabel(); + if (!is_valid) return false; // not a valid probe - auto path_it = path_table_.find(path_label); + // if we are here the producer is active + producer_is_active_ = true; - // update production rate and last_seq_nacked like in case of a nack - struct nack_packet_t *probe_pkt = - (struct nack_packet_t *)probe.getPayload()->data(); - uint64_t sender_timestamp = probe_pkt->getTimestamp(); - uint32_t production_seq = probe_pkt->getProductionSegement(); - uint32_t production_rate = probe_pkt->getProductionRate(); + // Like for data and nacks update the path stats. Here the RTT is computed + // by the probe handler. Both probes for rtt and bw are good to estimate + // info on the path. + uint32_t path_label = probe.getPathLabel(); + auto path_it = path_table_.find(path_label); if (path_it == path_table_.end()) { // found a new path @@ -344,27 +420,17 @@ bool RTCState::onProbePacketReceived(const core::ContentObject &probe) { auto path = path_it->second; - path->insertRttSample(rtt); + path->insertRttSample(utils::SteadyTime::Milliseconds(rtt), true); path->receivedNack(); - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + uint64_t now = utils::SteadyTime::nowMs().count(); - int64_t OWD = now - sender_timestamp; + int64_t OWD = now - params.timestamp; path->insertOwdSample(OWD); - if (last_prod_update_ < sender_timestamp) { - last_production_seq_ = production_seq; - last_prod_update_ = sender_timestamp; - production_rate_ = (double)production_rate; - } - - // the producer is responding - // we consider it active only if the production rate is not 0 - // or the production sequence numner is not 1 - if (production_rate_ != 0 || production_seq != 1) { - producer_is_active_ = true; + if (last_prod_update_seq_ < params.prod_seg) { + last_production_seq_ = params.prod_seg; + production_rate_ = (double)params.prod_rate; } // check for init RTT. if received_probes_ is equal to 0 schedule a timer to @@ -375,7 +441,7 @@ bool RTCState::onProbePacketReceived(const core::ContentObject &probe) { if (!init_rtt_ && received_probes_ <= INIT_RTT_PROBES) { if (received_probes_ == 1) { // we got the first probe, wait at most INIT_RTT_PROBE_WAIT sec for the - // others + // others. main_path_ = path; setInitRttTimer(INIT_RTT_PROBE_WAIT); } @@ -393,11 +459,21 @@ bool RTCState::onProbePacketReceived(const core::ContentObject &probe) { return true; } -void RTCState::onNewRound(double round_len, bool in_sync) { - // XXX - // here we take into account only the single path case so we assume that we - // don't use two paths in parellel for this single flow +void RTCState::onJumpForward(uint32_t next_seq) { + for (uint32_t seq = highest_seq_received_in_order_ + 1; seq < next_seq; + seq++) { + PacketState packet_state = getPacketState(seq); + if (packet_state != PacketState::RECEIVED && + packet_state != PacketState::DEFINITELY_LOST) { + // here we considere the packet as definitely lost whitout increase the + // lost packet counter because this loss is not due to the network + // condition but the transport wants to skip the packet + onPacketLost(seq); + } + } +} +void RTCState::onNewRound(double round_len, bool in_sync) { if (path_table_.empty()) return; double bytes_per_sec = @@ -407,33 +483,65 @@ void RTCState::onNewRound(double round_len, bool in_sync) { else received_rate_ = (received_rate_ * MOVING_AVG_ALPHA) + ((1 - MOVING_AVG_ALPHA) * bytes_per_sec); + double fec_bytes_per_sec = + ((double)received_fec_bytes_ * (MILLI_IN_A_SEC / round_len)); + + if (fec_received_rate_ == 0) + fec_received_rate_ = fec_bytes_per_sec; + else + fec_received_rate_ = (fec_received_rate_ * 0.8) + (0.2 * fec_bytes_per_sec); - // search for an active path. There should be only one active path (meaning a - // path that leads to the producer socket -no cache- and from which we are - // currently getting data packets) at any time. However it may happen that - // there are mulitple active paths in case of mobility (the old path will - // remain active for a short ammount of time). The main path is selected as - // the active path from where the consumer received the latest data packet + double fec_recovered_bytes_per_sec = + ((double)recovered_bytes_with_fec_ * (MILLI_IN_A_SEC / round_len)); - uint64_t last_packet_ts = 0; + if (fec_recovered_rate_ == 0) + fec_recovered_rate_ = fec_recovered_bytes_per_sec; + else + fec_recovered_rate_ = + (fec_recovered_rate_ * 0.8) + (0.2 * fec_recovered_bytes_per_sec); + + // search for an active path. Is it possible to have multiple path that are + // used at the same time. We use as reference path the one from where we gets + // more packets. This means that the path should have better lantecy or less + // channel losses + + uint32_t last_round_packets = 0; + uint64_t min_edge_rtt = UINT_MAX; + std::shared_ptr<RTCDataPath> old_main_path = main_path_; main_path_ = nullptr; + edge_path_ = nullptr; for (auto it = path_table_.begin(); it != path_table_.end(); it++) { - it->second->roundEnd(); - if (it->second->isActive()) { - uint64_t ts = it->second->getLastPacketTS(); - if (ts > last_packet_ts) { - last_packet_ts = ts; + if (it->second->isValidProducer()) { + uint32_t pkt = it->second->getPacketsLastRound(); + if (pkt > last_round_packets) { + last_round_packets = pkt; main_path_ = it->second; } + } else if (it->second->isActive() && !it->second->pathToProducer()) { + // this is a path to a cache from where we are receiving content + if (it->second->getMinRtt() < min_edge_rtt) { + min_edge_rtt = it->second->getMinRtt(); + edge_path_ = it->second; + } } + it->second->roundEnd(); } - // if (in_sync) updateLossRate(); - updateLossRate(); + if (main_path_ == nullptr) main_path_ = old_main_path; + if (edge_path_ == nullptr) edge_path_ = main_path_; + if (edge_path_->getMinRtt() >= main_path_->getMinRtt()) + edge_path_ = main_path_; + + // in case we get a new main path we reset the stats of the old one. this is + // beacuse, in case we need to switch back we don't what to take decisions on + // old stats that may be outdated. + if (main_path_ != old_main_path) old_main_path->clearRtt(); + + updateLossRate(in_sync); // handle nacks - if (!nack_on_last_round_ && received_bytes_ > 0) { + if (!past_nack_on_last_round_ && received_bytes_ > 0) { rounds_without_nacks_++; } else { rounds_without_nacks_ = 0; @@ -450,22 +558,16 @@ void RTCState::onNewRound(double round_len, bool in_sync) { } } - // compute cache/producer ratio - if (received_data_last_round_ != 0) { - double new_rate = - (double)received_data_from_cache_ / (double)received_data_last_round_; - data_from_cache_rate_ = data_from_cache_rate_ * MOVING_AVG_ALPHA + - (new_rate * (1 - MOVING_AVG_ALPHA)); - } - // reset counters received_bytes_ = 0; + received_fec_bytes_ = 0; + recovered_bytes_with_fec_ = 0; packets_lost_ = 0; definitely_lost_pkt_ = 0; losses_recovered_ = 0; first_seq_in_round_ = highest_seq_received_; - nack_on_last_round_ = false; + past_nack_on_last_round_ = false; received_nacks_last_round_ = 0; received_packets_last_round_ = 0; @@ -479,9 +581,15 @@ void RTCState::onNewRound(double round_len, bool in_sync) { rounds_++; } -void RTCState::updateReceivedBytes(const core::ContentObject &content_object) { - received_bytes_ += - (uint32_t)(content_object.headerSize() + content_object.payloadSize()); +void RTCState::updateReceivedBytes(const core::ContentObject &content_object, + bool isFec) { + if (isFec) { + received_fec_bytes_ += + (uint32_t)(content_object.headerSize() + content_object.payloadSize()); + } else { + received_bytes_ += + (uint32_t)(content_object.headerSize() + content_object.payloadSize()); + } } void RTCState::updatePacketSize(const core::ContentObject &content_object) { @@ -516,20 +624,16 @@ void RTCState::updatePathStats(const core::ContentObject &content_object, // it means that we are processing an interest // that is not pending - uint64_t now = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + uint64_t now = utils::SteadyTime::nowMs().count(); uint64_t RTT = now - interest_sent_time; - path->insertRttSample(RTT); + path->insertRttSample(utils::SteadyTime::Milliseconds(RTT), false); // compute OWD (the first part of the nack and data packet header are the // same, so we cast to data data packet) - struct data_packet_t *packet = - (struct data_packet_t *)content_object.getPayload()->data(); - uint64_t sender_timestamp = packet->getTimestamp(); - int64_t OWD = now - sender_timestamp; + core::ParamsRTC params = RTCState::getDataParams(content_object); + int64_t OWD = now - params.timestamp; path->insertOwdSample(OWD); // compute IAT or set path to producer @@ -543,59 +647,110 @@ void RTCState::updatePathStats(const core::ContentObject &content_object, } } -void RTCState::updateLossRate() { +void RTCState::updateLossRate(bool in_sync) { last_round_loss_rate_ = loss_rate_; loss_rate_ = 0.0; - residual_loss_rate_ = 0.0; uint32_t number_theorically_received_packets_ = highest_seq_received_ - first_seq_in_round_; - // in this case no new packet was recevied after the previuos round, avoid - // division by 0 - if (number_theorically_received_packets_ == 0) return; - // XXX this may be quite inefficient if the rate is high // maybe is better to iterate over the set? - for (uint32_t i = first_seq_in_round_; i < highest_seq_received_; i++) { - auto it = skipped_interests_.find(i); - if (it != skipped_interests_.end()) { + + uint32_t fec_packets = 0; + for (uint32_t i = (first_seq_in_round_ + 1); i < highest_seq_received_; i++) { + PacketState state = getPacketState(i); + if (state == PacketState::SKIPPED) { if (number_theorically_received_packets_ > 0) number_theorically_received_packets_--; - skipped_interests_.erase(it); } + if (indexer_->isFec(i)) fec_packets++; } + if (indexer_->isFec(highest_seq_received_)) fec_packets++; - loss_rate_ = (double)((double)(packets_lost_) / - (double)number_theorically_received_packets_); + // in this case no new packet was received after the previous round, avoid + // division by 0 + if (number_theorically_received_packets_ == 0 && packets_lost_ == 0) return; - if (rounds_ % 15 == 0) max_loss_rate_ = 0; // reset every 3 sec - if (loss_rate_ > max_loss_rate_) max_loss_rate_ = loss_rate_; + if (number_theorically_received_packets_ != 0) + loss_rate_ = (double)((double)(packets_lost_) / + (double)number_theorically_received_packets_); + else + // we didn't receive anything except NACKs that triggered losses + loss_rate_ = 1.0; - if (avg_loss_rate_ == 0) + if (avg_loss_rate_ == -1.0) avg_loss_rate_ = loss_rate_; else avg_loss_rate_ = avg_loss_rate_ * MOVING_AVG_ALPHA + loss_rate_ * (1 - MOVING_AVG_ALPHA); - residual_loss_rate_ = (double)((double)(packets_lost_ - losses_recovered_) / - (double)number_theorically_received_packets_); + // update counters for loss rate per second + total_expected_packets_ += number_theorically_received_packets_; + lost_per_sec_ += packets_lost_; + + if (in_sync) { + // update counters for residual losses + // fec packets are not sent to the app so we don't want to count them here + expected_packets_ += + ((highest_seq_received_ - first_seq_in_round_) - fec_packets); + } else { + expected_packets_ = 0; + packets_sent_to_app_ = 0; + } + + if (rounds_from_last_compute_ >= (MILLI_IN_A_SEC / ROUND_LEN)) { + // compute loss rate per second + if (lost_per_sec_ > total_expected_packets_) + lost_per_sec_ = total_expected_packets_; + + if (total_expected_packets_ == 0) + per_sec_loss_rate_ = 0; + else + per_sec_loss_rate_ = + (double)((double)(lost_per_sec_) / (double)total_expected_packets_); + + loss_history_.pushBack(per_sec_loss_rate_); + + if (in_sync && expected_packets_ != 0) { + // compute residual loss rate + if (packets_sent_to_app_ > expected_packets_) { + // this may happen if we get packet from the prev bin that get recovered + // on the current one + packets_sent_to_app_ = expected_packets_; + } + + residual_loss_rate_ = + 1.0 - ((double)packets_sent_to_app_ / (double)expected_packets_); + if (residual_loss_rate_ < 0.0) residual_loss_rate_ = 0.0; + } - if (residual_loss_rate_ < 0) residual_loss_rate_ = 0; + lost_per_sec_ = 0; + total_expected_packets_ = 0; + expected_packets_ = 0; + packets_sent_to_app_ = 0; + rounds_from_last_compute_ = 0; + } + + rounds_from_last_compute_++; +} + +void RTCState::dataToBeReceived(uint32_t seq) { + addToPacketCache(seq, PacketState::TO_BE_RECEIVED); +} + +void RTCState::updateHighestSeqReceived(uint32_t seq) { + if (seq > highest_seq_received_) highest_seq_received_ = seq; } void RTCState::addRecvOrLost(uint32_t seq, PacketState state) { - if (indexer_->isFec(seq)) { - pending_fec_pkt_--; + auto it = pending_interests_.find(seq); + if (it != pending_interests_.end()) { + pending_interests_.erase(it); + if (indexer_->isFec(seq)) pending_fec_pkt_--; } - pending_interests_.erase(seq); - if (received_or_lost_packets_.size() >= MAX_CACHED_PACKETS) { - received_or_lost_packets_.erase(received_or_lost_packets_.begin()); - } - // notice that it may happen that a packet that we consider lost arrives after - // some time, in this case we simply overwrite the packet state. - received_or_lost_packets_[seq] = state; + addToPacketCache(seq, state); // keep track of the last packet received/lost // without holes. @@ -608,16 +763,27 @@ void RTCState::addRecvOrLost(uint32_t seq, PacketState state) { } else if (seq <= highest_seq_received_in_order_) { // here we do nothing } else if (seq > highest_seq_received_in_order_) { - // 1) there is a gap in the sequence so we do not update largest_in_seq_ - // 2) all the packets from largest_in_seq_ to seq are in - // received_or_lost_packets_ an we upate largest_in_seq_ - // or are FEC packets + // 1) there is a gap in the sequence so we do not update + // highest_seq_received_in_order_ + // 2) all the packets from highest_seq_received_in_order_ to seq are + // received or lost or are fec packetis. In this case we increase + // highest_seq_received_in_order_ until we find an hole in the sequence for (uint32_t i = highest_seq_received_in_order_ + 1; i <= seq; i++) { - if (received_or_lost_packets_.find(i) == - received_or_lost_packets_.end() && - !indexer_->isFec(i)) { - break; + PacketState state = getPacketState(i); + if ((state == PacketState::UNKNOWN || state == PacketState::LOST)) { + if (indexer_->isFec(i)) { + // this is a fec packet and we don't care to receive it + // however we may need to increse the number or lost packets + // XXX: in case we want to use rtx to recover fec packets, + // this may prevent to detect a packet loss and no rtx will be sent + if (TRANSPORT_EXPECT_TRUE(i >= first_interest_sent_seq_)) { + onLossDetected(i); + } + } else { + // this is a data packet and we need to get it + break; + } } // this packet is in order so we can update the // highest_seq_received_in_order_ @@ -629,36 +795,104 @@ void RTCState::addRecvOrLost(uint32_t seq, PacketState state) { void RTCState::setInitRttTimer(uint32_t wait) { init_rtt_timer_->cancel(); init_rtt_timer_->expires_from_now(std::chrono::milliseconds(wait)); - init_rtt_timer_->async_wait([this](std::error_code ec) { + + std::weak_ptr<RTCState> self = shared_from_this(); + init_rtt_timer_->async_wait([self](const std::error_code &ec) { if (ec) return; - checkInitRttTimer(); + + if (auto ptr = self.lock()) { + ptr->checkInitRttTimer(); + } }); } void RTCState::checkInitRttTimer() { - if (received_probes_ < INIT_RTT_MIN_PROBES_TO_RECV) { - // we didn't received enough probes, restart + if (received_probes_ < INIT_RTT_MIN_PROBES_TO_RECV || + probe_handler_->getProbeLossRate() == 1.0) { + // we didn't received enough probes or they were not valid, restart received_probes_ = 0; - rtt_probes_->setProbes(INIT_RTT_PROBE_INTERVAL, INIT_RTT_PROBES); - rtt_probes_->sendProbes(); + probe_handler_->setSuffixRange(MIN_INIT_PROBE_SEQ, MAX_INIT_PROBE_SEQ); + probe_handler_->setProbes(INIT_RTT_PROBE_INTERVAL, INIT_RTT_PROBES); + probe_handler_->sendProbes(); setInitRttTimer(INIT_RTT_PROBE_RESTART); return; } + init_rtt_ = true; main_path_->roundEnd(); - rtt_probes_->setProbes(RTT_PROBE_INTERVAL, 0); - rtt_probes_->sendProbes(); + loss_history_.pushBack(probe_handler_->getProbeLossRate()); + + probe_handler_->setSuffixRange(MIN_RTT_PROBE_SEQ, MAX_RTT_PROBE_SEQ); + probe_handler_->setProbes(RTT_PROBE_INTERVAL, 0); + probe_handler_->sendProbes(); // init last_seq_nacked_. skip packets that may come from the cache double prod_rate = getProducerRate(); - double rtt = (double)getRTT() / MILLI_IN_A_SEC; + double rtt = (double)getMinRTT() / MILLI_IN_A_SEC; double packet_size = getAveragePacketSize(); - uint32_t pkt_in_rtt_ = std::floor(((prod_rate / packet_size) * rtt) * 0.8); + uint32_t pkt_in_rtt_ = std::floor(((prod_rate / packet_size) * rtt)); last_seq_nacked_ = last_production_seq_ + pkt_in_rtt_; discovered_rtt_callback_(); } +core::ParamsRTC RTCState::getProbeParams(const core::ContentObject &probe) { + uint32_t seq = probe.getName().getSuffix(); + core::ParamsRTC params; + + switch (ProbeHandler::getProbeType(seq)) { + case ProbeType::INIT: { + core::ContentObjectManifest manifest( + const_cast<core::ContentObject &>(probe).shared_from_this()); + manifest.decode(); + params = manifest.getParamsRTC(); + break; + } + case ProbeType::RTT: { + struct nack_packet_t *probe_pkt = + (struct nack_packet_t *)probe.getPayload()->data(); + params = core::ParamsRTC{ + .timestamp = probe_pkt->getTimestamp(), + .prod_rate = probe_pkt->getProductionRate(), + .prod_seg = probe_pkt->getProductionSegment(), + }; + break; + } + default: + break; + } + + return params; +} + +core::ParamsRTC RTCState::getDataParams(const core::ContentObject &data) { + core::ParamsRTC params; + + switch (data.getPayloadType()) { + case core::PayloadType::DATA: { + struct data_packet_t *data_pkt = + (struct data_packet_t *)data.getPayload()->data(); + params = core::ParamsRTC{ + .timestamp = data_pkt->getTimestamp(), + .prod_rate = data_pkt->getProductionRate(), + .prod_seg = data.getName().getSuffix(), + }; + break; + } + case core::PayloadType::MANIFEST: { + core::ContentObjectManifest manifest( + const_cast<core::ContentObject &>(data).shared_from_this()); + manifest.decode(); + params = manifest.getParamsRTC(); + break; + } + default: + break; + } + + return params; +} + } // namespace rtc } // namespace protocol diff --git a/libtransport/src/protocols/rtc/rtc_state.h b/libtransport/src/protocols/rtc/rtc_state.h index 729ba7a1b..ac3cc621f 100644 --- a/libtransport/src/protocols/rtc/rtc_state.h +++ b/libtransport/src/protocols/rtc/rtc_state.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -14,13 +14,16 @@ */ #pragma once +#include <core/facade.h> #include <hicn/transport/config.h> #include <hicn/transport/core/asio_wrapper.h> #include <hicn/transport/core/content_object.h> #include <hicn/transport/core/name.h> +#include <hicn/transport/utils/rtc_quality_score.h> #include <protocols/indexer.h> #include <protocols/rtc/probe_handler.h> #include <protocols/rtc/rtc_data_path.h> +#include <utils/max_filter.h> #include <map> #include <set> @@ -31,34 +34,62 @@ namespace protocol { namespace rtc { -enum class PacketState : uint8_t { RECEIVED, LOST, UNKNOWN }; +// packet state +// RECEIVED: the packet was already received +// LOST: the packet is marked as lost but can be recovered +// DEFINITELY_LOST: the packet is lost and cannot be recovered +// TO_BE_RECEIVED: when a packet is received is sent to the FEC decoder. the fec +// decoder may decide to send the packet directly to the app. to avoid +// duplicated the packet is marked with this state +// SKIPPED: an interest that was not sent, only for FEC packets +// UNKNOWN: unknown state +enum class PacketState : uint8_t { + RECEIVED, + TO_BE_RECEIVED, + LOST, + DEFINITELY_LOST, + SKIPPED, + UNKNOWN +}; + +class RTCState : public std::enable_shared_from_this<RTCState> { + using PendingInterestsMap = std::map<uint32_t, uint64_t>; + + private: + const double MAX_CACHED_PACKETS = 8192; // XXX this value may be too small + // for high rate apps -class RTCState : std::enable_shared_from_this<RTCState> { public: using DiscoveredRttCallback = std::function<void()>; public: - RTCState(Indexer *indexer, - ProbeHandler::SendProbeCallback &&rtt_probes_callback, + RTCState(Indexer *indexer, ProbeHandler::SendProbeCallback &&probe_callback, DiscoveredRttCallback &&discovered_rtt_callback, asio::io_service &io_service); ~RTCState(); + // initialization + void initParams(); + // packet events void onSendNewInterest(const core::Name *interest_name); void onTimeout(uint32_t seq, bool lost); void onLossDetected(uint32_t seq); void onRetransmission(uint32_t seq); + void onPossibleLossWithNoRtx(uint32_t seq); void onDataPacketReceived(const core::ContentObject &content_object, bool compute_stats); void onFecPacketReceived(const core::ContentObject &content_object); void onNackPacketReceived(const core::ContentObject &nack, bool compute_stats); void onPacketLost(uint32_t seq); - void onPacketRecoveredRtx(uint32_t seq); - void onPacketRecoveredFec(uint32_t seq); + void onPacketRecoveredRtx(const core::ContentObject &content_object, + uint64_t rtt); + void onFecPacketRecoveredRtx(const core::ContentObject &content_object); + void onPacketRecoveredFec(uint32_t seq, uint32_t size); bool onProbePacketReceived(const core::ContentObject &probe); + void onJumpForward(uint32_t next_seq); // protocol state void onNewRound(double round_len, bool in_sync); @@ -72,10 +103,26 @@ class RTCState : std::enable_shared_from_this<RTCState> { // delay metrics bool isRttDiscovered() const { return init_rtt_; } - uint64_t getRTT() const { + uint64_t getMinRTT() const { if (mainPathIsValid()) return main_path_->getMinRtt(); return 0; } + + uint64_t getAvgRTT() const { + if (mainPathIsValid()) return main_path_->getAvgRtt(); + return 0; + } + + uint64_t getMaxRTT() const { + if (mainPathIsValid()) return main_path_->getMaxRtt(); + return 0; + } + + uint64_t getEdgeRtt() const { + if (edge_path_ != nullptr) return edge_path_->getMinRtt(); + return 0; + } + void resetRttStats() { if (mainPathIsValid()) main_path_->clearRtt(); } @@ -98,6 +145,7 @@ class RTCState : std::enable_shared_from_this<RTCState> { uint64_t getInterestSentTime(uint32_t seq) { auto it = pending_interests_.find(seq); if (it != pending_interests_.end()) return it->second; + return 0; } @@ -107,19 +155,24 @@ class RTCState : std::enable_shared_from_this<RTCState> { } uint32_t getPendingInterestNumber() const { - return pending_interests_.size(); + return (uint32_t)pending_interests_.size(); } - PacketState isReceivedOrLost(uint32_t seq) { - auto it = received_or_lost_packets_.find(seq); - if (it != received_or_lost_packets_.end()) return it->second; + PacketState getPacketState(uint32_t seq) { + auto it = packet_cache_.find(seq); + if (it != packet_cache_.end()) return it->second; return PacketState::UNKNOWN; } // loss rate - double getLossRate() const { return loss_rate_; } + double getPerRoundLossRate() const { return loss_rate_; } + double getPerSecondLossRate() const { return per_sec_loss_rate_; } double getAvgLossRate() const { return avg_loss_rate_; } - double getMaxLossRate() const { return max_loss_rate_; } + double getMaxLossRate() const { + if (loss_history_.size() != 0) return loss_history_.begin(); + return 0; + } + double getLastRoundLossRate() const { return last_round_loss_rate_; } double getResidualLossRate() const { return residual_loss_rate_; } @@ -140,9 +193,14 @@ class RTCState : std::enable_shared_from_this<RTCState> { // generic stats uint32_t getReceivedBytesInRound() const { return received_bytes_; } + uint32_t getReceivedFecBytesInRound() const { return received_fec_bytes_; } + uint32_t getRecoveredFecBytesInRound() const { + return recovered_bytes_with_fec_; + } uint32_t getReceivedNacksInRound() const { return received_nacks_last_round_; } + uint32_t getReceivedDataInRound() const { return received_data_last_round_; } uint32_t getSentInterestInRound() const { return sent_interests_last_round_; } uint32_t getSentRtxInRound() const { return sent_rtx_last_round_; } @@ -150,6 +208,9 @@ class RTCState : std::enable_shared_from_this<RTCState> { double getAvailableBw() const { return 0.0; }; // TODO double getProducerRate() const { return production_rate_; } double getReceivedRate() const { return received_rate_; } + double getReceivedFecRate() const { return fec_received_rate_; } + double getRecoveredFecRate() const { return fec_recovered_rate_; } + double getAveragePacketSize() const { return avg_packet_size_; } // nacks @@ -160,24 +221,57 @@ class RTCState : std::enable_shared_from_this<RTCState> { bool isProducerActive() const { return producer_is_active_; } // packets from cache - double getPacketFromCacheRatio() const { return data_from_cache_rate_; } + // this should be called at the end of a round beacuse otherwise we may have + // not enough packets to get a good stat + double getPacketFromCacheRatio() const { + if (received_data_last_round_ == 0) return 0; + return (double)received_data_from_cache_ / + (double)received_data_last_round_; + } - std::map<uint32_t, uint64_t>::iterator getPendingInterestsMapBegin() { + PendingInterestsMap::iterator getPendingInterestsMapBegin() { return pending_interests_.begin(); } - std::map<uint32_t, uint64_t>::iterator getPendingInterestsMapEnd() { + PendingInterestsMap::iterator getPendingInterestsMapEnd() { return pending_interests_.end(); } + // quality + uint8_t getQualityScore() { + uint8_t qs = quality_score_.getQualityScore( + getMaxRTT(), std::round(getResidualLossRate() * 100)); + return qs; + } + + // We received a data pkt that will be set to RECEIVED, but first we have to + // go through FEC. We do not want to consider this pkt as recovered, thus we + // set it as TO_BE_RECEIVED. + void dataToBeReceived(uint32_t seq); + + void updateHighestSeqReceived(uint32_t seq); + + // Extract RTC parameters from probes (init or RTT probes) and data packets. + static core::ParamsRTC getProbeParams(const core::ContentObject &probe); + static core::ParamsRTC getDataParams(const core::ContentObject &data); + private: - void initParams(); + void addToPacketCache(uint32_t seq, PacketState state) { + // this function adds or updates the current state + if (packet_cache_.size() >= MAX_CACHED_PACKETS) { + packet_cache_.erase(packet_cache_.begin()); + } + packet_cache_[seq] = state; + } + + void eraseFromPacketCache(uint32_t seq) { packet_cache_.erase(seq); } // update stats void updateState(); - void updateReceivedBytes(const core::ContentObject &content_object); + void updateReceivedBytes(const core::ContentObject &content_object, + bool isFec); void updatePacketSize(const core::ContentObject &content_object); void updatePathStats(const core::ContentObject &content_object, bool is_nack); - void updateLossRate(); + void updateLossRate(bool in_sycn); void addRecvOrLost(uint32_t seq, PacketState state); @@ -209,36 +303,51 @@ class RTCState : std::enable_shared_from_this<RTCState> { uint32_t last_seq_nacked_; // segment for which we got an oldNack double loss_rate_; double avg_loss_rate_; - double max_loss_rate_; double last_round_loss_rate_; + utils::MaxFilter<double> loss_history_; + + // per second loss rate + uint32_t lost_per_sec_; + uint32_t total_expected_packets_; + double per_sec_loss_rate_; + + // conunters for residual losses + // residual losses are computed every second and are used + // as feedback to the upper levels (e.g application) + uint32_t expected_packets_; + uint32_t packets_sent_to_app_; + uint32_t rounds_from_last_compute_; double residual_loss_rate_; // bw counters uint32_t received_bytes_; + uint32_t received_fec_bytes_; + uint32_t recovered_bytes_with_fec_; double avg_packet_size_; - double production_rate_; // rate communicated by the producer using nacks - double received_rate_; // rate recevied by the consumer - - // nack counter - // the bool takes tracks only about the valid nacks (no rtx) and it is used to - // switch between the states. Instead received_nacks_last_round_ logs all the - // nacks for statistics - bool nack_on_last_round_; + double production_rate_; // rate communicated by the producer using nacks + double received_rate_; // rate recevied by the consumer (only data) + double fec_received_rate_; // fec rate recevied by the consumer + double fec_recovered_rate_; // rate recovered using fec + + // nack counters + // the bool takes tracks only about the valid past nacks (no rtx) and it is + // used to switch between the states. Instead received_nacks_last_round_ logs + // all the nacks for statistics + bool past_nack_on_last_round_; uint32_t received_nacks_last_round_; - // packets counter + // packets counters uint32_t received_packets_last_round_; uint32_t received_data_last_round_; uint32_t received_data_from_cache_; - double data_from_cache_rate_; uint32_t sent_interests_last_round_; uint32_t sent_rtx_last_round_; - // fec counter + // fec counters uint32_t received_fec_pkt_; uint32_t pending_fec_pkt_; - // round conunters + // round counters uint32_t rounds_; uint32_t rounds_without_nacks_; uint32_t rounds_without_packets_; @@ -250,34 +359,46 @@ class RTCState : std::enable_shared_from_this<RTCState> { // producer state bool producer_is_active_; // the prodcuer is active if we receive some packets - uint32_t - last_production_seq_; // last production seq received by the producer - uint64_t last_prod_update_; // timestamp of the last packets used to update - // stats from the producer + uint32_t last_production_seq_; // last production seq received by the + // producer used to init the sync protcol + uint32_t last_prod_update_seq_; // seq number of the last packet used to + // update the update from the producer. + // assumption: the highest seq number carries + // the most up to date info. in case of + // probes we look at the produced seq number // paths stats std::unordered_map<uint32_t, std::shared_ptr<RTCDataPath>> path_table_; - std::shared_ptr<RTCDataPath> main_path_; + std::shared_ptr<RTCDataPath> main_path_; // this is the path that connects + // the consumer to the producer. in + // case of multipath the trasnport + // uses the most active path + std::shared_ptr<RTCDataPath> edge_path_; // path to the closest cache if it + // exists // packet received // cache where to store info about the last MAX_CACHED_PACKETS - std::map<uint32_t, PacketState> received_or_lost_packets_; + // these are packets that are received or lost or definitely lost and are not + // anymore in the pending intetest list + std::map<uint32_t, PacketState> packet_cache_; // pending interests - std::map<uint32_t, uint64_t> pending_interests_; + PendingInterestsMap pending_interests_; // indexer Indexer *indexer_; - // skipped interests + // used to keep track of the skipped interests uint32_t last_interest_sent_; - std::unordered_set<uint32_t> skipped_interests_; // probes - std::shared_ptr<ProbeHandler> rtt_probes_; + std::shared_ptr<ProbeHandler> probe_handler_; bool init_rtt_; std::unique_ptr<asio::steady_timer> init_rtt_timer_; + // quality score + RTCQualityScore quality_score_; + // callbacks DiscoveredRttCallback discovered_rtt_callback_; }; diff --git a/libtransport/src/protocols/rtc/rtc_verifier.cc b/libtransport/src/protocols/rtc/rtc_verifier.cc new file mode 100644 index 000000000..60fce92a5 --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_verifier.cc @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2017-2022 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <core/facade.h> +#include <protocols/rtc/rtc_packet.h> +#include <protocols/rtc/rtc_verifier.h> + +namespace transport { +namespace protocol { +namespace rtc { + +RTCVerifier::RTCVerifier(std::shared_ptr<auth::Verifier> verifier, + uint32_t factor_relevant, uint32_t factor_alert) + : verifier_(verifier), + factor_relevant_(factor_relevant), + factor_alert_(factor_alert), + manifest_max_capacity_(std::numeric_limits<uint8_t>::max()) {} + +void RTCVerifier::setState(std::shared_ptr<RTCState> rtc_state) { + rtc_state_ = rtc_state; +} + +void RTCVerifier::setVerifier(std::shared_ptr<auth::Verifier> verifier) { + verifier_ = verifier; +} + +void RTCVerifier::setFactorRelevant(uint32_t factor_relevant) { + factor_relevant_ = factor_relevant; +} + +void RTCVerifier::setFactorAlert(uint32_t factor_alert) { + factor_alert_ = factor_alert; +} + +auth::VerificationPolicy RTCVerifier::verify(core::Interest &interest) { + return verifier_->verifyPackets(&interest); +} + +auth::VerificationPolicy RTCVerifier::verify( + core::ContentObject &content_object, bool is_fec) { + auth::Suffix suffix = content_object.getName().getSuffix(); + auth::VerificationPolicy default_policy = auth::VerificationPolicy::ABORT; + + core::PayloadType payload_type = content_object.getPayloadType(); + bool is_probe = ProbeHandler::getProbeType(suffix) != ProbeType::NOT_PROBE; + bool is_nack = !is_probe && content_object.payloadSize() == NACK_HEADER_SIZE; + bool is_manifest = !is_probe && !is_nack && !is_fec && + payload_type == core::PayloadType::MANIFEST; + bool is_data = !is_probe && !is_nack && !is_fec && + payload_type == core::PayloadType::DATA; + + if (is_probe) return verifyProbe(content_object); + if (is_nack) return verifyNack(content_object); + if (is_fec) return verifyFec(content_object); + if (is_data) return verifyData(content_object); + if (is_manifest) return verifyManifest(content_object); + + verifier_->callVerificationFailedCallback(suffix, default_policy); + return default_policy; +} + +auth::VerificationPolicy RTCVerifier::verifyProbe( + core::ContentObject &content_object) { + auth::Suffix suffix = content_object.getName().getSuffix(); + auth::VerificationPolicy policy = auth::VerificationPolicy::ABORT; + + switch (ProbeHandler::getProbeType(suffix)) { + case ProbeType::INIT: + policy = verifyManifest(content_object); + if (policy == auth::VerificationPolicy::ACCEPT) { + policy = processManifest(content_object); + } + break; + case ProbeType::RTT: + policy = verifyNack(content_object); + break; + default: + verifier_->callVerificationFailedCallback(suffix, policy); + break; + } + + return policy; +} + +auth::VerificationPolicy RTCVerifier::verifyNack( + core::ContentObject &content_object) { + return verifier_->verifyPackets(&content_object); +} + +auth::VerificationPolicy RTCVerifier::verifyFec( + core::ContentObject &content_object) { + return verifier_->verifyPackets(&content_object); +} + +auth::VerificationPolicy RTCVerifier::verifyData( + core::ContentObject &content_object) { + if (HICN_PACKET_FORMAT_IS_AH(content_object.getFormat())) { + return verifier_->verifyPackets(&content_object); + } + + auth::Suffix suffix = content_object.getName().getSuffix(); + auth::VerificationPolicy policy = auth::VerificationPolicy::ABORT; + + uint32_t threshold_relevant = factor_relevant_ * manifest_max_capacity_; + uint32_t threshold_alert = factor_alert_ * manifest_max_capacity_; + + // Flush packets outside relevance window + for (auto it = packets_unverif_.set().begin(); + it != packets_unverif_.set().end();) { + if (it->first > current_index_ - threshold_relevant) { + break; + } + packets_unverif_erased_.insert((unsigned int)it->first); + it = packets_unverif_.remove(it); + } + + // Add packet to set of unverified packets + packets_unverif_.add({current_index_, suffix}, + content_object.computeDigest(manifest_hash_algo_)); + current_index_++; + + // Check that the number of unverified packets is below the alert threshold + if (packets_unverif_.set().size() <= threshold_alert) { + policy = auth::VerificationPolicy::ACCEPT; + } + + verifier_->callVerificationFailedCallback(suffix, policy); + return policy; +} + +auth::VerificationPolicy RTCVerifier::verifyManifest( + core::ContentObject &content_object) { + return verifier_->verifyPackets(&content_object); +} + +auth::VerificationPolicy RTCVerifier::processManifest( + core::ContentObject &content_object) { + auth::Suffix suffix = content_object.getName().getSuffix(); + auth::VerificationPolicy accept_policy = auth::VerificationPolicy::ACCEPT; + + // Decode manifest + core::ContentObjectManifest manifest(content_object.shared_from_this()); + manifest.decode(); + + // Extract manifest data + manifest_max_capacity_ = manifest.getMaxCapacity(); + manifest_hash_algo_ = manifest.getHashAlgorithm(); + auth::Verifier::SuffixMap suffix_map = manifest.getSuffixMap(); + + // Return early if the manifest is empty + if (suffix_map.empty()) { + verifier_->callVerificationFailedCallback(suffix, accept_policy); + return accept_policy; + } + + // Add hashes to map of all manifest hashes + manifest_digests_.insert(suffix_map.begin(), suffix_map.end()); + + // Remove discarded and definitely lost packets from digest map + for (auto it = manifest_digests_.begin(); it != manifest_digests_.end();) { + auto it_erased = packets_unverif_erased_.find(it->first); + + if (it_erased != packets_unverif_erased_.end()) { + packets_unverif_erased_.erase(it_erased); + it = manifest_digests_.erase(it); + continue; + } + + if (rtc_state_->getPacketState(it->first) == PacketState::DEFINITELY_LOST) { + it = manifest_digests_.erase(it); + continue; + } + + ++it; + } + + // Verify packets + auth::Verifier::PolicyMap policies = + verifier_->verifyHashes(packets_unverif_.suffixMap(), manifest_digests_); + + for (const auto &p : policies) { + switch (p.second) { + case auth::VerificationPolicy::ACCEPT: { + packets_unverif_.remove(packets_unverif_.packet(p.first)); + manifest_digests_.erase(p.first); + break; + } + case auth::VerificationPolicy::UNKNOWN: + break; + case auth::VerificationPolicy::DROP: + case auth::VerificationPolicy::ABORT: + return p.second; + } + } + + verifier_->callVerificationFailedCallback(suffix, accept_policy); + return accept_policy; +} + +void RTCVerifier::onDataRecoveredFec(uint32_t suffix) { + manifest_digests_.erase(suffix); +} + +std::pair<RTCVerifier::PacketSet::iterator, bool> RTCVerifier::Packets::add( + const Packet &packet, const auth::CryptoHash &digest) { + auto inserted = packets_.insert(packet); + if (inserted.second) { + packets_map_[packet.second] = inserted.first; + suffix_map_[packet.second] = digest; + } + return inserted; +} + +RTCVerifier::PacketSet::iterator RTCVerifier::Packets::remove( + PacketSet::iterator packet_it) { + packets_map_.erase(packet_it->second); + suffix_map_.erase(packet_it->second); + return packets_.erase(packet_it); +} + +const std::set<RTCVerifier::Packet> &RTCVerifier::Packets::set() const { + return packets_; +}; + +RTCVerifier::PacketSet::iterator RTCVerifier::Packets::packet( + auth::Suffix suffix) { + return packets_map_.at(suffix); +}; + +const auth::Verifier::SuffixMap &RTCVerifier::Packets::suffixMap() const { + return suffix_map_; +} + +} // end namespace rtc +} // end namespace protocol +} // end namespace transport diff --git a/libtransport/src/protocols/rtc/rtc_verifier.h b/libtransport/src/protocols/rtc/rtc_verifier.h new file mode 100644 index 000000000..c83faf08a --- /dev/null +++ b/libtransport/src/protocols/rtc/rtc_verifier.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2017-2022 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <core/facade.h> +#include <hicn/transport/auth/verifier.h> +#include <hicn/transport/core/content_object.h> +#include <protocols/rtc/rtc_state.h> + +namespace transport { +namespace protocol { +namespace rtc { + +class RTCVerifier { + public: + explicit RTCVerifier(std::shared_ptr<auth::Verifier> verifier, + uint32_t factor_relevant, uint32_t factor_alert); + + virtual ~RTCVerifier() = default; + + void setState(std::shared_ptr<RTCState> rtc_state); + void setVerifier(std::shared_ptr<auth::Verifier> verifier); + void setFactorRelevant(uint32_t factor_relevant); + void setFactorAlert(uint32_t factor_alert); + + auth::VerificationPolicy verify(core::Interest &interest); + auth::VerificationPolicy verify(core::ContentObject &content_object, + bool is_fec = false); + auth::VerificationPolicy verifyProbe(core::ContentObject &content_object); + auth::VerificationPolicy verifyNack(core::ContentObject &content_object); + auth::VerificationPolicy verifyFec(core::ContentObject &content_object); + auth::VerificationPolicy verifyData(core::ContentObject &content_object); + auth::VerificationPolicy verifyManifest(core::ContentObject &content_object); + + auth::VerificationPolicy processManifest(core::ContentObject &content_object); + + void onDataRecoveredFec(uint32_t suffix); + + protected: + using Index = uint64_t; + using Packet = std::pair<Index, auth::Suffix>; + using PacketSet = std::set<Packet>; + + class Packets { + public: + std::pair<PacketSet::iterator, bool> add(const Packet &packet, + const auth::CryptoHash &digest); + PacketSet::iterator remove(PacketSet::iterator packet_it); + const PacketSet &set() const; + PacketSet::iterator packet(auth::Suffix suffix); + const auth::Verifier::SuffixMap &suffixMap() const; + + private: + PacketSet packets_; + std::unordered_map<auth::Suffix, PacketSet::iterator> packets_map_; + auth::Verifier::SuffixMap suffix_map_; + }; + + // The RTC state. + std::shared_ptr<RTCState> rtc_state_; + // The verifier instance. + std::shared_ptr<auth::Verifier> verifier_; + // Used to compute the relevance windows size (in packets). + uint32_t factor_relevant_; + // Used to compute the alert threshold (in packets). + uint32_t factor_alert_; + // The maximum number of entries a manifest can contain. + uint8_t manifest_max_capacity_; + // Hash algorithm used by manifests. + auth::CryptoHashType manifest_hash_algo_; + // Digests extracted from all manifests received. + auth::Verifier::SuffixMap manifest_digests_; + // The number of data packets processed. + Index current_index_; + // Unverified packets with index in relevance window. + Packets packets_unverif_; + // Unverified erased packets with index outside relevance window. + std::unordered_set<auth::Suffix> packets_unverif_erased_; +}; + +} // namespace rtc +} // namespace protocol +} // namespace transport diff --git a/libtransport/src/protocols/transport_protocol.cc b/libtransport/src/protocols/transport_protocol.cc index d6954ac37..29d140454 100644 --- a/libtransport/src/protocols/transport_protocol.cc +++ b/libtransport/src/protocols/transport_protocol.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -25,7 +25,8 @@ using namespace interface; TransportProtocol::TransportProtocol(implementation::ConsumerSocket *icn_socket, Indexer *indexer, Reassembly *reassembly) - : socket_(icn_socket), + : Protocol(), + socket_(icn_socket), indexer_verifier_(indexer), reassembly_(reassembly), fec_decoder_(nullptr), @@ -36,90 +37,77 @@ TransportProtocol::TransportProtocol(implementation::ConsumerSocket *icn_socket, on_interest_satisfied_(VOID_HANDLER), on_content_object_input_(VOID_HANDLER), stats_summary_(VOID_HANDLER), + on_fwd_strategy_(VOID_HANDLER), + on_rec_strategy_(VOID_HANDLER), on_payload_(VOID_HANDLER), - fec_type_(fec::FECType::UNKNOWN), - is_running_(false) { + fec_type_(fec::FECType::UNKNOWN) { socket_->getSocketOption(GeneralTransportOptions::PORTAL, portal_); socket_->getSocketOption(OtherOptions::STATISTICS, &stats_); - // Set this transport protocol as portal's consumer callback - portal_->setConsumerCallback(this); - indexer_verifier_->setReassembly(reassembly_.get()); reassembly->setIndexer(indexer_verifier_.get()); } +TransportProtocol::~TransportProtocol() {} + int TransportProtocol::start() { // If the protocol is already running, return otherwise set as running - if (is_running_) return -1; - - // Get all callbacks references - socket_->getSocketOption(ConsumerCallbacksOptions::INTEREST_RETRANSMISSION, - &on_interest_retransmission_); - socket_->getSocketOption(ConsumerCallbacksOptions::INTEREST_OUTPUT, - &on_interest_output_); - socket_->getSocketOption(ConsumerCallbacksOptions::INTEREST_EXPIRED, - &on_interest_timeout_); - socket_->getSocketOption(ConsumerCallbacksOptions::INTEREST_SATISFIED, - &on_interest_satisfied_); - socket_->getSocketOption(ConsumerCallbacksOptions::CONTENT_OBJECT_INPUT, - &on_content_object_input_); - socket_->getSocketOption(ConsumerCallbacksOptions::STATS_SUMMARY, - &stats_summary_); - socket_->getSocketOption(ConsumerCallbacksOptions::READ_CALLBACK, - &on_payload_); - - socket_->getSocketOption(GeneralTransportOptions::ASYNC_MODE, is_async_); - - // Set it is the first time we schedule an interest - is_first_ = true; - - // Reset the protocol state machine - reset(); - - // Schedule next interests - scheduleNextInterests(); - - is_first_ = false; - - // Set the protocol as running - is_running_ = true; - - if (!is_async_) { - // Start Event loop - portal_->runEventsLoop(); - - // Not running anymore - is_running_ = false; + if (isRunning()) { + return -1; } - return 0; -} - -void TransportProtocol::stop() { - is_running_ = false; + // Start protocol on its own thread + portal_->getThread().add([this]() { + // Get all callbacks references + socket_->getSocketOption(ConsumerCallbacksOptions::INTEREST_RETRANSMISSION, + &on_interest_retransmission_); + socket_->getSocketOption(ConsumerCallbacksOptions::INTEREST_OUTPUT, + &on_interest_output_); + socket_->getSocketOption(ConsumerCallbacksOptions::INTEREST_EXPIRED, + &on_interest_timeout_); + socket_->getSocketOption(ConsumerCallbacksOptions::INTEREST_SATISFIED, + &on_interest_satisfied_); + socket_->getSocketOption(ConsumerCallbacksOptions::CONTENT_OBJECT_INPUT, + &on_content_object_input_); + socket_->getSocketOption(ConsumerCallbacksOptions::STATS_SUMMARY, + &stats_summary_); + socket_->getSocketOption(ConsumerCallbacksOptions::FWD_STRATEGY_CHANGE, + &on_fwd_strategy_); + socket_->getSocketOption(ConsumerCallbacksOptions::REC_STRATEGY_CHANGE, + &on_rec_strategy_); + socket_->getSocketOption(ConsumerCallbacksOptions::READ_CALLBACK, + &on_payload_); + + socket_->getSocketOption(GeneralTransportOptions::ASYNC_MODE, is_async_); + socket_->getSocketOption(GeneralTransportOptions::SIGNER, signer_); + + // Set it is the first time we schedule an interest + is_first_ = true; + + // Reset the protocol state machine + reset(); + + // Set this transport protocol as portal's consumer callback + portal_->registerTransportCallback(this); + + // Schedule next interests + scheduleNextInterests(); + + is_first_ = false; + + // Set the protocol as running + setRunning(); + }); - if (!is_async_) { - portal_->stopEventsLoop(); - } else { - portal_->clear(); - } + return 0; } void TransportProtocol::resume() { if (isRunning()) return; - is_running_ = true; - - scheduleNextInterests(); - - if (!is_async_) { - // Start Event loop - portal_->runEventsLoop(); + setRunning(); - // Not running anymore - is_running_ = false; - } + portal_->getThread().tryRunHandlerNow([this]() { scheduleNextInterests(); }); } void TransportProtocol::reset() { @@ -130,7 +118,7 @@ void TransportProtocol::reset() { } } -void TransportProtocol::onContentReassembled(std::error_code ec) { +void TransportProtocol::onContentReassembled(const std::error_code &ec) { stop(); if (!on_payload_) { @@ -153,7 +141,19 @@ void TransportProtocol::sendInterest( uint32_t len) { DLOG_IF(INFO, VLOG_IS_ON(3)) << "Sending interest for name " << interest_name; - auto interest = core::PacketManager<>::getInstance().getPacket<Interest>(); + Packet::Format format; + socket_->getSocketOption(interface::GeneralTransportOptions::PACKET_FORMAT, + format); + size_t signature_size = 0; + + // If aggregated interest, add spapce for signature + if (len > 0) { + format = Packet::toAHFormat(format); + signature_size = signer_->getSignatureFieldSize(); + } + + auto interest = core::PacketManager<>::getInstance().getPacket<Interest>( + format, signature_size); interest->setName(interest_name); for (uint32_t i = 0; i < len; i++) { @@ -173,7 +173,24 @@ void TransportProtocol::sendInterest( return; } - portal_->sendInterest(std::move(interest)); + bool content_sharing_mode; + socket_->getSocketOption(RtcTransportOptions::CONTENT_SHARING_MODE, + content_sharing_mode); + if (content_sharing_mode) lifetime = ceil((double)lifetime * 0.9); + + // Compute signature + bool is_ah = HICN_PACKET_FORMAT_IS_AH(interest->getFormat()); + if (is_ah) signer_->signPacket(interest.get()); + + portal_->sendInterest(interest, lifetime); +} + +void TransportProtocol::onError(const std::error_code &ec) { + // error from portal: stop socket + stop(); + + // signal error to application + on_payload_->readError(ec); } void TransportProtocol::onTimeout(Interest::Ptr &i, const Name &n) { diff --git a/libtransport/src/protocols/transport_protocol.h b/libtransport/src/protocols/transport_protocol.h index 1008a238b..e71992561 100644 --- a/libtransport/src/protocols/transport_protocol.h +++ b/libtransport/src/protocols/transport_protocol.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Copyright (c) 2021 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -19,10 +19,10 @@ #include <hicn/transport/interfaces/socket_consumer.h> #include <hicn/transport/interfaces/statistics.h> #include <hicn/transport/utils/object_pool.h> -#include <implementation/socket.h> #include <protocols/data_processing_events.h> #include <protocols/fec_base.h> #include <protocols/indexer.h> +#include <protocols/protocol.h> #include <protocols/reassembly.h> #include <array> @@ -38,8 +38,10 @@ class IndexVerificationManager; using ReadCallback = interface::ConsumerSocket::ReadCallback; -class TransportProtocol : public core::Portal::ConsumerCallback, - public ContentObjectProcessingEventCallback { +class TransportProtocol + : public ContentObjectProcessingEventCallback, + public Protocol, + public std::enable_shared_from_this<TransportProtocol> { static constexpr std::size_t interest_pool_size = 4096; friend class ManifestIndexManager; @@ -48,13 +50,11 @@ class TransportProtocol : public core::Portal::ConsumerCallback, TransportProtocol(implementation::ConsumerSocket *icn_socket, Indexer *indexer, Reassembly *reassembly); - virtual ~TransportProtocol() = default; - - TRANSPORT_ALWAYS_INLINE bool isRunning() { return is_running_; } + virtual ~TransportProtocol(); virtual int start(); - virtual void stop(); + using Protocol::stop; virtual void resume(); @@ -64,12 +64,12 @@ class TransportProtocol : public core::Portal::ConsumerCallback, * * @return The header length in bytes. */ - virtual std::size_t transportHeaderLength() { return 0; } + virtual std::size_t transportHeaderLength(bool isFEC) { return 0; } virtual void scheduleNextInterests() = 0; // Events generated by the indexing - virtual void onContentReassembled(std::error_code ec); + virtual void onContentReassembled(const std::error_code &ec); virtual void onPacketDropped(Interest &interest, ContentObject &content_object, const std::error_code &ec) override = 0; @@ -90,7 +90,8 @@ class TransportProtocol : public core::Portal::ConsumerCallback, AllocatorHandler &&allocator_handler) { if (!fec_decoder_) { // Try to get FEC from environment - if (const char *fec_str = std::getenv("TRANSPORT_FEC_TYPE")) { + const char *fec_str = std::getenv("TRANSPORT_FEC_TYPE"); + if (fec_str && (fec_type_ == fec::FECType::UNKNOWN)) { LOG(INFO) << "Using FEC " << fec_str; fec_type_ = fec::FECUtils::fecTypeFromString(fec_str); } @@ -114,14 +115,13 @@ class TransportProtocol : public core::Portal::ConsumerCallback, // Consumer Callback void onContentObject(Interest &i, ContentObject &c) override; void onTimeout(Interest::Ptr &i, const Name &n) override; - void onError(std::error_code ec) override {} + void onError(const std::error_code &ec) override; protected: implementation::ConsumerSocket *socket_; std::unique_ptr<Indexer> indexer_verifier_; std::unique_ptr<Reassembly> reassembly_; std::unique_ptr<fec::ConsumerFEC> fec_decoder_; - std::shared_ptr<core::Portal> portal_; // True if it si the first time we schedule an interest std::atomic<bool> is_first_; interface::TransportStatistics *stats_; @@ -134,14 +134,16 @@ class TransportProtocol : public core::Portal::ConsumerCallback, interface::ConsumerContentObjectCallback *on_content_object_input_; interface::ConsumerContentObjectCallback *on_content_object_; interface::ConsumerTimerCallback *stats_summary_; + interface::StrategyCallback *on_fwd_strategy_; + interface::StrategyCallback *on_rec_strategy_; ReadCallback *on_payload_; bool is_async_; fec::FECType fec_type_; - private: - std::atomic<bool> is_running_; + // Signer for aggregated interests + std::shared_ptr<auth::Signer> signer_; }; } // end namespace protocol |