summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--dpdk/Makefile7
-rw-r--r--dpdk/dpdk-2.2.0_patches/0001-e1000-Set-VLAN-Rx-Offload-tag-correctly.patch75
-rw-r--r--dpdk/dpdk-2.2.0_patches/0002-ixgbe-Wait-a-bit-longer-for-autonegotiation-to-leave.patch25
-rw-r--r--dpdk/dpdk-2.2.0_patches/0003-virtio-Cleanup-virtio-pmd-debug-log-output-reset.patch76
-rw-r--r--dpdk/dpdk-2.2.0_patches/0004-mbuf-rearrange-rte_mbuf-metadata-to-suit-vpp.patch83
-rw-r--r--dpdk/dpdk-2.2.0_patches/0005-missing-include.patch24
-rw-r--r--vnet/vnet/devices/dpdk/device.c19
-rw-r--r--vnet/vnet/devices/dpdk/dpdk_priv.h22
8 files changed, 319 insertions, 12 deletions
diff --git a/dpdk/Makefile b/dpdk/Makefile
index c33cf63ccf5..b95ed7668c7 100644
--- a/dpdk/Makefile
+++ b/dpdk/Makefile
@@ -23,10 +23,11 @@ DPDK_DEBUG ?= n
B := $(DPDK_BUILD_DIR)
I := $(DPDK_INSTALL_DIR)
-DPDK_VERSION := 2.1.0
+DPDK_VERSION ?= 2.1.0
DPDK_TARBALL := dpdk-$(DPDK_VERSION).tar.gz
DPDK_TAR_URL := http://dpdk.org/browse/dpdk/snapshot/$(DPDK_TARBALL)
-DPDK_TARBALL_MD5_CKSUM := 205a0d12bfd6eb717d57506272f43519
+DPDK_2.1.0_TARBALL_MD5_CKSUM := 205a0d12bfd6eb717d57506272f43519
+DPDK_2.2.0_TARBALL_MD5_CKSUM := 22e2fd68cd5504f43fe9a5a6fd6dd938
DPDK_SOURCE := $(B)/dpdk-$(DPDK_VERSION)
DPDK_TARGET := x86_64-native-linuxapp-gcc
JOBS := $(shell grep processor /proc/cpuinfo | wc -l)
@@ -125,7 +126,7 @@ $(CURDIR)/$(DPDK_TARBALL):
$(B)/.download.ok: $(CURDIR)/$(DPDK_TARBALL)
@openssl md5 $< | cut -f 2 -d " " - > $(B)/$(DPDK_TARBALL).md5sum
- @([ "$$(<$(B)/$(DPDK_TARBALL).md5sum)" = "$(DPDK_TARBALL_MD5_CKSUM)" ] || \
+ @([ "$$(<$(B)/$(DPDK_TARBALL).md5sum)" = "$(DPDK_$(DPDK_VERSION)_TARBALL_MD5_CKSUM)" ] || \
( echo "Bad Checksum! Please remove $< and retry" && \
rm $(B)/$(DPDK_TARBALL).md5sum && false ))
@touch $@
diff --git a/dpdk/dpdk-2.2.0_patches/0001-e1000-Set-VLAN-Rx-Offload-tag-correctly.patch b/dpdk/dpdk-2.2.0_patches/0001-e1000-Set-VLAN-Rx-Offload-tag-correctly.patch
new file mode 100644
index 00000000000..6ed2fc61930
--- /dev/null
+++ b/dpdk/dpdk-2.2.0_patches/0001-e1000-Set-VLAN-Rx-Offload-tag-correctly.patch
@@ -0,0 +1,75 @@
+From 4a599535445d16a1c55fac0bd71edc443c6c23f2 Mon Sep 17 00:00:00 2001
+From: Damjan Marion <damarion@cisco.com>
+Date: Wed, 16 Dec 2015 03:21:21 +0100
+Subject: [PATCH 1/4] e1000: Set VLAN Rx Offload tag correctly
+
+---
+ drivers/net/e1000/igb_rxtx.c | 30 ++++++++++++++++++++++++++++++
+ lib/librte_ether/rte_ether.h | 3 +++
+ 2 files changed, 33 insertions(+)
+
+diff --git a/drivers/net/e1000/igb_rxtx.c b/drivers/net/e1000/igb_rxtx.c
+index 996e7da..cbe80a1 100644
+--- a/drivers/net/e1000/igb_rxtx.c
++++ b/drivers/net/e1000/igb_rxtx.c
+@@ -910,6 +910,21 @@ eth_igb_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+ pkt_flags = rx_desc_hlen_type_rss_to_pkt_flags(rxq, hlen_type_rss);
+ pkt_flags = pkt_flags | rx_desc_status_to_pkt_flags(staterr);
+ pkt_flags = pkt_flags | rx_desc_error_to_pkt_flags(staterr);
++ {
++ /*
++ * Check packet for VLAN ethernet types and set
++ * RX Offload flag PKT_RX_VLAN_PKT accordingly.
++ */
++ struct ether_hdr *eth_hdr =
++ rte_pktmbuf_mtod(rxm, struct ether_hdr *);
++ u16 eth_type = rte_be_to_cpu_16(eth_hdr->ether_type);
++
++ if ((eth_type == ETHER_TYPE_VLAN) ||
++ (eth_type == ETHER_TYPE_VLAN_AD) ||
++ (eth_type == ETHER_TYPE_VLAN_9100) ||
++ (eth_type == ETHER_TYPE_VLAN_9200))
++ pkt_flags |= PKT_RX_VLAN_PKT;
++ }
+ rxm->ol_flags = pkt_flags;
+ rxm->packet_type = igb_rxd_pkt_info_to_pkt_type(rxd.wb.lower.
+ lo_dword.hs_rss.pkt_info);
+@@ -1146,6 +1161,21 @@ eth_igb_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+ pkt_flags = rx_desc_hlen_type_rss_to_pkt_flags(rxq, hlen_type_rss);
+ pkt_flags = pkt_flags | rx_desc_status_to_pkt_flags(staterr);
+ pkt_flags = pkt_flags | rx_desc_error_to_pkt_flags(staterr);
++ {
++ /*
++ * Check packet for VLAN ethernet types and set
++ * RX Offload flag PKT_RX_VLAN_PKT accordingly.
++ */
++ struct ether_hdr *eth_hdr =
++ rte_pktmbuf_mtod(rxm, struct ether_hdr *);
++ u16 eth_type = rte_be_to_cpu_16(eth_hdr->ether_type);
++
++ if ((eth_type == ETHER_TYPE_VLAN) ||
++ (eth_type == ETHER_TYPE_VLAN_AD) ||
++ (eth_type == ETHER_TYPE_VLAN_9100) ||
++ (eth_type == ETHER_TYPE_VLAN_9200))
++ pkt_flags |= PKT_RX_VLAN_PKT;
++ }
+ first_seg->ol_flags = pkt_flags;
+ first_seg->packet_type = igb_rxd_pkt_info_to_pkt_type(rxd.wb.
+ lower.lo_dword.hs_rss.pkt_info);
+diff --git a/lib/librte_ether/rte_ether.h b/lib/librte_ether/rte_ether.h
+index 07c17d7..fd646ec 100644
+--- a/lib/librte_ether/rte_ether.h
++++ b/lib/librte_ether/rte_ether.h
+@@ -332,6 +332,9 @@ struct vxlan_hdr {
+ #define ETHER_TYPE_1588 0x88F7 /**< IEEE 802.1AS 1588 Precise Time Protocol. */
+ #define ETHER_TYPE_SLOW 0x8809 /**< Slow protocols (LACP and Marker). */
+ #define ETHER_TYPE_TEB 0x6558 /**< Transparent Ethernet Bridging. */
++#define ETHER_TYPE_VLAN_AD 0x88a8 /**< IEEE 802.1AD VLAN tagging. */
++#define ETHER_TYPE_VLAN_9100 0x9100 /**< VLAN 0x9100 tagging. */
++#define ETHER_TYPE_VLAN_9200 0x9200 /**< VLAN 0x9200 tagging. */
+
+ #define ETHER_VXLAN_HLEN (sizeof(struct udp_hdr) + sizeof(struct vxlan_hdr))
+ /**< VXLAN tunnel header length. */
+--
+2.5.0
+
diff --git a/dpdk/dpdk-2.2.0_patches/0002-ixgbe-Wait-a-bit-longer-for-autonegotiation-to-leave.patch b/dpdk/dpdk-2.2.0_patches/0002-ixgbe-Wait-a-bit-longer-for-autonegotiation-to-leave.patch
new file mode 100644
index 00000000000..b7a50298b35
--- /dev/null
+++ b/dpdk/dpdk-2.2.0_patches/0002-ixgbe-Wait-a-bit-longer-for-autonegotiation-to-leave.patch
@@ -0,0 +1,25 @@
+From 009cd67e5b1ed0592de0fb6ae2fa662ffc172dde Mon Sep 17 00:00:00 2001
+From: Damjan Marion <damarion@cisco.com>
+Date: Wed, 16 Dec 2015 03:22:11 +0100
+Subject: [PATCH 2/4] ixgbe: Wait a bit longer for autonegotiation to leave
+
+---
+ drivers/net/ixgbe/base/ixgbe_82599.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/drivers/net/ixgbe/base/ixgbe_82599.c b/drivers/net/ixgbe/base/ixgbe_82599.c
+index f0deb59..ae66380 100644
+--- a/drivers/net/ixgbe/base/ixgbe_82599.c
++++ b/drivers/net/ixgbe/base/ixgbe_82599.c
+@@ -2442,7 +2442,7 @@ s32 ixgbe_reset_pipeline_82599(struct ixgbe_hw *hw)
+ IXGBE_WRITE_REG(hw, IXGBE_AUTOC,
+ autoc_reg ^ (0x4 << IXGBE_AUTOC_LMS_SHIFT));
+ /* Wait for AN to leave state 0 */
+- for (i = 0; i < 10; i++) {
++ for (i = 0; i < 50; i++) {
+ msec_delay(4);
+ anlp1_reg = IXGBE_READ_REG(hw, IXGBE_ANLP1);
+ if (anlp1_reg & IXGBE_ANLP1_AN_STATE_MASK)
+--
+2.5.0
+
diff --git a/dpdk/dpdk-2.2.0_patches/0003-virtio-Cleanup-virtio-pmd-debug-log-output-reset.patch b/dpdk/dpdk-2.2.0_patches/0003-virtio-Cleanup-virtio-pmd-debug-log-output-reset.patch
new file mode 100644
index 00000000000..874f666b9ec
--- /dev/null
+++ b/dpdk/dpdk-2.2.0_patches/0003-virtio-Cleanup-virtio-pmd-debug-log-output-reset.patch
@@ -0,0 +1,76 @@
+From e2592eb622c33791d8ae51153360bd8249bdd056 Mon Sep 17 00:00:00 2001
+From: Damjan Marion <damarion@cisco.com>
+Date: Wed, 16 Dec 2015 03:29:22 +0100
+Subject: [PATCH 3/4] virtio: Cleanup virtio pmd debug log output, reset
+
+---
+ drivers/net/virtio/virtio_ethdev.c | 10 +++++-----
+ drivers/net/virtio/virtio_rxtx.c | 4 +++-
+ 2 files changed, 8 insertions(+), 6 deletions(-)
+
+diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c
+index d928339..2fa1587 100644
+--- a/drivers/net/virtio/virtio_ethdev.c
++++ b/drivers/net/virtio/virtio_ethdev.c
+@@ -1635,24 +1635,24 @@ virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complet
+ link.link_speed = SPEED_10G;
+
+ if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) {
+- PMD_INIT_LOG(DEBUG, "Get link status from hw");
+ vtpci_read_dev_config(hw,
+ offsetof(struct virtio_net_config, status),
+ &status, sizeof(status));
+ if ((status & VIRTIO_NET_S_LINK_UP) == 0) {
+ link.link_status = 0;
+- PMD_INIT_LOG(DEBUG, "Port %d is down",
+- dev->data->port_id);
+ } else {
+ link.link_status = 1;
+- PMD_INIT_LOG(DEBUG, "Port %d is up",
+- dev->data->port_id);
+ }
+ } else {
+ link.link_status = 1; /* Link up */
+ }
+ virtio_dev_atomic_write_link_status(dev, &link);
+
++ /* This message is far too noisy for normal use */
++ if (0)
++ PMD_INIT_LOG(DEBUG, "Port %d is %s\n", dev->data->port_id,
++ link.link_status ? "up" : "down");
++
+ return (old.link_status == link.link_status) ? -1 : 0;
+ }
+
+diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
+index 74b39ef..2512bce 100644
+--- a/drivers/net/virtio/virtio_rxtx.c
++++ b/drivers/net/virtio/virtio_rxtx.c
+@@ -618,6 +618,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+ rxm->next = NULL;
+ rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
+ rxm->data_len = (uint16_t)(len[i] - hdr_size);
++ rxm->ol_flags = 0;
+
+ if (hw->vlan_strip)
+ rte_vlan_strip(rxm);
+@@ -737,6 +738,7 @@ virtio_recv_mergeable_pkts(void *rx_queue,
+ rxm->vlan_tci = 0;
+ rxm->pkt_len = (uint32_t)(len[0] - hdr_size);
+ rxm->data_len = (uint16_t)(len[0] - hdr_size);
++ rxm->ol_flags = 0;
+
+ rxm->port = rxvq->port_id;
+ rx_pkts[nb_rx] = rxm;
+@@ -838,7 +840,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+ if (unlikely(nb_pkts < 1))
+ return nb_pkts;
+
+- PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
++ PMD_TX_LOG(DEBUG, "%d packets to xmit\n", nb_pkts);
+ nb_used = VIRTQUEUE_NUSED(txvq);
+
+ virtio_rmb();
+--
+2.5.0
+
diff --git a/dpdk/dpdk-2.2.0_patches/0004-mbuf-rearrange-rte_mbuf-metadata-to-suit-vpp.patch b/dpdk/dpdk-2.2.0_patches/0004-mbuf-rearrange-rte_mbuf-metadata-to-suit-vpp.patch
new file mode 100644
index 00000000000..bee64dff781
--- /dev/null
+++ b/dpdk/dpdk-2.2.0_patches/0004-mbuf-rearrange-rte_mbuf-metadata-to-suit-vpp.patch
@@ -0,0 +1,83 @@
+From b8b575a3398c480f6e02525a0933e5e057139b78 Mon Sep 17 00:00:00 2001
+From: Damjan Marion <damarion@cisco.com>
+Date: Wed, 16 Dec 2015 04:25:23 +0100
+Subject: [PATCH 4/4] mbuf: rearrange rte_mbuf metadata to suit vpp
+
+---
+ .../linuxapp/eal/include/exec-env/rte_kni_common.h | 5 +++--
+ lib/librte_mbuf/rte_mbuf.h | 20 ++++++++++++--------
+ 2 files changed, 15 insertions(+), 10 deletions(-)
+
+diff --git a/lib/librte_eal/linuxapp/eal/include/exec-env/rte_kni_common.h b/lib/librte_eal/linuxapp/eal/include/exec-env/rte_kni_common.h
+index bd1cc09..a68a949 100644
+--- a/lib/librte_eal/linuxapp/eal/include/exec-env/rte_kni_common.h
++++ b/lib/librte_eal/linuxapp/eal/include/exec-env/rte_kni_common.h
+@@ -120,11 +120,12 @@ struct rte_kni_mbuf {
+ char pad2[4];
+ uint32_t pkt_len; /**< Total pkt len: sum of all segment data_len. */
+ uint16_t data_len; /**< Amount of data in segment buffer. */
++ char pad3[8];
++ void *next;
+
+ /* fields on second cache line */
+- char pad3[8] __attribute__((__aligned__(RTE_CACHE_LINE_SIZE)));
++ char pad4[16] __attribute__((__aligned__(RTE_CACHE_LINE_SIZE)));
+ void *pool;
+- void *next;
+ };
+
+ /*
+diff --git a/lib/librte_mbuf/rte_mbuf.h b/lib/librte_mbuf/rte_mbuf.h
+index f234ac9..a1b4405 100644
+--- a/lib/librte_mbuf/rte_mbuf.h
++++ b/lib/librte_mbuf/rte_mbuf.h
+@@ -734,6 +734,12 @@ struct rte_mbuf_offload;
+ /**
+ * The generic rte_mbuf, containing a packet mbuf.
+ */
++/*
++ * offload in the second cache line, next in the first. Better for vpp
++ * at least as of right now.
++ * If you change this structure, you must change the user-mode
++ * version in rte_mbuf.h
++ */
+ struct rte_mbuf {
+ MARKER cacheline0;
+
+@@ -786,6 +792,12 @@ struct rte_mbuf {
+ uint32_t pkt_len; /**< Total pkt len: sum of all segments. */
+ uint16_t data_len; /**< Amount of data in segment buffer. */
+ uint16_t vlan_tci; /**< VLAN Tag Control Identifier (CPU order) */
++ uint32_t seqn; /**< Sequence number. See also rte_reorder_insert() */
++ uint16_t vlan_tci_outer; /**< Outer VLAN Tag Control Identifier (CPU order) */
++ struct rte_mbuf *next; /**< Next segment of scattered packet. */
++
++ /* second cache line - fields only used in slow path or on TX */
++ MARKER cacheline1 __rte_cache_aligned;
+
+ union {
+ uint32_t rss; /**< RSS hash result if RSS enabled */
+@@ -809,20 +821,12 @@ struct rte_mbuf {
+ uint32_t usr; /**< User defined tags. See rte_distributor_process() */
+ } hash; /**< hash information */
+
+- uint32_t seqn; /**< Sequence number. See also rte_reorder_insert() */
+-
+- uint16_t vlan_tci_outer; /**< Outer VLAN Tag Control Identifier (CPU order) */
+-
+- /* second cache line - fields only used in slow path or on TX */
+- MARKER cacheline1 __rte_cache_aligned;
+-
+ union {
+ void *userdata; /**< Can be used for external metadata */
+ uint64_t udata64; /**< Allow 8-byte userdata on 32-bit */
+ };
+
+ struct rte_mempool *pool; /**< Pool from which mbuf was allocated. */
+- struct rte_mbuf *next; /**< Next segment of scattered packet. */
+
+ /* fields to support TX offloads */
+ union {
+--
+2.5.0
+
diff --git a/dpdk/dpdk-2.2.0_patches/0005-missing-include.patch b/dpdk/dpdk-2.2.0_patches/0005-missing-include.patch
new file mode 100644
index 00000000000..c6211cd4077
--- /dev/null
+++ b/dpdk/dpdk-2.2.0_patches/0005-missing-include.patch
@@ -0,0 +1,24 @@
+From a8767269f3ee545141e83e5a5f62ff24c29248a9 Mon Sep 17 00:00:00 2001
+From: Damjan Marion <damarion@cisco.com>
+Date: Wed, 16 Dec 2015 04:43:40 +0100
+Subject: [PATCH 5/5] missing include
+
+---
+ lib/librte_eal/linuxapp/eal/eal_timer.c | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/lib/librte_eal/linuxapp/eal/eal_timer.c b/lib/librte_eal/linuxapp/eal/eal_timer.c
+index 9ceff33..d0792be 100644
+--- a/lib/librte_eal/linuxapp/eal/eal_timer.c
++++ b/lib/librte_eal/linuxapp/eal/eal_timer.c
+@@ -51,6 +51,7 @@
+ #include <rte_memzone.h>
+ #include <rte_eal.h>
+ #include <rte_debug.h>
++#include <rte_lcore.h>
+
+ #include "eal_private.h"
+ #include "eal_internal_cfg.h"
+--
+2.5.0
+
diff --git a/vnet/vnet/devices/dpdk/device.c b/vnet/vnet/devices/dpdk/device.c
index a19c3131ef9..781fff46782 100644
--- a/vnet/vnet/devices/dpdk/device.c
+++ b/vnet/vnet/devices/dpdk/device.c
@@ -988,15 +988,16 @@ static u8 * format_dpdk_device (u8 * s, va_list * args)
rte_eth_dev_rss_hash_conf_get(xd->device_index, &rss_conf);
pci = di.pci_dev;
- s = format(s, "%Upci id: device %04x:%04x subsystem %04x:%04x\n"
- "%Upci address: %04x:%02x:%02x.%02x\n",
- format_white_space, indent + 2,
- pci->id.vendor_id, pci->id.device_id,
- pci->id.subsystem_vendor_id,
- pci->id.subsystem_device_id,
- format_white_space, indent + 2,
- pci->addr.domain, pci->addr.bus,
- pci->addr.devid, pci->addr.function);
+ if (pci)
+ s = format(s, "%Upci id: device %04x:%04x subsystem %04x:%04x\n"
+ "%Upci address: %04x:%02x:%02x.%02x\n",
+ format_white_space, indent + 2,
+ pci->id.vendor_id, pci->id.device_id,
+ pci->id.subsystem_vendor_id,
+ pci->id.subsystem_device_id,
+ format_white_space, indent + 2,
+ pci->addr.domain, pci->addr.bus,
+ pci->addr.devid, pci->addr.function);
s = format(s, "%Umax rx packet len: %d\n",
format_white_space, indent + 2, di.max_rx_pktlen);
s = format(s, "%Upromiscuous: unicast %s all-multicast %s\n",
diff --git a/vnet/vnet/devices/dpdk/dpdk_priv.h b/vnet/vnet/devices/dpdk/dpdk_priv.h
index e452e02d90d..314e280636a 100644
--- a/vnet/vnet/devices/dpdk/dpdk_priv.h
+++ b/vnet/vnet/devices/dpdk/dpdk_priv.h
@@ -22,6 +22,22 @@
#define DPDK_NB_RX_DESC_40GE (4096-128)
#define DPDK_NB_TX_DESC_40GE 2048
+#if RTE_VERSION >= RTE_VERSION_NUM(2, 2, 0, 0)
+#define foreach_dpdk_counter \
+ _ (tx_frames_ok, opackets) \
+ _ (tx_bytes_ok, obytes) \
+ _ (tx_errors, oerrors) \
+ _ (tx_loopback_frames_ok, olbpackets) \
+ _ (tx_loopback_bytes_ok, olbbytes) \
+ _ (rx_frames_ok, ipackets) \
+ _ (rx_bytes_ok, ibytes) \
+ _ (rx_errors, ierrors) \
+ _ (rx_missed, imissed) \
+ _ (rx_multicast_frames_ok, imcasts) \
+ _ (rx_no_bufs, rx_nombuf) \
+ _ (rx_loopback_frames_ok, ilbpackets) \
+ _ (rx_loopback_bytes_ok, ilbbytes)
+#else
#define foreach_dpdk_counter \
_ (tx_frames_ok, opackets) \
_ (tx_bytes_ok, obytes) \
@@ -44,6 +60,7 @@
_ (rx_pause_xoff, rx_pause_xoff) \
_ (rx_loopback_frames_ok, ilbpackets) \
_ (rx_loopback_bytes_ok, ilbbytes)
+#endif
#define foreach_dpdk_q_counter \
_ (rx_frames_ok, q_ipackets) \
@@ -412,10 +429,15 @@ dpdk_update_counters (dpdk_device_t * xd, f64 now)
xd->stats.imissed -
xd->last_stats.imissed);
}
+#if RTE_VERSION >= RTE_VERSION_NUM(2, 2, 0, 0)
+ rxerrors = xd->stats.ierrors;
+ last_rxerrors = xd->last_stats.ierrors;
+#else
rxerrors = xd->stats.ibadcrc
+ xd->stats.ibadlen + xd->stats.ierrors;
last_rxerrors = xd->last_stats.ibadcrc
+ xd->last_stats.ibadlen + xd->last_stats.ierrors;
+#endif
if (PREDICT_FALSE (rxerrors != last_rxerrors))
{