diff options
Diffstat (limited to 'examples/vm_power_manager')
| -rw-r--r-- | examples/vm_power_manager/Makefile | 16 | ||||
| -rw-r--r-- | examples/vm_power_manager/channel_manager.c | 67 | ||||
| -rw-r--r-- | examples/vm_power_manager/channel_manager.h | 25 | ||||
| -rw-r--r-- | examples/vm_power_manager/channel_monitor.c | 336 | ||||
| -rw-r--r-- | examples/vm_power_manager/channel_monitor.h | 18 | ||||
| -rw-r--r-- | examples/vm_power_manager/guest_cli/vm_power_cli_guest.c | 108 | ||||
| -rw-r--r-- | examples/vm_power_manager/guest_cli/vm_power_cli_guest.h | 6 | ||||
| -rw-r--r-- | examples/vm_power_manager/main.c | 261 | ||||
| -rw-r--r-- | examples/vm_power_manager/power_manager.c | 68 | ||||
| -rw-r--r-- | examples/vm_power_manager/power_manager.h | 65 | ||||
| -rw-r--r-- | examples/vm_power_manager/vm_power_cli.c | 21 |
11 files changed, 963 insertions, 28 deletions
diff --git a/examples/vm_power_manager/Makefile b/examples/vm_power_manager/Makefile index 59a96417..9cf20a28 100644 --- a/examples/vm_power_manager/Makefile +++ b/examples/vm_power_manager/Makefile @@ -54,6 +54,22 @@ CFLAGS += $(WERROR_FLAGS) LDLIBS += -lvirt +ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),y) + +ifeq ($(CONFIG_RTE_LIBRTE_IXGBE_PMD),y) +LDLIBS += -lrte_pmd_ixgbe +endif + +ifeq ($(CONFIG_RTE_LIBRTE_I40E_PMD),y) +LDLIBS += -lrte_pmd_i40e +endif + +ifeq ($(CONFIG_RTE_LIBRTE_BNXT_PMD),y) +LDLIBS += -lrte_pmd_bnxt +endif + +endif + # workaround for a gcc bug with noreturn attribute # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603 ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) diff --git a/examples/vm_power_manager/channel_manager.c b/examples/vm_power_manager/channel_manager.c index e068ae28..ab856bdb 100644 --- a/examples/vm_power_manager/channel_manager.c +++ b/examples/vm_power_manager/channel_manager.c @@ -574,6 +574,73 @@ set_channel_status(const char *vm_name, unsigned *channel_list, return num_channels_changed; } +void +get_all_vm(int *num_vm, int *num_vcpu) +{ + + virNodeInfo node_info; + virDomainPtr *domptr; + uint64_t mask; + int i, ii, numVcpus[MAX_VCPUS], cpu, n_vcpus; + unsigned int jj; + const char *vm_name; + unsigned int domain_flags = VIR_CONNECT_LIST_DOMAINS_RUNNING | + VIR_CONNECT_LIST_DOMAINS_PERSISTENT; + unsigned int domain_flag = VIR_DOMAIN_VCPU_CONFIG; + + + memset(global_cpumaps, 0, CHANNEL_CMDS_MAX_CPUS*global_maplen); + if (virNodeGetInfo(global_vir_conn_ptr, &node_info)) { + RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n"); + return; + } + + /* Returns number of pcpus */ + global_n_host_cpus = (unsigned int)node_info.cpus; + + /* Returns number of active domains */ + *num_vm = virConnectListAllDomains(global_vir_conn_ptr, &domptr, + domain_flags); + if (*num_vm <= 0) { + RTE_LOG(ERR, CHANNEL_MANAGER, "No Active Domains Running\n"); + return; + } + + for (i = 0; i < *num_vm; i++) { + + /* Get Domain Names */ + vm_name = virDomainGetName(domptr[i]); + lvm_info[i].vm_name = vm_name; + + /* Get Number of Vcpus */ + numVcpus[i] = virDomainGetVcpusFlags(domptr[i], domain_flag); + + /* Get Number of VCpus & VcpuPinInfo */ + n_vcpus = virDomainGetVcpuPinInfo(domptr[i], + numVcpus[i], global_cpumaps, + global_maplen, domain_flag); + + if ((int)n_vcpus > 0) { + *num_vcpu = n_vcpus; + lvm_info[i].num_cpus = n_vcpus; + } + + /* Save pcpu in use by libvirt VMs */ + for (ii = 0; ii < n_vcpus; ii++) { + mask = 0; + for (jj = 0; jj < global_n_host_cpus; jj++) { + if (VIR_CPU_USABLE(global_cpumaps, + global_maplen, ii, jj) > 0) { + mask |= 1ULL << jj; + } + } + ITERATIVE_BITMASK_CHECK_64(mask, cpu) { + lvm_info[i].pcpus[ii] = cpu; + } + } + } +} + int get_info_vm(const char *vm_name, struct vm_info *info) { diff --git a/examples/vm_power_manager/channel_manager.h b/examples/vm_power_manager/channel_manager.h index 47c3b9cd..358fb8f2 100644 --- a/examples/vm_power_manager/channel_manager.h +++ b/examples/vm_power_manager/channel_manager.h @@ -66,6 +66,17 @@ struct sockaddr_un _sockaddr_un; #define UNIX_PATH_MAX sizeof(_sockaddr_un.sun_path) #endif +#define MAX_VMS 4 +#define MAX_VCPUS 20 + + +struct libvirt_vm_info { + const char *vm_name; + unsigned int pcpus[MAX_VCPUS]; + uint8_t num_cpus; +}; + +struct libvirt_vm_info lvm_info[MAX_VMS]; /* Communication Channel Status */ enum channel_status { CHANNEL_MGR_CHANNEL_DISCONNECTED = 0, CHANNEL_MGR_CHANNEL_CONNECTED, @@ -319,6 +330,20 @@ int set_channel_status(const char *vm_name, unsigned *channel_list, */ int get_info_vm(const char *vm_name, struct vm_info *info); +/** + * Populates a table with all domains running and their physical cpu. + * All information is gathered through libvirt api. + * + * @param num_vm + * modified to store number of active VMs + * + * @param num_vcpu + modified to store number of vcpus active + * + * @return + * void + */ +void get_all_vm(int *num_vm, int *num_vcpu); #ifdef __cplusplus } #endif diff --git a/examples/vm_power_manager/channel_monitor.c b/examples/vm_power_manager/channel_monitor.c index e7f5cc4a..37e71ed9 100644 --- a/examples/vm_power_manager/channel_monitor.c +++ b/examples/vm_power_manager/channel_monitor.c @@ -41,13 +41,17 @@ #include <sys/types.h> #include <sys/epoll.h> #include <sys/queue.h> +#include <sys/time.h> #include <rte_log.h> #include <rte_memory.h> #include <rte_malloc.h> #include <rte_atomic.h> +#include <rte_cycles.h> +#include <rte_ethdev.h> +#include <rte_pmd_i40e.h> - +#include <libvirt/libvirt.h> #include "channel_monitor.h" #include "channel_commands.h" #include "channel_manager.h" @@ -57,10 +61,15 @@ #define MAX_EVENTS 256 +uint64_t vsi_pkt_count_prev[384]; +uint64_t rdtsc_prev[384]; +double time_period_ms = 1; static volatile unsigned run_loop = 1; static int global_event_fd; +static unsigned int policy_is_set; static struct epoll_event *global_events_list; +static struct policy policies[MAX_VMS]; void channel_monitor_exit(void) { @@ -68,6 +77,287 @@ void channel_monitor_exit(void) rte_free(global_events_list); } +static void +core_share(int pNo, int z, int x, int t) +{ + if (policies[pNo].core_share[z].pcpu == lvm_info[x].pcpus[t]) { + if (strcmp(policies[pNo].pkt.vm_name, + lvm_info[x].vm_name) != 0) { + policies[pNo].core_share[z].status = 1; + power_manager_scale_core_max( + policies[pNo].core_share[z].pcpu); + } + } +} + +static void +core_share_status(int pNo) +{ + + int noVms, noVcpus, z, x, t; + + get_all_vm(&noVms, &noVcpus); + + /* Reset Core Share Status. */ + for (z = 0; z < noVcpus; z++) + policies[pNo].core_share[z].status = 0; + + /* Foreach vcpu in a policy. */ + for (z = 0; z < policies[pNo].pkt.num_vcpu; z++) { + /* Foreach VM on the platform. */ + for (x = 0; x < noVms; x++) { + /* Foreach vcpu of VMs on platform. */ + for (t = 0; t < lvm_info[x].num_cpus; t++) + core_share(pNo, z, x, t); + } + } +} + +static void +get_pcpu_to_control(struct policy *pol) +{ + + /* Convert vcpu to pcpu. */ + struct vm_info info; + int pcpu, count; + uint64_t mask_u64b; + + RTE_LOG(INFO, CHANNEL_MONITOR, "Looking for pcpu for %s\n", + pol->pkt.vm_name); + get_info_vm(pol->pkt.vm_name, &info); + + for (count = 0; count < pol->pkt.num_vcpu; count++) { + mask_u64b = info.pcpu_mask[pol->pkt.vcpu_to_control[count]]; + for (pcpu = 0; mask_u64b; mask_u64b &= ~(1ULL << pcpu++)) { + if ((mask_u64b >> pcpu) & 1) + pol->core_share[count].pcpu = pcpu; + } + } +} + +static int +get_pfid(struct policy *pol) +{ + + int i, x, ret = 0, nb_ports; + + nb_ports = rte_eth_dev_count(); + for (i = 0; i < pol->pkt.nb_mac_to_monitor; i++) { + + for (x = 0; x < nb_ports; x++) { + ret = rte_pmd_i40e_query_vfid_by_mac(x, + (struct ether_addr *)&(pol->pkt.vfid[i])); + if (ret != -EINVAL) { + pol->port[i] = x; + break; + } + } + if (ret == -EINVAL || ret == -ENOTSUP || ret == ENODEV) { + RTE_LOG(INFO, CHANNEL_MONITOR, + "Error with Policy. MAC not found on " + "attached ports "); + pol->enabled = 0; + return ret; + } + pol->pfid[i] = ret; + } + return 1; +} + +static int +update_policy(struct channel_packet *pkt) +{ + + unsigned int updated = 0; + int i; + + for (i = 0; i < MAX_VMS; i++) { + if (strcmp(policies[i].pkt.vm_name, pkt->vm_name) == 0) { + policies[i].pkt = *pkt; + get_pcpu_to_control(&policies[i]); + if (get_pfid(&policies[i]) == -1) { + updated = 1; + break; + } + core_share_status(i); + policies[i].enabled = 1; + updated = 1; + } + } + if (!updated) { + for (i = 0; i < MAX_VMS; i++) { + if (policies[i].enabled == 0) { + policies[i].pkt = *pkt; + get_pcpu_to_control(&policies[i]); + if (get_pfid(&policies[i]) == -1) + break; + core_share_status(i); + policies[i].enabled = 1; + break; + } + } + } + return 0; +} + +static uint64_t +get_pkt_diff(struct policy *pol) +{ + + uint64_t vsi_pkt_count, + vsi_pkt_total = 0, + vsi_pkt_count_prev_total = 0; + double rdtsc_curr, rdtsc_diff, diff; + int x; + struct rte_eth_stats vf_stats; + + for (x = 0; x < pol->pkt.nb_mac_to_monitor; x++) { + + /*Read vsi stats*/ + if (rte_pmd_i40e_get_vf_stats(x, pol->pfid[x], &vf_stats) == 0) + vsi_pkt_count = vf_stats.ipackets; + else + vsi_pkt_count = -1; + + vsi_pkt_total += vsi_pkt_count; + + vsi_pkt_count_prev_total += vsi_pkt_count_prev[pol->pfid[x]]; + vsi_pkt_count_prev[pol->pfid[x]] = vsi_pkt_count; + } + + rdtsc_curr = rte_rdtsc_precise(); + rdtsc_diff = rdtsc_curr - rdtsc_prev[pol->pfid[x-1]]; + rdtsc_prev[pol->pfid[x-1]] = rdtsc_curr; + + diff = (vsi_pkt_total - vsi_pkt_count_prev_total) * + ((double)rte_get_tsc_hz() / rdtsc_diff); + + return diff; +} + +static void +apply_traffic_profile(struct policy *pol) +{ + + int count; + uint64_t diff = 0; + + diff = get_pkt_diff(pol); + + RTE_LOG(INFO, CHANNEL_MONITOR, "Applying traffic profile\n"); + + if (diff >= (pol->pkt.traffic_policy.max_max_packet_thresh)) { + for (count = 0; count < pol->pkt.num_vcpu; count++) { + if (pol->core_share[count].status != 1) + power_manager_scale_core_max( + pol->core_share[count].pcpu); + } + } else if (diff >= (pol->pkt.traffic_policy.avg_max_packet_thresh)) { + for (count = 0; count < pol->pkt.num_vcpu; count++) { + if (pol->core_share[count].status != 1) + power_manager_scale_core_med( + pol->core_share[count].pcpu); + } + } else if (diff < (pol->pkt.traffic_policy.avg_max_packet_thresh)) { + for (count = 0; count < pol->pkt.num_vcpu; count++) { + if (pol->core_share[count].status != 1) + power_manager_scale_core_min( + pol->core_share[count].pcpu); + } + } +} + +static void +apply_time_profile(struct policy *pol) +{ + + int count, x; + struct timeval tv; + struct tm *ptm; + char time_string[40]; + + /* Obtain the time of day, and convert it to a tm struct. */ + gettimeofday(&tv, NULL); + ptm = localtime(&tv.tv_sec); + /* Format the date and time, down to a single second. */ + strftime(time_string, sizeof(time_string), "%Y-%m-%d %H:%M:%S", ptm); + + for (x = 0; x < HOURS; x++) { + + if (ptm->tm_hour == pol->pkt.timer_policy.busy_hours[x]) { + for (count = 0; count < pol->pkt.num_vcpu; count++) { + if (pol->core_share[count].status != 1) { + power_manager_scale_core_max( + pol->core_share[count].pcpu); + RTE_LOG(INFO, CHANNEL_MONITOR, + "Scaling up core %d to max\n", + pol->core_share[count].pcpu); + } + } + break; + } else if (ptm->tm_hour == + pol->pkt.timer_policy.quiet_hours[x]) { + for (count = 0; count < pol->pkt.num_vcpu; count++) { + if (pol->core_share[count].status != 1) { + power_manager_scale_core_min( + pol->core_share[count].pcpu); + RTE_LOG(INFO, CHANNEL_MONITOR, + "Scaling down core %d to min\n", + pol->core_share[count].pcpu); + } + } + break; + } else if (ptm->tm_hour == + pol->pkt.timer_policy.hours_to_use_traffic_profile[x]) { + apply_traffic_profile(pol); + break; + } + } +} + +static void +apply_workload_profile(struct policy *pol) +{ + + int count; + + if (pol->pkt.workload == HIGH) { + for (count = 0; count < pol->pkt.num_vcpu; count++) { + if (pol->core_share[count].status != 1) + power_manager_scale_core_max( + pol->core_share[count].pcpu); + } + } else if (pol->pkt.workload == MEDIUM) { + for (count = 0; count < pol->pkt.num_vcpu; count++) { + if (pol->core_share[count].status != 1) + power_manager_scale_core_med( + pol->core_share[count].pcpu); + } + } else if (pol->pkt.workload == LOW) { + for (count = 0; count < pol->pkt.num_vcpu; count++) { + if (pol->core_share[count].status != 1) + power_manager_scale_core_min( + pol->core_share[count].pcpu); + } + } +} + +static void +apply_policy(struct policy *pol) +{ + + struct channel_packet *pkt = &pol->pkt; + + /*Check policy to use*/ + if (pkt->policy_to_use == TRAFFIC) + apply_traffic_profile(pol); + else if (pkt->policy_to_use == TIME) + apply_time_profile(pol); + else if (pkt->policy_to_use == WORKLOAD) + apply_workload_profile(pol); +} + + static int process_request(struct channel_packet *pkt, struct channel_info *chan_info) { @@ -105,6 +395,12 @@ process_request(struct channel_packet *pkt, struct channel_info *chan_info) case(CPU_POWER_SCALE_UP): power_manager_scale_core_up(core_num); break; + case(CPU_POWER_ENABLE_TURBO): + power_manager_enable_turbo_core(core_num); + break; + case(CPU_POWER_DISABLE_TURBO): + power_manager_disable_turbo_core(core_num); + break; default: break; } @@ -122,12 +418,25 @@ process_request(struct channel_packet *pkt, struct channel_info *chan_info) case(CPU_POWER_SCALE_UP): power_manager_scale_mask_up(core_mask); break; + case(CPU_POWER_ENABLE_TURBO): + power_manager_enable_turbo_mask(core_mask); + break; + case(CPU_POWER_DISABLE_TURBO): + power_manager_disable_turbo_mask(core_mask); + break; default: break; } } } + + if (pkt->command == PKT_POLICY) { + RTE_LOG(INFO, CHANNEL_MONITOR, "\nProcessing Policy request from Guest\n"); + update_policy(pkt); + policy_is_set = 1; + } + /* Return is not checked as channel status may have been set to DISABLED * from management thread */ @@ -197,9 +506,10 @@ run_channel_monitor(void) struct channel_info *chan_info = (struct channel_info *) global_events_list[i].data.ptr; if ((global_events_list[i].events & EPOLLERR) || - (global_events_list[i].events & EPOLLHUP)) { + (global_events_list[i].events & EPOLLHUP)) { RTE_LOG(DEBUG, CHANNEL_MONITOR, "Remote closed connection for " - "channel '%s'\n", chan_info->channel_path); + "channel '%s'\n", + chan_info->channel_path); remove_channel(&chan_info); continue; } @@ -211,14 +521,17 @@ run_channel_monitor(void) int buffer_len = sizeof(pkt); while (buffer_len > 0) { - n_bytes = read(chan_info->fd, buffer, buffer_len); + n_bytes = read(chan_info->fd, + buffer, buffer_len); if (n_bytes == buffer_len) break; if (n_bytes == -1) { err = errno; - RTE_LOG(DEBUG, CHANNEL_MONITOR, "Received error on " - "channel '%s' read: %s\n", - chan_info->channel_path, strerror(err)); + RTE_LOG(DEBUG, CHANNEL_MONITOR, + "Received error on " + "channel '%s' read: %s\n", + chan_info->channel_path, + strerror(err)); remove_channel(&chan_info); break; } @@ -229,5 +542,14 @@ run_channel_monitor(void) process_request(&pkt, chan_info); } } + rte_delay_us(time_period_ms*1000); + if (policy_is_set) { + int j; + + for (j = 0; j < MAX_VMS; j++) { + if (policies[j].enabled == 1) + apply_policy(&policies[j]); + } + } } } diff --git a/examples/vm_power_manager/channel_monitor.h b/examples/vm_power_manager/channel_monitor.h index c1386079..b52c1fca 100644 --- a/examples/vm_power_manager/channel_monitor.h +++ b/examples/vm_power_manager/channel_monitor.h @@ -35,6 +35,24 @@ #define CHANNEL_MONITOR_H_ #include "channel_manager.h" +#include "channel_commands.h" + +struct core_share { + unsigned int pcpu; + /* + * 1 CORE SHARE + * 0 NOT SHARED + */ + int status; +}; + +struct policy { + struct channel_packet pkt; + uint32_t pfid[MAX_VFS]; + uint32_t port[MAX_VFS]; + unsigned int enabled; + struct core_share core_share[MAX_VCPU_PER_VM]; +}; #ifdef __cplusplus extern "C" { diff --git a/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c b/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c index 7931135e..63f711e0 100644 --- a/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c +++ b/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c @@ -45,8 +45,10 @@ #include <cmdline.h> #include <rte_log.h> #include <rte_lcore.h> +#include <rte_ethdev.h> #include <rte_power.h> +#include <guest_channel.h> #include "vm_power_cli_guest.h" @@ -108,6 +110,10 @@ cmd_set_cpu_freq_parsed(void *parsed_result, struct cmdline *cl, ret = rte_power_freq_min(res->lcore_id); else if (!strcmp(res->cmd , "max")) ret = rte_power_freq_max(res->lcore_id); + else if (!strcmp(res->cmd, "enable_turbo")) + ret = rte_power_freq_enable_turbo(res->lcore_id); + else if (!strcmp(res->cmd, "disable_turbo")) + ret = rte_power_freq_disable_turbo(res->lcore_id); if (ret != 1) cmdline_printf(cl, "Error sending message: %s\n", strerror(ret)); } @@ -120,13 +126,14 @@ cmdline_parse_token_string_t cmd_set_cpu_freq_core_num = lcore_id, UINT8); cmdline_parse_token_string_t cmd_set_cpu_freq_cmd_cmd = TOKEN_STRING_INITIALIZER(struct cmd_set_cpu_freq_result, - cmd, "up#down#min#max"); + cmd, "up#down#min#max#enable_turbo#disable_turbo"); cmdline_parse_inst_t cmd_set_cpu_freq_set = { .f = cmd_set_cpu_freq_parsed, .data = NULL, - .help_str = "set_cpu_freq <core_num> <up|down|min|max>, Set the current " - "frequency for the specified core by scaling up/down/min/max", + .help_str = "set_cpu_freq <core_num> " + "<up|down|min|max|enable_turbo|disable_turbo>, " + "adjust the frequency for the specified core.", .tokens = { (void *)&cmd_set_cpu_freq, (void *)&cmd_set_cpu_freq_core_num, @@ -135,8 +142,103 @@ cmdline_parse_inst_t cmd_set_cpu_freq_set = { }, }; +struct cmd_send_policy_result { + cmdline_fixed_string_t send_policy; + cmdline_fixed_string_t cmd; +}; + +union PFID { + struct ether_addr addr; + uint64_t pfid; +}; + +static inline int +send_policy(void) +{ + struct channel_packet pkt; + int ret; + + union PFID pfid; + /* Use port MAC address as the vfid */ + rte_eth_macaddr_get(0, &pfid.addr); + printf("Port %u MAC: %02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 ":" + "%02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 "\n", + 1, + pfid.addr.addr_bytes[0], pfid.addr.addr_bytes[1], + pfid.addr.addr_bytes[2], pfid.addr.addr_bytes[3], + pfid.addr.addr_bytes[4], pfid.addr.addr_bytes[5]); + pkt.vfid[0] = pfid.pfid; + + pkt.nb_mac_to_monitor = 1; + pkt.t_boost_status.tbEnabled = false; + + pkt.vcpu_to_control[0] = 0; + pkt.vcpu_to_control[1] = 1; + pkt.num_vcpu = 2; + /* Dummy Population. */ + pkt.traffic_policy.min_packet_thresh = 96000; + pkt.traffic_policy.avg_max_packet_thresh = 1800000; + pkt.traffic_policy.max_max_packet_thresh = 2000000; + + pkt.timer_policy.busy_hours[0] = 3; + pkt.timer_policy.busy_hours[1] = 4; + pkt.timer_policy.busy_hours[2] = 5; + pkt.timer_policy.quiet_hours[0] = 11; + pkt.timer_policy.quiet_hours[1] = 12; + pkt.timer_policy.quiet_hours[2] = 13; + + pkt.timer_policy.hours_to_use_traffic_profile[0] = 8; + pkt.timer_policy.hours_to_use_traffic_profile[1] = 10; + + pkt.workload = LOW; + pkt.policy_to_use = TIME; + pkt.command = PKT_POLICY; + strcpy(pkt.vm_name, "ubuntu2"); + ret = rte_power_guest_channel_send_msg(&pkt, 1); + if (ret == 0) + return 1; + RTE_LOG(DEBUG, POWER, "Error sending message: %s\n", + ret > 0 ? strerror(ret) : "channel not connected"); + return -1; +} + +static void +cmd_send_policy_parsed(void *parsed_result, struct cmdline *cl, + __attribute__((unused)) void *data) +{ + int ret = -1; + struct cmd_send_policy_result *res = parsed_result; + + if (!strcmp(res->cmd, "now")) { + printf("Sending Policy down now!\n"); + ret = send_policy(); + } + if (ret != 1) + cmdline_printf(cl, "Error sending message: %s\n", + strerror(ret)); +} + +cmdline_parse_token_string_t cmd_send_policy = + TOKEN_STRING_INITIALIZER(struct cmd_send_policy_result, + send_policy, "send_policy"); +cmdline_parse_token_string_t cmd_send_policy_cmd_cmd = + TOKEN_STRING_INITIALIZER(struct cmd_send_policy_result, + cmd, "now"); + +cmdline_parse_inst_t cmd_send_policy_set = { + .f = cmd_send_policy_parsed, + .data = NULL, + .help_str = "send_policy now", + .tokens = { + (void *)&cmd_send_policy, + (void *)&cmd_send_policy_cmd_cmd, + NULL, + }, +}; + cmdline_parse_ctx_t main_ctx[] = { (cmdline_parse_inst_t *)&cmd_quit, + (cmdline_parse_inst_t *)&cmd_send_policy_set, (cmdline_parse_inst_t *)&cmd_set_cpu_freq_set, NULL, }; diff --git a/examples/vm_power_manager/guest_cli/vm_power_cli_guest.h b/examples/vm_power_manager/guest_cli/vm_power_cli_guest.h index 0c4bdd5b..277eab3d 100644 --- a/examples/vm_power_manager/guest_cli/vm_power_cli_guest.h +++ b/examples/vm_power_manager/guest_cli/vm_power_cli_guest.h @@ -40,12 +40,6 @@ extern "C" { #include "channel_commands.h" -int guest_channel_host_connect(unsigned lcore_id); - -int guest_channel_send_msg(struct channel_packet *pkt, unsigned lcore_id); - -void guest_channel_host_disconnect(unsigned lcore_id); - void run_cli(__attribute__((unused)) void *arg); #ifdef __cplusplus diff --git a/examples/vm_power_manager/main.c b/examples/vm_power_manager/main.c index c33fcc93..399fbdd4 100644 --- a/examples/vm_power_manager/main.c +++ b/examples/vm_power_manager/main.c @@ -49,14 +49,206 @@ #include <rte_log.h> #include <rte_per_lcore.h> #include <rte_lcore.h> +#include <rte_ethdev.h> +#include <getopt.h> +#include <rte_cycles.h> #include <rte_debug.h> #include "channel_manager.h" #include "channel_monitor.h" #include "power_manager.h" #include "vm_power_cli.h" +#include <rte_pmd_ixgbe.h> +#include <rte_pmd_i40e.h> +#include <rte_pmd_bnxt.h> + +#define RX_RING_SIZE 512 +#define TX_RING_SIZE 512 + +#define NUM_MBUFS 8191 +#define MBUF_CACHE_SIZE 250 +#define BURST_SIZE 32 + +static uint32_t enabled_port_mask; +static volatile bool force_quit; + +/****************/ +static const struct rte_eth_conf port_conf_default = { + .rxmode = { .max_rx_pkt_len = ETHER_MAX_LEN } +}; + +static inline int +port_init(uint16_t port, struct rte_mempool *mbuf_pool) +{ + struct rte_eth_conf port_conf = port_conf_default; + const uint16_t rx_rings = 1, tx_rings = 1; + int retval; + uint16_t q; + + if (port >= rte_eth_dev_count()) + return -1; + + /* Configure the Ethernet device. */ + retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf); + if (retval != 0) + return retval; + + /* Allocate and set up 1 RX queue per Ethernet port. */ + for (q = 0; q < rx_rings; q++) { + retval = rte_eth_rx_queue_setup(port, q, RX_RING_SIZE, + rte_eth_dev_socket_id(port), NULL, mbuf_pool); + if (retval < 0) + return retval; + } + + /* Allocate and set up 1 TX queue per Ethernet port. */ + for (q = 0; q < tx_rings; q++) { + retval = rte_eth_tx_queue_setup(port, q, TX_RING_SIZE, + rte_eth_dev_socket_id(port), NULL); + if (retval < 0) + return retval; + } + + /* Start the Ethernet port. */ + retval = rte_eth_dev_start(port); + if (retval < 0) + return retval; + + /* Display the port MAC address. */ + struct ether_addr addr; + rte_eth_macaddr_get(port, &addr); + printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 + " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", + (unsigned int)port, + addr.addr_bytes[0], addr.addr_bytes[1], + addr.addr_bytes[2], addr.addr_bytes[3], + addr.addr_bytes[4], addr.addr_bytes[5]); + + /* Enable RX in promiscuous mode for the Ethernet device. */ + rte_eth_promiscuous_enable(port); + + + return 0; +} static int +parse_portmask(const char *portmask) +{ + char *end = NULL; + unsigned long pm; + + /* parse hexadecimal string */ + pm = strtoul(portmask, &end, 16); + if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) + return -1; + + if (pm == 0) + return -1; + + return pm; +} +/* Parse the argument given in the command line of the application */ +static int +parse_args(int argc, char **argv) +{ + int opt, ret; + char **argvopt; + int option_index; + char *prgname = argv[0]; + static struct option lgopts[] = { + { "mac-updating", no_argument, 0, 1}, + { "no-mac-updating", no_argument, 0, 0}, + {NULL, 0, 0, 0} + }; + argvopt = argv; + + while ((opt = getopt_long(argc, argvopt, "p:q:T:", + lgopts, &option_index)) != EOF) { + + switch (opt) { + /* portmask */ + case 'p': + enabled_port_mask = parse_portmask(optarg); + if (enabled_port_mask == 0) { + printf("invalid portmask\n"); + return -1; + } + break; + /* long options */ + case 0: + break; + + default: + return -1; + } + } + + if (optind >= 0) + argv[optind-1] = prgname; + + ret = optind-1; + optind = 0; /* reset getopt lib */ + return ret; +} + +static void +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) +{ +#define CHECK_INTERVAL 100 /* 100ms */ +#define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ + uint16_t portid, count, all_ports_up, print_flag = 0; + struct rte_eth_link link; + + printf("\nChecking link status"); + fflush(stdout); + for (count = 0; count <= MAX_CHECK_TIME; count++) { + if (force_quit) + return; + all_ports_up = 1; + for (portid = 0; portid < port_num; portid++) { + if (force_quit) + return; + if ((port_mask & (1 << portid)) == 0) + continue; + memset(&link, 0, sizeof(link)); + rte_eth_link_get_nowait(portid, &link); + /* print link status if flag set */ + if (print_flag == 1) { + if (link.link_status) + printf("Port %d Link Up - speed %u " + "Mbps - %s\n", (uint16_t)portid, + (unsigned int)link.link_speed, + (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? + ("full-duplex") : ("half-duplex\n")); + else + printf("Port %d Link Down\n", + (uint16_t)portid); + continue; + } + /* clear all_ports_up flag if any link down */ + if (link.link_status == ETH_LINK_DOWN) { + all_ports_up = 0; + break; + } + } + /* after finally printing all link status, get out */ + if (print_flag == 1) + break; + + if (all_ports_up == 0) { + printf("."); + fflush(stdout); + rte_delay_ms(CHECK_INTERVAL); + } + + /* set the print_flag if all ports up or timeout */ + if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { + print_flag = 1; + printf("done\n"); + } + } +} +static int run_monitor(__attribute__((unused)) void *arg) { if (channel_monitor_init() < 0) { @@ -82,6 +274,10 @@ main(int argc, char **argv) { int ret; unsigned lcore_id; + unsigned int nb_ports; + struct rte_mempool *mbuf_pool; + uint16_t portid; + ret = rte_eal_init(argc, argv); if (ret < 0) @@ -90,12 +286,77 @@ main(int argc, char **argv) signal(SIGINT, sig_handler); signal(SIGTERM, sig_handler); + argc -= ret; + argv += ret; + + /* parse application arguments (after the EAL ones) */ + ret = parse_args(argc, argv); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Invalid arguments\n"); + + nb_ports = rte_eth_dev_count(); + + mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * nb_ports, + MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); + + if (mbuf_pool == NULL) + rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); + + /* Initialize ports. */ + for (portid = 0; portid < nb_ports; portid++) { + struct ether_addr eth; + int w, j; + int ret = -ENOTSUP; + + if ((enabled_port_mask & (1 << portid)) == 0) + continue; + + eth.addr_bytes[0] = 0xe0; + eth.addr_bytes[1] = 0xe0; + eth.addr_bytes[2] = 0xe0; + eth.addr_bytes[3] = 0xe0; + eth.addr_bytes[4] = portid + 0xf0; + + if (port_init(portid, mbuf_pool) != 0) + rte_exit(EXIT_FAILURE, "Cannot init port %"PRIu8 "\n", + portid); + + for (w = 0; w < MAX_VFS; w++) { + eth.addr_bytes[5] = w + 0xf0; + + if (ret == -ENOTSUP) + ret = rte_pmd_ixgbe_set_vf_mac_addr(portid, + w, ð); + if (ret == -ENOTSUP) + ret = rte_pmd_i40e_set_vf_mac_addr(portid, + w, ð); + if (ret == -ENOTSUP) + ret = rte_pmd_bnxt_set_vf_mac_addr(portid, + w, ð); + + switch (ret) { + case 0: + printf("Port %d VF %d MAC: ", + portid, w); + for (j = 0; j < 6; j++) { + printf("%02x", eth.addr_bytes[j]); + if (j < 5) + printf(":"); + } + printf("\n"); + break; + } + } + } + lcore_id = rte_get_next_lcore(-1, 1, 0); if (lcore_id == RTE_MAX_LCORE) { RTE_LOG(ERR, EAL, "A minimum of two cores are required to run " "application\n"); return 0; } + + check_all_ports_link_status(nb_ports, enabled_port_mask); rte_eal_remote_launch(run_monitor, NULL, lcore_id); if (power_manager_init() < 0) { diff --git a/examples/vm_power_manager/power_manager.c b/examples/vm_power_manager/power_manager.c index 2644fce6..1834a823 100644 --- a/examples/vm_power_manager/power_manager.c +++ b/examples/vm_power_manager/power_manager.c @@ -108,7 +108,7 @@ set_host_cpus_mask(void) int power_manager_init(void) { - unsigned i, num_cpus; + unsigned int i, num_cpus, num_freqs; uint64_t cpu_mask; int ret = 0; @@ -121,15 +121,21 @@ power_manager_init(void) rte_power_set_env(PM_ENV_ACPI_CPUFREQ); cpu_mask = global_enabled_cpus; for (i = 0; cpu_mask; cpu_mask &= ~(1 << i++)) { - if (rte_power_init(i) < 0 || rte_power_freqs(i, - global_core_freq_info[i].freqs, - RTE_MAX_LCORE_FREQS) == 0) { - RTE_LOG(ERR, POWER_MANAGER, "Unable to initialize power manager " + if (rte_power_init(i) < 0) + RTE_LOG(ERR, POWER_MANAGER, + "Unable to initialize power manager " "for core %u\n", i); + num_freqs = rte_power_freqs(i, global_core_freq_info[i].freqs, + RTE_MAX_LCORE_FREQS); + if (num_freqs == 0) { + RTE_LOG(ERR, POWER_MANAGER, + "Unable to get frequency list for core %u\n", + i); global_enabled_cpus &= ~(1 << i); num_cpus--; ret = -1; } + global_core_freq_info[i].num_freqs = num_freqs; rte_spinlock_init(&global_core_freq_info[i].power_sl); } RTE_LOG(INFO, POWER_MANAGER, "Detected %u host CPUs , enabled core mask:" @@ -216,6 +222,24 @@ power_manager_scale_mask_max(uint64_t core_mask) } int +power_manager_enable_turbo_mask(uint64_t core_mask) +{ + int ret = 0; + + POWER_SCALE_MASK(enable_turbo, core_mask, ret); + return ret; +} + +int +power_manager_disable_turbo_mask(uint64_t core_mask) +{ + int ret = 0; + + POWER_SCALE_MASK(disable_turbo, core_mask, ret); + return ret; +} + +int power_manager_scale_core_up(unsigned core_num) { int ret = 0; @@ -250,3 +274,37 @@ power_manager_scale_core_max(unsigned core_num) POWER_SCALE_CORE(max, core_num, ret); return ret; } + +int +power_manager_enable_turbo_core(unsigned int core_num) +{ + int ret = 0; + + POWER_SCALE_CORE(enable_turbo, core_num, ret); + return ret; +} + +int +power_manager_disable_turbo_core(unsigned int core_num) +{ + int ret = 0; + + POWER_SCALE_CORE(disable_turbo, core_num, ret); + return ret; +} + +int +power_manager_scale_core_med(unsigned int core_num) +{ + int ret = 0; + + if (core_num >= POWER_MGR_MAX_CPUS) + return -1; + if (!(global_enabled_cpus & (1ULL << core_num))) + return -1; + rte_spinlock_lock(&global_core_freq_info[core_num].power_sl); + ret = rte_power_set_freq(core_num, + global_core_freq_info[core_num].num_freqs / 2); + rte_spinlock_unlock(&global_core_freq_info[core_num].power_sl); + return ret; +} diff --git a/examples/vm_power_manager/power_manager.h b/examples/vm_power_manager/power_manager.h index 1b45babf..b52fb4c7 100644 --- a/examples/vm_power_manager/power_manager.h +++ b/examples/vm_power_manager/power_manager.h @@ -113,6 +113,32 @@ int power_manager_scale_mask_min(uint64_t core_mask); int power_manager_scale_mask_max(uint64_t core_mask); /** + * Enable Turbo Boost on the cores specified in core_mask. + * It is thread-safe. + * + * @param core_mask + * The uint64_t bit-mask of cores to change frequency. + * + * @return + * - 1 on success. + * - Negative on error. + */ +int power_manager_enable_turbo_mask(uint64_t core_mask); + +/** + * Disable Turbo Boost on the cores specified in core_mask. + * It is thread-safe. + * + * @param core_mask + * The uint64_t bit-mask of cores to change frequency. + * + * @return + * - 1 on success. + * - Negative on error. + */ +int power_manager_disable_turbo_mask(uint64_t core_mask); + +/** * Scale up frequency for the core specified by core_num. * It is thread-safe. * @@ -168,6 +194,32 @@ int power_manager_scale_core_min(unsigned core_num); int power_manager_scale_core_max(unsigned core_num); /** + * Enable Turbo Boost for the core specified by core_num. + * It is thread-safe. + * + * @param core_num + * The core number to boost + * + * @return + * - 1 on success. + * - Negative on error. + */ +int power_manager_enable_turbo_core(unsigned int core_num); + +/** + * Disable Turbo Boost for the core specified by core_num. + * It is thread-safe. + * + * @param core_num + * The core number to boost + * + * @return + * - 1 on success. + * - Negative on error. + */ +int power_manager_disable_turbo_core(unsigned int core_num); + +/** * Get the current freuency of the core specified by core_num * * @param core_num @@ -179,6 +231,19 @@ int power_manager_scale_core_max(unsigned core_num); */ uint32_t power_manager_get_current_frequency(unsigned core_num); +/** + * Scale to medium frequency for the core specified by core_num. + * It is thread-safe. + * + * @param core_num + * The core number to change frequency + * + * @return + * - 1 on success. + * - 0 if frequency not changed. + * - Negative on error. + */ +int power_manager_scale_core_med(unsigned int core_num); #ifdef __cplusplus } diff --git a/examples/vm_power_manager/vm_power_cli.c b/examples/vm_power_manager/vm_power_cli.c index c5e8d934..6f234fb7 100644 --- a/examples/vm_power_manager/vm_power_cli.c +++ b/examples/vm_power_manager/vm_power_cli.c @@ -520,6 +520,10 @@ cmd_set_cpu_freq_mask_parsed(void *parsed_result, struct cmdline *cl, ret = power_manager_scale_mask_min(res->core_mask); else if (!strcmp(res->cmd , "max")) ret = power_manager_scale_mask_max(res->core_mask); + else if (!strcmp(res->cmd, "enable_turbo")) + ret = power_manager_enable_turbo_mask(res->core_mask); + else if (!strcmp(res->cmd, "disable_turbo")) + ret = power_manager_disable_turbo_mask(res->core_mask); if (ret < 0) { cmdline_printf(cl, "Error scaling core_mask(0x%"PRIx64") '%s' , not " "all cores specified have been scaled\n", @@ -535,14 +539,13 @@ cmdline_parse_token_num_t cmd_set_cpu_freq_mask_core_mask = core_mask, UINT64); cmdline_parse_token_string_t cmd_set_cpu_freq_mask_result = TOKEN_STRING_INITIALIZER(struct cmd_set_cpu_freq_mask_result, - cmd, "up#down#min#max"); + cmd, "up#down#min#max#enable_turbo#disable_turbo"); cmdline_parse_inst_t cmd_set_cpu_freq_mask_set = { .f = cmd_set_cpu_freq_mask_parsed, .data = NULL, - .help_str = "set_cpu_freq <core_mask> <up|down|min|max>, Set the current " - "frequency for the cores specified in <core_mask> by scaling " - "each up/down/min/max.", + .help_str = "set_cpu_freq <core_mask> <up|down|min|max|enable_turbo|disable_turbo>, adjust the current " + "frequency for the cores specified in <core_mask>", .tokens = { (void *)&cmd_set_cpu_freq_mask, (void *)&cmd_set_cpu_freq_mask_core_mask, @@ -614,6 +617,10 @@ cmd_set_cpu_freq_parsed(void *parsed_result, struct cmdline *cl, ret = power_manager_scale_core_min(res->core_num); else if (!strcmp(res->cmd , "max")) ret = power_manager_scale_core_max(res->core_num); + else if (!strcmp(res->cmd, "enable_turbo")) + ret = power_manager_enable_turbo_core(res->core_num); + else if (!strcmp(res->cmd, "disable_turbo")) + ret = power_manager_disable_turbo_core(res->core_num); if (ret < 0) { cmdline_printf(cl, "Error scaling core(%u) '%s'\n", res->core_num, res->cmd); @@ -628,13 +635,13 @@ cmdline_parse_token_num_t cmd_set_cpu_freq_core_num = core_num, UINT8); cmdline_parse_token_string_t cmd_set_cpu_freq_cmd_cmd = TOKEN_STRING_INITIALIZER(struct cmd_set_cpu_freq_result, - cmd, "up#down#min#max"); + cmd, "up#down#min#max#enable_turbo#disable_turbo"); cmdline_parse_inst_t cmd_set_cpu_freq_set = { .f = cmd_set_cpu_freq_parsed, .data = NULL, - .help_str = "set_cpu_freq <core_num> <up|down|min|max>, Set the current " - "frequency for the specified core by scaling up/down/min/max", + .help_str = "set_cpu_freq <core_num> <up|down|min|max|enable_turbo|disable_turbo>, adjust the current " + "frequency for the specified core", .tokens = { (void *)&cmd_set_cpu_freq, (void *)&cmd_set_cpu_freq_core_num, |