From 0e9325d31b6028ca6ff16257e2568da103111fcc Mon Sep 17 00:00:00 2001 From: Mihai Carabas Date: Wed, 12 Feb 2014 13:10:38 +0200 Subject: [PATCH] CPU Topology: add support for Compute Units on AMD processors Detect shared compute units between cores on AMD processors and downgrade them to THREAD_LEVEL in the logical CPU topology used by the scheduler. --- sys/cpu/x86_64/include/specialreg.h | 1 + sys/kern/subr_cpu_topology.c | 110 ++++++++++++++++++++++---- sys/kern/usched_dfly.c | 18 ++--- sys/platform/pc64/include/smp.h | 3 + sys/platform/pc64/x86_64/mp_machdep.c | 33 ++++++++ sys/sys/cpu_topology.h | 5 +- 6 files changed, 145 insertions(+), 25 deletions(-) diff --git a/sys/cpu/x86_64/include/specialreg.h b/sys/cpu/x86_64/include/specialreg.h index 55d57040d2..d2ec515816 100644 --- a/sys/cpu/x86_64/include/specialreg.h +++ b/sys/cpu/x86_64/include/specialreg.h @@ -175,6 +175,7 @@ #define AMDID2_SSE5 0x00000800 #define AMDID2_SKINIT 0x00001000 #define AMDID2_WDT 0x00002000 +#define AMDID2_TOPOEXT 0x00400000 /* * CPUID instruction 1 eax info diff --git a/sys/kern/subr_cpu_topology.c b/sys/kern/subr_cpu_topology.c index 5294b3e469..a1f4da577a 100644 --- a/sys/kern/subr_cpu_topology.c +++ b/sys/kern/subr_cpu_topology.c @@ -109,38 +109,48 @@ build_topology_tree(int *children_no_per_level, node->child_no = children_no_per_level[cur_level]; node->type = level_types[cur_level]; node->members = 0; + node->compute_unit_id = -1; if (node->child_no == 0) { - node->child_node = NULL; *apicid = get_next_valid_apicid(*apicid); node->members = CPUMASK(get_cpuid_from_apicid(*apicid)); return; } - node->child_node = *last_free_node; - (*last_free_node) += node->child_no; if (node->parent_node == NULL) root_cpu_node = node; for (i = 0; i < node->child_no; i++) { - node->child_node[i].parent_node = node; + node->child_node[i] = *last_free_node; + (*last_free_node)++; + + node->child_node[i]->parent_node = node; build_topology_tree(children_no_per_level, level_types, cur_level + 1, - &(node->child_node[i]), + node->child_node[i], last_free_node, apicid); - node->members |= node->child_node[i].members; + node->members |= node->child_node[i]->members; } } +static void migrate_elements(cpu_node_t **a, int n, int pos) { + int i; + + for (i = pos; i < n - 1 ; i++) { + a[i] = a[i+1]; + } + a[i] = NULL; +} + /* Build CPU topology. The detection is made by comparing the * chip, core and logical IDs of each CPU with the IDs of the * BSP. When we found a match, at that level the CPUs are siblings. */ -static cpu_node_t * +static void build_cpu_topology(void) { detect_cpu_topology(); @@ -243,7 +253,67 @@ build_cpu_topology(void) } - return root; + cpu_root_node = root; + + +#if defined(__x86_64__) + if (fix_amd_topology() == 0) { + int visited[MAXCPU], i, j, pos, cpuid; + cpu_node_t *leaf, *parent; + + bzero(visited, MAXCPU * sizeof(int)); + + for (i = 0; i < ncpus; i++) { + if (visited[i] == 0) { + pos = 0; + visited[i] = 1; + leaf = get_cpu_node_by_cpuid(i); + + if (leaf->type == CORE_LEVEL) { + parent = leaf->parent_node; + + last_free_node->child_node[0] = leaf; + last_free_node->child_no = 1; + last_free_node->members = leaf->members; + last_free_node->compute_unit_id = leaf->compute_unit_id; + last_free_node->parent_node = parent; + last_free_node->type = CORE_LEVEL; + + + for (j = 0; j < parent->child_no; j++) { + if (parent->child_node[j] != leaf) { + + cpuid = BSFCPUMASK(parent->child_node[j]->members); + if (visited[cpuid] == 0 && + parent->child_node[j]->compute_unit_id == leaf->compute_unit_id) { + + last_free_node->child_node[last_free_node->child_no] = parent->child_node[j]; + last_free_node->child_no++; + last_free_node->members |= parent->child_node[j]->members; + + parent->child_node[j]->type = THREAD_LEVEL; + parent->child_node[j]->parent_node = last_free_node; + visited[cpuid] = 1; + + migrate_elements(parent->child_node, parent->child_no, j); + parent->child_no--; + j--; + } + } else { + pos = j; + } + } + if (last_free_node->child_no > 1) { + parent->child_node[pos] = last_free_node; + leaf->type = THREAD_LEVEL; + leaf->parent_node = last_free_node; + last_free_node++; + } + } + } + } + } +#endif } /* Recursive function helper to print the CPU topology tree */ @@ -276,11 +346,21 @@ print_cpu_topology_tree_sysctl_helper(cpu_node_t *node, sbuf_printf(sb,"CHIP ID %d: ", get_chip_ID(bsr_member)); } else if (node->type == CORE_LEVEL) { - sbuf_printf(sb,"CORE ID %d: ", - get_core_number_within_chip(bsr_member)); + if (node->compute_unit_id != -1) { + sbuf_printf(sb,"Compute Unit ID %d: ", + node->compute_unit_id); + } else { + sbuf_printf(sb,"CORE ID %d: ", + get_core_number_within_chip(bsr_member)); + } } else if (node->type == THREAD_LEVEL) { - sbuf_printf(sb,"THREAD ID %d: ", - get_logical_CPU_number_within_core(bsr_member)); + if (node->compute_unit_id != -1) { + sbuf_printf(sb,"CORE ID %d: ", + get_core_number_within_chip(bsr_member)); + } else { + sbuf_printf(sb,"THREAD ID %d: ", + get_logical_CPU_number_within_core(bsr_member)); + } } else { sbuf_printf(sb,"UNKNOWN: "); } @@ -291,7 +371,7 @@ print_cpu_topology_tree_sysctl_helper(cpu_node_t *node, sbuf_printf(sb,"\n"); for (i = 0; i < node->child_no; i++) { - print_cpu_topology_tree_sysctl_helper(&(node->child_node[i]), + print_cpu_topology_tree_sysctl_helper(node->child_node[i], sb, buf, buf_len, i == (node->child_no -1)); } } @@ -364,7 +444,7 @@ get_cpu_node_by_cpumask(cpu_node_t * node, } for (i = 0; i < node->child_no; i++) { - found = get_cpu_node_by_cpumask(&(node->child_node[i]), mask); + found = get_cpu_node_by_cpumask(node->child_node[i], mask); if (found != NULL) { return found; } @@ -560,7 +640,7 @@ build_sysctl_cpu_topology(void) static void init_cpu_topology(void) { - cpu_root_node = build_cpu_topology(); + build_cpu_topology(); init_pcpu_topology_sysctl(); build_sysctl_cpu_topology(); diff --git a/sys/kern/usched_dfly.c b/sys/kern/usched_dfly.c index d09105252d..fc12dae632 100644 --- a/sys/kern/usched_dfly.c +++ b/sys/kern/usched_dfly.c @@ -1468,15 +1468,15 @@ dfly_choose_best_queue(struct lwp *lp) /* * Degenerate case super-root */ - if (cpup->child_node && cpup->child_no == 1) { - cpup = cpup->child_node; + if (cpup->child_no == 1) { + cpup = cpup->child_node[0]; continue; } /* * Terminal cpunode */ - if (cpup->child_node == NULL) { + if (cpup->child_no == 0) { rdd = &dfly_pcpu[BSFCPUMASK(cpup->members)]; break; } @@ -1489,7 +1489,7 @@ dfly_choose_best_queue(struct lwp *lp) * Accumulate load information for all cpus * which are members of this node. */ - cpun = &cpup->child_node[n]; + cpun = cpup->child_node[n]; mask = cpun->members & usched_global_cpumask & smp_active_mask & lp->lwp_cpumask; if (mask == 0) @@ -1557,7 +1557,7 @@ dfly_choose_best_queue(struct lwp *lp) * the weight1 factor for the all-but-one nodes. */ if (cpun->members & wakemask) { - if (cpun->child_node != NULL) { + if (cpun->child_no != 0) { /* advantage */ load -= usched_dfly_weight2; } else { @@ -1635,15 +1635,15 @@ dfly_choose_worst_queue(dfly_pcpu_t dd) /* * Degenerate case super-root */ - if (cpup->child_node && cpup->child_no == 1) { - cpup = cpup->child_node; + if (cpup->child_no == 1) { + cpup = cpup->child_node[0]; continue; } /* * Terminal cpunode */ - if (cpup->child_node == NULL) { + if (cpup->child_no == 0) { rdd = &dfly_pcpu[BSFCPUMASK(cpup->members)]; break; } @@ -1656,7 +1656,7 @@ dfly_choose_worst_queue(dfly_pcpu_t dd) * Accumulate load information for all cpus * which are members of this node. */ - cpun = &cpup->child_node[n]; + cpun = cpup->child_node[n]; mask = cpun->members & usched_global_cpumask & smp_active_mask; if (mask == 0) diff --git a/sys/platform/pc64/include/smp.h b/sys/platform/pc64/include/smp.h index ff04c5301b..fef3375f60 100644 --- a/sys/platform/pc64/include/smp.h +++ b/sys/platform/pc64/include/smp.h @@ -81,6 +81,9 @@ extern cpumask_t smp_active_mask; /* Detect CPU topology bits */ void detect_cpu_topology(void); +/* Fix AMD CPU topology related to compute units */ +int fix_amd_topology(void); + /* Interface functions for IDs calculation */ int get_chip_ID(int cpuid); int get_core_number_within_chip(int cpuid); diff --git a/sys/platform/pc64/x86_64/mp_machdep.c b/sys/platform/pc64/x86_64/mp_machdep.c index 5810bb9b65..589b450bfd 100644 --- a/sys/platform/pc64/x86_64/mp_machdep.c +++ b/sys/platform/pc64/x86_64/mp_machdep.c @@ -35,6 +35,7 @@ #include #include /* cngetc() */ #include +#include #include @@ -1248,6 +1249,38 @@ detect_amd_topology(int count_htt_cores) } } +static void +amd_get_compute_unit_id(void *arg) +{ + u_int regs[4]; + + do_cpuid(0x8000001e, regs); + cpu_node_t * mynode = get_cpu_node_by_cpuid(mycpuid); + /* + * AMD - CPUID Specification September 2010 + * page 34 - //ComputeUnitID = ebx[0:7]// + */ + mynode->compute_unit_id = regs[1] & 0xff; +} + +int +fix_amd_topology(void) +{ + if (cpu_vendor_id != CPU_VENDOR_AMD) + return -1; + if ((amd_feature2 & AMDID2_TOPOEXT) == 0) + return -1; + + lwkt_cpusync_simple(-1, amd_get_compute_unit_id, NULL); + + kprintf("Compute unit iDS:\n"); + int i; + for (i = 0; i < ncpus; i++) + kprintf("%d-%d; \n", i, get_cpu_node_by_cpuid(i)->compute_unit_id); + + return 0; +} + /* Calculate * - logical_CPU_bits * - core_bits diff --git a/sys/sys/cpu_topology.h b/sys/sys/cpu_topology.h index a7a2342e25..182498c5af 100644 --- a/sys/sys/cpu_topology.h +++ b/sys/sys/cpu_topology.h @@ -6,10 +6,13 @@ /* CPU TOPOLOGY DATA AND FUNCTIONS */ struct cpu_node { struct cpu_node * parent_node; - struct cpu_node * child_node; + struct cpu_node * child_node[MAXCPU]; uint32_t child_no; cpumask_t members; uint8_t type; +#if defined(__x86_64__) + uint8_t compute_unit_id; /* AMD compute unit ID */ +#endif }; typedef struct cpu_node cpu_node_t; -- 2.41.0