2 * Copyright (c) 2012 The DragonFly Project. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in
12 * the documentation and/or other materials provided with the
14 * 3. Neither the name of The DragonFly Project nor the names of its
15 * contributors may be used to endorse or promote products derived
16 * from this software without specific, prior written permission.
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
21 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
22 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
24 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
28 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/kernel.h>
36 #include <sys/sysctl.h>
38 #include <sys/cpu_topology.h>
40 #include <machine/smp.h>
46 #define INDENT_BUF_SIZE LEVEL_NO*3
49 /* Per-cpu sysctl nodes and info */
50 struct per_cpu_sysctl_info {
51 struct sysctl_ctx_list sysctl_ctx;
52 struct sysctl_oid *sysctl_tree;
56 char physical_siblings[8*MAXCPU];
57 char core_siblings[8*MAXCPU];
59 typedef struct per_cpu_sysctl_info per_cpu_sysctl_info_t;
61 static cpu_node_t cpu_topology_nodes[MAXCPU]; /* Memory for topology */
62 static cpu_node_t *cpu_root_node; /* Root node pointer */
64 static struct sysctl_ctx_list cpu_topology_sysctl_ctx;
65 static struct sysctl_oid *cpu_topology_sysctl_tree;
66 static char cpu_topology_members[8*MAXCPU];
67 static per_cpu_sysctl_info_t pcpu_sysctl[MAXCPU];
69 int cpu_topology_levels_number = 1;
70 cpu_node_t *root_cpu_node;
72 /* Get the next valid apicid starting
73 * from current apicid (curr_apicid
76 get_next_valid_apicid(int curr_apicid)
78 int next_apicid = curr_apicid;
82 while(get_cpuid_from_apicid(next_apicid) == -1 &&
83 next_apicid < NAPICID);
84 if (next_apicid == NAPICID) {
85 kprintf("Warning: No next valid APICID found. Returning -1\n");
91 /* Generic topology tree. The parameters have the following meaning:
92 * - children_no_per_level : the number of children on each level
93 * - level_types : the type of the level (THREAD, CORE, CHIP, etc)
94 * - cur_level : the current level of the tree
95 * - node : the current node
96 * - last_free_node : the last free node in the global array.
97 * - cpuid : basicly this are the ids of the leafs
100 build_topology_tree(int *children_no_per_level,
101 uint8_t *level_types,
104 cpu_node_t **last_free_node,
109 node->child_no = children_no_per_level[cur_level];
110 node->type = level_types[cur_level];
111 CPUMASK_ASSZERO(node->members);
112 node->compute_unit_id = -1;
114 if (node->child_no == 0) {
115 *apicid = get_next_valid_apicid(*apicid);
116 CPUMASK_ASSBIT(node->members, get_cpuid_from_apicid(*apicid));
120 if (node->parent_node == NULL)
121 root_cpu_node = node;
123 for (i = 0; i < node->child_no; i++) {
124 node->child_node[i] = *last_free_node;
127 node->child_node[i]->parent_node = node;
129 build_topology_tree(children_no_per_level,
136 CPUMASK_ORMASK(node->members, node->child_node[i]->members);
140 #if defined(__x86_64__) && !defined(_KERNEL_VIRTUAL)
142 migrate_elements(cpu_node_t **a, int n, int pos)
146 for (i = pos; i < n - 1 ; i++) {
153 /* Build CPU topology. The detection is made by comparing the
154 * chip, core and logical IDs of each CPU with the IDs of the
155 * BSP. When we found a match, at that level the CPUs are siblings.
158 build_cpu_topology(void)
160 detect_cpu_topology();
163 int threads_per_core = 0;
164 int cores_per_chip = 0;
165 int chips_per_package = 0;
166 int children_no_per_level[LEVEL_NO];
167 uint8_t level_types[LEVEL_NO];
170 cpu_node_t *root = &cpu_topology_nodes[0];
171 cpu_node_t *last_free_node = root + 1;
173 /* Assume that the topology is uniform.
174 * Find the number of siblings within chip
175 * and witin core to build up the topology
177 for (i = 0; i < ncpus; i++) {
180 CPUMASK_ASSBIT(mask, i);
182 if (CPUMASK_TESTMASK(mask, smp_active_mask) == 0)
185 if (get_chip_ID(BSPID) == get_chip_ID(i))
190 if (get_core_number_within_chip(BSPID) ==
191 get_core_number_within_chip(i))
195 cores_per_chip /= threads_per_core;
196 chips_per_package = ncpus / (cores_per_chip * threads_per_core);
199 kprintf("CPU Topology: cores_per_chip: %d; threads_per_core: %d; chips_per_package: %d;\n",
200 cores_per_chip, threads_per_core, chips_per_package);
202 if (threads_per_core > 1) { /* HT available - 4 levels */
204 children_no_per_level[0] = chips_per_package;
205 children_no_per_level[1] = cores_per_chip;
206 children_no_per_level[2] = threads_per_core;
207 children_no_per_level[3] = 0;
209 level_types[0] = PACKAGE_LEVEL;
210 level_types[1] = CHIP_LEVEL;
211 level_types[2] = CORE_LEVEL;
212 level_types[3] = THREAD_LEVEL;
214 build_topology_tree(children_no_per_level,
221 cpu_topology_levels_number = 4;
223 } else if (cores_per_chip > 1) { /* No HT available - 3 levels */
225 children_no_per_level[0] = chips_per_package;
226 children_no_per_level[1] = cores_per_chip;
227 children_no_per_level[2] = 0;
229 level_types[0] = PACKAGE_LEVEL;
230 level_types[1] = CHIP_LEVEL;
231 level_types[2] = CORE_LEVEL;
233 build_topology_tree(children_no_per_level,
240 cpu_topology_levels_number = 3;
242 } else { /* No HT and no Multi-Core - 2 levels */
244 children_no_per_level[0] = chips_per_package;
245 children_no_per_level[1] = 0;
247 level_types[0] = PACKAGE_LEVEL;
248 level_types[1] = CHIP_LEVEL;
250 build_topology_tree(children_no_per_level,
257 cpu_topology_levels_number = 2;
261 cpu_root_node = root;
264 #if defined(__x86_64__) && !defined(_KERNEL_VIRTUAL)
265 if (fix_amd_topology() == 0) {
266 int visited[MAXCPU], i, j, pos, cpuid;
267 cpu_node_t *leaf, *parent;
269 bzero(visited, MAXCPU * sizeof(int));
271 for (i = 0; i < ncpus; i++) {
272 if (visited[i] == 0) {
275 leaf = get_cpu_node_by_cpuid(i);
277 if (leaf->type == CORE_LEVEL) {
278 parent = leaf->parent_node;
280 last_free_node->child_node[0] = leaf;
281 last_free_node->child_no = 1;
282 last_free_node->members = leaf->members;
283 last_free_node->compute_unit_id = leaf->compute_unit_id;
284 last_free_node->parent_node = parent;
285 last_free_node->type = CORE_LEVEL;
288 for (j = 0; j < parent->child_no; j++) {
289 if (parent->child_node[j] != leaf) {
291 cpuid = BSFCPUMASK(parent->child_node[j]->members);
292 if (visited[cpuid] == 0 &&
293 parent->child_node[j]->compute_unit_id == leaf->compute_unit_id) {
295 last_free_node->child_node[last_free_node->child_no] = parent->child_node[j];
296 last_free_node->child_no++;
297 CPUMASK_ORMASK(last_free_node->members, parent->child_node[j]->members);
299 parent->child_node[j]->type = THREAD_LEVEL;
300 parent->child_node[j]->parent_node = last_free_node;
303 migrate_elements(parent->child_node, parent->child_no, j);
311 if (last_free_node->child_no > 1) {
312 parent->child_node[pos] = last_free_node;
313 leaf->type = THREAD_LEVEL;
314 leaf->parent_node = last_free_node;
324 /* Recursive function helper to print the CPU topology tree */
326 print_cpu_topology_tree_sysctl_helper(cpu_node_t *node,
335 sbuf_bcat(sb, buf, buf_len);
337 sbuf_printf(sb, "\\-");
338 buf[buf_len] = ' ';buf_len++;
339 buf[buf_len] = ' ';buf_len++;
341 sbuf_printf(sb, "|-");
342 buf[buf_len] = '|';buf_len++;
343 buf[buf_len] = ' ';buf_len++;
346 bsr_member = BSRCPUMASK(node->members);
348 if (node->type == PACKAGE_LEVEL) {
349 sbuf_printf(sb,"PACKAGE MEMBERS: ");
350 } else if (node->type == CHIP_LEVEL) {
351 sbuf_printf(sb,"CHIP ID %d: ",
352 get_chip_ID(bsr_member));
353 } else if (node->type == CORE_LEVEL) {
354 if (node->compute_unit_id != (uint8_t)-1) {
355 sbuf_printf(sb,"Compute Unit ID %d: ",
356 node->compute_unit_id);
358 sbuf_printf(sb,"CORE ID %d: ",
359 get_core_number_within_chip(bsr_member));
361 } else if (node->type == THREAD_LEVEL) {
362 if (node->compute_unit_id != (uint8_t)-1) {
363 sbuf_printf(sb,"CORE ID %d: ",
364 get_core_number_within_chip(bsr_member));
366 sbuf_printf(sb,"THREAD ID %d: ",
367 get_logical_CPU_number_within_core(bsr_member));
370 sbuf_printf(sb,"UNKNOWN: ");
372 CPUSET_FOREACH(i, node->members) {
373 sbuf_printf(sb,"cpu%d ", i);
376 sbuf_printf(sb,"\n");
378 for (i = 0; i < node->child_no; i++) {
379 print_cpu_topology_tree_sysctl_helper(node->child_node[i],
380 sb, buf, buf_len, i == (node->child_no -1));
384 /* SYSCTL PROCEDURE for printing the CPU Topology tree */
386 print_cpu_topology_tree_sysctl(SYSCTL_HANDLER_ARGS)
390 char buf[INDENT_BUF_SIZE];
392 KASSERT(cpu_root_node != NULL, ("cpu_root_node isn't initialized"));
394 sb = sbuf_new(NULL, NULL, 500, SBUF_AUTOEXTEND);
398 sbuf_printf(sb,"\n");
399 print_cpu_topology_tree_sysctl_helper(cpu_root_node, sb, buf, 0, 1);
403 ret = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb));
410 /* SYSCTL PROCEDURE for printing the CPU Topology level description */
412 print_cpu_topology_level_description_sysctl(SYSCTL_HANDLER_ARGS)
417 sb = sbuf_new(NULL, NULL, 500, SBUF_AUTOEXTEND);
421 if (cpu_topology_levels_number == 4) /* HT available */
422 sbuf_printf(sb, "0 - thread; 1 - core; 2 - socket; 3 - anything");
423 else if (cpu_topology_levels_number == 3) /* No HT available */
424 sbuf_printf(sb, "0 - core; 1 - socket; 2 - anything");
425 else if (cpu_topology_levels_number == 2) /* No HT and no Multi-Core */
426 sbuf_printf(sb, "0 - socket; 1 - anything");
428 sbuf_printf(sb, "Unknown");
432 ret = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb));
439 /* Find a cpu_node_t by a mask */
441 get_cpu_node_by_cpumask(cpu_node_t * node,
444 cpu_node_t * found = NULL;
447 if (CPUMASK_CMPMASKEQ(node->members, mask))
450 for (i = 0; i < node->child_no; i++) {
451 found = get_cpu_node_by_cpumask(node->child_node[i], mask);
460 get_cpu_node_by_cpuid(int cpuid) {
463 CPUMASK_ASSBIT(mask, cpuid);
465 KASSERT(cpu_root_node != NULL, ("cpu_root_node isn't initialized"));
467 return get_cpu_node_by_cpumask(cpu_root_node, mask);
470 /* Get the mask of siblings for level_type of a cpuid */
472 get_cpumask_from_level(int cpuid,
478 CPUMASK_ASSBIT(mask, cpuid);
480 KASSERT(cpu_root_node != NULL, ("cpu_root_node isn't initialized"));
482 node = get_cpu_node_by_cpumask(cpu_root_node, mask);
485 CPUMASK_ASSZERO(mask);
489 while (node != NULL) {
490 if (node->type == level_type) {
491 return node->members;
493 node = node->parent_node;
495 CPUMASK_ASSZERO(mask);
500 /* init pcpu_sysctl structure info */
502 init_pcpu_topology_sysctl(void)
509 for (i = 0; i < ncpus; i++) {
511 sbuf_new(&sb, pcpu_sysctl[i].cpu_name,
512 sizeof(pcpu_sysctl[i].cpu_name), SBUF_FIXEDLEN);
513 sbuf_printf(&sb,"cpu%d", i);
517 /* Get physical siblings */
518 mask = get_cpumask_from_level(i, CHIP_LEVEL);
519 if (CPUMASK_TESTZERO(mask)) {
520 pcpu_sysctl[i].physical_id = INVALID_ID;
524 sbuf_new(&sb, pcpu_sysctl[i].physical_siblings,
525 sizeof(pcpu_sysctl[i].physical_siblings), SBUF_FIXEDLEN);
526 CPUSET_FOREACH(cpu, mask) {
527 sbuf_printf(&sb,"cpu%d ", cpu);
532 pcpu_sysctl[i].physical_id = get_chip_ID(i);
534 /* Get core siblings */
535 mask = get_cpumask_from_level(i, CORE_LEVEL);
536 if (CPUMASK_TESTZERO(mask)) {
537 pcpu_sysctl[i].core_id = INVALID_ID;
541 sbuf_new(&sb, pcpu_sysctl[i].core_siblings,
542 sizeof(pcpu_sysctl[i].core_siblings), SBUF_FIXEDLEN);
543 CPUSET_FOREACH(cpu, mask) {
544 sbuf_printf(&sb,"cpu%d ", cpu);
549 pcpu_sysctl[i].core_id = get_core_number_within_chip(i);
554 /* Build SYSCTL structure for revealing
555 * the CPU Topology to user-space.
558 build_sysctl_cpu_topology(void)
563 /* SYSCTL new leaf for "cpu_topology" */
564 sysctl_ctx_init(&cpu_topology_sysctl_ctx);
565 cpu_topology_sysctl_tree = SYSCTL_ADD_NODE(&cpu_topology_sysctl_ctx,
566 SYSCTL_STATIC_CHILDREN(_hw),
571 /* SYSCTL cpu_topology "tree" entry */
572 SYSCTL_ADD_PROC(&cpu_topology_sysctl_ctx,
573 SYSCTL_CHILDREN(cpu_topology_sysctl_tree),
574 OID_AUTO, "tree", CTLTYPE_STRING | CTLFLAG_RD,
575 NULL, 0, print_cpu_topology_tree_sysctl, "A",
576 "Tree print of CPU topology");
578 /* SYSCTL cpu_topology "level_description" entry */
579 SYSCTL_ADD_PROC(&cpu_topology_sysctl_ctx,
580 SYSCTL_CHILDREN(cpu_topology_sysctl_tree),
581 OID_AUTO, "level_description", CTLTYPE_STRING | CTLFLAG_RD,
582 NULL, 0, print_cpu_topology_level_description_sysctl, "A",
583 "Level description of CPU topology");
585 /* SYSCTL cpu_topology "members" entry */
586 sbuf_new(&sb, cpu_topology_members,
587 sizeof(cpu_topology_members), SBUF_FIXEDLEN);
588 CPUSET_FOREACH(i, cpu_root_node->members) {
589 sbuf_printf(&sb,"cpu%d ", i);
593 SYSCTL_ADD_STRING(&cpu_topology_sysctl_ctx,
594 SYSCTL_CHILDREN(cpu_topology_sysctl_tree),
595 OID_AUTO, "members", CTLFLAG_RD,
596 cpu_topology_members, 0,
597 "Members of the CPU Topology");
599 /* SYSCTL per_cpu info */
600 for (i = 0; i < ncpus; i++) {
601 /* New leaf : hw.cpu_topology.cpux */
602 sysctl_ctx_init(&pcpu_sysctl[i].sysctl_ctx);
603 pcpu_sysctl[i].sysctl_tree = SYSCTL_ADD_NODE(&pcpu_sysctl[i].sysctl_ctx,
604 SYSCTL_CHILDREN(cpu_topology_sysctl_tree),
606 pcpu_sysctl[i].cpu_name,
609 /* Check if the physical_id found is valid */
610 if (pcpu_sysctl[i].physical_id == INVALID_ID) {
614 /* Add physical id info */
615 SYSCTL_ADD_INT(&pcpu_sysctl[i].sysctl_ctx,
616 SYSCTL_CHILDREN(pcpu_sysctl[i].sysctl_tree),
617 OID_AUTO, "physical_id", CTLFLAG_RD,
618 &pcpu_sysctl[i].physical_id, 0,
621 /* Add physical siblings */
622 SYSCTL_ADD_STRING(&pcpu_sysctl[i].sysctl_ctx,
623 SYSCTL_CHILDREN(pcpu_sysctl[i].sysctl_tree),
624 OID_AUTO, "physical_siblings", CTLFLAG_RD,
625 pcpu_sysctl[i].physical_siblings, 0,
626 "Physical siblings");
628 /* Check if the core_id found is valid */
629 if (pcpu_sysctl[i].core_id == INVALID_ID) {
633 /* Add core id info */
634 SYSCTL_ADD_INT(&pcpu_sysctl[i].sysctl_ctx,
635 SYSCTL_CHILDREN(pcpu_sysctl[i].sysctl_tree),
636 OID_AUTO, "core_id", CTLFLAG_RD,
637 &pcpu_sysctl[i].core_id, 0,
640 /*Add core siblings */
641 SYSCTL_ADD_STRING(&pcpu_sysctl[i].sysctl_ctx,
642 SYSCTL_CHILDREN(pcpu_sysctl[i].sysctl_tree),
643 OID_AUTO, "core_siblings", CTLFLAG_RD,
644 pcpu_sysctl[i].core_siblings, 0,
649 /* Build the CPU Topology and SYSCTL Topology tree */
651 init_cpu_topology(void)
653 build_cpu_topology();
655 init_pcpu_topology_sysctl();
656 build_sysctl_cpu_topology();
658 SYSINIT(cpu_topology, SI_BOOT2_CPU_TOPOLOGY, SI_ORDER_FIRST,
659 init_cpu_topology, NULL)