From 9bea61140575a5730974980a453d1b7b4057b3e4 Mon Sep 17 00:00:00 2001 From: Mihai Carabas Date: Wed, 22 Aug 2012 10:03:12 +0000 Subject: [PATCH] vkernel{,64} - CPU topology support * Part of "Add SMT/HT awareness to DragonFly BSD scheduler" GSoC project. * Details at: http://leaf.dragonflybsd.org/mailarchive/kernel/2012-08/msg00009.html Mentored-by: Alex Hornung (alexh@) Sponsored-by: Google Summer of Code 2012 --- share/man/man7/vkernel.7 | 19 ++++++++++-- sys/platform/vkernel/i386/mp.c | 36 +++++++++++++++++++++++ sys/platform/vkernel/include/smp.h | 14 +++++++++ sys/platform/vkernel/platform/init.c | 50 ++++++++++++++++++++++++++++---- sys/platform/vkernel64/include/smp.h | 14 +++++++++ sys/platform/vkernel64/platform/init.c | 50 ++++++++++++++++++++++++++++---- sys/platform/vkernel64/x86_64/mp.c | 37 +++++++++++++++++++++++ 7 files changed, 205 insertions(+), 15 deletions(-) diff --git a/share/man/man7/vkernel.7 b/share/man/man7/vkernel.7 index faae0b3..5fa6cb4 100644 --- a/share/man/man7/vkernel.7 +++ b/share/man/man7/vkernel.7 @@ -53,7 +53,7 @@ .Op Fl I Ar interface Ns Op Ar :address1 Ns Oo Ar :address2 Oc Ns Oo Ar /netmask Oc .Op Fl l Ar cpulock .Op Fl m Ar size -.Op Fl n Ar numcpus +.Op Fl n Ar numcpus Ns Op Ar :lbits Ns Oo Ar :cbits Oc .Op Fl p Ar pidfile .Op Fl r Ar file .Sh DESCRIPTION @@ -190,12 +190,25 @@ Lowercase versions of and .Cm G are allowed. -.It Fl n Ar numcpus -Specify the number of CPUs you wish to emulate. +.It Fl n Ar numcpus Ns Op Ar :lbits Ns Oo Ar :cbits Oc +.Ar numcpus +specifies the number of CPUs you wish to emulate. Up to 16 CPUs are supported. The virtual kernel must be built with .Cd options SMP to use this option and will default to 2 CPUs unless otherwise specified. +.Ar lbits +specifies the number of bits within APICID(=CPUID) needed for representing +the logical ID. +Controls the number of threads/core (0bits - 1 thread, 1bit - 2 threads). +This parameter is optional (mandatory only if +.Ar cbits +is specified). +.Ar cbits +specifies the number of bits within APICID(=CPUID) needed for representing +the core ID. +Controls the number of core/package (0bits - 1 core, 1bit - 2 cores). +This parameter is optional. .It Fl p Ar pidfile Specify a pidfile in which to store the process ID. Scripts can use this file to locate the vkernel pid for the purpose of diff --git a/sys/platform/vkernel/i386/mp.c b/sys/platform/vkernel/i386/mp.c index a1e459f..ffafd8c 100644 --- a/sys/platform/vkernel/i386/mp.c +++ b/sys/platform/vkernel/i386/mp.c @@ -68,6 +68,10 @@ static cpumask_t smp_startup_mask = 1; /* which cpus have been started */ int mp_naps; /* # of Applications processors */ static int mp_finish; +/* Local data for detecting CPU TOPOLOGY */ +static int core_bits = 0; +static int logical_CPU_bits = 0; + /* function prototypes XXX these should go elsewhere */ void bootstrap_idle(void); void single_cpu_ipi(int, int, int); @@ -455,3 +459,35 @@ start_all_aps(u_int boot_addr) return(ncpus - 1); } + +/* + * CPU TOPOLOGY DETECTION FUNCTIONS. + */ + +void +detect_cpu_topology(void) +{ + logical_CPU_bits = vkernel_b_arg; + core_bits = vkernel_B_arg; +} + +int +get_chip_ID(int cpuid) +{ + return get_apicid_from_cpuid(cpuid) >> + (logical_CPU_bits + core_bits); +} + +int +get_core_number_within_chip(int cpuid) +{ + return (get_apicid_from_cpuid(cpuid) >> logical_CPU_bits) & + ( (1 << core_bits) -1); +} + +int +get_logical_CPU_number_within_core(int cpuid) +{ + return get_apicid_from_cpuid(cpuid) & + ( (1 << logical_CPU_bits) -1); +} diff --git a/sys/platform/vkernel/include/smp.h b/sys/platform/vkernel/include/smp.h index 0a0b416..2df73db 100644 --- a/sys/platform/vkernel/include/smp.h +++ b/sys/platform/vkernel/include/smp.h @@ -51,6 +51,8 @@ void bootMP (void); /* global data in apic_vector.s */ extern volatile cpumask_t stopped_cpus; extern int optcpus; /* from main() */ +extern int vkernel_b_arg; /* arg from main() */ +extern int vkernel_B_arg; /* arg from main() */ #if 0 extern volatile cpumask_t started_cpus; @@ -165,6 +167,18 @@ int cpu_send_ipiq_passive (int); /* global data in init_smp.c */ extern cpumask_t smp_active_mask; +/* Detect CPU topology bits */ +void detect_cpu_topology(void); + +/* Interface functions for IDs calculation */ +int get_chip_ID(int cpuid); +int get_core_number_within_chip(int cpuid); +int get_logical_CPU_number_within_core(int cpuid); + +/* Assume that APICID = CPUID for virtual processors */ +#define get_cpuid_from_apicid(cpuid) cpuid +#define get_apicid_from_cpuid(cpuid) cpuid + #endif /* !LOCORE */ #else /* !SMP */ diff --git a/sys/platform/vkernel/platform/init.c b/sys/platform/vkernel/platform/init.c index 5b649ed..6acae97 100644 --- a/sys/platform/vkernel/platform/init.c +++ b/sys/platform/vkernel/platform/init.c @@ -112,6 +112,8 @@ int optcpus; /* number of cpus - see mp_start() */ int lwp_cpu_lock; /* if/how to lock virtual CPUs to real CPUs */ int real_ncpus; /* number of real CPUs */ int next_cpu; /* next real CPU to lock a virtual CPU to */ +int vkernel_b_arg; /* -b argument - no of logical CPU bits - only SMP */ +int vkernel_B_arg; /* -B argument - no of core bits - only SMP */ int via_feature_xcrypt = 0; /* XXX */ int via_feature_rng = 0; /* XXX */ @@ -149,6 +151,7 @@ main(int ac, char **av) char *suffix; char *endp; char *tmp; + char *tok; int netifFileNum = 0; int diskFileNum = 0; int cdFileNum = 0; @@ -178,6 +181,8 @@ main(int ac, char **av) kernel_mem_readonly = 1; #ifdef SMP optcpus = 2; + vkernel_b_arg = 0; + vkernel_B_arg = 0; #endif lwp_cpu_lock = LCL_NONE; @@ -197,7 +202,7 @@ main(int ac, char **av) if (ac < 2) usage_help(false); - while ((c = getopt(ac, av, "c:hsvl:m:n:r:e:i:p:I:U")) != -1) { + while ((c = getopt(ac, av, "c:hsvl:m:n:r:e:i:p:I:Ub:B:")) != -1) { switch(c) { case 'e': /* @@ -317,19 +322,46 @@ main(int ac, char **av) * This value is set up by mp_start(), don't just * set ncpus here. */ + tok = strtok(optarg, ":"); #ifdef SMP - optcpus = strtol(optarg, NULL, 0); + optcpus = strtol(tok, NULL, 0); if (optcpus < 1 || optcpus > MAXCPU) usage_err("Bad ncpus, valid range is 1-%d", MAXCPU); + + /* :core_bits argument */ + tok = strtok(NULL, ":"); + if (tok != NULL) { + vkernel_b_arg = strtol(tok, NULL, 0); + + /* :logical_CPU_bits argument */ + tok = strtok(NULL, ":"); + if (tok != NULL) { + vkernel_B_arg = strtol(tok, NULL, 0); + } + + } + #else - if (strtol(optarg, NULL, 0) != 1) { + if (strtol(tok, NULL, 0) != 1) { usage_err("You built a UP vkernel, only 1 cpu!"); } + + /* :logical_CPU_bits argument */ + tok = strtok(NULL, ":"); + if (tok != NULL) { + usage_err("You built a UP vkernel. No CPU topology available"); + + /* :core_bits argument */ + tok = strtok(NULL, ":"); + if (tok != NULL) { + usage_err("You built a UP vkernel. No CPU topology available"); + } + } #endif break; case 'p': - pid_file = optarg; + pid_file = optarg; break; case 'U': kernel_mem_readonly = 0; @@ -1310,7 +1342,8 @@ usage_help(_Bool help) { fprintf(stderr, "Usage: %s [-hsUv] [-c file] [-e name=value:name=value:...]\n" "\t[-i file] [-I interface[:address1[:address2][/netmask]]] [-l cpulock]\n" - "\t[-m size] [-n numcpus] [-p file] [-r file]\n", save_av[0]); + "\t[-m size] [-n numcpus[:lbits[:cbits]]]\n" + "\t[-p file] [-r file]\n", save_av[0]); if (help) fprintf(stderr, "\nArguments:\n" @@ -1321,7 +1354,12 @@ usage_help(_Bool help) "\t-I\tCreate a virtual network device.\n" "\t-l\tSpecify which, if any, real CPUs to lock virtual CPUs to.\n" "\t-m\tSpecify the amount of memory to be used by the kernel in bytes.\n" - "\t-n\tSpecify the number of CPUs you wish to emulate.\n" + "\t-n\tSpecify the number of CPUs and the topology you wish to emulate:\n" + "\t \t- numcpus - number of cpus\n" + "\t \t- :lbits - specify the number of bits within APICID(=CPUID) needed for representing\n" + "\t \tthe logical ID. Controls the number of threads/core (0bits - 1 thread, 1bit - 2 threads).\n" + "\t \t- :cbits - specify the number of bits within APICID(=CPUID) needed for representing\n" + "\t \tthe core ID. Controls the number of core/package (0bits - 1 core, 1bit - 2 cores).\n" "\t-p\tSpecify a file in which to store the process ID.\n" "\t-r\tSpecify a R/W disk image file to be used by the kernel.\n" "\t-s\tBoot into single-user mode.\n" diff --git a/sys/platform/vkernel64/include/smp.h b/sys/platform/vkernel64/include/smp.h index b84d0a4..1156c5b 100644 --- a/sys/platform/vkernel64/include/smp.h +++ b/sys/platform/vkernel64/include/smp.h @@ -51,6 +51,8 @@ void bootMP (void); /* global data in apic_vector.s */ extern volatile cpumask_t stopped_cpus; extern int optcpus; /* from main() */ +extern int vkernel_b_arg; /* arg from main() */ +extern int vkernel_B_arg; /* arg from main() */ #if 0 extern volatile cpumask_t started_cpus; @@ -165,6 +167,18 @@ int cpu_send_ipiq_passive (int); /* global data in init_smp.c */ extern cpumask_t smp_active_mask; +/* Detect CPU topology bits */ +void detect_cpu_topology(void); + +/* Interface functions for IDs calculation */ +int get_chip_ID(int cpuid); +int get_core_number_within_chip(int cpuid); +int get_logical_CPU_number_within_core(int cpuid); + +/* Assume that APICID = CPUID for virtual processors */ +#define get_cpuid_from_apicid(cpuid) cpuid +#define get_apicid_from_cpuid(cpuid) cpuid + #endif /* !LOCORE */ #else /* !SMP */ diff --git a/sys/platform/vkernel64/platform/init.c b/sys/platform/vkernel64/platform/init.c index 7be297a..a8c984d 100644 --- a/sys/platform/vkernel64/platform/init.c +++ b/sys/platform/vkernel64/platform/init.c @@ -112,6 +112,8 @@ int optcpus; /* number of cpus - see mp_start() */ int lwp_cpu_lock; /* if/how to lock virtual CPUs to real CPUs */ int real_ncpus; /* number of real CPUs */ int next_cpu; /* next real CPU to lock a virtual CPU to */ +int vkernel_b_arg; /* -b argument - no of logical CPU bits - only SMP */ +int vkernel_B_arg; /* -B argument - no of core bits - only SMP */ struct privatespace *CPU_prvspace; @@ -146,6 +148,7 @@ main(int ac, char **av) char *suffix; char *endp; char *tmp; + char *tok; int netifFileNum = 0; int diskFileNum = 0; int cdFileNum = 0; @@ -175,6 +178,8 @@ main(int ac, char **av) kernel_mem_readonly = 1; #ifdef SMP optcpus = 2; + vkernel_b_arg = 0; + vkernel_B_arg = 0; #endif lwp_cpu_lock = LCL_NONE; @@ -194,7 +199,7 @@ main(int ac, char **av) if (ac < 2) usage_help(false); - while ((c = getopt(ac, av, "c:hsvl:m:n:r:e:i:p:I:U")) != -1) { + while ((c = getopt(ac, av, "c:hsvl:m:n:r:e:i:p:I:Ub:B:")) != -1) { switch(c) { case 'e': /* @@ -314,17 +319,44 @@ main(int ac, char **av) * This value is set up by mp_start(), don't just * set ncpus here. */ + tok = strtok(optarg, ":"); #ifdef SMP - optcpus = strtol(optarg, NULL, 0); + optcpus = strtol(tok, NULL, 0); if (optcpus < 1 || optcpus > MAXCPU) usage_err("Bad ncpus, valid range is 1-%d", MAXCPU); + + /* :lbits argument */ + tok = strtok(NULL, ":"); + if (tok != NULL) { + vkernel_b_arg = strtol(tok, NULL, 0); + + /* :cbits argument */ + tok = strtok(NULL, ":"); + if (tok != NULL) { + vkernel_B_arg = strtol(tok, NULL, 0); + } + + } + #else - if (strtol(optarg, NULL, 0) != 1) { + if (strtol(tok, NULL, 0) != 1) { usage_err("You built a UP vkernel, only 1 cpu!"); } + + /* :lbits argument */ + tok = strtok(NULL, ":"); + if (tok != NULL) { + usage_err("You built a UP vkernel. No CPU topology available"); + + /* :cbits argument */ + tok = strtok(NULL, ":"); + if (tok != NULL) { + usage_err("You built a UP vkernel. No CPU topology available"); + } + } #endif - break; + case 'p': pid_file = optarg; break; @@ -1271,7 +1303,8 @@ usage_help(_Bool help) { fprintf(stderr, "Usage: %s [-hsUv] [-c file] [-e name=value:name=value:...]\n" "\t[-i file] [-I interface[:address1[:address2][/netmask]]] [-l cpulock]\n" - "\t[-m size] [-n numcpus] [-p file] [-r file]\n", save_av[0]); + "\t[-m size] [-n numcpus[:lbits[:cbits]]]\n" + "\t[-p file] [-r file]\n", save_av[0]); if (help) fprintf(stderr, "\nArguments:\n" @@ -1282,7 +1315,12 @@ usage_help(_Bool help) "\t-I\tCreate a virtual network device.\n" "\t-l\tSpecify which, if any, real CPUs to lock virtual CPUs to.\n" "\t-m\tSpecify the amount of memory to be used by the kernel in bytes.\n" - "\t-n\tSpecify the number of CPUs you wish to emulate.\n" + "\t-n\tSpecify the number of CPUs and the topology you wish to emulate:\n" + "\t \t- numcpus - number of cpus\n" + "\t \t- :lbits - specify the number of bits within APICID(=CPUID) needed for representing\n" + "\t \t the logical ID. Controls the number of threads/core (0bits - 1 thread, 1bit - 2 threads).\n" + "\t \t- :cbits - specify the number of bits within APICID(=CPUID) needed for representing\n" + "\t \t the core ID. Controls the number of core/package (0bits - 1 core, 1bit - 2 cores).\n" "\t-p\tSpecify a file in which to store the process ID.\n" "\t-r\tSpecify a R/W disk image file to be used by the kernel.\n" "\t-s\tBoot into single-user mode.\n" diff --git a/sys/platform/vkernel64/x86_64/mp.c b/sys/platform/vkernel64/x86_64/mp.c index 7358034..3da0394 100644 --- a/sys/platform/vkernel64/x86_64/mp.c +++ b/sys/platform/vkernel64/x86_64/mp.c @@ -68,6 +68,10 @@ static cpumask_t smp_startup_mask = 1; /* which cpus have been started */ int mp_naps; /* # of Applications processors */ static int mp_finish; +/* Local data for detecting CPU TOPOLOGY */ +static int core_bits = 0; +static int logical_CPU_bits = 0; + /* function prototypes XXX these should go elsewhere */ void bootstrap_idle(void); void single_cpu_ipi(int, int, int); @@ -457,3 +461,36 @@ start_all_aps(u_int boot_addr) return(ncpus - 1); } + +/* + * CPU TOPOLOGY DETECTION FUNCTIONS. + */ + +void +detect_cpu_topology(void) +{ + logical_CPU_bits = vkernel_b_arg; + core_bits = vkernel_B_arg; +} + +int +get_chip_ID(int cpuid) +{ + return get_apicid_from_cpuid(cpuid) >> + (logical_CPU_bits + core_bits); +} + +int +get_core_number_within_chip(int cpuid) +{ + return (get_apicid_from_cpuid(cpuid) >> logical_CPU_bits) & + ( (1 << core_bits) -1); +} + +int +get_logical_CPU_number_within_core(int cpuid) +{ + return get_apicid_from_cpuid(cpuid) & + ( (1 << logical_CPU_bits) -1); +} + -- 1.7.7.2