sys/dev/acpica/acpi_cpu_pstate.c

   1 /*
   2  * Copyright (c) 2009 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Sepherosa Ziehau <sepherosa@gmail.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 #include "opt_acpi.h"
  36
  37 #include <sys/param.h>
  38 #include <sys/bus.h>
  39 #include <sys/kernel.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/rman.h>
  43 #include <sys/sysctl.h>
  44 #include <sys/msgport2.h>
  45 #include <sys/cpu_topology.h>
  46
  47 #include <net/netisr2.h>
  48 #include <net/netmsg2.h>
  49 #include <net/if_var.h>
  50
  51 #include "acpi.h"
  52 #include "acpivar.h"
  53 #include "acpi_cpu.h"
  54 #include "acpi_cpu_pstate.h"
  55
  56 #define ACPI_NPSTATE_MAX        32
  57
  58 #define ACPI_PSS_PX_NENTRY      6
  59
  60 #define ACPI_PSD_COORD_SWALL    0xfc
  61 #define ACPI_PSD_COORD_SWANY    0xfd
  62 #define ACPI_PSD_COORD_HWALL    0xfe
  63 #define ACPI_PSD_COORD_VALID(coord) \
  64         ((coord) == ACPI_PSD_COORD_SWALL || \
  65          (coord) == ACPI_PSD_COORD_SWANY || \
  66          (coord) == ACPI_PSD_COORD_HWALL)
  67
  68 struct acpi_pst_softc;
  69 LIST_HEAD(acpi_pst_list, acpi_pst_softc);
  70
  71 struct netmsg_acpi_pst {
  72         struct netmsg_base base;
  73         const struct acpi_pst_res *ctrl;
  74         const struct acpi_pst_res *status;
  75 };
  76
  77 struct acpi_pst_domain {
  78         uint32_t                pd_dom;
  79         uint32_t                pd_coord;
  80         uint32_t                pd_nproc;
  81         LIST_ENTRY(acpi_pst_domain) pd_link;
  82
  83         uint32_t                pd_flags;
  84
  85         struct lwkt_serialize   pd_serialize;
  86
  87         int                     pd_state;
  88         struct acpi_pst_list    pd_pstlist;
  89
  90         struct sysctl_ctx_list  pd_sysctl_ctx;
  91         struct sysctl_oid       *pd_sysctl_tree;
  92 };
  93 LIST_HEAD(acpi_pst_domlist, acpi_pst_domain);
  94
  95 #define ACPI_PSTDOM_FLAG_STUB   0x1     /* stub domain, no _PSD */
  96 #define ACPI_PSTDOM_FLAG_DEAD   0x2     /* domain can't be started */
  97 #define ACPI_PSTDOM_FLAG_INT    0x4     /* domain created from Integer _PSD */
  98
  99 struct acpi_pst_softc {
 100         device_t                pst_dev;
 101         struct acpi_cpu_softc   *pst_parent;
 102         struct acpi_pst_domain  *pst_domain;
 103         struct acpi_pst_res     pst_creg;
 104         struct acpi_pst_res     pst_sreg;
 105
 106         int                     pst_state;
 107         int                     pst_cpuid;
 108
 109         uint32_t                pst_flags;
 110
 111         ACPI_HANDLE             pst_handle;
 112
 113         LIST_ENTRY(acpi_pst_softc) pst_link;
 114 };
 115
 116 #define ACPI_PST_FLAG_PPC       0x1
 117 #define ACPI_PST_FLAG_PDL       0x2
 118
 119 static int      acpi_pst_probe(device_t dev);
 120 static int      acpi_pst_attach(device_t dev);
 121 static void     acpi_pst_notify(device_t dev);
 122
 123 static void     acpi_pst_postattach(void *);
 124 static struct acpi_pst_domain *
 125                 acpi_pst_domain_create_int(device_t, uint32_t);
 126 static struct acpi_pst_domain *
 127                 acpi_pst_domain_create_pkg(device_t, ACPI_OBJECT *);
 128 static struct acpi_pst_domain *
 129                 acpi_pst_domain_find(uint32_t);
 130 static struct acpi_pst_domain *
 131                 acpi_pst_domain_alloc(uint32_t, uint32_t, uint32_t);
 132 static void     acpi_pst_domain_set_pstate_locked(struct acpi_pst_domain *,
 133                     int, int *);
 134 static void     acpi_pst_domain_set_pstate(struct acpi_pst_domain *, int,
 135                     int *);
 136 static void     acpi_pst_domain_check_nproc(device_t, struct acpi_pst_domain *);
 137 static void     acpi_pst_global_set_pstate(int);
 138 static void     acpi_pst_global_fixup_pstate(void);
 139
 140 static int      acpi_pst_check_csr(struct acpi_pst_softc *);
 141 static int      acpi_pst_check_pstates(struct acpi_pst_softc *);
 142 static int      acpi_pst_init(struct acpi_pst_softc *);
 143 static int      acpi_pst_set_pstate(struct acpi_pst_softc *,
 144                     const struct acpi_pstate *);
 145 static const struct acpi_pstate *
 146                 acpi_pst_get_pstate(struct acpi_pst_softc *);
 147 static int      acpi_pst_alloc_resource(device_t, ACPI_OBJECT *, int,
 148                     struct acpi_pst_res *);
 149 static int      acpi_pst_eval_ppc(struct acpi_pst_softc *, int *);
 150 static int      acpi_pst_eval_pdl(struct acpi_pst_softc *, int *);
 151
 152 static void     acpi_pst_check_csr_handler(netmsg_t);
 153 static void     acpi_pst_check_pstates_handler(netmsg_t);
 154 static void     acpi_pst_init_handler(netmsg_t);
 155 static void     acpi_pst_set_pstate_handler(netmsg_t);
 156 static void     acpi_pst_get_pstate_handler(netmsg_t);
 157
 158 static int      acpi_pst_sysctl_freqs(SYSCTL_HANDLER_ARGS);
 159 static int      acpi_pst_sysctl_freqs_bin(SYSCTL_HANDLER_ARGS);
 160 static int      acpi_pst_sysctl_power(SYSCTL_HANDLER_ARGS);
 161 static int      acpi_pst_sysctl_members(SYSCTL_HANDLER_ARGS);
 162 static int      acpi_pst_sysctl_select(SYSCTL_HANDLER_ARGS);
 163 static int      acpi_pst_sysctl_global(SYSCTL_HANDLER_ARGS);
 164
 165 static struct acpi_pst_domlist  acpi_pst_domains =
 166         LIST_HEAD_INITIALIZER(acpi_pst_domains);
 167 static int                      acpi_pst_domain_id;
 168
 169 static int                      acpi_pst_global_state;
 170
 171 static int                      acpi_pstate_start = -1;
 172 static int                      acpi_pstate_count;
 173 static int                      acpi_npstates;
 174 static struct acpi_pstate       *acpi_pstates;
 175
 176 static const struct acpi_pst_md *acpi_pst_md;
 177
 178 static int                      acpi_pst_pdl = -1;
 179 TUNABLE_INT("hw.acpi.cpu.pst.pdl", &acpi_pst_pdl);
 180
 181 static int                      acpi_pst_ht_reuse_domain = 1;
 182 TUNABLE_INT("hw.acpi.cpu.pst.ht_reuse_domain", &acpi_pst_ht_reuse_domain);
 183
 184 static int                      acpi_pst_force_pkg_domain = 0;
 185 TUNABLE_INT("hw.acpi.cpu.pst.force_pkg_domain", &acpi_pst_force_pkg_domain);
 186
 187 static int                      acpi_pst_handle_notify = 1;
 188 TUNABLE_INT("hw.acpi.cpu.pst.handle_notify", &acpi_pst_handle_notify);
 189
 190 /*
 191  * Force CPU package power domain for Intel CPUs.
 192  *
 193  * As of this write (14 July 2015), all Intel CPUs only have CPU package
 194  * power domain.
 195  */
 196 static int                      acpi_pst_intel_pkg_domain = 1;
 197 TUNABLE_INT("hw.acpi.cpu.pst.intel_pkg_domain", &acpi_pst_intel_pkg_domain);
 198
 199 static device_method_t acpi_pst_methods[] = {
 200         /* Device interface */
 201         DEVMETHOD(device_probe,                 acpi_pst_probe),
 202         DEVMETHOD(device_attach,                acpi_pst_attach),
 203         DEVMETHOD(device_detach,                bus_generic_detach),
 204         DEVMETHOD(device_shutdown,              bus_generic_shutdown),
 205         DEVMETHOD(device_suspend,               bus_generic_suspend),
 206         DEVMETHOD(device_resume,                bus_generic_resume),
 207
 208         /* Bus interface */
 209         DEVMETHOD(bus_add_child,                bus_generic_add_child),
 210         DEVMETHOD(bus_print_child,              bus_generic_print_child),
 211         DEVMETHOD(bus_read_ivar,                bus_generic_read_ivar),
 212         DEVMETHOD(bus_write_ivar,               bus_generic_write_ivar),
 213         DEVMETHOD(bus_get_resource_list,        bus_generic_get_resource_list),
 214         DEVMETHOD(bus_set_resource,             bus_generic_rl_set_resource),
 215         DEVMETHOD(bus_get_resource,             bus_generic_rl_get_resource),
 216         DEVMETHOD(bus_alloc_resource,           bus_generic_alloc_resource),
 217         DEVMETHOD(bus_release_resource,         bus_generic_release_resource),
 218         DEVMETHOD(bus_driver_added,             bus_generic_driver_added),
 219         DEVMETHOD(bus_activate_resource,        bus_generic_activate_resource),
 220         DEVMETHOD(bus_deactivate_resource,      bus_generic_deactivate_resource),
 221         DEVMETHOD(bus_setup_intr,               bus_generic_setup_intr),
 222         DEVMETHOD(bus_teardown_intr,            bus_generic_teardown_intr),
 223
 224         DEVMETHOD_END
 225 };
 226
 227 static driver_t acpi_pst_driver = {
 228         "cpu_pst",
 229         acpi_pst_methods,
 230         sizeof(struct acpi_pst_softc)
 231 };
 232
 233 static devclass_t acpi_pst_devclass;
 234 DRIVER_MODULE(cpu_pst, cpu, acpi_pst_driver, acpi_pst_devclass, NULL, NULL);
 235 MODULE_DEPEND(cpu_pst, acpi, 1, 1, 1);
 236
 237 static __inline int
 238 acpi_pst_freq2index(int freq)
 239 {
 240         int i;
 241
 242         for (i = 0; i < acpi_npstates; ++i) {
 243                 if (acpi_pstates[i].st_freq == freq)
 244                         return i;
 245         }
 246         return -1;
 247 }
 248
 249 static int
 250 acpi_pst_probe(device_t dev)
 251 {
 252         ACPI_BUFFER buf;
 253         ACPI_HANDLE handle;
 254         ACPI_STATUS status;
 255         ACPI_OBJECT *obj;
 256
 257         if (acpi_disabled("cpu_pst") ||
 258             acpi_get_type(dev) != ACPI_TYPE_PROCESSOR)
 259                 return ENXIO;
 260
 261         if (acpi_pst_md == NULL)
 262                 acpi_pst_md = acpi_pst_md_probe();
 263
 264         handle = acpi_get_handle(dev);
 265
 266         /*
 267          * Check _PSD package
 268          *
 269          * NOTE:
 270          * Some BIOSes do not expose _PCT for the second thread of
 271          * CPU cores.  In this case, _PSD should be enough to get the
 272          * P-state of the second thread working, since it must have
 273          * the same _PCT and _PSS as the first thread in the same
 274          * core.
 275          */
 276         buf.Pointer = NULL;
 277         buf.Length = ACPI_ALLOCATE_BUFFER;
 278         status = AcpiEvaluateObject(handle, "_PSD", NULL, &buf);
 279         if (!ACPI_FAILURE(status)) {
 280                 AcpiOsFree((ACPI_OBJECT *)buf.Pointer);
 281                 goto done;
 282         }
 283
 284         /*
 285          * Check _PCT package
 286          */
 287         buf.Pointer = NULL;
 288         buf.Length = ACPI_ALLOCATE_BUFFER;
 289         status = AcpiEvaluateObject(handle, "_PCT", NULL, &buf);
 290         if (ACPI_FAILURE(status)) {
 291                 if (bootverbose) {
 292                         device_printf(dev, "Can't get _PCT package - %s\n",
 293                                       AcpiFormatException(status));
 294                 }
 295                 return ENXIO;
 296         }
 297
 298         obj = (ACPI_OBJECT *)buf.Pointer;
 299         if (!ACPI_PKG_VALID_EQ(obj, 2)) {
 300                 device_printf(dev, "Invalid _PCT package\n");
 301                 AcpiOsFree(obj);
 302                 return ENXIO;
 303         }
 304         AcpiOsFree(obj);
 305
 306         /*
 307          * Check _PSS package
 308          */
 309         buf.Pointer = NULL;
 310         buf.Length = ACPI_ALLOCATE_BUFFER;
 311         status = AcpiEvaluateObject(handle, "_PSS", NULL, &buf);
 312         if (ACPI_FAILURE(status)) {
 313                 device_printf(dev, "Can't get _PSS package - %s\n",
 314                               AcpiFormatException(status));
 315                 return ENXIO;
 316         }
 317
 318         obj = (ACPI_OBJECT *)buf.Pointer;
 319         if (!ACPI_PKG_VALID(obj, 1)) {
 320                 device_printf(dev, "Invalid _PSS package\n");
 321                 AcpiOsFree(obj);
 322                 return ENXIO;
 323         }
 324         AcpiOsFree(obj);
 325
 326 done:
 327         device_set_desc(dev, "ACPI CPU P-State");
 328         return 0;
 329 }
 330
 331 static int
 332 acpi_pst_attach(device_t dev)
 333 {
 334         struct acpi_pst_softc *sc = device_get_softc(dev);
 335         struct acpi_pst_domain *dom = NULL;
 336         ACPI_BUFFER buf;
 337         ACPI_STATUS status;
 338         ACPI_OBJECT *obj;
 339         struct acpi_pstate *pstate, *p;
 340         int i, npstate, error, sstart, scount;
 341
 342         sc->pst_dev = dev;
 343         sc->pst_parent = device_get_softc(device_get_parent(dev));
 344         sc->pst_handle = acpi_get_handle(dev);
 345         sc->pst_cpuid = acpi_get_magic(dev);
 346
 347         /*
 348          * If there is a _PSD, then we create procossor domain
 349          * accordingly.  If there is no _PSD, we just fake a
 350          * default processor domain0.
 351          */
 352         buf.Pointer = NULL;
 353         buf.Length = ACPI_ALLOCATE_BUFFER;
 354         status = AcpiEvaluateObject(sc->pst_handle, "_PSD", NULL, &buf);
 355         if (!ACPI_FAILURE(status)) {
 356                 obj = (ACPI_OBJECT *)buf.Pointer;
 357
 358                 if (acpi_pst_domain_id > 0) {
 359                         device_printf(dev, "Missing _PSD for certain CPUs\n");
 360                         AcpiOsFree(obj);
 361                         return ENXIO;
 362                 }
 363                 acpi_pst_domain_id = -1;
 364
 365                 if (ACPI_PKG_VALID_EQ(obj, 1)) {
 366                         dom = acpi_pst_domain_create_pkg(dev,
 367                                 &obj->Package.Elements[0]);
 368                         if (dom == NULL) {
 369                                 AcpiOsFree(obj);
 370                                 return ENXIO;
 371                         }
 372                 } else {
 373                         if (obj->Type != ACPI_TYPE_INTEGER) {
 374                                 device_printf(dev,
 375                                     "Invalid _PSD package, Type 0x%x\n",
 376                                     obj->Type);
 377                                 AcpiOsFree(obj);
 378                                 return ENXIO;
 379                         } else {
 380                                 device_printf(dev, "Integer _PSD %ju\n",
 381                                     (uintmax_t)obj->Integer.Value);
 382                                 dom = acpi_pst_domain_create_int(dev,
 383                                     obj->Integer.Value);
 384                                 if (dom == NULL) {
 385                                         AcpiOsFree(obj);
 386                                         return ENXIO;
 387                                 }
 388                         }
 389                 }
 390
 391                 /* Free _PSD */
 392                 AcpiOsFree(buf.Pointer);
 393         } else {
 394                 if (acpi_pst_domain_id < 0) {
 395                         device_printf(dev, "Missing _PSD for cpu%d\n",
 396                             sc->pst_cpuid);
 397                         return ENXIO;
 398                 }
 399
 400                 /*
 401                  * Create a stub one processor domain for each processor
 402                  */
 403                 dom = acpi_pst_domain_alloc(acpi_pst_domain_id,
 404                         ACPI_PSD_COORD_SWANY, 1);
 405                 dom->pd_flags |= ACPI_PSTDOM_FLAG_STUB;
 406
 407                 ++acpi_pst_domain_id;
 408         }
 409
 410         /* Make sure that adding us will not overflow our domain */
 411         acpi_pst_domain_check_nproc(dev, dom);
 412
 413         /*
 414          * Get control/status registers from _PCT
 415          */
 416         buf.Pointer = NULL;
 417         buf.Length = ACPI_ALLOCATE_BUFFER;
 418         status = AcpiEvaluateObject(sc->pst_handle, "_PCT", NULL, &buf);
 419         if (ACPI_FAILURE(status)) {
 420                 struct acpi_pst_softc *pst;
 421
 422                 /*
 423                  * No _PCT.  See the comment in acpi_pst_probe() near
 424                  * _PSD check.
 425                  *
 426                  * Use control/status registers of another CPU in the
 427                  * same domain, or in the same core, if the type of
 428                  * these registers are "Fixed Hardware", e.g. on most
 429                  * of the model Intel CPUs.
 430                  */
 431                 pst = LIST_FIRST(&dom->pd_pstlist);
 432                 if (pst == NULL) {
 433                         cpumask_t mask;
 434
 435                         mask = get_cpumask_from_level(sc->pst_cpuid,
 436                             CORE_LEVEL);
 437                         if (CPUMASK_TESTNZERO(mask)) {
 438                                 struct acpi_pst_domain *dom1;
 439
 440                                 LIST_FOREACH(dom1, &acpi_pst_domains, pd_link) {
 441                                         LIST_FOREACH(pst, &dom1->pd_pstlist,
 442                                             pst_link) {
 443                                                 if (CPUMASK_TESTBIT(mask,
 444                                                     pst->pst_cpuid))
 445                                                         break;
 446                                         }
 447                                         if (pst != NULL)
 448                                                 break;
 449                                 }
 450                                 if (pst != NULL && acpi_pst_ht_reuse_domain) {
 451                                         /*
 452                                          * Use the same domain for CPUs in the
 453                                          * same core.
 454                                          */
 455                                         device_printf(dev, "Destroy domain%u, "
 456                                             "reuse domain%u\n",
 457                                             dom->pd_dom, dom1->pd_dom);
 458                                         LIST_REMOVE(dom, pd_link);
 459                                         kfree(dom, M_DEVBUF);
 460                                         dom = dom1;
 461                                         /*
 462                                          * Make sure that adding us will not
 463                                          * overflow the domain containing
 464                                          * siblings in the same core.
 465                                          */
 466                                         acpi_pst_domain_check_nproc(dev, dom);
 467                                 }
 468                         }
 469                 }
 470                 if (pst != NULL &&
 471                     pst->pst_creg.pr_res == NULL &&
 472                     pst->pst_creg.pr_rid == 0 &&
 473                     pst->pst_creg.pr_gas.SpaceId ==
 474                     ACPI_ADR_SPACE_FIXED_HARDWARE &&
 475                     pst->pst_sreg.pr_res == NULL &&
 476                     pst->pst_sreg.pr_rid == 0 &&
 477                     pst->pst_sreg.pr_gas.SpaceId ==
 478                     ACPI_ADR_SPACE_FIXED_HARDWARE) {
 479                         sc->pst_creg = pst->pst_creg;
 480                         sc->pst_sreg = pst->pst_sreg;
 481                         device_printf(dev,
 482                             "No _PCT; reuse %s control/status regs\n",
 483                             device_get_nameunit(pst->pst_dev));
 484                         goto fetch_pss;
 485                 }
 486                 device_printf(dev, "Can't get _PCT package - %s\n",
 487                               AcpiFormatException(status));
 488                 return ENXIO;
 489         }
 490
 491         obj = (ACPI_OBJECT *)buf.Pointer;
 492         if (!ACPI_PKG_VALID_EQ(obj, 2)) {
 493                 device_printf(dev, "Invalid _PCT package\n");
 494                 AcpiOsFree(obj);
 495                 return ENXIO;
 496         }
 497
 498         /* Save and try allocating control register */
 499         error = acpi_pst_alloc_resource(dev, obj, 0, &sc->pst_creg);
 500         if (error) {
 501                 AcpiOsFree(obj);
 502                 return error;
 503         }
 504         if (bootverbose) {
 505                 device_printf(dev, "control reg %d %jx\n",
 506                               sc->pst_creg.pr_gas.SpaceId,
 507                               (uintmax_t)sc->pst_creg.pr_gas.Address);
 508         }
 509
 510         /* Save and try allocating status register */
 511         error = acpi_pst_alloc_resource(dev, obj, 1, &sc->pst_sreg);
 512         if (error) {
 513                 AcpiOsFree(obj);
 514                 return error;
 515         }
 516         if (bootverbose) {
 517                 device_printf(dev, "status reg %d %jx\n",
 518                               sc->pst_sreg.pr_gas.SpaceId,
 519                               (uintmax_t)sc->pst_sreg.pr_gas.Address);
 520         }
 521
 522         /* Free _PCT */
 523         AcpiOsFree(obj);
 524
 525 fetch_pss:
 526         /*
 527          * Create P-State table according to _PSS
 528          */
 529         buf.Pointer = NULL;
 530         buf.Length = ACPI_ALLOCATE_BUFFER;
 531         status = AcpiEvaluateObject(sc->pst_handle, "_PSS", NULL, &buf);
 532         if (ACPI_FAILURE(status)) {
 533                 /*
 534                  * No _PSS.  See the comment in acpi_pst_probe() near
 535                  * _PSD check.
 536                  *
 537                  * Assume _PSS are same across all CPUs; well, they
 538                  * should/have to be so.
 539                  */
 540                 if (acpi_npstates > 0 && acpi_pstates != NULL) {
 541                         device_printf(dev, "No _PSS\n");
 542                         goto fetch_ppc;
 543                 }
 544                 device_printf(dev, "Can't get _PSS package - %s\n",
 545                               AcpiFormatException(status));
 546                 return ENXIO;
 547         }
 548
 549         obj = (ACPI_OBJECT *)buf.Pointer;
 550         if (!ACPI_PKG_VALID(obj, 1)) {
 551                 device_printf(dev, "Invalid _PSS package\n");
 552                 AcpiOsFree(obj);
 553                 return ENXIO;
 554         }
 555
 556         /* Don't create too many P-States */
 557         npstate = obj->Package.Count;
 558         if (npstate > ACPI_NPSTATE_MAX) {
 559                 device_printf(dev, "Too many P-States, %d->%d\n",
 560                               npstate, ACPI_NPSTATE_MAX);
 561                 npstate = ACPI_NPSTATE_MAX;
 562         }
 563
 564         /*
 565          * If we have already created P-State table,
 566          * we must make sure that number of entries
 567          * is consistent.
 568          */
 569         if (acpi_pstates != NULL && acpi_npstates != npstate) {
 570                 device_printf(dev, "Inconsistent # of P-States "
 571                               "cross Processor objects\n");
 572                 AcpiOsFree(obj);
 573                 return ENXIO;
 574         }
 575
 576         /*
 577          * Create a temporary P-State table
 578          */
 579         pstate = kmalloc(sizeof(*pstate) * npstate, M_TEMP, M_WAITOK);
 580         for (i = 0, p = pstate; i < npstate; ++i, ++p) {
 581                 ACPI_OBJECT *pkg;
 582                 uint32_t *ptr[ACPI_PSS_PX_NENTRY] = {
 583                         &p->st_freq, &p->st_power, &p->st_xsit_lat,
 584                         &p->st_bm_lat, &p->st_cval, &p->st_sval
 585                 };
 586                 int j;
 587
 588                 pkg = &obj->Package.Elements[i];
 589                 if (!ACPI_PKG_VALID(pkg, ACPI_PSS_PX_NENTRY)) {
 590                         device_printf(dev, "Invalud _PSS P%d\n", i);
 591                         AcpiOsFree(obj);
 592                         kfree(pstate, M_TEMP);
 593                         return ENXIO;
 594                 }
 595                 for (j = 0; j < ACPI_PSS_PX_NENTRY; ++j) {
 596                         if (acpi_PkgInt32(pkg, j, ptr[j]) != 0) {
 597                                 device_printf(dev, "Can't extract "
 598                                               "_PSS P%d %dth entry\n", i, j);
 599                                 AcpiOsFree(obj);
 600                                 kfree(pstate, M_TEMP);
 601                                 return ENXIO;
 602                         }
 603                 }
 604         }
 605
 606         /* Free _PSS */
 607         AcpiOsFree(obj);
 608
 609         if (acpi_pstates == NULL) {
 610                 /*
 611                  * If no P-State table is created yet,
 612                  * save the temporary one we just created.
 613                  */
 614                 acpi_pstates = pstate;
 615                 acpi_npstates = npstate;
 616                 pstate = NULL;
 617
 618                 if (bootverbose) {
 619                         for (i = 0; i < acpi_npstates; ++i) {
 620                                 device_printf(dev,
 621                                 "freq %u, pwr %u, xlat %u, blat %u, "
 622                                 "cv %08x, sv %08x\n",
 623                                 acpi_pstates[i].st_freq,
 624                                 acpi_pstates[i].st_power,
 625                                 acpi_pstates[i].st_xsit_lat,
 626                                 acpi_pstates[i].st_bm_lat,
 627                                 acpi_pstates[i].st_cval,
 628                                 acpi_pstates[i].st_sval);
 629                         }
 630                 }
 631         } else {
 632                 /*
 633                  * Make sure that P-State tables are same
 634                  * for all processors.
 635                  */
 636                 if (memcmp(pstate, acpi_pstates,
 637                            sizeof(*pstate) * npstate) != 0) {
 638                         device_printf(dev, "Inconsistent _PSS "
 639                                       "cross Processor objects\n");
 640 #if 0
 641                         /*
 642                          * Some BIOSes create different P-State tables;
 643                          * just trust the one from the BSP and move on.
 644                          */
 645                         kfree(pstate, M_TEMP);
 646                         return ENXIO;
 647 #endif
 648                 }
 649                 kfree(pstate, M_TEMP);
 650         }
 651
 652 fetch_ppc:
 653         /* By default, we start from P-State table's first entry */
 654         sstart = 0;
 655
 656         /*
 657          * Adjust the usable first entry of P-State table,
 658          * if there is _PPC object.
 659          */
 660         error = acpi_pst_eval_ppc(sc, &sstart);
 661         if (error && error != ENOENT)
 662                 return error;
 663         else if (!error)
 664                 sc->pst_flags |= ACPI_PST_FLAG_PPC;
 665         if (acpi_pstate_start < 0) {
 666                 acpi_pstate_start = sstart;
 667         } else if (acpi_pstate_start != sstart) {
 668                 device_printf(dev, "_PPC mismatch, was %d, now %d\n",
 669                     acpi_pstate_start, sstart);
 670                 if (acpi_pstate_start < sstart) {
 671                         device_printf(dev, "_PPC %d -> %d\n",
 672                             acpi_pstate_start, sstart);
 673                         acpi_pstate_start = sstart;
 674                 }
 675         }
 676
 677         /*
 678          * By default, we assume number of usable P-States is same as
 679          * number of P-States.
 680          */
 681         scount = acpi_npstates;
 682
 683         /*
 684          * Allow users to override or set _PDL
 685          */
 686         if (acpi_pst_pdl >= 0) {
 687                 if (acpi_pst_pdl < acpi_npstates) {
 688                         if (bootverbose) {
 689                                 device_printf(dev, "_PDL override %d\n",
 690                                     acpi_pst_pdl);
 691                         }
 692                         scount = acpi_pst_pdl + 1;
 693                         goto proc_pdl;
 694                 } else {
 695                         device_printf(dev, "Invalid _PDL override %d, "
 696                             "must be less than %d\n", acpi_pst_pdl,
 697                             acpi_npstates);
 698                 }
 699         }
 700
 701         /*
 702          * Adjust the number of usable entries in P-State table,
 703          * if there is _PDL object.
 704          */
 705         error = acpi_pst_eval_pdl(sc, &scount);
 706         if (error && error != ENOENT)
 707                 return error;
 708         else if (!error)
 709                 sc->pst_flags |= ACPI_PST_FLAG_PDL;
 710 proc_pdl:
 711         if (acpi_pstate_count == 0) {
 712                 acpi_pstate_count = scount;
 713         } else if (acpi_pstate_count != scount) {
 714                 device_printf(dev, "_PDL mismatch, was %d, now %d\n",
 715                     acpi_pstate_count, scount);
 716                 if (acpi_pstate_count > scount) {
 717                         device_printf(dev, "_PDL %d -> %d\n",
 718                             acpi_pstate_count, scount);
 719                         acpi_pstate_count = scount;
 720                 }
 721         }
 722
 723         /*
 724          * Some CPUs only have package P-states, but some BIOSes put each
 725          * hyperthread to its own P-state domain; allow user to override.
 726          */
 727         if (LIST_EMPTY(&dom->pd_pstlist) &&
 728             (acpi_pst_force_pkg_domain ||
 729              (cpu_vendor_id == CPU_VENDOR_INTEL &&
 730               acpi_pst_intel_pkg_domain))) {
 731                 cpumask_t mask;
 732
 733                 mask = get_cpumask_from_level(sc->pst_cpuid, CHIP_LEVEL);
 734                 if (CPUMASK_TESTNZERO(mask)) {
 735                         struct acpi_pst_softc *pst = NULL;
 736                         struct acpi_pst_domain *dom1;
 737
 738                         LIST_FOREACH(dom1, &acpi_pst_domains, pd_link) {
 739                                 LIST_FOREACH(pst, &dom1->pd_pstlist,
 740                                     pst_link) {
 741                                         if (CPUMASK_TESTBIT(mask,
 742                                             pst->pst_cpuid))
 743                                                 break;
 744                                 }
 745                                 if (pst != NULL)
 746                                         break;
 747                         }
 748                         if (pst != NULL &&
 749                             memcmp(&pst->pst_creg, &sc->pst_creg,
 750                                 sizeof(sc->pst_creg)) == 0 &&
 751                             memcmp(&pst->pst_sreg, &sc->pst_sreg,
 752                                 sizeof(sc->pst_sreg)) == 0) {
 753                                 /*
 754                                  * Use the same domain for CPUs in the
 755                                  * same package.
 756                                  */
 757                                 device_printf(dev, "Destroy domain%u, "
 758                                     "force pkg domain%u\n",
 759                                     dom->pd_dom, dom1->pd_dom);
 760                                 LIST_REMOVE(dom, pd_link);
 761                                 kfree(dom, M_DEVBUF);
 762                                 dom = dom1;
 763                                 /*
 764                                  * Make sure that adding us will not
 765                                  * overflow the domain containing
 766                                  * siblings in the same package.
 767                                  */
 768                                 acpi_pst_domain_check_nproc(dev, dom);
 769                         }
 770                 }
 771         }
 772
 773         /* Link us with the domain */
 774         sc->pst_domain = dom;
 775         LIST_INSERT_HEAD(&dom->pd_pstlist, sc, pst_link);
 776
 777         if (device_get_unit(dev) == 0)
 778                 AcpiOsExecute(OSL_NOTIFY_HANDLER, acpi_pst_postattach, NULL);
 779
 780         if (sc->pst_flags & (ACPI_PST_FLAG_PPC | ACPI_PST_FLAG_PDL))
 781                 sc->pst_parent->cpu_pst_notify = acpi_pst_notify;
 782
 783         return 0;
 784 }
 785
 786 static struct acpi_pst_domain *
 787 acpi_pst_domain_create_pkg(device_t dev, ACPI_OBJECT *obj)
 788 {
 789         struct acpi_pst_domain *dom;
 790         uint32_t val, domain, coord, nproc;
 791
 792         if (!ACPI_PKG_VALID_EQ(obj, 5)) {
 793                 device_printf(dev, "Invalid _PSD package\n");
 794                 return NULL;
 795         }
 796
 797         /* NumberOfEntries */
 798         if (acpi_PkgInt32(obj, 0, &val) != 0 || val != 5) {
 799                 device_printf(dev, "Invalid _PSD NumberOfEntries\n");
 800                 return NULL;
 801         }
 802
 803         /* Revision */
 804         if (acpi_PkgInt32(obj, 1, &val) != 0 || val != 0) {
 805                 device_printf(dev, "Invalid _PSD Revision\n");
 806                 return NULL;
 807         }
 808
 809         if (acpi_PkgInt32(obj, 2, &domain) != 0 ||
 810             acpi_PkgInt32(obj, 3, &coord) != 0 ||
 811             acpi_PkgInt32(obj, 4, &nproc) != 0) {
 812                 device_printf(dev, "Can't extract _PSD package\n");
 813                 return NULL;
 814         }
 815
 816         if (!ACPI_PSD_COORD_VALID(coord)) {
 817                 device_printf(dev, "Invalid _PSD CoordType (%#x)\n", coord);
 818                 return NULL;
 819         }
 820
 821         if (nproc > MAXCPU) {
 822                 /*
 823                  * If NumProcessors is greater than MAXCPU
 824                  * and domain's coordination is SWALL, then
 825                  * we will never be able to start all CPUs
 826                  * within this domain, and power state
 827                  * transition will never be completed, so we
 828                  * just bail out here.
 829                  */
 830                 if (coord == ACPI_PSD_COORD_SWALL) {
 831                         device_printf(dev, "Unsupported _PSD NumProcessors "
 832                                       "(%d)\n", nproc);
 833                         return NULL;
 834                 }
 835         } else if (nproc == 0) {
 836                 device_printf(dev, "_PSD NumProcessors are zero\n");
 837                 return NULL;
 838         }
 839
 840         dom = acpi_pst_domain_find(domain);
 841         if (dom != NULL) {
 842                 if (dom->pd_flags & ACPI_PSTDOM_FLAG_INT) {
 843                         device_printf(dev, "Mixed Integer _PSD and "
 844                             "Package _PSD\n");
 845                         return NULL;
 846                 }
 847                 if (dom->pd_coord != coord) {
 848                         device_printf(dev, "Inconsistent _PSD coord "
 849                             "information cross Processor objects\n");
 850                         return NULL;
 851                 }
 852                 if (dom->pd_nproc != nproc) {
 853                         device_printf(dev, "Inconsistent _PSD nproc "
 854                             "information cross Processor objects\n");
 855                         /*
 856                          * Some stupid BIOSes will set wrong "# of processors",
 857                          * e.g. 1 for CPU w/ hyperthreading; Be lenient here.
 858                          */
 859                 }
 860                 return dom;
 861         }
 862
 863         dom = acpi_pst_domain_alloc(domain, coord, nproc);
 864         if (bootverbose) {
 865                 device_printf(dev, "create pkg domain%u, coord %#x\n",
 866                     dom->pd_dom, dom->pd_coord);
 867         }
 868
 869         return dom;
 870 }
 871
 872 static struct acpi_pst_domain *
 873 acpi_pst_domain_create_int(device_t dev, uint32_t domain)
 874 {
 875         struct acpi_pst_domain *dom;
 876
 877         dom = acpi_pst_domain_find(domain);
 878         if (dom != NULL) {
 879                 if ((dom->pd_flags & ACPI_PSTDOM_FLAG_INT) == 0) {
 880                         device_printf(dev, "Mixed Package _PSD and "
 881                             "Integer _PSD\n");
 882                         return NULL;
 883                 }
 884                 KKASSERT(dom->pd_coord == ACPI_PSD_COORD_SWALL);
 885
 886                 dom->pd_nproc++;
 887                 return dom;
 888         }
 889
 890         dom = acpi_pst_domain_alloc(domain, ACPI_PSD_COORD_SWALL, 1);
 891         dom->pd_flags |= ACPI_PSTDOM_FLAG_INT;
 892
 893         if (bootverbose)
 894                 device_printf(dev, "create int domain%u\n", dom->pd_dom);
 895
 896         return dom;
 897 }
 898
 899 static struct acpi_pst_domain *
 900 acpi_pst_domain_find(uint32_t domain)
 901 {
 902         struct acpi_pst_domain *dom;
 903
 904         LIST_FOREACH(dom, &acpi_pst_domains, pd_link) {
 905                 if (dom->pd_flags & ACPI_PSTDOM_FLAG_STUB)
 906                         continue;
 907                 if (dom->pd_dom == domain)
 908                         return dom;
 909         }
 910         return NULL;
 911 }
 912
 913 static struct acpi_pst_domain *
 914 acpi_pst_domain_alloc(uint32_t domain, uint32_t coord, uint32_t nproc)
 915 {
 916         struct acpi_pst_domain *dom;
 917
 918         dom = kmalloc(sizeof(*dom), M_DEVBUF, M_WAITOK | M_ZERO);
 919         dom->pd_dom = domain;
 920         dom->pd_coord = coord;
 921         dom->pd_nproc = nproc;
 922         LIST_INIT(&dom->pd_pstlist);
 923         lwkt_serialize_init(&dom->pd_serialize);
 924
 925         LIST_INSERT_HEAD(&acpi_pst_domains, dom, pd_link);
 926
 927         return dom;
 928 }
 929
 930 static void
 931 acpi_pst_domain_set_pstate_locked(struct acpi_pst_domain *dom, int i, int *global)
 932 {
 933         const struct acpi_pstate *pstate;
 934         struct acpi_pst_softc *sc;
 935         int done, error;
 936
 937         ASSERT_SERIALIZED(&dom->pd_serialize);
 938
 939         KKASSERT(i >= 0 && i < acpi_npstates);
 940         pstate = &acpi_pstates[i];
 941
 942         done = 0;
 943         LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
 944                 if (!done) {
 945                         error = acpi_pst_set_pstate(sc, pstate);
 946                         if (error) {
 947                                 device_printf(sc->pst_dev, "can't set "
 948                                               "freq %d\n", pstate->st_freq);
 949                                 /* XXX error cleanup? */
 950                         }
 951                         if (dom->pd_coord == ACPI_PSD_COORD_SWANY)
 952                                 done = 1;
 953                 }
 954                 sc->pst_state = i;
 955         }
 956         dom->pd_state = i;
 957
 958         if (global != NULL)
 959                 *global = i;
 960 }
 961
 962 static void
 963 acpi_pst_domain_set_pstate(struct acpi_pst_domain *dom, int i, int *global)
 964 {
 965         lwkt_serialize_enter(&dom->pd_serialize);
 966         acpi_pst_domain_set_pstate_locked(dom, i, global);
 967         lwkt_serialize_exit(&dom->pd_serialize);
 968 }
 969
 970 static void
 971 acpi_pst_global_set_pstate(int i)
 972 {
 973         struct acpi_pst_domain *dom;
 974         int *global = &acpi_pst_global_state;
 975
 976         LIST_FOREACH(dom, &acpi_pst_domains, pd_link) {
 977                 /* Skip dead domain */
 978                 if (dom->pd_flags & ACPI_PSTDOM_FLAG_DEAD)
 979                         continue;
 980                 acpi_pst_domain_set_pstate(dom, i, global);
 981                 global = NULL;
 982         }
 983 }
 984
 985 static void
 986 acpi_pst_global_fixup_pstate(void)
 987 {
 988         struct acpi_pst_domain *dom;
 989         int *global = &acpi_pst_global_state;
 990         int sstart, scount;
 991
 992         sstart = acpi_pstate_start;
 993         scount = acpi_pstate_count;
 994
 995         LIST_FOREACH(dom, &acpi_pst_domains, pd_link) {
 996                 int i = -1;
 997
 998                 /* Skip dead domain */
 999                 if (dom->pd_flags & ACPI_PSTDOM_FLAG_DEAD)
1000                         continue;
1001
1002                 lwkt_serialize_enter(&dom->pd_serialize);
1003
1004                 if (global != NULL) {
1005                         if (*global < sstart)
1006                                 *global = sstart;
1007                         else if (*global >= scount)
1008                                 *global = scount - 1;
1009                         global = NULL;
1010                 }
1011                 if (dom->pd_state < sstart)
1012                         i = sstart;
1013                 else if (dom->pd_state >= scount)
1014                         i = scount - 1;
1015                 if (i >= 0)
1016                         acpi_pst_domain_set_pstate_locked(dom, i, NULL);
1017
1018                 lwkt_serialize_exit(&dom->pd_serialize);
1019         }
1020 }
1021
1022 static void
1023 acpi_pst_postattach(void *arg __unused)
1024 {
1025         struct acpi_pst_domain *dom;
1026         struct acpi_cpu_softc *cpu;
1027         device_t *devices;
1028         int i, ndevices, error, has_domain;
1029
1030         devices = NULL;
1031         ndevices = 0;
1032         error = devclass_get_devices(acpi_pst_devclass, &devices, &ndevices);
1033         if (error)
1034                 return;
1035
1036         if (ndevices == 0)
1037                 return;
1038
1039         cpu = NULL;
1040         for (i = 0; i < ndevices; ++i) {
1041                 cpu = device_get_softc(device_get_parent(devices[i]));
1042                 if (cpu->glob_sysctl_tree != NULL)
1043                         break;
1044         }
1045         kfree(devices, M_TEMP);
1046         KKASSERT(cpu != NULL);
1047
1048         if (acpi_pst_md == NULL)
1049                 kprintf("ACPI: no P-State CPU driver\n");
1050
1051         has_domain = 0;
1052         LIST_FOREACH(dom, &acpi_pst_domains, pd_link) {
1053                 struct acpi_pst_softc *sc;
1054                 char buf[32];
1055
1056                 dom->pd_state = acpi_pstate_start;
1057
1058                 /*
1059                  * Make sure that all processors belonging to this
1060                  * domain are located.
1061                  */
1062                 i = 0;
1063                 LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
1064                         sc->pst_state = acpi_pstate_start;
1065                         ++i;
1066                 }
1067                 if (i != dom->pd_nproc) {
1068                         KKASSERT(i < dom->pd_nproc);
1069
1070                         kprintf("ACPI: domain%u misses processors, "
1071                                 "should be %d, got %d\n", dom->pd_dom,
1072                                 dom->pd_nproc, i);
1073                         if (dom->pd_coord == ACPI_PSD_COORD_SWALL) {
1074                                 /*
1075                                  * If this domain's coordination is
1076                                  * SWALL and we don't see all of the
1077                                  * member CPUs of this domain, then
1078                                  * the P-State transition will never
1079                                  * be completed, so just leave this
1080                                  * domain out.
1081                                  */
1082                                 dom->pd_flags |= ACPI_PSTDOM_FLAG_DEAD;
1083                                 continue;
1084                         }
1085                         dom->pd_nproc = i;
1086                 }
1087
1088                 /*
1089                  * Validate P-State configurations for this domain
1090                  */
1091                 LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
1092                         error = acpi_pst_check_csr(sc);
1093                         if (error)
1094                                 break;
1095
1096                         error = acpi_pst_check_pstates(sc);
1097                         if (error)
1098                                 break;
1099                 }
1100                 if (sc != NULL) {
1101                         kprintf("ACPI: domain%u P-State configuration "
1102                                 "check failed\n", dom->pd_dom);
1103                         dom->pd_flags |= ACPI_PSTDOM_FLAG_DEAD;
1104                         continue;
1105                 }
1106
1107                 /*
1108                  * Do necssary P-State initialization
1109                  */
1110                 LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
1111                         error = acpi_pst_init(sc);
1112                         if (error)
1113                                 break;
1114                 }
1115                 if (sc != NULL) {
1116                         kprintf("ACPI: domain%u P-State initialization "
1117                                 "check failed\n", dom->pd_dom);
1118                         dom->pd_flags |= ACPI_PSTDOM_FLAG_DEAD;
1119                         continue;
1120                 }
1121
1122                 has_domain = 1;
1123
1124                 ksnprintf(buf, sizeof(buf), "px_dom%u", dom->pd_dom);
1125
1126                 sysctl_ctx_init(&dom->pd_sysctl_ctx);
1127                 dom->pd_sysctl_tree =
1128                 SYSCTL_ADD_NODE(&dom->pd_sysctl_ctx,
1129                         SYSCTL_CHILDREN(cpu->glob_sysctl_tree),
1130                         OID_AUTO, buf, CTLFLAG_RD, 0,
1131                         "P-State domain");
1132                 if (dom->pd_sysctl_tree == NULL) {
1133                         kprintf("ACPI: Can't create sysctl tree for domain%u",
1134                                 dom->pd_dom);
1135                         continue;
1136                 }
1137
1138                 SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
1139                                 SYSCTL_CHILDREN(dom->pd_sysctl_tree),
1140                                 OID_AUTO, "available",
1141                                 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP,
1142                                 dom, 0, acpi_pst_sysctl_freqs, "A",
1143                                 "available frequencies");
1144
1145                 SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
1146                                 SYSCTL_CHILDREN(dom->pd_sysctl_tree),
1147                                 OID_AUTO, "avail",
1148                                 CTLTYPE_OPAQUE | CTLFLAG_RD,
1149                                 dom, 0, acpi_pst_sysctl_freqs_bin, "IU",
1150                                 "available frequencies");
1151
1152                 SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
1153                                 SYSCTL_CHILDREN(dom->pd_sysctl_tree),
1154                                 OID_AUTO, "power",
1155                                 CTLTYPE_OPAQUE | CTLFLAG_RD,
1156                                 dom, 0, acpi_pst_sysctl_power, "IU",
1157                                 "power of available frequencies");
1158
1159                 SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
1160                                 SYSCTL_CHILDREN(dom->pd_sysctl_tree),
1161                                 OID_AUTO, "members",
1162                                 CTLTYPE_STRING | CTLFLAG_RD,
1163                                 dom, 0, acpi_pst_sysctl_members, "A",
1164                                 "member cpus");
1165
1166                 if (acpi_pst_md != NULL &&
1167                     acpi_pst_md->pmd_set_pstate != NULL) {
1168                         SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
1169                                         SYSCTL_CHILDREN(dom->pd_sysctl_tree),
1170                                         OID_AUTO, "select",
1171                                         CTLTYPE_UINT | CTLFLAG_RW,
1172                                         dom, 0, acpi_pst_sysctl_select,
1173                                         "IU", "select freq");
1174                 }
1175         }
1176
1177         if (has_domain && acpi_pst_md != NULL &&
1178             acpi_pst_md->pmd_set_pstate != NULL) {
1179                 SYSCTL_ADD_PROC(&cpu->glob_sysctl_ctx,
1180                                 SYSCTL_CHILDREN(cpu->glob_sysctl_tree),
1181                                 OID_AUTO, "px_global",
1182                                 CTLTYPE_UINT | CTLFLAG_RW,
1183                                 NULL, 0, acpi_pst_sysctl_global,
1184                                 "IU", "select freq for all domains");
1185                 SYSCTL_ADD_INT(&cpu->glob_sysctl_ctx,
1186                                SYSCTL_CHILDREN(cpu->glob_sysctl_tree),
1187                                OID_AUTO, "px_handle_notify", CTLFLAG_RW,
1188                                &acpi_pst_handle_notify, 0,
1189                                "handle type 0x80 notify");
1190
1191                 acpi_pst_global_set_pstate(acpi_pstate_start);
1192         }
1193 }
1194
1195 static int
1196 acpi_pst_sysctl_freqs(SYSCTL_HANDLER_ARGS)
1197 {
1198         int i, error, sstart, scount;
1199
1200         error = 0;
1201         sstart = acpi_pstate_start;
1202         scount = acpi_pstate_count;
1203         for (i = 0; i < acpi_npstates; ++i) {
1204                 if (error == 0 && i)
1205                         error = SYSCTL_OUT(req, " ", 1);
1206                 if (error == 0) {
1207                         const char *pat;
1208                         char buf[32];
1209
1210                         if (i < sstart || i >= scount)
1211                                 pat = "(%u)";
1212                         else
1213                                 pat = "%u";
1214
1215                         ksnprintf(buf, sizeof(buf), pat,
1216                                   acpi_pstates[i].st_freq);
1217                         error = SYSCTL_OUT(req, buf, strlen(buf));
1218                 }
1219         }
1220         return error;
1221 }
1222
1223 static int
1224 acpi_pst_sysctl_freqs_bin(SYSCTL_HANDLER_ARGS)
1225 {
1226         uint32_t freqs[ACPI_NPSTATE_MAX];
1227         int cnt, i, sstart, scount;
1228
1229         sstart = acpi_pstate_start;
1230         scount = acpi_pstate_count;
1231
1232         cnt = scount - sstart;
1233         for (i = 0; i < cnt; ++i)
1234                 freqs[i] = acpi_pstates[sstart + i].st_freq;
1235
1236         return sysctl_handle_opaque(oidp, freqs, cnt * sizeof(freqs[0]), req);
1237 }
1238
1239 static int
1240 acpi_pst_sysctl_power(SYSCTL_HANDLER_ARGS)
1241 {
1242         uint32_t power[ACPI_NPSTATE_MAX];
1243         int cnt, i, sstart, scount;
1244
1245         sstart = acpi_pstate_start;
1246         scount = acpi_pstate_count;
1247
1248         cnt = scount - sstart;
1249         for (i = 0; i < cnt; ++i)
1250                 power[i] = acpi_pstates[sstart + i].st_power;
1251
1252         return sysctl_handle_opaque(oidp, power, cnt * sizeof(power[0]), req);
1253 }
1254
1255 static int
1256 acpi_pst_sysctl_members(SYSCTL_HANDLER_ARGS)
1257 {
1258         struct acpi_pst_domain *dom = arg1;
1259         struct acpi_pst_softc *sc;
1260         int loop, error;
1261
1262         loop = error = 0;
1263         LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
1264                 char buf[32];
1265
1266                 if (error == 0 && loop)
1267                         error = SYSCTL_OUT(req, " ", 1);
1268                 if (error == 0) {
1269                         ksnprintf(buf, sizeof(buf), "cpu%d", sc->pst_cpuid);
1270                         error = SYSCTL_OUT(req, buf, strlen(buf));
1271                 }
1272
1273                 if (error == 0 && acpi_pst_md && acpi_pst_md->pmd_get_pstate) {
1274                         const struct acpi_pstate *pstate;
1275                         const char *str;
1276
1277                         pstate = acpi_pst_get_pstate(sc);
1278                         if (pstate == NULL) {
1279                                 str = "(*)";
1280                         } else {
1281                                 ksnprintf(buf, sizeof(buf), "(%d)",
1282                                           pstate->st_freq);
1283                                 str = buf;
1284                         }
1285                         error = SYSCTL_OUT(req, str, strlen(str));
1286                 }
1287                 ++loop;
1288         }
1289         return error;
1290 }
1291
1292 static int
1293 acpi_pst_sysctl_select(SYSCTL_HANDLER_ARGS)
1294 {
1295         struct acpi_pst_domain *dom = arg1;
1296         int error, i, freq;
1297
1298         KKASSERT(dom->pd_state >= 0 && dom->pd_state < acpi_npstates);
1299
1300         freq = acpi_pstates[dom->pd_state].st_freq;
1301
1302         error = sysctl_handle_int(oidp, &freq, 0, req);
1303         if (error || req->newptr == NULL)
1304                 return error;
1305
1306         i = acpi_pst_freq2index(freq);
1307         if (i < 0)
1308                 return EINVAL;
1309
1310         acpi_pst_domain_set_pstate(dom, i, NULL);
1311         return 0;
1312 }
1313
1314 static int
1315 acpi_pst_sysctl_global(SYSCTL_HANDLER_ARGS)
1316 {
1317         int error, i, freq;
1318
1319         KKASSERT(acpi_pst_global_state >= 0 &&
1320                  acpi_pst_global_state < acpi_npstates);
1321
1322         freq = acpi_pstates[acpi_pst_global_state].st_freq;
1323
1324         error = sysctl_handle_int(oidp, &freq, 0, req);
1325         if (error || req->newptr == NULL)
1326                 return error;
1327
1328         i = acpi_pst_freq2index(freq);
1329         if (i < 0)
1330                 return EINVAL;
1331
1332         acpi_pst_global_set_pstate(i);
1333
1334         return 0;
1335 }
1336
1337 static void
1338 acpi_pst_check_csr_handler(netmsg_t msg)
1339 {
1340         struct netmsg_acpi_pst *rmsg = (struct netmsg_acpi_pst *)msg;
1341         int error;
1342
1343         error = acpi_pst_md->pmd_check_csr(rmsg->ctrl, rmsg->status);
1344         lwkt_replymsg(&rmsg->base.lmsg, error);
1345 }
1346
1347 static int
1348 acpi_pst_check_csr(struct acpi_pst_softc *sc)
1349 {
1350         struct netmsg_acpi_pst msg;
1351
1352         if (acpi_pst_md == NULL)
1353                 return 0;
1354
1355         netmsg_init(&msg.base, NULL, &curthread->td_msgport,
1356                     MSGF_PRIORITY, acpi_pst_check_csr_handler);
1357         msg.ctrl = &sc->pst_creg;
1358         msg.status = &sc->pst_sreg;
1359
1360         return lwkt_domsg(netisr_cpuport(sc->pst_cpuid), &msg.base.lmsg, 0);
1361 }
1362
1363 static void
1364 acpi_pst_check_pstates_handler(netmsg_t msg)
1365 {
1366         int error;
1367
1368         error = acpi_pst_md->pmd_check_pstates(acpi_pstates, acpi_npstates);
1369         lwkt_replymsg(&msg->lmsg, error);
1370 }
1371
1372 static int
1373 acpi_pst_check_pstates(struct acpi_pst_softc *sc)
1374 {
1375         struct netmsg_base msg;
1376
1377         if (acpi_pst_md == NULL)
1378                 return 0;
1379
1380         netmsg_init(&msg, NULL, &curthread->td_msgport,
1381                     MSGF_PRIORITY, acpi_pst_check_pstates_handler);
1382
1383         return lwkt_domsg(netisr_cpuport(sc->pst_cpuid), &msg.lmsg, 0);
1384 }
1385
1386 static void
1387 acpi_pst_init_handler(netmsg_t msg)
1388 {
1389         struct netmsg_acpi_pst *rmsg = (struct netmsg_acpi_pst *)msg;
1390         int error;
1391
1392         error = acpi_pst_md->pmd_init(rmsg->ctrl, rmsg->status);
1393         lwkt_replymsg(&rmsg->base.lmsg, error);
1394 }
1395
1396 static int
1397 acpi_pst_init(struct acpi_pst_softc *sc)
1398 {
1399         struct netmsg_acpi_pst msg;
1400
1401         if (acpi_pst_md == NULL)
1402                 return 0;
1403
1404         netmsg_init(&msg.base, NULL, &curthread->td_msgport,
1405                     MSGF_PRIORITY, acpi_pst_init_handler);
1406         msg.ctrl = &sc->pst_creg;
1407         msg.status = &sc->pst_sreg;
1408
1409         return lwkt_domsg(netisr_cpuport(sc->pst_cpuid), &msg.base.lmsg, 0);
1410 }
1411
1412 static void
1413 acpi_pst_set_pstate_handler(netmsg_t msg)
1414 {
1415         struct netmsg_acpi_pst *rmsg = (struct netmsg_acpi_pst *)msg;
1416         int error;
1417
1418         error = acpi_pst_md->pmd_set_pstate(rmsg->ctrl, rmsg->status,
1419                                             rmsg->base.lmsg.u.ms_resultp);
1420         lwkt_replymsg(&rmsg->base.lmsg, error);
1421 }
1422
1423 static int
1424 acpi_pst_set_pstate(struct acpi_pst_softc *sc, const struct acpi_pstate *pstate)
1425 {
1426         struct netmsg_acpi_pst msg;
1427
1428         KKASSERT(acpi_pst_md != NULL);
1429
1430         netmsg_init(&msg.base, NULL, &curthread->td_msgport,
1431                     MSGF_PRIORITY, acpi_pst_set_pstate_handler);
1432         msg.base.lmsg.u.ms_resultp = __DECONST(void *, pstate);
1433         msg.ctrl = &sc->pst_creg;
1434         msg.status = &sc->pst_sreg;
1435
1436         return lwkt_domsg(netisr_cpuport(sc->pst_cpuid), &msg.base.lmsg, 0);
1437 }
1438
1439 static void
1440 acpi_pst_get_pstate_handler(netmsg_t msg)
1441 {
1442         struct netmsg_acpi_pst *rmsg = (struct netmsg_acpi_pst *)msg;
1443         const struct acpi_pstate *pstate;
1444
1445         pstate = acpi_pst_md->pmd_get_pstate(rmsg->status, acpi_pstates,
1446                                              acpi_npstates);
1447         rmsg->base.lmsg.u.ms_resultp = __DECONST(void *, pstate);
1448         lwkt_replymsg(&rmsg->base.lmsg, 0);
1449 }
1450
1451 static const struct acpi_pstate *
1452 acpi_pst_get_pstate(struct acpi_pst_softc *sc)
1453 {
1454         struct netmsg_acpi_pst msg;
1455
1456         if (acpi_pst_md == NULL)
1457                 return 0;
1458
1459         netmsg_init(&msg.base, NULL, &curthread->td_msgport,
1460                     MSGF_PRIORITY, acpi_pst_get_pstate_handler);
1461         msg.status = &sc->pst_sreg;
1462
1463         lwkt_domsg(netisr_cpuport(sc->pst_cpuid), &msg.base.lmsg, 0);
1464         return msg.base.lmsg.u.ms_resultp;
1465 }
1466
1467 static int
1468 acpi_pst_alloc_resource(device_t dev, ACPI_OBJECT *obj, int idx,
1469                         struct acpi_pst_res *res)
1470 {
1471         struct acpi_pst_softc *sc = device_get_softc(dev);
1472         int error, type;
1473
1474         /* Save GAS */
1475         error = acpi_PkgRawGas(obj, idx, &res->pr_gas);
1476         if (error)
1477                 return error;
1478
1479         /* Allocate resource, if possible */
1480         res->pr_rid = sc->pst_parent->cpu_next_rid;
1481         acpi_bus_alloc_gas(dev, &type, &res->pr_rid, &res->pr_gas, &res->pr_res, 0);
1482         if (res->pr_res != NULL) {
1483                 sc->pst_parent->cpu_next_rid++;
1484                 res->pr_bt = rman_get_bustag(res->pr_res);
1485                 res->pr_bh = rman_get_bushandle(res->pr_res);
1486         } else {
1487                 res->pr_rid = 0;
1488         }
1489         return 0;
1490 }
1491
1492 static void
1493 acpi_pst_domain_check_nproc(device_t dev, struct acpi_pst_domain *dom)
1494 {
1495         struct acpi_pst_softc *pst;
1496         int i;
1497
1498         i = 0;
1499         LIST_FOREACH(pst, &dom->pd_pstlist, pst_link)
1500                 ++i;
1501         if (i == dom->pd_nproc) {
1502                 /*
1503                  * Some stupid BIOSes will set wrong "# of processors",
1504                  * e.g. 1 for CPU w/ hyperthreading; Be lenient here.
1505                  */
1506                 if (bootverbose) {
1507                         device_printf(dev, "domain%u already contains %d "
1508                             "P-States\n", dom->pd_dom, dom->pd_nproc);
1509                 }
1510                 dom->pd_nproc++;
1511         }
1512         KKASSERT(i < dom->pd_nproc);
1513 }
1514
1515 static int
1516 acpi_pst_eval_ppc(struct acpi_pst_softc *sc, int *sstart)
1517 {
1518         ACPI_BUFFER buf;
1519         ACPI_STATUS status;
1520         ACPI_OBJECT *obj;
1521
1522         buf.Pointer = NULL;
1523         buf.Length = ACPI_ALLOCATE_BUFFER;
1524         status = AcpiEvaluateObject(sc->pst_handle, "_PPC", NULL, &buf);
1525         if (!ACPI_FAILURE(status)) {
1526                 ACPI_OBJECT_LIST arglist;
1527                 ACPI_OBJECT arg[2];
1528
1529                 obj = (ACPI_OBJECT *)buf.Pointer;
1530                 if (obj->Type == ACPI_TYPE_INTEGER) {
1531                         if (obj->Integer.Value >= acpi_npstates) {
1532                                 device_printf(sc->pst_dev,
1533                                     "Invalid _PPC value\n");
1534                                 AcpiOsFree(obj);
1535                                 return ENXIO;
1536                         }
1537                         *sstart = obj->Integer.Value;
1538                         if (bootverbose) {
1539                                 device_printf(sc->pst_dev, "_PPC %d\n",
1540                                     *sstart);
1541                         }
1542                 } else {
1543                         device_printf(sc->pst_dev, "Invalid _PPC object\n");
1544                         AcpiOsFree(obj);
1545                         return ENXIO;
1546                 }
1547
1548                 /* Free _PPC */
1549                 AcpiOsFree(obj);
1550
1551                 /* _PPC has been successfully processed */
1552                 arglist.Pointer = arg;
1553                 arglist.Count = 2;
1554                 arg[0].Type = ACPI_TYPE_INTEGER;
1555                 arg[0].Integer.Value = 0x80;
1556                 arg[1].Type = ACPI_TYPE_INTEGER;
1557                 arg[1].Integer.Value = 0;
1558                 AcpiEvaluateObject(sc->pst_handle, "_OST", &arglist, NULL);
1559
1560                 return 0;
1561         }
1562         return ENOENT;
1563 }
1564
1565 static int
1566 acpi_pst_eval_pdl(struct acpi_pst_softc *sc, int *scount)
1567 {
1568         ACPI_BUFFER buf;
1569         ACPI_STATUS status;
1570         ACPI_OBJECT *obj;
1571
1572         buf.Pointer = NULL;
1573         buf.Length = ACPI_ALLOCATE_BUFFER;
1574         status = AcpiEvaluateObject(sc->pst_handle, "_PDL", NULL, &buf);
1575         if (!ACPI_FAILURE(status)) {
1576                 obj = (ACPI_OBJECT *)buf.Pointer;
1577                 if (obj->Type == ACPI_TYPE_INTEGER) {
1578                         if (obj->Integer.Value >= acpi_npstates) {
1579                                 device_printf(sc->pst_dev,
1580                                     "Invalid _PDL value\n");
1581                                 AcpiOsFree(obj);
1582                                 return ENXIO;
1583                         }
1584                         if (obj->Integer.Value >= acpi_pstate_start) {
1585                                 *scount = obj->Integer.Value + 1;
1586                                 if (bootverbose) {
1587                                         device_printf(sc->pst_dev, "_PDL %d\n",
1588                                             *scount);
1589                                 }
1590                         } else {
1591                                 /* Prefer _PPC as stated in ACPI 5.1 8.4.4.6 */
1592                                 device_printf(sc->pst_dev, "conflict _PDL %ju "
1593                                     "and _PPC %d, ignore\n",
1594                                     (uintmax_t)obj->Integer.Value,
1595                                     acpi_pstate_start);
1596                         }
1597                 } else {
1598                         device_printf(sc->pst_dev, "Invalid _PDL object\n");
1599                         AcpiOsFree(obj);
1600                         return ENXIO;
1601                 }
1602
1603                 /* Free _PDL */
1604                 AcpiOsFree(obj);
1605
1606                 return 0;
1607         }
1608         return ENOENT;
1609 }
1610
1611 /*
1612  * Notify is serialized by acpi task thread.
1613  */
1614 static void
1615 acpi_pst_notify(device_t dev)
1616 {
1617         struct acpi_pst_softc *sc = device_get_softc(dev);
1618         boolean_t fixup = FALSE;
1619
1620         if (!acpi_pst_handle_notify)
1621                 return;
1622
1623         /*
1624          * NOTE:
1625          * _PPC and _PDL evaluation order is critical.  _PDL
1626          * evaluation depends on _PPC evaluation.
1627          */
1628         if (sc->pst_flags & ACPI_PST_FLAG_PPC) {
1629                 int sstart = acpi_pstate_start;
1630
1631                 acpi_pst_eval_ppc(sc, &sstart);
1632                 if (acpi_pstate_start != sstart && sc->pst_cpuid == 0) {
1633                         acpi_pstate_start = sstart;
1634                         fixup = TRUE;
1635                 }
1636         }
1637         if (sc->pst_flags & ACPI_PST_FLAG_PDL) {
1638                 int scount = acpi_pstate_count;
1639
1640                 acpi_pst_eval_pdl(sc, &scount);
1641                 if (acpi_pstate_count != scount && sc->pst_cpuid == 0) {
1642                         acpi_pstate_count = scount;
1643                         fixup = TRUE;
1644                 }
1645         }
1646
1647         if (fixup && acpi_pst_md != NULL &&
1648             acpi_pst_md->pmd_set_pstate != NULL)
1649                 acpi_pst_global_fixup_pstate();
1650 }