sys/dev/acpica/acpi_cpu_pstate.c

   1 /*
   2  * Copyright (c) 2009 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Sepherosa Ziehau <sepherosa@gmail.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 #include "opt_acpi.h"
  36
  37 #include <sys/param.h>
  38 #include <sys/bus.h>
  39 #include <sys/kernel.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/rman.h>
  43 #include <sys/sysctl.h>
  44 #include <sys/msgport2.h>
  45 #include <sys/cpu_topology.h>
  46
  47 #include <net/netisr2.h>
  48 #include <net/netmsg2.h>
  49 #include <net/if_var.h>
  50
  51 #include "acpi.h"
  52 #include "acpivar.h"
  53 #include "acpi_cpu.h"
  54 #include "acpi_cpu_pstate.h"
  55
  56 #define ACPI_NPSTATE_MAX        32
  57
  58 #define ACPI_PSS_PX_NENTRY      6
  59
  60 #define ACPI_PSD_COORD_SWALL    0xfc
  61 #define ACPI_PSD_COORD_SWANY    0xfd
  62 #define ACPI_PSD_COORD_HWALL    0xfe
  63 #define ACPI_PSD_COORD_VALID(coord) \
  64         ((coord) == ACPI_PSD_COORD_SWALL || \
  65          (coord) == ACPI_PSD_COORD_SWANY || \
  66          (coord) == ACPI_PSD_COORD_HWALL)
  67
  68 struct acpi_pst_softc;
  69 LIST_HEAD(acpi_pst_list, acpi_pst_softc);
  70
  71 struct netmsg_acpi_pst {
  72         struct netmsg_base base;
  73         const struct acpi_pst_res *ctrl;
  74         const struct acpi_pst_res *status;
  75 };
  76
  77 struct acpi_pst_domain {
  78         uint32_t                pd_dom;
  79         uint32_t                pd_coord;
  80         uint32_t                pd_nproc;
  81         LIST_ENTRY(acpi_pst_domain) pd_link;
  82
  83         uint32_t                pd_flags;
  84
  85         struct lwkt_serialize   pd_serialize;
  86
  87         int                     pd_state;
  88         struct acpi_pst_list    pd_pstlist;
  89
  90         struct sysctl_ctx_list  pd_sysctl_ctx;
  91         struct sysctl_oid       *pd_sysctl_tree;
  92 };
  93 LIST_HEAD(acpi_pst_domlist, acpi_pst_domain);
  94
  95 #define ACPI_PSTDOM_FLAG_STUB   0x1     /* stub domain, no _PSD */
  96 #define ACPI_PSTDOM_FLAG_DEAD   0x2     /* domain can't be started */
  97 #define ACPI_PSTDOM_FLAG_INT    0x4     /* domain created from Integer _PSD */
  98
  99 struct acpi_pst_softc {
 100         device_t                pst_dev;
 101         struct acpi_cpu_softc   *pst_parent;
 102         struct acpi_pst_domain  *pst_domain;
 103         struct acpi_pst_res     pst_creg;
 104         struct acpi_pst_res     pst_sreg;
 105
 106         int                     pst_state;
 107         int                     pst_cpuid;
 108
 109         uint32_t                pst_flags;
 110
 111         ACPI_HANDLE             pst_handle;
 112
 113         LIST_ENTRY(acpi_pst_softc) pst_link;
 114 };
 115
 116 #define ACPI_PST_FLAG_PPC       0x1
 117 #define ACPI_PST_FLAG_PDL       0x2
 118
 119 static int      acpi_pst_probe(device_t dev);
 120 static int      acpi_pst_attach(device_t dev);
 121 static void     acpi_pst_notify(device_t dev);
 122
 123 static void     acpi_pst_postattach(void *);
 124 static struct acpi_pst_domain *
 125                 acpi_pst_domain_create_int(device_t, uint32_t);
 126 static struct acpi_pst_domain *
 127                 acpi_pst_domain_create_pkg(device_t, ACPI_OBJECT *);
 128 static struct acpi_pst_domain *
 129                 acpi_pst_domain_find(uint32_t);
 130 static struct acpi_pst_domain *
 131                 acpi_pst_domain_alloc(uint32_t, uint32_t, uint32_t);
 132 static void     acpi_pst_domain_set_pstate_locked(struct acpi_pst_domain *,
 133                     int, int *);
 134 static void     acpi_pst_domain_set_pstate(struct acpi_pst_domain *, int,
 135                     int *);
 136 static void     acpi_pst_domain_check_nproc(device_t, struct acpi_pst_domain *);
 137 static void     acpi_pst_global_set_pstate(int);
 138 static void     acpi_pst_global_fixup_pstate(void);
 139
 140 static int      acpi_pst_check_csr(struct acpi_pst_softc *);
 141 static int      acpi_pst_check_pstates(struct acpi_pst_softc *);
 142 static int      acpi_pst_init(struct acpi_pst_softc *);
 143 static int      acpi_pst_set_pstate(struct acpi_pst_softc *,
 144                     const struct acpi_pstate *);
 145 static const struct acpi_pstate *
 146                 acpi_pst_get_pstate(struct acpi_pst_softc *);
 147 static int      acpi_pst_alloc_resource(device_t, ACPI_OBJECT *, int,
 148                     struct acpi_pst_res *);
 149 static int      acpi_pst_eval_ppc(struct acpi_pst_softc *, int *);
 150 static int      acpi_pst_eval_pdl(struct acpi_pst_softc *, int *);
 151
 152 static void     acpi_pst_check_csr_handler(netmsg_t);
 153 static void     acpi_pst_check_pstates_handler(netmsg_t);
 154 static void     acpi_pst_init_handler(netmsg_t);
 155 static void     acpi_pst_set_pstate_handler(netmsg_t);
 156 static void     acpi_pst_get_pstate_handler(netmsg_t);
 157
 158 static int      acpi_pst_sysctl_freqs(SYSCTL_HANDLER_ARGS);
 159 static int      acpi_pst_sysctl_freqs_bin(SYSCTL_HANDLER_ARGS);
 160 static int      acpi_pst_sysctl_power(SYSCTL_HANDLER_ARGS);
 161 static int      acpi_pst_sysctl_members(SYSCTL_HANDLER_ARGS);
 162 static int      acpi_pst_sysctl_select(SYSCTL_HANDLER_ARGS);
 163 static int      acpi_pst_sysctl_global(SYSCTL_HANDLER_ARGS);
 164
 165 static struct acpi_pst_domlist  acpi_pst_domains =
 166         LIST_HEAD_INITIALIZER(acpi_pst_domains);
 167 static int                      acpi_pst_domain_id;
 168
 169 static int                      acpi_pst_global_state;
 170
 171 static int                      acpi_pstate_start = -1;
 172 static int                      acpi_pstate_count;
 173 static int                      acpi_npstates;
 174 static struct acpi_pstate       *acpi_pstates;
 175
 176 static const struct acpi_pst_md *acpi_pst_md;
 177
 178 static int                      acpi_pst_pdl = -1;
 179 TUNABLE_INT("hw.acpi.cpu.pst.pdl", &acpi_pst_pdl);
 180
 181 static int                      acpi_pst_ht_reuse_domain = 1;
 182 TUNABLE_INT("hw.acpi.cpu.pst.ht_reuse_domain", &acpi_pst_ht_reuse_domain);
 183
 184 static int                      acpi_pst_force_pkg_domain = 0;
 185 TUNABLE_INT("hw.acpi.cpu.pst.force_pkg_domain", &acpi_pst_force_pkg_domain);
 186
 187 /*
 188  * Force CPU package power domain for Intel CPUs.
 189  *
 190  * As of this write (14 July 2015), all Intel CPUs only have CPU package
 191  * power domain.
 192  */
 193 static int                      acpi_pst_intel_pkg_domain = 1;
 194 TUNABLE_INT("hw.acpi.cpu.pst.intel_pkg_domain", &acpi_pst_intel_pkg_domain);
 195
 196 static device_method_t acpi_pst_methods[] = {
 197         /* Device interface */
 198         DEVMETHOD(device_probe,                 acpi_pst_probe),
 199         DEVMETHOD(device_attach,                acpi_pst_attach),
 200         DEVMETHOD(device_detach,                bus_generic_detach),
 201         DEVMETHOD(device_shutdown,              bus_generic_shutdown),
 202         DEVMETHOD(device_suspend,               bus_generic_suspend),
 203         DEVMETHOD(device_resume,                bus_generic_resume),
 204
 205         /* Bus interface */
 206         DEVMETHOD(bus_add_child,                bus_generic_add_child),
 207         DEVMETHOD(bus_print_child,              bus_generic_print_child),
 208         DEVMETHOD(bus_read_ivar,                bus_generic_read_ivar),
 209         DEVMETHOD(bus_write_ivar,               bus_generic_write_ivar),
 210         DEVMETHOD(bus_get_resource_list,        bus_generic_get_resource_list),
 211         DEVMETHOD(bus_set_resource,             bus_generic_rl_set_resource),
 212         DEVMETHOD(bus_get_resource,             bus_generic_rl_get_resource),
 213         DEVMETHOD(bus_alloc_resource,           bus_generic_alloc_resource),
 214         DEVMETHOD(bus_release_resource,         bus_generic_release_resource),
 215         DEVMETHOD(bus_driver_added,             bus_generic_driver_added),
 216         DEVMETHOD(bus_activate_resource,        bus_generic_activate_resource),
 217         DEVMETHOD(bus_deactivate_resource,      bus_generic_deactivate_resource),
 218         DEVMETHOD(bus_setup_intr,               bus_generic_setup_intr),
 219         DEVMETHOD(bus_teardown_intr,            bus_generic_teardown_intr),
 220
 221         DEVMETHOD_END
 222 };
 223
 224 static driver_t acpi_pst_driver = {
 225         "cpu_pst",
 226         acpi_pst_methods,
 227         sizeof(struct acpi_pst_softc)
 228 };
 229
 230 static devclass_t acpi_pst_devclass;
 231 DRIVER_MODULE(cpu_pst, cpu, acpi_pst_driver, acpi_pst_devclass, NULL, NULL);
 232 MODULE_DEPEND(cpu_pst, acpi, 1, 1, 1);
 233
 234 static __inline int
 235 acpi_pst_freq2index(int freq)
 236 {
 237         int i;
 238
 239         for (i = 0; i < acpi_npstates; ++i) {
 240                 if (acpi_pstates[i].st_freq == freq)
 241                         return i;
 242         }
 243         return -1;
 244 }
 245
 246 static int
 247 acpi_pst_probe(device_t dev)
 248 {
 249         ACPI_BUFFER buf;
 250         ACPI_HANDLE handle;
 251         ACPI_STATUS status;
 252         ACPI_OBJECT *obj;
 253
 254         if (acpi_disabled("cpu_pst") ||
 255             acpi_get_type(dev) != ACPI_TYPE_PROCESSOR)
 256                 return ENXIO;
 257
 258         if (acpi_pst_md == NULL)
 259                 acpi_pst_md = acpi_pst_md_probe();
 260
 261         handle = acpi_get_handle(dev);
 262
 263         /*
 264          * Check _PSD package
 265          *
 266          * NOTE:
 267          * Some BIOSes do not expose _PCT for the second thread of
 268          * CPU cores.  In this case, _PSD should be enough to get the
 269          * P-state of the second thread working, since it must have
 270          * the same _PCT and _PSS as the first thread in the same
 271          * core.
 272          */
 273         buf.Pointer = NULL;
 274         buf.Length = ACPI_ALLOCATE_BUFFER;
 275         status = AcpiEvaluateObject(handle, "_PSD", NULL, &buf);
 276         if (!ACPI_FAILURE(status)) {
 277                 AcpiOsFree((ACPI_OBJECT *)buf.Pointer);
 278                 goto done;
 279         }
 280
 281         /*
 282          * Check _PCT package
 283          */
 284         buf.Pointer = NULL;
 285         buf.Length = ACPI_ALLOCATE_BUFFER;
 286         status = AcpiEvaluateObject(handle, "_PCT", NULL, &buf);
 287         if (ACPI_FAILURE(status)) {
 288                 if (bootverbose) {
 289                         device_printf(dev, "Can't get _PCT package - %s\n",
 290                                       AcpiFormatException(status));
 291                 }
 292                 return ENXIO;
 293         }
 294
 295         obj = (ACPI_OBJECT *)buf.Pointer;
 296         if (!ACPI_PKG_VALID_EQ(obj, 2)) {
 297                 device_printf(dev, "Invalid _PCT package\n");
 298                 AcpiOsFree(obj);
 299                 return ENXIO;
 300         }
 301         AcpiOsFree(obj);
 302
 303         /*
 304          * Check _PSS package
 305          */
 306         buf.Pointer = NULL;
 307         buf.Length = ACPI_ALLOCATE_BUFFER;
 308         status = AcpiEvaluateObject(handle, "_PSS", NULL, &buf);
 309         if (ACPI_FAILURE(status)) {
 310                 device_printf(dev, "Can't get _PSS package - %s\n",
 311                               AcpiFormatException(status));
 312                 return ENXIO;
 313         }
 314
 315         obj = (ACPI_OBJECT *)buf.Pointer;
 316         if (!ACPI_PKG_VALID(obj, 1)) {
 317                 device_printf(dev, "Invalid _PSS package\n");
 318                 AcpiOsFree(obj);
 319                 return ENXIO;
 320         }
 321         AcpiOsFree(obj);
 322
 323 done:
 324         device_set_desc(dev, "ACPI CPU P-State");
 325         return 0;
 326 }
 327
 328 static int
 329 acpi_pst_attach(device_t dev)
 330 {
 331         struct acpi_pst_softc *sc = device_get_softc(dev);
 332         struct acpi_pst_domain *dom = NULL;
 333         ACPI_BUFFER buf;
 334         ACPI_STATUS status;
 335         ACPI_OBJECT *obj;
 336         struct acpi_pstate *pstate, *p;
 337         int i, npstate, error, sstart, scount;
 338
 339         sc->pst_dev = dev;
 340         sc->pst_parent = device_get_softc(device_get_parent(dev));
 341         sc->pst_handle = acpi_get_handle(dev);
 342         sc->pst_cpuid = acpi_get_magic(dev);
 343
 344         /*
 345          * If there is a _PSD, then we create procossor domain
 346          * accordingly.  If there is no _PSD, we just fake a
 347          * default processor domain0.
 348          */
 349         buf.Pointer = NULL;
 350         buf.Length = ACPI_ALLOCATE_BUFFER;
 351         status = AcpiEvaluateObject(sc->pst_handle, "_PSD", NULL, &buf);
 352         if (!ACPI_FAILURE(status)) {
 353                 obj = (ACPI_OBJECT *)buf.Pointer;
 354
 355                 if (acpi_pst_domain_id > 0) {
 356                         device_printf(dev, "Missing _PSD for certain CPUs\n");
 357                         AcpiOsFree(obj);
 358                         return ENXIO;
 359                 }
 360                 acpi_pst_domain_id = -1;
 361
 362                 if (ACPI_PKG_VALID_EQ(obj, 1)) {
 363                         dom = acpi_pst_domain_create_pkg(dev,
 364                                 &obj->Package.Elements[0]);
 365                         if (dom == NULL) {
 366                                 AcpiOsFree(obj);
 367                                 return ENXIO;
 368                         }
 369                 } else {
 370                         if (obj->Type != ACPI_TYPE_INTEGER) {
 371                                 device_printf(dev,
 372                                     "Invalid _PSD package, Type 0x%x\n",
 373                                     obj->Type);
 374                                 AcpiOsFree(obj);
 375                                 return ENXIO;
 376                         } else {
 377                                 device_printf(dev, "Integer _PSD %ju\n",
 378                                     (uintmax_t)obj->Integer.Value);
 379                                 dom = acpi_pst_domain_create_int(dev,
 380                                     obj->Integer.Value);
 381                                 if (dom == NULL) {
 382                                         AcpiOsFree(obj);
 383                                         return ENXIO;
 384                                 }
 385                         }
 386                 }
 387
 388                 /* Free _PSD */
 389                 AcpiOsFree(buf.Pointer);
 390         } else {
 391                 if (acpi_pst_domain_id < 0) {
 392                         device_printf(dev, "Missing _PSD for cpu%d\n",
 393                             sc->pst_cpuid);
 394                         return ENXIO;
 395                 }
 396
 397                 /*
 398                  * Create a stub one processor domain for each processor
 399                  */
 400                 dom = acpi_pst_domain_alloc(acpi_pst_domain_id,
 401                         ACPI_PSD_COORD_SWANY, 1);
 402                 dom->pd_flags |= ACPI_PSTDOM_FLAG_STUB;
 403
 404                 ++acpi_pst_domain_id;
 405         }
 406
 407         /* Make sure that adding us will not overflow our domain */
 408         acpi_pst_domain_check_nproc(dev, dom);
 409
 410         /*
 411          * Get control/status registers from _PCT
 412          */
 413         buf.Pointer = NULL;
 414         buf.Length = ACPI_ALLOCATE_BUFFER;
 415         status = AcpiEvaluateObject(sc->pst_handle, "_PCT", NULL, &buf);
 416         if (ACPI_FAILURE(status)) {
 417                 struct acpi_pst_softc *pst;
 418
 419                 /*
 420                  * No _PCT.  See the comment in acpi_pst_probe() near
 421                  * _PSD check.
 422                  *
 423                  * Use control/status registers of another CPU in the
 424                  * same domain, or in the same core, if the type of
 425                  * these registers are "Fixed Hardware", e.g. on most
 426                  * of the model Intel CPUs.
 427                  */
 428                 pst = LIST_FIRST(&dom->pd_pstlist);
 429                 if (pst == NULL) {
 430                         cpumask_t mask;
 431
 432                         mask = get_cpumask_from_level(sc->pst_cpuid,
 433                             CORE_LEVEL);
 434                         if (CPUMASK_TESTNZERO(mask)) {
 435                                 struct acpi_pst_domain *dom1;
 436
 437                                 LIST_FOREACH(dom1, &acpi_pst_domains, pd_link) {
 438                                         LIST_FOREACH(pst, &dom1->pd_pstlist,
 439                                             pst_link) {
 440                                                 if (CPUMASK_TESTBIT(mask,
 441                                                     pst->pst_cpuid))
 442                                                         break;
 443                                         }
 444                                         if (pst != NULL)
 445                                                 break;
 446                                 }
 447                                 if (pst != NULL && acpi_pst_ht_reuse_domain) {
 448                                         /*
 449                                          * Use the same domain for CPUs in the
 450                                          * same core.
 451                                          */
 452                                         device_printf(dev, "Destroy domain%u, "
 453                                             "reuse domain%u\n",
 454                                             dom->pd_dom, dom1->pd_dom);
 455                                         LIST_REMOVE(dom, pd_link);
 456                                         kfree(dom, M_DEVBUF);
 457                                         dom = dom1;
 458                                         /*
 459                                          * Make sure that adding us will not
 460                                          * overflow the domain containing
 461                                          * siblings in the same core.
 462                                          */
 463                                         acpi_pst_domain_check_nproc(dev, dom);
 464                                 }
 465                         }
 466                 }
 467                 if (pst != NULL &&
 468                     pst->pst_creg.pr_res == NULL &&
 469                     pst->pst_creg.pr_rid == 0 &&
 470                     pst->pst_creg.pr_gas.SpaceId ==
 471                     ACPI_ADR_SPACE_FIXED_HARDWARE &&
 472                     pst->pst_sreg.pr_res == NULL &&
 473                     pst->pst_sreg.pr_rid == 0 &&
 474                     pst->pst_sreg.pr_gas.SpaceId ==
 475                     ACPI_ADR_SPACE_FIXED_HARDWARE) {
 476                         sc->pst_creg = pst->pst_creg;
 477                         sc->pst_sreg = pst->pst_sreg;
 478                         device_printf(dev,
 479                             "No _PCT; reuse %s control/status regs\n",
 480                             device_get_nameunit(pst->pst_dev));
 481                         goto fetch_pss;
 482                 }
 483                 device_printf(dev, "Can't get _PCT package - %s\n",
 484                               AcpiFormatException(status));
 485                 return ENXIO;
 486         }
 487
 488         obj = (ACPI_OBJECT *)buf.Pointer;
 489         if (!ACPI_PKG_VALID_EQ(obj, 2)) {
 490                 device_printf(dev, "Invalid _PCT package\n");
 491                 AcpiOsFree(obj);
 492                 return ENXIO;
 493         }
 494
 495         /* Save and try allocating control register */
 496         error = acpi_pst_alloc_resource(dev, obj, 0, &sc->pst_creg);
 497         if (error) {
 498                 AcpiOsFree(obj);
 499                 return error;
 500         }
 501         if (bootverbose) {
 502                 device_printf(dev, "control reg %d %jx\n",
 503                               sc->pst_creg.pr_gas.SpaceId,
 504                               (uintmax_t)sc->pst_creg.pr_gas.Address);
 505         }
 506
 507         /* Save and try allocating status register */
 508         error = acpi_pst_alloc_resource(dev, obj, 1, &sc->pst_sreg);
 509         if (error) {
 510                 AcpiOsFree(obj);
 511                 return error;
 512         }
 513         if (bootverbose) {
 514                 device_printf(dev, "status reg %d %jx\n",
 515                               sc->pst_sreg.pr_gas.SpaceId,
 516                               (uintmax_t)sc->pst_sreg.pr_gas.Address);
 517         }
 518
 519         /* Free _PCT */
 520         AcpiOsFree(obj);
 521
 522 fetch_pss:
 523         /*
 524          * Create P-State table according to _PSS
 525          */
 526         buf.Pointer = NULL;
 527         buf.Length = ACPI_ALLOCATE_BUFFER;
 528         status = AcpiEvaluateObject(sc->pst_handle, "_PSS", NULL, &buf);
 529         if (ACPI_FAILURE(status)) {
 530                 /*
 531                  * No _PSS.  See the comment in acpi_pst_probe() near
 532                  * _PSD check.
 533                  *
 534                  * Assume _PSS are same across all CPUs; well, they
 535                  * should/have to be so.
 536                  */
 537                 if (acpi_npstates > 0 && acpi_pstates != NULL) {
 538                         device_printf(dev, "No _PSS\n");
 539                         goto fetch_ppc;
 540                 }
 541                 device_printf(dev, "Can't get _PSS package - %s\n",
 542                               AcpiFormatException(status));
 543                 return ENXIO;
 544         }
 545
 546         obj = (ACPI_OBJECT *)buf.Pointer;
 547         if (!ACPI_PKG_VALID(obj, 1)) {
 548                 device_printf(dev, "Invalid _PSS package\n");
 549                 AcpiOsFree(obj);
 550                 return ENXIO;
 551         }
 552
 553         /* Don't create too many P-States */
 554         npstate = obj->Package.Count;
 555         if (npstate > ACPI_NPSTATE_MAX) {
 556                 device_printf(dev, "Too many P-States, %d->%d\n",
 557                               npstate, ACPI_NPSTATE_MAX);
 558                 npstate = ACPI_NPSTATE_MAX;
 559         }
 560
 561         /*
 562          * If we have already created P-State table,
 563          * we must make sure that number of entries
 564          * is consistent.
 565          */
 566         if (acpi_pstates != NULL && acpi_npstates != npstate) {
 567                 device_printf(dev, "Inconsistent # of P-States "
 568                               "cross Processor objects\n");
 569                 AcpiOsFree(obj);
 570                 return ENXIO;
 571         }
 572
 573         /*
 574          * Create a temporary P-State table
 575          */
 576         pstate = kmalloc(sizeof(*pstate) * npstate, M_TEMP, M_WAITOK);
 577         for (i = 0, p = pstate; i < npstate; ++i, ++p) {
 578                 ACPI_OBJECT *pkg;
 579                 uint32_t *ptr[ACPI_PSS_PX_NENTRY] = {
 580                         &p->st_freq, &p->st_power, &p->st_xsit_lat,
 581                         &p->st_bm_lat, &p->st_cval, &p->st_sval
 582                 };
 583                 int j;
 584
 585                 pkg = &obj->Package.Elements[i];
 586                 if (!ACPI_PKG_VALID(pkg, ACPI_PSS_PX_NENTRY)) {
 587                         device_printf(dev, "Invalud _PSS P%d\n", i);
 588                         AcpiOsFree(obj);
 589                         kfree(pstate, M_TEMP);
 590                         return ENXIO;
 591                 }
 592                 for (j = 0; j < ACPI_PSS_PX_NENTRY; ++j) {
 593                         if (acpi_PkgInt32(pkg, j, ptr[j]) != 0) {
 594                                 device_printf(dev, "Can't extract "
 595                                               "_PSS P%d %dth entry\n", i, j);
 596                                 AcpiOsFree(obj);
 597                                 kfree(pstate, M_TEMP);
 598                                 return ENXIO;
 599                         }
 600                 }
 601         }
 602
 603         /* Free _PSS */
 604         AcpiOsFree(obj);
 605
 606         if (acpi_pstates == NULL) {
 607                 /*
 608                  * If no P-State table is created yet,
 609                  * save the temporary one we just created.
 610                  */
 611                 acpi_pstates = pstate;
 612                 acpi_npstates = npstate;
 613                 pstate = NULL;
 614
 615                 if (bootverbose) {
 616                         for (i = 0; i < acpi_npstates; ++i) {
 617                                 device_printf(dev,
 618                                 "freq %u, pwr %u, xlat %u, blat %u, "
 619                                 "cv %08x, sv %08x\n",
 620                                 acpi_pstates[i].st_freq,
 621                                 acpi_pstates[i].st_power,
 622                                 acpi_pstates[i].st_xsit_lat,
 623                                 acpi_pstates[i].st_bm_lat,
 624                                 acpi_pstates[i].st_cval,
 625                                 acpi_pstates[i].st_sval);
 626                         }
 627                 }
 628         } else {
 629                 /*
 630                  * Make sure that P-State tables are same
 631                  * for all processors.
 632                  */
 633                 if (memcmp(pstate, acpi_pstates,
 634                            sizeof(*pstate) * npstate) != 0) {
 635                         device_printf(dev, "Inconsistent _PSS "
 636                                       "cross Processor objects\n");
 637 #if 0
 638                         /*
 639                          * Some BIOSes create different P-State tables;
 640                          * just trust the one from the BSP and move on.
 641                          */
 642                         kfree(pstate, M_TEMP);
 643                         return ENXIO;
 644 #endif
 645                 }
 646                 kfree(pstate, M_TEMP);
 647         }
 648
 649 fetch_ppc:
 650         /* By default, we start from P-State table's first entry */
 651         sstart = 0;
 652
 653         /*
 654          * Adjust the usable first entry of P-State table,
 655          * if there is _PPC object.
 656          */
 657         error = acpi_pst_eval_ppc(sc, &sstart);
 658         if (error && error != ENOENT)
 659                 return error;
 660         else if (!error)
 661                 sc->pst_flags |= ACPI_PST_FLAG_PPC;
 662         if (acpi_pstate_start < 0) {
 663                 acpi_pstate_start = sstart;
 664         } else if (acpi_pstate_start != sstart) {
 665                 device_printf(dev, "_PPC mismatch, was %d, now %d\n",
 666                     acpi_pstate_start, sstart);
 667                 if (acpi_pstate_start < sstart) {
 668                         device_printf(dev, "_PPC %d -> %d\n",
 669                             acpi_pstate_start, sstart);
 670                         acpi_pstate_start = sstart;
 671                 }
 672         }
 673
 674         /*
 675          * By default, we assume number of usable P-States is same as
 676          * number of P-States.
 677          */
 678         scount = acpi_npstates;
 679
 680         /*
 681          * Allow users to override or set _PDL
 682          */
 683         if (acpi_pst_pdl >= 0) {
 684                 if (acpi_pst_pdl < acpi_npstates) {
 685                         if (bootverbose) {
 686                                 device_printf(dev, "_PDL override %d\n",
 687                                     acpi_pst_pdl);
 688                         }
 689                         scount = acpi_pst_pdl + 1;
 690                         goto proc_pdl;
 691                 } else {
 692                         device_printf(dev, "Invalid _PDL override %d, "
 693                             "must be less than %d\n", acpi_pst_pdl,
 694                             acpi_npstates);
 695                 }
 696         }
 697
 698         /*
 699          * Adjust the number of usable entries in P-State table,
 700          * if there is _PDL object.
 701          */
 702         error = acpi_pst_eval_pdl(sc, &scount);
 703         if (error && error != ENOENT)
 704                 return error;
 705         else if (!error)
 706                 sc->pst_flags |= ACPI_PST_FLAG_PDL;
 707 proc_pdl:
 708         if (acpi_pstate_count == 0) {
 709                 acpi_pstate_count = scount;
 710         } else if (acpi_pstate_count != scount) {
 711                 device_printf(dev, "_PDL mismatch, was %d, now %d\n",
 712                     acpi_pstate_count, scount);
 713                 if (acpi_pstate_count > scount) {
 714                         device_printf(dev, "_PDL %d -> %d\n",
 715                             acpi_pstate_count, scount);
 716                         acpi_pstate_count = scount;
 717                 }
 718         }
 719
 720         /*
 721          * Some CPUs only have package P-states, but some BIOSes put each
 722          * hyperthread to its own P-state domain; allow user to override.
 723          */
 724         if (LIST_EMPTY(&dom->pd_pstlist) &&
 725             (acpi_pst_force_pkg_domain ||
 726              (cpu_vendor_id == CPU_VENDOR_INTEL &&
 727               acpi_pst_intel_pkg_domain))) {
 728                 cpumask_t mask;
 729
 730                 mask = get_cpumask_from_level(sc->pst_cpuid, CHIP_LEVEL);
 731                 if (CPUMASK_TESTNZERO(mask)) {
 732                         struct acpi_pst_softc *pst = NULL;
 733                         struct acpi_pst_domain *dom1;
 734
 735                         LIST_FOREACH(dom1, &acpi_pst_domains, pd_link) {
 736                                 LIST_FOREACH(pst, &dom1->pd_pstlist,
 737                                     pst_link) {
 738                                         if (CPUMASK_TESTBIT(mask,
 739                                             pst->pst_cpuid))
 740                                                 break;
 741                                 }
 742                                 if (pst != NULL)
 743                                         break;
 744                         }
 745                         if (pst != NULL &&
 746                             memcmp(&pst->pst_creg, &sc->pst_creg,
 747                                 sizeof(sc->pst_creg)) == 0 &&
 748                             memcmp(&pst->pst_sreg, &sc->pst_sreg,
 749                                 sizeof(sc->pst_sreg)) == 0) {
 750                                 /*
 751                                  * Use the same domain for CPUs in the
 752                                  * same package.
 753                                  */
 754                                 device_printf(dev, "Destroy domain%u, "
 755                                     "force pkg domain%u\n",
 756                                     dom->pd_dom, dom1->pd_dom);
 757                                 LIST_REMOVE(dom, pd_link);
 758                                 kfree(dom, M_DEVBUF);
 759                                 dom = dom1;
 760                                 /*
 761                                  * Make sure that adding us will not
 762                                  * overflow the domain containing
 763                                  * siblings in the same package.
 764                                  */
 765                                 acpi_pst_domain_check_nproc(dev, dom);
 766                         }
 767                 }
 768         }
 769
 770         /* Link us with the domain */
 771         sc->pst_domain = dom;
 772         LIST_INSERT_HEAD(&dom->pd_pstlist, sc, pst_link);
 773
 774         if (device_get_unit(dev) == 0)
 775                 AcpiOsExecute(OSL_NOTIFY_HANDLER, acpi_pst_postattach, NULL);
 776
 777         if (sc->pst_flags & (ACPI_PST_FLAG_PPC | ACPI_PST_FLAG_PDL))
 778                 sc->pst_parent->cpu_pst_notify = acpi_pst_notify;
 779
 780         return 0;
 781 }
 782
 783 static struct acpi_pst_domain *
 784 acpi_pst_domain_create_pkg(device_t dev, ACPI_OBJECT *obj)
 785 {
 786         struct acpi_pst_domain *dom;
 787         uint32_t val, domain, coord, nproc;
 788
 789         if (!ACPI_PKG_VALID_EQ(obj, 5)) {
 790                 device_printf(dev, "Invalid _PSD package\n");
 791                 return NULL;
 792         }
 793
 794         /* NumberOfEntries */
 795         if (acpi_PkgInt32(obj, 0, &val) != 0 || val != 5) {
 796                 device_printf(dev, "Invalid _PSD NumberOfEntries\n");
 797                 return NULL;
 798         }
 799
 800         /* Revision */
 801         if (acpi_PkgInt32(obj, 1, &val) != 0 || val != 0) {
 802                 device_printf(dev, "Invalid _PSD Revision\n");
 803                 return NULL;
 804         }
 805
 806         if (acpi_PkgInt32(obj, 2, &domain) != 0 ||
 807             acpi_PkgInt32(obj, 3, &coord) != 0 ||
 808             acpi_PkgInt32(obj, 4, &nproc) != 0) {
 809                 device_printf(dev, "Can't extract _PSD package\n");
 810                 return NULL;
 811         }
 812
 813         if (!ACPI_PSD_COORD_VALID(coord)) {
 814                 device_printf(dev, "Invalid _PSD CoordType (%#x)\n", coord);
 815                 return NULL;
 816         }
 817
 818         if (nproc > MAXCPU) {
 819                 /*
 820                  * If NumProcessors is greater than MAXCPU
 821                  * and domain's coordination is SWALL, then
 822                  * we will never be able to start all CPUs
 823                  * within this domain, and power state
 824                  * transition will never be completed, so we
 825                  * just bail out here.
 826                  */
 827                 if (coord == ACPI_PSD_COORD_SWALL) {
 828                         device_printf(dev, "Unsupported _PSD NumProcessors "
 829                                       "(%d)\n", nproc);
 830                         return NULL;
 831                 }
 832         } else if (nproc == 0) {
 833                 device_printf(dev, "_PSD NumProcessors are zero\n");
 834                 return NULL;
 835         }
 836
 837         dom = acpi_pst_domain_find(domain);
 838         if (dom != NULL) {
 839                 if (dom->pd_flags & ACPI_PSTDOM_FLAG_INT) {
 840                         device_printf(dev, "Mixed Integer _PSD and "
 841                             "Package _PSD\n");
 842                         return NULL;
 843                 }
 844                 if (dom->pd_coord != coord) {
 845                         device_printf(dev, "Inconsistent _PSD coord "
 846                             "information cross Processor objects\n");
 847                         return NULL;
 848                 }
 849                 if (dom->pd_nproc != nproc) {
 850                         device_printf(dev, "Inconsistent _PSD nproc "
 851                             "information cross Processor objects\n");
 852                         /*
 853                          * Some stupid BIOSes will set wrong "# of processors",
 854                          * e.g. 1 for CPU w/ hyperthreading; Be lenient here.
 855                          */
 856                 }
 857                 return dom;
 858         }
 859
 860         dom = acpi_pst_domain_alloc(domain, coord, nproc);
 861         if (bootverbose) {
 862                 device_printf(dev, "create pkg domain%u, coord %#x\n",
 863                     dom->pd_dom, dom->pd_coord);
 864         }
 865
 866         return dom;
 867 }
 868
 869 static struct acpi_pst_domain *
 870 acpi_pst_domain_create_int(device_t dev, uint32_t domain)
 871 {
 872         struct acpi_pst_domain *dom;
 873
 874         dom = acpi_pst_domain_find(domain);
 875         if (dom != NULL) {
 876                 if ((dom->pd_flags & ACPI_PSTDOM_FLAG_INT) == 0) {
 877                         device_printf(dev, "Mixed Package _PSD and "
 878                             "Integer _PSD\n");
 879                         return NULL;
 880                 }
 881                 KKASSERT(dom->pd_coord == ACPI_PSD_COORD_SWALL);
 882
 883                 dom->pd_nproc++;
 884                 return dom;
 885         }
 886
 887         dom = acpi_pst_domain_alloc(domain, ACPI_PSD_COORD_SWALL, 1);
 888         dom->pd_flags |= ACPI_PSTDOM_FLAG_INT;
 889
 890         if (bootverbose)
 891                 device_printf(dev, "create int domain%u\n", dom->pd_dom);
 892
 893         return dom;
 894 }
 895
 896 static struct acpi_pst_domain *
 897 acpi_pst_domain_find(uint32_t domain)
 898 {
 899         struct acpi_pst_domain *dom;
 900
 901         LIST_FOREACH(dom, &acpi_pst_domains, pd_link) {
 902                 if (dom->pd_flags & ACPI_PSTDOM_FLAG_STUB)
 903                         continue;
 904                 if (dom->pd_dom == domain)
 905                         return dom;
 906         }
 907         return NULL;
 908 }
 909
 910 static struct acpi_pst_domain *
 911 acpi_pst_domain_alloc(uint32_t domain, uint32_t coord, uint32_t nproc)
 912 {
 913         struct acpi_pst_domain *dom;
 914
 915         dom = kmalloc(sizeof(*dom), M_DEVBUF, M_WAITOK | M_ZERO);
 916         dom->pd_dom = domain;
 917         dom->pd_coord = coord;
 918         dom->pd_nproc = nproc;
 919         LIST_INIT(&dom->pd_pstlist);
 920         lwkt_serialize_init(&dom->pd_serialize);
 921
 922         LIST_INSERT_HEAD(&acpi_pst_domains, dom, pd_link);
 923
 924         return dom;
 925 }
 926
 927 static void
 928 acpi_pst_domain_set_pstate_locked(struct acpi_pst_domain *dom, int i, int *global)
 929 {
 930         const struct acpi_pstate *pstate;
 931         struct acpi_pst_softc *sc;
 932         int done, error;
 933
 934         ASSERT_SERIALIZED(&dom->pd_serialize);
 935
 936         KKASSERT(i >= 0 && i < acpi_npstates);
 937         pstate = &acpi_pstates[i];
 938
 939         done = 0;
 940         LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
 941                 if (!done) {
 942                         error = acpi_pst_set_pstate(sc, pstate);
 943                         if (error) {
 944                                 device_printf(sc->pst_dev, "can't set "
 945                                               "freq %d\n", pstate->st_freq);
 946                                 /* XXX error cleanup? */
 947                         }
 948                         if (dom->pd_coord == ACPI_PSD_COORD_SWANY)
 949                                 done = 1;
 950                 }
 951                 sc->pst_state = i;
 952         }
 953         dom->pd_state = i;
 954
 955         if (global != NULL)
 956                 *global = i;
 957 }
 958
 959 static void
 960 acpi_pst_domain_set_pstate(struct acpi_pst_domain *dom, int i, int *global)
 961 {
 962         lwkt_serialize_enter(&dom->pd_serialize);
 963         acpi_pst_domain_set_pstate_locked(dom, i, global);
 964         lwkt_serialize_exit(&dom->pd_serialize);
 965 }
 966
 967 static void
 968 acpi_pst_global_set_pstate(int i)
 969 {
 970         struct acpi_pst_domain *dom;
 971         int *global = &acpi_pst_global_state;
 972
 973         LIST_FOREACH(dom, &acpi_pst_domains, pd_link) {
 974                 /* Skip dead domain */
 975                 if (dom->pd_flags & ACPI_PSTDOM_FLAG_DEAD)
 976                         continue;
 977                 acpi_pst_domain_set_pstate(dom, i, global);
 978                 global = NULL;
 979         }
 980 }
 981
 982 static void
 983 acpi_pst_global_fixup_pstate(void)
 984 {
 985         struct acpi_pst_domain *dom;
 986         int *global = &acpi_pst_global_state;
 987         int sstart, scount;
 988
 989         sstart = acpi_pstate_start;
 990         scount = acpi_pstate_count;
 991
 992         LIST_FOREACH(dom, &acpi_pst_domains, pd_link) {
 993                 int i = -1;
 994
 995                 /* Skip dead domain */
 996                 if (dom->pd_flags & ACPI_PSTDOM_FLAG_DEAD)
 997                         continue;
 998
 999                 lwkt_serialize_enter(&dom->pd_serialize);
1000
1001                 if (global != NULL) {
1002                         if (*global < sstart)
1003                                 *global = sstart;
1004                         else if (*global >= scount)
1005                                 *global = scount - 1;
1006                         global = NULL;
1007                 }
1008                 if (dom->pd_state < sstart)
1009                         i = sstart;
1010                 else if (dom->pd_state >= scount)
1011                         i = scount - 1;
1012                 if (i >= 0)
1013                         acpi_pst_domain_set_pstate_locked(dom, i, NULL);
1014
1015                 lwkt_serialize_exit(&dom->pd_serialize);
1016         }
1017 }
1018
1019 static void
1020 acpi_pst_postattach(void *arg __unused)
1021 {
1022         struct acpi_pst_domain *dom;
1023         struct acpi_cpu_softc *cpu;
1024         device_t *devices;
1025         int i, ndevices, error, has_domain;
1026
1027         devices = NULL;
1028         ndevices = 0;
1029         error = devclass_get_devices(acpi_pst_devclass, &devices, &ndevices);
1030         if (error)
1031                 return;
1032
1033         if (ndevices == 0)
1034                 return;
1035
1036         cpu = NULL;
1037         for (i = 0; i < ndevices; ++i) {
1038                 cpu = device_get_softc(device_get_parent(devices[i]));
1039                 if (cpu->glob_sysctl_tree != NULL)
1040                         break;
1041         }
1042         kfree(devices, M_TEMP);
1043         KKASSERT(cpu != NULL);
1044
1045         if (acpi_pst_md == NULL)
1046                 kprintf("ACPI: no P-State CPU driver\n");
1047
1048         has_domain = 0;
1049         LIST_FOREACH(dom, &acpi_pst_domains, pd_link) {
1050                 struct acpi_pst_softc *sc;
1051                 char buf[32];
1052
1053                 dom->pd_state = acpi_pstate_start;
1054
1055                 /*
1056                  * Make sure that all processors belonging to this
1057                  * domain are located.
1058                  */
1059                 i = 0;
1060                 LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
1061                         sc->pst_state = acpi_pstate_start;
1062                         ++i;
1063                 }
1064                 if (i != dom->pd_nproc) {
1065                         KKASSERT(i < dom->pd_nproc);
1066
1067                         kprintf("ACPI: domain%u misses processors, "
1068                                 "should be %d, got %d\n", dom->pd_dom,
1069                                 dom->pd_nproc, i);
1070                         if (dom->pd_coord == ACPI_PSD_COORD_SWALL) {
1071                                 /*
1072                                  * If this domain's coordination is
1073                                  * SWALL and we don't see all of the
1074                                  * member CPUs of this domain, then
1075                                  * the P-State transition will never
1076                                  * be completed, so just leave this
1077                                  * domain out.
1078                                  */
1079                                 dom->pd_flags |= ACPI_PSTDOM_FLAG_DEAD;
1080                                 continue;
1081                         }
1082                         dom->pd_nproc = i;
1083                 }
1084
1085                 /*
1086                  * Validate P-State configurations for this domain
1087                  */
1088                 LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
1089                         error = acpi_pst_check_csr(sc);
1090                         if (error)
1091                                 break;
1092
1093                         error = acpi_pst_check_pstates(sc);
1094                         if (error)
1095                                 break;
1096                 }
1097                 if (sc != NULL) {
1098                         kprintf("ACPI: domain%u P-State configuration "
1099                                 "check failed\n", dom->pd_dom);
1100                         dom->pd_flags |= ACPI_PSTDOM_FLAG_DEAD;
1101                         continue;
1102                 }
1103
1104                 /*
1105                  * Do necssary P-State initialization
1106                  */
1107                 LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
1108                         error = acpi_pst_init(sc);
1109                         if (error)
1110                                 break;
1111                 }
1112                 if (sc != NULL) {
1113                         kprintf("ACPI: domain%u P-State initialization "
1114                                 "check failed\n", dom->pd_dom);
1115                         dom->pd_flags |= ACPI_PSTDOM_FLAG_DEAD;
1116                         continue;
1117                 }
1118
1119                 has_domain = 1;
1120
1121                 ksnprintf(buf, sizeof(buf), "px_dom%u", dom->pd_dom);
1122
1123                 sysctl_ctx_init(&dom->pd_sysctl_ctx);
1124                 dom->pd_sysctl_tree =
1125                 SYSCTL_ADD_NODE(&dom->pd_sysctl_ctx,
1126                         SYSCTL_CHILDREN(cpu->glob_sysctl_tree),
1127                         OID_AUTO, buf, CTLFLAG_RD, 0,
1128                         "P-State domain");
1129                 if (dom->pd_sysctl_tree == NULL) {
1130                         kprintf("ACPI: Can't create sysctl tree for domain%u",
1131                                 dom->pd_dom);
1132                         continue;
1133                 }
1134
1135                 SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
1136                                 SYSCTL_CHILDREN(dom->pd_sysctl_tree),
1137                                 OID_AUTO, "available",
1138                                 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP,
1139                                 dom, 0, acpi_pst_sysctl_freqs, "A",
1140                                 "available frequencies");
1141
1142                 SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
1143                                 SYSCTL_CHILDREN(dom->pd_sysctl_tree),
1144                                 OID_AUTO, "avail",
1145                                 CTLTYPE_OPAQUE | CTLFLAG_RD,
1146                                 dom, 0, acpi_pst_sysctl_freqs_bin, "IU",
1147                                 "available frequencies");
1148
1149                 SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
1150                                 SYSCTL_CHILDREN(dom->pd_sysctl_tree),
1151                                 OID_AUTO, "power",
1152                                 CTLTYPE_OPAQUE | CTLFLAG_RD,
1153                                 dom, 0, acpi_pst_sysctl_power, "IU",
1154                                 "power of available frequencies");
1155
1156                 SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
1157                                 SYSCTL_CHILDREN(dom->pd_sysctl_tree),
1158                                 OID_AUTO, "members",
1159                                 CTLTYPE_STRING | CTLFLAG_RD,
1160                                 dom, 0, acpi_pst_sysctl_members, "A",
1161                                 "member cpus");
1162
1163                 if (acpi_pst_md != NULL &&
1164                     acpi_pst_md->pmd_set_pstate != NULL) {
1165                         SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
1166                                         SYSCTL_CHILDREN(dom->pd_sysctl_tree),
1167                                         OID_AUTO, "select",
1168                                         CTLTYPE_UINT | CTLFLAG_RW,
1169                                         dom, 0, acpi_pst_sysctl_select,
1170                                         "IU", "select freq");
1171                 }
1172         }
1173
1174         if (has_domain && acpi_pst_md != NULL &&
1175             acpi_pst_md->pmd_set_pstate != NULL) {
1176                 SYSCTL_ADD_PROC(&cpu->glob_sysctl_ctx,
1177                                 SYSCTL_CHILDREN(cpu->glob_sysctl_tree),
1178                                 OID_AUTO, "px_global",
1179                                 CTLTYPE_UINT | CTLFLAG_RW,
1180                                 NULL, 0, acpi_pst_sysctl_global,
1181                                 "IU", "select freq for all domains");
1182
1183                 acpi_pst_global_set_pstate(acpi_pstate_start);
1184         }
1185 }
1186
1187 static int
1188 acpi_pst_sysctl_freqs(SYSCTL_HANDLER_ARGS)
1189 {
1190         int i, error, sstart, scount;
1191
1192         error = 0;
1193         sstart = acpi_pstate_start;
1194         scount = acpi_pstate_count;
1195         for (i = 0; i < acpi_npstates; ++i) {
1196                 if (error == 0 && i)
1197                         error = SYSCTL_OUT(req, " ", 1);
1198                 if (error == 0) {
1199                         const char *pat;
1200                         char buf[32];
1201
1202                         if (i < sstart || i >= scount)
1203                                 pat = "(%u)";
1204                         else
1205                                 pat = "%u";
1206
1207                         ksnprintf(buf, sizeof(buf), pat,
1208                                   acpi_pstates[i].st_freq);
1209                         error = SYSCTL_OUT(req, buf, strlen(buf));
1210                 }
1211         }
1212         return error;
1213 }
1214
1215 static int
1216 acpi_pst_sysctl_freqs_bin(SYSCTL_HANDLER_ARGS)
1217 {
1218         uint32_t freqs[ACPI_NPSTATE_MAX];
1219         int cnt, i, sstart, scount;
1220
1221         sstart = acpi_pstate_start;
1222         scount = acpi_pstate_count;
1223
1224         cnt = scount - sstart;
1225         for (i = 0; i < cnt; ++i)
1226                 freqs[i] = acpi_pstates[sstart + i].st_freq;
1227
1228         return sysctl_handle_opaque(oidp, freqs, cnt * sizeof(freqs[0]), req);
1229 }
1230
1231 static int
1232 acpi_pst_sysctl_power(SYSCTL_HANDLER_ARGS)
1233 {
1234         uint32_t power[ACPI_NPSTATE_MAX];
1235         int cnt, i, sstart, scount;
1236
1237         sstart = acpi_pstate_start;
1238         scount = acpi_pstate_count;
1239
1240         cnt = scount - sstart;
1241         for (i = 0; i < cnt; ++i)
1242                 power[i] = acpi_pstates[sstart + i].st_power;
1243
1244         return sysctl_handle_opaque(oidp, power, cnt * sizeof(power[0]), req);
1245 }
1246
1247 static int
1248 acpi_pst_sysctl_members(SYSCTL_HANDLER_ARGS)
1249 {
1250         struct acpi_pst_domain *dom = arg1;
1251         struct acpi_pst_softc *sc;
1252         int loop, error;
1253
1254         loop = error = 0;
1255         LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
1256                 char buf[32];
1257
1258                 if (error == 0 && loop)
1259                         error = SYSCTL_OUT(req, " ", 1);
1260                 if (error == 0) {
1261                         ksnprintf(buf, sizeof(buf), "cpu%d", sc->pst_cpuid);
1262                         error = SYSCTL_OUT(req, buf, strlen(buf));
1263                 }
1264
1265                 if (error == 0 && acpi_pst_md && acpi_pst_md->pmd_get_pstate) {
1266                         const struct acpi_pstate *pstate;
1267                         const char *str;
1268
1269                         pstate = acpi_pst_get_pstate(sc);
1270                         if (pstate == NULL) {
1271                                 str = "(*)";
1272                         } else {
1273                                 ksnprintf(buf, sizeof(buf), "(%d)",
1274                                           pstate->st_freq);
1275                                 str = buf;
1276                         }
1277                         error = SYSCTL_OUT(req, str, strlen(str));
1278                 }
1279                 ++loop;
1280         }
1281         return error;
1282 }
1283
1284 static int
1285 acpi_pst_sysctl_select(SYSCTL_HANDLER_ARGS)
1286 {
1287         struct acpi_pst_domain *dom = arg1;
1288         int error, i, freq;
1289
1290         KKASSERT(dom->pd_state >= 0 && dom->pd_state < acpi_npstates);
1291
1292         freq = acpi_pstates[dom->pd_state].st_freq;
1293
1294         error = sysctl_handle_int(oidp, &freq, 0, req);
1295         if (error || req->newptr == NULL)
1296                 return error;
1297
1298         i = acpi_pst_freq2index(freq);
1299         if (i < 0)
1300                 return EINVAL;
1301
1302         acpi_pst_domain_set_pstate(dom, i, NULL);
1303         return 0;
1304 }
1305
1306 static int
1307 acpi_pst_sysctl_global(SYSCTL_HANDLER_ARGS)
1308 {
1309         int error, i, freq;
1310
1311         KKASSERT(acpi_pst_global_state >= 0 &&
1312                  acpi_pst_global_state < acpi_npstates);
1313
1314         freq = acpi_pstates[acpi_pst_global_state].st_freq;
1315
1316         error = sysctl_handle_int(oidp, &freq, 0, req);
1317         if (error || req->newptr == NULL)
1318                 return error;
1319
1320         i = acpi_pst_freq2index(freq);
1321         if (i < 0)
1322                 return EINVAL;
1323
1324         acpi_pst_global_set_pstate(i);
1325
1326         return 0;
1327 }
1328
1329 static void
1330 acpi_pst_check_csr_handler(netmsg_t msg)
1331 {
1332         struct netmsg_acpi_pst *rmsg = (struct netmsg_acpi_pst *)msg;
1333         int error;
1334
1335         error = acpi_pst_md->pmd_check_csr(rmsg->ctrl, rmsg->status);
1336         lwkt_replymsg(&rmsg->base.lmsg, error);
1337 }
1338
1339 static int
1340 acpi_pst_check_csr(struct acpi_pst_softc *sc)
1341 {
1342         struct netmsg_acpi_pst msg;
1343
1344         if (acpi_pst_md == NULL)
1345                 return 0;
1346
1347         netmsg_init(&msg.base, NULL, &curthread->td_msgport,
1348                     MSGF_PRIORITY, acpi_pst_check_csr_handler);
1349         msg.ctrl = &sc->pst_creg;
1350         msg.status = &sc->pst_sreg;
1351
1352         return lwkt_domsg(netisr_cpuport(sc->pst_cpuid), &msg.base.lmsg, 0);
1353 }
1354
1355 static void
1356 acpi_pst_check_pstates_handler(netmsg_t msg)
1357 {
1358         int error;
1359
1360         error = acpi_pst_md->pmd_check_pstates(acpi_pstates, acpi_npstates);
1361         lwkt_replymsg(&msg->lmsg, error);
1362 }
1363
1364 static int
1365 acpi_pst_check_pstates(struct acpi_pst_softc *sc)
1366 {
1367         struct netmsg_base msg;
1368
1369         if (acpi_pst_md == NULL)
1370                 return 0;
1371
1372         netmsg_init(&msg, NULL, &curthread->td_msgport,
1373                     MSGF_PRIORITY, acpi_pst_check_pstates_handler);
1374
1375         return lwkt_domsg(netisr_cpuport(sc->pst_cpuid), &msg.lmsg, 0);
1376 }
1377
1378 static void
1379 acpi_pst_init_handler(netmsg_t msg)
1380 {
1381         struct netmsg_acpi_pst *rmsg = (struct netmsg_acpi_pst *)msg;
1382         int error;
1383
1384         error = acpi_pst_md->pmd_init(rmsg->ctrl, rmsg->status);
1385         lwkt_replymsg(&rmsg->base.lmsg, error);
1386 }
1387
1388 static int
1389 acpi_pst_init(struct acpi_pst_softc *sc)
1390 {
1391         struct netmsg_acpi_pst msg;
1392
1393         if (acpi_pst_md == NULL)
1394                 return 0;
1395
1396         netmsg_init(&msg.base, NULL, &curthread->td_msgport,
1397                     MSGF_PRIORITY, acpi_pst_init_handler);
1398         msg.ctrl = &sc->pst_creg;
1399         msg.status = &sc->pst_sreg;
1400
1401         return lwkt_domsg(netisr_cpuport(sc->pst_cpuid), &msg.base.lmsg, 0);
1402 }
1403
1404 static void
1405 acpi_pst_set_pstate_handler(netmsg_t msg)
1406 {
1407         struct netmsg_acpi_pst *rmsg = (struct netmsg_acpi_pst *)msg;
1408         int error;
1409
1410         error = acpi_pst_md->pmd_set_pstate(rmsg->ctrl, rmsg->status,
1411                                             rmsg->base.lmsg.u.ms_resultp);
1412         lwkt_replymsg(&rmsg->base.lmsg, error);
1413 }
1414
1415 static int
1416 acpi_pst_set_pstate(struct acpi_pst_softc *sc, const struct acpi_pstate *pstate)
1417 {
1418         struct netmsg_acpi_pst msg;
1419
1420         KKASSERT(acpi_pst_md != NULL);
1421
1422         netmsg_init(&msg.base, NULL, &curthread->td_msgport,
1423                     MSGF_PRIORITY, acpi_pst_set_pstate_handler);
1424         msg.base.lmsg.u.ms_resultp = __DECONST(void *, pstate);
1425         msg.ctrl = &sc->pst_creg;
1426         msg.status = &sc->pst_sreg;
1427
1428         return lwkt_domsg(netisr_cpuport(sc->pst_cpuid), &msg.base.lmsg, 0);
1429 }
1430
1431 static void
1432 acpi_pst_get_pstate_handler(netmsg_t msg)
1433 {
1434         struct netmsg_acpi_pst *rmsg = (struct netmsg_acpi_pst *)msg;
1435         const struct acpi_pstate *pstate;
1436
1437         pstate = acpi_pst_md->pmd_get_pstate(rmsg->status, acpi_pstates,
1438                                              acpi_npstates);
1439         rmsg->base.lmsg.u.ms_resultp = __DECONST(void *, pstate);
1440         lwkt_replymsg(&rmsg->base.lmsg, 0);
1441 }
1442
1443 static const struct acpi_pstate *
1444 acpi_pst_get_pstate(struct acpi_pst_softc *sc)
1445 {
1446         struct netmsg_acpi_pst msg;
1447
1448         if (acpi_pst_md == NULL)
1449                 return 0;
1450
1451         netmsg_init(&msg.base, NULL, &curthread->td_msgport,
1452                     MSGF_PRIORITY, acpi_pst_get_pstate_handler);
1453         msg.status = &sc->pst_sreg;
1454
1455         lwkt_domsg(netisr_cpuport(sc->pst_cpuid), &msg.base.lmsg, 0);
1456         return msg.base.lmsg.u.ms_resultp;
1457 }
1458
1459 static int
1460 acpi_pst_alloc_resource(device_t dev, ACPI_OBJECT *obj, int idx,
1461                         struct acpi_pst_res *res)
1462 {
1463         struct acpi_pst_softc *sc = device_get_softc(dev);
1464         int error, type;
1465
1466         /* Save GAS */
1467         error = acpi_PkgRawGas(obj, idx, &res->pr_gas);
1468         if (error)
1469                 return error;
1470
1471         /* Allocate resource, if possible */
1472         res->pr_rid = sc->pst_parent->cpu_next_rid;
1473         acpi_bus_alloc_gas(dev, &type, &res->pr_rid, &res->pr_gas, &res->pr_res, 0);
1474         if (res->pr_res != NULL) {
1475                 sc->pst_parent->cpu_next_rid++;
1476                 res->pr_bt = rman_get_bustag(res->pr_res);
1477                 res->pr_bh = rman_get_bushandle(res->pr_res);
1478         } else {
1479                 res->pr_rid = 0;
1480         }
1481         return 0;
1482 }
1483
1484 static void
1485 acpi_pst_domain_check_nproc(device_t dev, struct acpi_pst_domain *dom)
1486 {
1487         struct acpi_pst_softc *pst;
1488         int i;
1489
1490         i = 0;
1491         LIST_FOREACH(pst, &dom->pd_pstlist, pst_link)
1492                 ++i;
1493         if (i == dom->pd_nproc) {
1494                 /*
1495                  * Some stupid BIOSes will set wrong "# of processors",
1496                  * e.g. 1 for CPU w/ hyperthreading; Be lenient here.
1497                  */
1498                 if (bootverbose) {
1499                         device_printf(dev, "domain%u already contains %d "
1500                             "P-States\n", dom->pd_dom, dom->pd_nproc);
1501                 }
1502                 dom->pd_nproc++;
1503         }
1504         KKASSERT(i < dom->pd_nproc);
1505 }
1506
1507 static int
1508 acpi_pst_eval_ppc(struct acpi_pst_softc *sc, int *sstart)
1509 {
1510         ACPI_BUFFER buf;
1511         ACPI_STATUS status;
1512         ACPI_OBJECT *obj;
1513
1514         buf.Pointer = NULL;
1515         buf.Length = ACPI_ALLOCATE_BUFFER;
1516         status = AcpiEvaluateObject(sc->pst_handle, "_PPC", NULL, &buf);
1517         if (!ACPI_FAILURE(status)) {
1518                 ACPI_OBJECT_LIST arglist;
1519                 ACPI_OBJECT arg[2];
1520
1521                 obj = (ACPI_OBJECT *)buf.Pointer;
1522                 if (obj->Type == ACPI_TYPE_INTEGER) {
1523                         if (obj->Integer.Value >= acpi_npstates) {
1524                                 device_printf(sc->pst_dev,
1525                                     "Invalid _PPC value\n");
1526                                 AcpiOsFree(obj);
1527                                 return ENXIO;
1528                         }
1529                         *sstart = obj->Integer.Value;
1530                         if (bootverbose) {
1531                                 device_printf(sc->pst_dev, "_PPC %d\n",
1532                                     *sstart);
1533                         }
1534                 } else {
1535                         device_printf(sc->pst_dev, "Invalid _PPC object\n");
1536                         AcpiOsFree(obj);
1537                         return ENXIO;
1538                 }
1539
1540                 /* Free _PPC */
1541                 AcpiOsFree(obj);
1542
1543                 /* _PPC has been successfully processed */
1544                 arglist.Pointer = arg;
1545                 arglist.Count = 2;
1546                 arg[0].Type = ACPI_TYPE_INTEGER;
1547                 arg[0].Integer.Value = 0x80;
1548                 arg[1].Type = ACPI_TYPE_INTEGER;
1549                 arg[1].Integer.Value = 0;
1550                 AcpiEvaluateObject(sc->pst_handle, "_OST", &arglist, NULL);
1551
1552                 return 0;
1553         }
1554         return ENOENT;
1555 }
1556
1557 static int
1558 acpi_pst_eval_pdl(struct acpi_pst_softc *sc, int *scount)
1559 {
1560         ACPI_BUFFER buf;
1561         ACPI_STATUS status;
1562         ACPI_OBJECT *obj;
1563
1564         buf.Pointer = NULL;
1565         buf.Length = ACPI_ALLOCATE_BUFFER;
1566         status = AcpiEvaluateObject(sc->pst_handle, "_PDL", NULL, &buf);
1567         if (!ACPI_FAILURE(status)) {
1568                 obj = (ACPI_OBJECT *)buf.Pointer;
1569                 if (obj->Type == ACPI_TYPE_INTEGER) {
1570                         if (obj->Integer.Value >= acpi_npstates) {
1571                                 device_printf(sc->pst_dev,
1572                                     "Invalid _PDL value\n");
1573                                 AcpiOsFree(obj);
1574                                 return ENXIO;
1575                         }
1576                         if (obj->Integer.Value >= acpi_pstate_start) {
1577                                 *scount = obj->Integer.Value + 1;
1578                                 if (bootverbose) {
1579                                         device_printf(sc->pst_dev, "_PDL %d\n",
1580                                             *scount);
1581                                 }
1582                         } else {
1583                                 /* Prefer _PPC as stated in ACPI 5.1 8.4.4.6 */
1584                                 device_printf(sc->pst_dev, "conflict _PDL %ju "
1585                                     "and _PPC %d, ignore\n",
1586                                     (uintmax_t)obj->Integer.Value,
1587                                     acpi_pstate_start);
1588                         }
1589                 } else {
1590                         device_printf(sc->pst_dev, "Invalid _PDL object\n");
1591                         AcpiOsFree(obj);
1592                         return ENXIO;
1593                 }
1594
1595                 /* Free _PDL */
1596                 AcpiOsFree(obj);
1597
1598                 return 0;
1599         }
1600         return ENOENT;
1601 }
1602
1603 /*
1604  * Notify is serialized by acpi task thread.
1605  */
1606 static void
1607 acpi_pst_notify(device_t dev)
1608 {
1609         struct acpi_pst_softc *sc = device_get_softc(dev);
1610         boolean_t fixup = FALSE;
1611
1612         /*
1613          * NOTE:
1614          * _PPC and _PDL evaluation order is critical.  _PDL
1615          * evaluation depends on _PPC evaluation.
1616          */
1617         if (sc->pst_flags & ACPI_PST_FLAG_PPC) {
1618                 int sstart = acpi_pstate_start;
1619
1620                 acpi_pst_eval_ppc(sc, &sstart);
1621                 if (acpi_pstate_start != sstart && sc->pst_cpuid == 0) {
1622                         acpi_pstate_start = sstart;
1623                         fixup = TRUE;
1624                 }
1625         }
1626         if (sc->pst_flags & ACPI_PST_FLAG_PDL) {
1627                 int scount = acpi_pstate_count;
1628
1629                 acpi_pst_eval_pdl(sc, &scount);
1630                 if (acpi_pstate_count != scount && sc->pst_cpuid == 0) {
1631                         acpi_pstate_count = scount;
1632                         fixup = TRUE;
1633                 }
1634         }
1635
1636         if (fixup && acpi_pst_md != NULL &&
1637             acpi_pst_md->pmd_set_pstate != NULL)
1638                 acpi_pst_global_fixup_pstate();
1639 }