sys/dev/acpica/acpi_cpu_pstate.c

   1 /*
   2  * Copyright (c) 2009 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Sepherosa Ziehau <sepherosa@gmail.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 #include "opt_acpi.h"
  36
  37 #include <sys/param.h>
  38 #include <sys/bus.h>
  39 #include <sys/cpuhelper.h>
  40 #include <sys/kernel.h>
  41 #include <sys/malloc.h>
  42 #include <sys/queue.h>
  43 #include <sys/rman.h>
  44 #include <sys/sysctl.h>
  45 #include <sys/msgport2.h>
  46 #include <sys/cpu_topology.h>
  47
  48 #include "acpi.h"
  49 #include "acpivar.h"
  50 #include "acpi_cpu.h"
  51 #include "acpi_cpu_pstate.h"
  52
  53 #define ACPI_NPSTATE_MAX        32
  54
  55 #define ACPI_PSS_PX_NENTRY      6
  56
  57 #define ACPI_PSD_COORD_SWALL    0xfc
  58 #define ACPI_PSD_COORD_SWANY    0xfd
  59 #define ACPI_PSD_COORD_HWALL    0xfe
  60 #define ACPI_PSD_COORD_VALID(coord) \
  61         ((coord) == ACPI_PSD_COORD_SWALL || \
  62          (coord) == ACPI_PSD_COORD_SWANY || \
  63          (coord) == ACPI_PSD_COORD_HWALL)
  64
  65 struct acpi_pst_softc;
  66 LIST_HEAD(acpi_pst_list, acpi_pst_softc);
  67
  68 struct acpi_pst_chmsg {
  69         struct cpuhelper_msg    ch_msg;
  70         const struct acpi_pst_res *ch_ctrl;
  71         const struct acpi_pst_res *ch_status;
  72 };
  73
  74 struct acpi_pst_domain {
  75         uint32_t                pd_dom;
  76         uint32_t                pd_coord;
  77         uint32_t                pd_nproc;
  78         LIST_ENTRY(acpi_pst_domain) pd_link;
  79
  80         uint32_t                pd_flags;
  81
  82         struct lwkt_serialize   pd_serialize;
  83
  84         int                     pd_state;
  85         struct acpi_pst_list    pd_pstlist;
  86
  87         struct sysctl_ctx_list  pd_sysctl_ctx;
  88         struct sysctl_oid       *pd_sysctl_tree;
  89 };
  90 LIST_HEAD(acpi_pst_domlist, acpi_pst_domain);
  91
  92 #define ACPI_PSTDOM_FLAG_STUB   0x1     /* stub domain, no _PSD */
  93 #define ACPI_PSTDOM_FLAG_DEAD   0x2     /* domain can't be started */
  94 #define ACPI_PSTDOM_FLAG_INT    0x4     /* domain created from Integer _PSD */
  95
  96 struct acpi_pst_softc {
  97         device_t                pst_dev;
  98         struct acpi_cpu_softc   *pst_parent;
  99         struct acpi_pst_domain  *pst_domain;
 100         struct acpi_pst_res     pst_creg;
 101         struct acpi_pst_res     pst_sreg;
 102
 103         int                     pst_state;
 104         int                     pst_cpuid;
 105
 106         uint32_t                pst_flags;
 107
 108         ACPI_HANDLE             pst_handle;
 109
 110         LIST_ENTRY(acpi_pst_softc) pst_link;
 111 };
 112
 113 #define ACPI_PST_FLAG_PPC       0x1
 114 #define ACPI_PST_FLAG_PDL       0x2
 115
 116 static int      acpi_pst_probe(device_t dev);
 117 static int      acpi_pst_attach(device_t dev);
 118 static void     acpi_pst_notify(device_t dev);
 119
 120 static void     acpi_pst_postattach(void *);
 121 static struct acpi_pst_domain *
 122                 acpi_pst_domain_create_int(device_t, uint32_t);
 123 static struct acpi_pst_domain *
 124                 acpi_pst_domain_create_pkg(device_t, ACPI_OBJECT *);
 125 static struct acpi_pst_domain *
 126                 acpi_pst_domain_find(uint32_t);
 127 static struct acpi_pst_domain *
 128                 acpi_pst_domain_alloc(uint32_t, uint32_t, uint32_t);
 129 static void     acpi_pst_domain_set_pstate_locked(struct acpi_pst_domain *,
 130                     int, int *);
 131 static void     acpi_pst_domain_set_pstate(struct acpi_pst_domain *, int,
 132                     int *);
 133 static void     acpi_pst_domain_check_nproc(device_t, struct acpi_pst_domain *);
 134 static void     acpi_pst_global_set_pstate(int);
 135 static void     acpi_pst_global_fixup_pstate(void);
 136
 137 static int      acpi_pst_check_csr(struct acpi_pst_softc *);
 138 static int      acpi_pst_check_pstates(struct acpi_pst_softc *);
 139 static int      acpi_pst_init(struct acpi_pst_softc *);
 140 static int      acpi_pst_set_pstate(struct acpi_pst_softc *,
 141                     const struct acpi_pstate *);
 142 static const struct acpi_pstate *
 143                 acpi_pst_get_pstate(struct acpi_pst_softc *);
 144 static int      acpi_pst_alloc_resource(device_t, ACPI_OBJECT *, int,
 145                     struct acpi_pst_res *);
 146 static int      acpi_pst_eval_ppc(struct acpi_pst_softc *, int *);
 147 static int      acpi_pst_eval_pdl(struct acpi_pst_softc *, int *);
 148
 149 static void     acpi_pst_check_csr_handler(struct cpuhelper_msg *);
 150 static void     acpi_pst_check_pstates_handler(struct cpuhelper_msg *);
 151 static void     acpi_pst_init_handler(struct cpuhelper_msg *);
 152 static void     acpi_pst_set_pstate_handler(struct cpuhelper_msg *);
 153 static void     acpi_pst_get_pstate_handler(struct cpuhelper_msg *);
 154
 155 static int      acpi_pst_sysctl_freqs(SYSCTL_HANDLER_ARGS);
 156 static int      acpi_pst_sysctl_freqs_bin(SYSCTL_HANDLER_ARGS);
 157 static int      acpi_pst_sysctl_power(SYSCTL_HANDLER_ARGS);
 158 static int      acpi_pst_sysctl_members(SYSCTL_HANDLER_ARGS);
 159 static int      acpi_pst_sysctl_select(SYSCTL_HANDLER_ARGS);
 160 static int      acpi_pst_sysctl_global(SYSCTL_HANDLER_ARGS);
 161
 162 static struct acpi_pst_domlist  acpi_pst_domains =
 163         LIST_HEAD_INITIALIZER(acpi_pst_domains);
 164 static int                      acpi_pst_domain_id;
 165
 166 static int                      acpi_pst_global_state;
 167
 168 static int                      acpi_pstate_start = -1;
 169 static int                      acpi_pstate_count;
 170 static int                      acpi_npstates;
 171 static struct acpi_pstate       *acpi_pstates;
 172
 173 static const struct acpi_pst_md *acpi_pst_md;
 174
 175 static int                      acpi_pst_pdl = -1;
 176 TUNABLE_INT("hw.acpi.cpu.pst.pdl", &acpi_pst_pdl);
 177
 178 static int                      acpi_pst_ht_reuse_domain = 1;
 179 TUNABLE_INT("hw.acpi.cpu.pst.ht_reuse_domain", &acpi_pst_ht_reuse_domain);
 180
 181 static int                      acpi_pst_force_pkg_domain = 0;
 182 TUNABLE_INT("hw.acpi.cpu.pst.force_pkg_domain", &acpi_pst_force_pkg_domain);
 183
 184 static int                      acpi_pst_handle_notify = 1;
 185 TUNABLE_INT("hw.acpi.cpu.pst.handle_notify", &acpi_pst_handle_notify);
 186
 187 /*
 188  * Force CPU package power domain for Intel CPUs.
 189  *
 190  * As of this write (14 July 2015), all Intel CPUs only have CPU package
 191  * power domain.
 192  */
 193 static int                      acpi_pst_intel_pkg_domain = 1;
 194 TUNABLE_INT("hw.acpi.cpu.pst.intel_pkg_domain", &acpi_pst_intel_pkg_domain);
 195
 196 static device_method_t acpi_pst_methods[] = {
 197         /* Device interface */
 198         DEVMETHOD(device_probe,                 acpi_pst_probe),
 199         DEVMETHOD(device_attach,                acpi_pst_attach),
 200         DEVMETHOD(device_detach,                bus_generic_detach),
 201         DEVMETHOD(device_shutdown,              bus_generic_shutdown),
 202         DEVMETHOD(device_suspend,               bus_generic_suspend),
 203         DEVMETHOD(device_resume,                bus_generic_resume),
 204
 205         /* Bus interface */
 206         DEVMETHOD(bus_add_child,                bus_generic_add_child),
 207         DEVMETHOD(bus_print_child,              bus_generic_print_child),
 208         DEVMETHOD(bus_read_ivar,                bus_generic_read_ivar),
 209         DEVMETHOD(bus_write_ivar,               bus_generic_write_ivar),
 210         DEVMETHOD(bus_get_resource_list,        bus_generic_get_resource_list),
 211         DEVMETHOD(bus_set_resource,             bus_generic_rl_set_resource),
 212         DEVMETHOD(bus_get_resource,             bus_generic_rl_get_resource),
 213         DEVMETHOD(bus_alloc_resource,           bus_generic_alloc_resource),
 214         DEVMETHOD(bus_release_resource,         bus_generic_release_resource),
 215         DEVMETHOD(bus_driver_added,             bus_generic_driver_added),
 216         DEVMETHOD(bus_activate_resource,        bus_generic_activate_resource),
 217         DEVMETHOD(bus_deactivate_resource,      bus_generic_deactivate_resource),
 218         DEVMETHOD(bus_setup_intr,               bus_generic_setup_intr),
 219         DEVMETHOD(bus_teardown_intr,            bus_generic_teardown_intr),
 220
 221         DEVMETHOD_END
 222 };
 223
 224 static driver_t acpi_pst_driver = {
 225         "cpu_pst",
 226         acpi_pst_methods,
 227         sizeof(struct acpi_pst_softc),
 228         .gpri = KOBJ_GPRI_ACPI+2
 229 };
 230
 231 static devclass_t acpi_pst_devclass;
 232 DRIVER_MODULE(cpu_pst, cpu, acpi_pst_driver, acpi_pst_devclass, NULL, NULL);
 233 MODULE_DEPEND(cpu_pst, acpi, 1, 1, 1);
 234
 235 static __inline int
 236 acpi_pst_freq2index(int freq)
 237 {
 238         int i;
 239
 240         for (i = 0; i < acpi_npstates; ++i) {
 241                 if (acpi_pstates[i].st_freq == freq)
 242                         return i;
 243         }
 244         return -1;
 245 }
 246
 247 static int
 248 acpi_pst_probe(device_t dev)
 249 {
 250         ACPI_BUFFER buf;
 251         ACPI_HANDLE handle;
 252         ACPI_STATUS status;
 253         ACPI_OBJECT *obj;
 254
 255         if (acpi_disabled("cpu_pst") ||
 256             acpi_get_type(dev) != ACPI_TYPE_PROCESSOR)
 257                 return ENXIO;
 258
 259         if (acpi_pst_md == NULL)
 260                 acpi_pst_md = acpi_pst_md_probe();
 261
 262         handle = acpi_get_handle(dev);
 263
 264         /*
 265          * Check _PSD package
 266          *
 267          * NOTE:
 268          * Some BIOSes do not expose _PCT for the second thread of
 269          * CPU cores.  In this case, _PSD should be enough to get the
 270          * P-state of the second thread working, since it must have
 271          * the same _PCT and _PSS as the first thread in the same
 272          * core.
 273          */
 274         buf.Pointer = NULL;
 275         buf.Length = ACPI_ALLOCATE_BUFFER;
 276         status = AcpiEvaluateObject(handle, "_PSD", NULL, &buf);
 277         if (!ACPI_FAILURE(status)) {
 278                 AcpiOsFree((ACPI_OBJECT *)buf.Pointer);
 279                 goto done;
 280         }
 281
 282         /*
 283          * Check _PCT package
 284          */
 285         buf.Pointer = NULL;
 286         buf.Length = ACPI_ALLOCATE_BUFFER;
 287         status = AcpiEvaluateObject(handle, "_PCT", NULL, &buf);
 288         if (ACPI_FAILURE(status)) {
 289                 if (bootverbose) {
 290                         device_printf(dev, "Can't get _PCT package - %s\n",
 291                                       AcpiFormatException(status));
 292                 }
 293                 return ENXIO;
 294         }
 295
 296         obj = (ACPI_OBJECT *)buf.Pointer;
 297         if (!ACPI_PKG_VALID_EQ(obj, 2)) {
 298                 device_printf(dev, "Invalid _PCT package\n");
 299                 AcpiOsFree(obj);
 300                 return ENXIO;
 301         }
 302         AcpiOsFree(obj);
 303
 304         /*
 305          * Check _PSS package
 306          */
 307         buf.Pointer = NULL;
 308         buf.Length = ACPI_ALLOCATE_BUFFER;
 309         status = AcpiEvaluateObject(handle, "_PSS", NULL, &buf);
 310         if (ACPI_FAILURE(status)) {
 311                 device_printf(dev, "Can't get _PSS package - %s\n",
 312                               AcpiFormatException(status));
 313                 return ENXIO;
 314         }
 315
 316         obj = (ACPI_OBJECT *)buf.Pointer;
 317         if (!ACPI_PKG_VALID(obj, 1)) {
 318                 device_printf(dev, "Invalid _PSS package\n");
 319                 AcpiOsFree(obj);
 320                 return ENXIO;
 321         }
 322         AcpiOsFree(obj);
 323
 324 done:
 325         device_set_desc(dev, "ACPI CPU P-State");
 326         return 0;
 327 }
 328
 329 static int
 330 acpi_pst_attach(device_t dev)
 331 {
 332         struct acpi_pst_softc *sc = device_get_softc(dev);
 333         struct acpi_pst_domain *dom = NULL;
 334         ACPI_BUFFER buf;
 335         ACPI_STATUS status;
 336         ACPI_OBJECT *obj;
 337         struct acpi_pstate *pstate, *p;
 338         int i, npstate, error, sstart, scount;
 339
 340         sc->pst_dev = dev;
 341         sc->pst_parent = device_get_softc(device_get_parent(dev));
 342         sc->pst_handle = acpi_get_handle(dev);
 343         sc->pst_cpuid = acpi_get_magic(dev);
 344
 345         /*
 346          * If there is a _PSD, then we create procossor domain
 347          * accordingly.  If there is no _PSD, we just fake a
 348          * default processor domain0.
 349          */
 350         buf.Pointer = NULL;
 351         buf.Length = ACPI_ALLOCATE_BUFFER;
 352         status = AcpiEvaluateObject(sc->pst_handle, "_PSD", NULL, &buf);
 353         if (!ACPI_FAILURE(status)) {
 354                 obj = (ACPI_OBJECT *)buf.Pointer;
 355
 356                 if (acpi_pst_domain_id > 0) {
 357                         device_printf(dev, "Missing _PSD for certain CPUs\n");
 358                         AcpiOsFree(obj);
 359                         return ENXIO;
 360                 }
 361                 acpi_pst_domain_id = -1;
 362
 363                 if (ACPI_PKG_VALID_EQ(obj, 1)) {
 364                         dom = acpi_pst_domain_create_pkg(dev,
 365                                 &obj->Package.Elements[0]);
 366                         if (dom == NULL) {
 367                                 AcpiOsFree(obj);
 368                                 return ENXIO;
 369                         }
 370                 } else {
 371                         if (obj->Type != ACPI_TYPE_INTEGER) {
 372                                 device_printf(dev,
 373                                     "Invalid _PSD package, Type 0x%x\n",
 374                                     obj->Type);
 375                                 AcpiOsFree(obj);
 376                                 return ENXIO;
 377                         } else {
 378                                 device_printf(dev, "Integer _PSD %ju\n",
 379                                     (uintmax_t)obj->Integer.Value);
 380                                 dom = acpi_pst_domain_create_int(dev,
 381                                     obj->Integer.Value);
 382                                 if (dom == NULL) {
 383                                         AcpiOsFree(obj);
 384                                         return ENXIO;
 385                                 }
 386                         }
 387                 }
 388
 389                 /* Free _PSD */
 390                 AcpiOsFree(buf.Pointer);
 391         } else {
 392                 if (acpi_pst_domain_id < 0) {
 393                         device_printf(dev, "Missing _PSD for cpu%d\n",
 394                             sc->pst_cpuid);
 395                         return ENXIO;
 396                 }
 397
 398                 /*
 399                  * Create a stub one processor domain for each processor
 400                  */
 401                 dom = acpi_pst_domain_alloc(acpi_pst_domain_id,
 402                         ACPI_PSD_COORD_SWANY, 1);
 403                 dom->pd_flags |= ACPI_PSTDOM_FLAG_STUB;
 404
 405                 ++acpi_pst_domain_id;
 406         }
 407
 408         /* Make sure that adding us will not overflow our domain */
 409         acpi_pst_domain_check_nproc(dev, dom);
 410
 411         /*
 412          * Get control/status registers from _PCT
 413          */
 414         buf.Pointer = NULL;
 415         buf.Length = ACPI_ALLOCATE_BUFFER;
 416         status = AcpiEvaluateObject(sc->pst_handle, "_PCT", NULL, &buf);
 417         if (ACPI_FAILURE(status)) {
 418                 struct acpi_pst_softc *pst;
 419
 420                 /*
 421                  * No _PCT.  See the comment in acpi_pst_probe() near
 422                  * _PSD check.
 423                  *
 424                  * Use control/status registers of another CPU in the
 425                  * same domain, or in the same core, if the type of
 426                  * these registers are "Fixed Hardware", e.g. on most
 427                  * of the model Intel CPUs.
 428                  */
 429                 pst = LIST_FIRST(&dom->pd_pstlist);
 430                 if (pst == NULL) {
 431                         cpumask_t mask;
 432
 433                         mask = get_cpumask_from_level(sc->pst_cpuid,
 434                             CORE_LEVEL);
 435                         if (CPUMASK_TESTNZERO(mask)) {
 436                                 struct acpi_pst_domain *dom1;
 437
 438                                 LIST_FOREACH(dom1, &acpi_pst_domains, pd_link) {
 439                                         LIST_FOREACH(pst, &dom1->pd_pstlist,
 440                                             pst_link) {
 441                                                 if (CPUMASK_TESTBIT(mask,
 442                                                     pst->pst_cpuid))
 443                                                         break;
 444                                         }
 445                                         if (pst != NULL)
 446                                                 break;
 447                                 }
 448                                 if (pst != NULL && acpi_pst_ht_reuse_domain) {
 449                                         /*
 450                                          * Use the same domain for CPUs in the
 451                                          * same core.
 452                                          */
 453                                         device_printf(dev, "Destroy domain%u, "
 454                                             "reuse domain%u\n",
 455                                             dom->pd_dom, dom1->pd_dom);
 456                                         LIST_REMOVE(dom, pd_link);
 457                                         kfree(dom, M_DEVBUF);
 458                                         dom = dom1;
 459                                         /*
 460                                          * Make sure that adding us will not
 461                                          * overflow the domain containing
 462                                          * siblings in the same core.
 463                                          */
 464                                         acpi_pst_domain_check_nproc(dev, dom);
 465                                 }
 466                         }
 467                 }
 468                 if (pst != NULL &&
 469                     pst->pst_creg.pr_res == NULL &&
 470                     pst->pst_creg.pr_rid == 0 &&
 471                     pst->pst_creg.pr_gas.SpaceId ==
 472                     ACPI_ADR_SPACE_FIXED_HARDWARE &&
 473                     pst->pst_sreg.pr_res == NULL &&
 474                     pst->pst_sreg.pr_rid == 0 &&
 475                     pst->pst_sreg.pr_gas.SpaceId ==
 476                     ACPI_ADR_SPACE_FIXED_HARDWARE) {
 477                         sc->pst_creg = pst->pst_creg;
 478                         sc->pst_sreg = pst->pst_sreg;
 479                         device_printf(dev,
 480                             "No _PCT; reuse %s control/status regs\n",
 481                             device_get_nameunit(pst->pst_dev));
 482                         goto fetch_pss;
 483                 }
 484                 device_printf(dev, "Can't get _PCT package - %s\n",
 485                               AcpiFormatException(status));
 486                 return ENXIO;
 487         }
 488
 489         obj = (ACPI_OBJECT *)buf.Pointer;
 490         if (!ACPI_PKG_VALID_EQ(obj, 2)) {
 491                 device_printf(dev, "Invalid _PCT package\n");
 492                 AcpiOsFree(obj);
 493                 return ENXIO;
 494         }
 495
 496         /* Save and try allocating control register */
 497         error = acpi_pst_alloc_resource(dev, obj, 0, &sc->pst_creg);
 498         if (error) {
 499                 AcpiOsFree(obj);
 500                 return error;
 501         }
 502         if (bootverbose) {
 503                 device_printf(dev, "control reg %d %jx\n",
 504                               sc->pst_creg.pr_gas.SpaceId,
 505                               (uintmax_t)sc->pst_creg.pr_gas.Address);
 506         }
 507
 508         /* Save and try allocating status register */
 509         error = acpi_pst_alloc_resource(dev, obj, 1, &sc->pst_sreg);
 510         if (error) {
 511                 AcpiOsFree(obj);
 512                 return error;
 513         }
 514         if (bootverbose) {
 515                 device_printf(dev, "status reg %d %jx\n",
 516                               sc->pst_sreg.pr_gas.SpaceId,
 517                               (uintmax_t)sc->pst_sreg.pr_gas.Address);
 518         }
 519
 520         /* Free _PCT */
 521         AcpiOsFree(obj);
 522
 523 fetch_pss:
 524         /*
 525          * Create P-State table according to _PSS
 526          */
 527         buf.Pointer = NULL;
 528         buf.Length = ACPI_ALLOCATE_BUFFER;
 529         status = AcpiEvaluateObject(sc->pst_handle, "_PSS", NULL, &buf);
 530         if (ACPI_FAILURE(status)) {
 531                 /*
 532                  * No _PSS.  See the comment in acpi_pst_probe() near
 533                  * _PSD check.
 534                  *
 535                  * Assume _PSS are same across all CPUs; well, they
 536                  * should/have to be so.
 537                  */
 538                 if (acpi_npstates > 0 && acpi_pstates != NULL) {
 539                         device_printf(dev, "No _PSS\n");
 540                         goto fetch_ppc;
 541                 }
 542                 device_printf(dev, "Can't get _PSS package - %s\n",
 543                               AcpiFormatException(status));
 544                 return ENXIO;
 545         }
 546
 547         obj = (ACPI_OBJECT *)buf.Pointer;
 548         if (!ACPI_PKG_VALID(obj, 1)) {
 549                 device_printf(dev, "Invalid _PSS package\n");
 550                 AcpiOsFree(obj);
 551                 return ENXIO;
 552         }
 553
 554         /* Don't create too many P-States */
 555         npstate = obj->Package.Count;
 556         if (npstate > ACPI_NPSTATE_MAX) {
 557                 device_printf(dev, "Too many P-States, %d->%d\n",
 558                               npstate, ACPI_NPSTATE_MAX);
 559                 npstate = ACPI_NPSTATE_MAX;
 560         }
 561
 562         /*
 563          * If we have already created P-State table,
 564          * we must make sure that number of entries
 565          * is consistent.
 566          */
 567         if (acpi_pstates != NULL && acpi_npstates != npstate) {
 568                 device_printf(dev, "Inconsistent # of P-States "
 569                               "cross Processor objects\n");
 570                 AcpiOsFree(obj);
 571                 return ENXIO;
 572         }
 573
 574         /*
 575          * Create a temporary P-State table
 576          */
 577         pstate = kmalloc(sizeof(*pstate) * npstate, M_TEMP, M_WAITOK);
 578         for (i = 0, p = pstate; i < npstate; ++i, ++p) {
 579                 ACPI_OBJECT *pkg;
 580                 uint32_t *ptr[ACPI_PSS_PX_NENTRY] = {
 581                         &p->st_freq, &p->st_power, &p->st_xsit_lat,
 582                         &p->st_bm_lat, &p->st_cval, &p->st_sval
 583                 };
 584                 int j;
 585
 586                 pkg = &obj->Package.Elements[i];
 587                 if (!ACPI_PKG_VALID(pkg, ACPI_PSS_PX_NENTRY)) {
 588                         device_printf(dev, "Invalud _PSS P%d\n", i);
 589                         AcpiOsFree(obj);
 590                         kfree(pstate, M_TEMP);
 591                         return ENXIO;
 592                 }
 593                 for (j = 0; j < ACPI_PSS_PX_NENTRY; ++j) {
 594                         if (acpi_PkgInt32(pkg, j, ptr[j]) != 0) {
 595                                 device_printf(dev, "Can't extract "
 596                                               "_PSS P%d %dth entry\n", i, j);
 597                                 AcpiOsFree(obj);
 598                                 kfree(pstate, M_TEMP);
 599                                 return ENXIO;
 600                         }
 601                 }
 602                 if (p->st_freq & 0x80000000) {
 603                         device_printf(dev, "Invalid _PSS P%d freq: 0x%08x\n",
 604                             i, p->st_freq);
 605                         AcpiOsFree(obj);
 606                         kfree(pstate, M_TEMP);
 607                         return ENXIO;
 608                 }
 609         }
 610
 611         /* Free _PSS */
 612         AcpiOsFree(obj);
 613
 614         if (acpi_pstates == NULL) {
 615                 /*
 616                  * If no P-State table is created yet,
 617                  * save the temporary one we just created.
 618                  */
 619                 acpi_pstates = pstate;
 620                 acpi_npstates = npstate;
 621                 pstate = NULL;
 622
 623                 if (bootverbose) {
 624                         for (i = 0; i < acpi_npstates; ++i) {
 625                                 device_printf(dev,
 626                                 "freq %u, pwr %u, xlat %u, blat %u, "
 627                                 "cv %08x, sv %08x\n",
 628                                 acpi_pstates[i].st_freq,
 629                                 acpi_pstates[i].st_power,
 630                                 acpi_pstates[i].st_xsit_lat,
 631                                 acpi_pstates[i].st_bm_lat,
 632                                 acpi_pstates[i].st_cval,
 633                                 acpi_pstates[i].st_sval);
 634                         }
 635                 }
 636         } else {
 637                 /*
 638                  * Make sure that P-State tables are same
 639                  * for all processors.
 640                  */
 641                 if (memcmp(pstate, acpi_pstates,
 642                            sizeof(*pstate) * npstate) != 0) {
 643                         device_printf(dev, "Inconsistent _PSS "
 644                                       "cross Processor objects\n");
 645 #if 0
 646                         /*
 647                          * Some BIOSes create different P-State tables;
 648                          * just trust the one from the BSP and move on.
 649                          */
 650                         kfree(pstate, M_TEMP);
 651                         return ENXIO;
 652 #endif
 653                 }
 654                 kfree(pstate, M_TEMP);
 655         }
 656
 657 fetch_ppc:
 658         /* By default, we start from P-State table's first entry */
 659         sstart = 0;
 660
 661         /*
 662          * Adjust the usable first entry of P-State table,
 663          * if there is _PPC object.
 664          */
 665         error = acpi_pst_eval_ppc(sc, &sstart);
 666         if (error && error != ENOENT)
 667                 return error;
 668         else if (!error)
 669                 sc->pst_flags |= ACPI_PST_FLAG_PPC;
 670         if (acpi_pstate_start < 0) {
 671                 acpi_pstate_start = sstart;
 672         } else if (acpi_pstate_start != sstart) {
 673                 device_printf(dev, "_PPC mismatch, was %d, now %d\n",
 674                     acpi_pstate_start, sstart);
 675                 if (acpi_pstate_start < sstart) {
 676                         device_printf(dev, "_PPC %d -> %d\n",
 677                             acpi_pstate_start, sstart);
 678                         acpi_pstate_start = sstart;
 679                 }
 680         }
 681
 682         /*
 683          * By default, we assume number of usable P-States is same as
 684          * number of P-States.
 685          */
 686         scount = acpi_npstates;
 687
 688         /*
 689          * Allow users to override or set _PDL
 690          */
 691         if (acpi_pst_pdl >= 0) {
 692                 if (acpi_pst_pdl < acpi_npstates) {
 693                         if (bootverbose) {
 694                                 device_printf(dev, "_PDL override %d\n",
 695                                     acpi_pst_pdl);
 696                         }
 697                         scount = acpi_pst_pdl + 1;
 698                         goto proc_pdl;
 699                 } else {
 700                         device_printf(dev, "Invalid _PDL override %d, "
 701                             "must be less than %d\n", acpi_pst_pdl,
 702                             acpi_npstates);
 703                 }
 704         }
 705
 706         /*
 707          * Adjust the number of usable entries in P-State table,
 708          * if there is _PDL object.
 709          */
 710         error = acpi_pst_eval_pdl(sc, &scount);
 711         if (error && error != ENOENT)
 712                 return error;
 713         else if (!error)
 714                 sc->pst_flags |= ACPI_PST_FLAG_PDL;
 715 proc_pdl:
 716         if (acpi_pstate_count == 0) {
 717                 acpi_pstate_count = scount;
 718         } else if (acpi_pstate_count != scount) {
 719                 device_printf(dev, "_PDL mismatch, was %d, now %d\n",
 720                     acpi_pstate_count, scount);
 721                 if (acpi_pstate_count > scount) {
 722                         device_printf(dev, "_PDL %d -> %d\n",
 723                             acpi_pstate_count, scount);
 724                         acpi_pstate_count = scount;
 725                 }
 726         }
 727
 728         /*
 729          * Some CPUs only have package P-states, but some BIOSes put each
 730          * hyperthread to its own P-state domain; allow user to override.
 731          */
 732         if (LIST_EMPTY(&dom->pd_pstlist) &&
 733             (acpi_pst_force_pkg_domain ||
 734              (cpu_vendor_id == CPU_VENDOR_INTEL &&
 735               acpi_pst_intel_pkg_domain))) {
 736                 cpumask_t mask;
 737
 738                 mask = get_cpumask_from_level(sc->pst_cpuid, CHIP_LEVEL);
 739                 if (CPUMASK_TESTNZERO(mask)) {
 740                         struct acpi_pst_softc *pst = NULL;
 741                         struct acpi_pst_domain *dom1;
 742
 743                         LIST_FOREACH(dom1, &acpi_pst_domains, pd_link) {
 744                                 LIST_FOREACH(pst, &dom1->pd_pstlist,
 745                                     pst_link) {
 746                                         if (CPUMASK_TESTBIT(mask,
 747                                             pst->pst_cpuid))
 748                                                 break;
 749                                 }
 750                                 if (pst != NULL)
 751                                         break;
 752                         }
 753                         if (pst != NULL &&
 754                             memcmp(&pst->pst_creg, &sc->pst_creg,
 755                                 sizeof(sc->pst_creg)) == 0 &&
 756                             memcmp(&pst->pst_sreg, &sc->pst_sreg,
 757                                 sizeof(sc->pst_sreg)) == 0) {
 758                                 /*
 759                                  * Use the same domain for CPUs in the
 760                                  * same package.
 761                                  */
 762                                 device_printf(dev, "Destroy domain%u, "
 763                                     "force pkg domain%u\n",
 764                                     dom->pd_dom, dom1->pd_dom);
 765                                 LIST_REMOVE(dom, pd_link);
 766                                 kfree(dom, M_DEVBUF);
 767                                 dom = dom1;
 768                                 /*
 769                                  * Make sure that adding us will not
 770                                  * overflow the domain containing
 771                                  * siblings in the same package.
 772                                  */
 773                                 acpi_pst_domain_check_nproc(dev, dom);
 774                         }
 775                 }
 776         }
 777
 778         /* Link us with the domain */
 779         sc->pst_domain = dom;
 780         LIST_INSERT_HEAD(&dom->pd_pstlist, sc, pst_link);
 781
 782         if (device_get_unit(dev) == 0)
 783                 AcpiOsExecute(OSL_NOTIFY_HANDLER, acpi_pst_postattach, NULL);
 784
 785         if (sc->pst_flags & (ACPI_PST_FLAG_PPC | ACPI_PST_FLAG_PDL))
 786                 sc->pst_parent->cpu_pst_notify = acpi_pst_notify;
 787
 788         return 0;
 789 }
 790
 791 static struct acpi_pst_domain *
 792 acpi_pst_domain_create_pkg(device_t dev, ACPI_OBJECT *obj)
 793 {
 794         struct acpi_pst_domain *dom;
 795         uint32_t val, domain, coord, nproc;
 796
 797         if (!ACPI_PKG_VALID_EQ(obj, 5)) {
 798                 device_printf(dev, "Invalid _PSD package\n");
 799                 return NULL;
 800         }
 801
 802         /* NumberOfEntries */
 803         if (acpi_PkgInt32(obj, 0, &val) != 0 || val != 5) {
 804                 device_printf(dev, "Invalid _PSD NumberOfEntries\n");
 805                 return NULL;
 806         }
 807
 808         /* Revision */
 809         if (acpi_PkgInt32(obj, 1, &val) != 0 || val != 0) {
 810                 device_printf(dev, "Invalid _PSD Revision\n");
 811                 return NULL;
 812         }
 813
 814         if (acpi_PkgInt32(obj, 2, &domain) != 0 ||
 815             acpi_PkgInt32(obj, 3, &coord) != 0 ||
 816             acpi_PkgInt32(obj, 4, &nproc) != 0) {
 817                 device_printf(dev, "Can't extract _PSD package\n");
 818                 return NULL;
 819         }
 820
 821         if (!ACPI_PSD_COORD_VALID(coord)) {
 822                 device_printf(dev, "Invalid _PSD CoordType (%#x)\n", coord);
 823                 return NULL;
 824         }
 825
 826         if (nproc > MAXCPU) {
 827                 /*
 828                  * If NumProcessors is greater than MAXCPU
 829                  * and domain's coordination is SWALL, then
 830                  * we will never be able to start all CPUs
 831                  * within this domain, and power state
 832                  * transition will never be completed, so we
 833                  * just bail out here.
 834                  */
 835                 if (coord == ACPI_PSD_COORD_SWALL) {
 836                         device_printf(dev, "Unsupported _PSD NumProcessors "
 837                                       "(%d)\n", nproc);
 838                         return NULL;
 839                 }
 840         } else if (nproc == 0) {
 841                 device_printf(dev, "_PSD NumProcessors are zero\n");
 842                 return NULL;
 843         }
 844
 845         dom = acpi_pst_domain_find(domain);
 846         if (dom != NULL) {
 847                 if (dom->pd_flags & ACPI_PSTDOM_FLAG_INT) {
 848                         device_printf(dev, "Mixed Integer _PSD and "
 849                             "Package _PSD\n");
 850                         return NULL;
 851                 }
 852                 if (dom->pd_coord != coord) {
 853                         device_printf(dev, "Inconsistent _PSD coord "
 854                             "information cross Processor objects\n");
 855                         return NULL;
 856                 }
 857                 if (dom->pd_nproc != nproc) {
 858                         device_printf(dev, "Inconsistent _PSD nproc "
 859                             "information cross Processor objects\n");
 860                         /*
 861                          * Some stupid BIOSes will set wrong "# of processors",
 862                          * e.g. 1 for CPU w/ hyperthreading; Be lenient here.
 863                          */
 864                 }
 865                 return dom;
 866         }
 867
 868         dom = acpi_pst_domain_alloc(domain, coord, nproc);
 869         if (bootverbose) {
 870                 device_printf(dev, "create pkg domain%u, coord %#x\n",
 871                     dom->pd_dom, dom->pd_coord);
 872         }
 873
 874         return dom;
 875 }
 876
 877 static struct acpi_pst_domain *
 878 acpi_pst_domain_create_int(device_t dev, uint32_t domain)
 879 {
 880         struct acpi_pst_domain *dom;
 881
 882         dom = acpi_pst_domain_find(domain);
 883         if (dom != NULL) {
 884                 if ((dom->pd_flags & ACPI_PSTDOM_FLAG_INT) == 0) {
 885                         device_printf(dev, "Mixed Package _PSD and "
 886                             "Integer _PSD\n");
 887                         return NULL;
 888                 }
 889                 KKASSERT(dom->pd_coord == ACPI_PSD_COORD_SWALL);
 890
 891                 dom->pd_nproc++;
 892                 return dom;
 893         }
 894
 895         dom = acpi_pst_domain_alloc(domain, ACPI_PSD_COORD_SWALL, 1);
 896         dom->pd_flags |= ACPI_PSTDOM_FLAG_INT;
 897
 898         if (bootverbose)
 899                 device_printf(dev, "create int domain%u\n", dom->pd_dom);
 900
 901         return dom;
 902 }
 903
 904 static struct acpi_pst_domain *
 905 acpi_pst_domain_find(uint32_t domain)
 906 {
 907         struct acpi_pst_domain *dom;
 908
 909         LIST_FOREACH(dom, &acpi_pst_domains, pd_link) {
 910                 if (dom->pd_flags & ACPI_PSTDOM_FLAG_STUB)
 911                         continue;
 912                 if (dom->pd_dom == domain)
 913                         return dom;
 914         }
 915         return NULL;
 916 }
 917
 918 static struct acpi_pst_domain *
 919 acpi_pst_domain_alloc(uint32_t domain, uint32_t coord, uint32_t nproc)
 920 {
 921         struct acpi_pst_domain *dom;
 922
 923         dom = kmalloc(sizeof(*dom), M_DEVBUF, M_WAITOK | M_ZERO);
 924         dom->pd_dom = domain;
 925         dom->pd_coord = coord;
 926         dom->pd_nproc = nproc;
 927         LIST_INIT(&dom->pd_pstlist);
 928         lwkt_serialize_init(&dom->pd_serialize);
 929
 930         LIST_INSERT_HEAD(&acpi_pst_domains, dom, pd_link);
 931
 932         return dom;
 933 }
 934
 935 static void
 936 acpi_pst_domain_set_pstate_locked(struct acpi_pst_domain *dom, int i, int *global)
 937 {
 938         const struct acpi_pstate *pstate;
 939         struct acpi_pst_softc *sc;
 940         int done, error;
 941
 942         ASSERT_SERIALIZED(&dom->pd_serialize);
 943
 944         KKASSERT(i >= 0 && i < acpi_npstates);
 945         pstate = &acpi_pstates[i];
 946
 947         done = 0;
 948         LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
 949                 if (!done) {
 950                         error = acpi_pst_set_pstate(sc, pstate);
 951                         if (error) {
 952                                 device_printf(sc->pst_dev, "can't set "
 953                                               "freq %d\n", pstate->st_freq);
 954                                 /* XXX error cleanup? */
 955                         }
 956                         if (dom->pd_coord == ACPI_PSD_COORD_SWANY)
 957                                 done = 1;
 958                 }
 959                 sc->pst_state = i;
 960         }
 961         dom->pd_state = i;
 962
 963         if (global != NULL)
 964                 *global = i;
 965 }
 966
 967 static void
 968 acpi_pst_domain_set_pstate(struct acpi_pst_domain *dom, int i, int *global)
 969 {
 970         lwkt_serialize_enter(&dom->pd_serialize);
 971         acpi_pst_domain_set_pstate_locked(dom, i, global);
 972         lwkt_serialize_exit(&dom->pd_serialize);
 973 }
 974
 975 static void
 976 acpi_pst_global_set_pstate(int i)
 977 {
 978         struct acpi_pst_domain *dom;
 979         int *global = &acpi_pst_global_state;
 980
 981         LIST_FOREACH(dom, &acpi_pst_domains, pd_link) {
 982                 /* Skip dead domain */
 983                 if (dom->pd_flags & ACPI_PSTDOM_FLAG_DEAD)
 984                         continue;
 985                 acpi_pst_domain_set_pstate(dom, i, global);
 986                 global = NULL;
 987         }
 988 }
 989
 990 static void
 991 acpi_pst_global_fixup_pstate(void)
 992 {
 993         struct acpi_pst_domain *dom;
 994         int *global = &acpi_pst_global_state;
 995         int sstart, scount;
 996
 997         sstart = acpi_pstate_start;
 998         scount = acpi_pstate_count;
 999
1000         LIST_FOREACH(dom, &acpi_pst_domains, pd_link) {
1001                 int i = -1;
1002
1003                 /* Skip dead domain */
1004                 if (dom->pd_flags & ACPI_PSTDOM_FLAG_DEAD)
1005                         continue;
1006
1007                 lwkt_serialize_enter(&dom->pd_serialize);
1008
1009                 if (global != NULL) {
1010                         if (*global < sstart)
1011                                 *global = sstart;
1012                         else if (*global >= scount)
1013                                 *global = scount - 1;
1014                         global = NULL;
1015                 }
1016                 if (dom->pd_state < sstart)
1017                         i = sstart;
1018                 else if (dom->pd_state >= scount)
1019                         i = scount - 1;
1020                 if (i >= 0)
1021                         acpi_pst_domain_set_pstate_locked(dom, i, NULL);
1022
1023                 lwkt_serialize_exit(&dom->pd_serialize);
1024         }
1025 }
1026
1027 static void
1028 acpi_pst_postattach(void *arg __unused)
1029 {
1030         struct acpi_pst_domain *dom;
1031         struct acpi_cpu_softc *cpu;
1032         device_t *devices;
1033         int i, ndevices, error, has_domain;
1034
1035         devices = NULL;
1036         ndevices = 0;
1037         error = devclass_get_devices(acpi_pst_devclass, &devices, &ndevices);
1038         if (error)
1039                 return;
1040
1041         if (ndevices == 0)
1042                 return;
1043
1044         cpu = NULL;
1045         for (i = 0; i < ndevices; ++i) {
1046                 cpu = device_get_softc(device_get_parent(devices[i]));
1047                 if (cpu->glob_sysctl_tree != NULL)
1048                         break;
1049         }
1050         kfree(devices, M_TEMP);
1051         KKASSERT(cpu != NULL);
1052
1053         if (acpi_pst_md == NULL)
1054                 kprintf("ACPI: no P-State CPU driver\n");
1055
1056         has_domain = 0;
1057         LIST_FOREACH(dom, &acpi_pst_domains, pd_link) {
1058                 struct acpi_pst_softc *sc;
1059                 char buf[32];
1060
1061                 dom->pd_state = acpi_pstate_start;
1062
1063                 /*
1064                  * Make sure that all processors belonging to this
1065                  * domain are located.
1066                  */
1067                 i = 0;
1068                 LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
1069                         sc->pst_state = acpi_pstate_start;
1070                         ++i;
1071                 }
1072                 if (i != dom->pd_nproc) {
1073                         KKASSERT(i < dom->pd_nproc);
1074
1075                         kprintf("ACPI: domain%u misses processors, "
1076                                 "should be %d, got %d\n", dom->pd_dom,
1077                                 dom->pd_nproc, i);
1078                         if (dom->pd_coord == ACPI_PSD_COORD_SWALL) {
1079                                 /*
1080                                  * If this domain's coordination is
1081                                  * SWALL and we don't see all of the
1082                                  * member CPUs of this domain, then
1083                                  * the P-State transition will never
1084                                  * be completed, so just leave this
1085                                  * domain out.
1086                                  */
1087                                 dom->pd_flags |= ACPI_PSTDOM_FLAG_DEAD;
1088                                 continue;
1089                         }
1090                         dom->pd_nproc = i;
1091                 }
1092
1093                 /*
1094                  * Validate P-State configurations for this domain
1095                  */
1096                 LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
1097                         error = acpi_pst_check_csr(sc);
1098                         if (error)
1099                                 break;
1100
1101                         error = acpi_pst_check_pstates(sc);
1102                         if (error)
1103                                 break;
1104                 }
1105                 if (sc != NULL) {
1106                         kprintf("ACPI: domain%u P-State configuration "
1107                                 "check failed\n", dom->pd_dom);
1108                         dom->pd_flags |= ACPI_PSTDOM_FLAG_DEAD;
1109                         continue;
1110                 }
1111
1112                 /*
1113                  * Do necssary P-State initialization
1114                  */
1115                 LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
1116                         error = acpi_pst_init(sc);
1117                         if (error)
1118                                 break;
1119                 }
1120                 if (sc != NULL) {
1121                         kprintf("ACPI: domain%u P-State initialization "
1122                                 "check failed\n", dom->pd_dom);
1123                         dom->pd_flags |= ACPI_PSTDOM_FLAG_DEAD;
1124                         continue;
1125                 }
1126
1127                 has_domain = 1;
1128
1129                 ksnprintf(buf, sizeof(buf), "px_dom%u", dom->pd_dom);
1130
1131                 sysctl_ctx_init(&dom->pd_sysctl_ctx);
1132                 dom->pd_sysctl_tree =
1133                 SYSCTL_ADD_NODE(&dom->pd_sysctl_ctx,
1134                         SYSCTL_CHILDREN(cpu->glob_sysctl_tree),
1135                         OID_AUTO, buf, CTLFLAG_RD, 0,
1136                         "P-State domain");
1137                 if (dom->pd_sysctl_tree == NULL) {
1138                         kprintf("ACPI: Can't create sysctl tree for domain%u",
1139                                 dom->pd_dom);
1140                         continue;
1141                 }
1142
1143                 SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
1144                                 SYSCTL_CHILDREN(dom->pd_sysctl_tree),
1145                                 OID_AUTO, "available",
1146                                 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP,
1147                                 dom, 0, acpi_pst_sysctl_freqs, "A",
1148                                 "available frequencies");
1149
1150                 SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
1151                                 SYSCTL_CHILDREN(dom->pd_sysctl_tree),
1152                                 OID_AUTO, "avail",
1153                                 CTLTYPE_OPAQUE | CTLFLAG_RD,
1154                                 dom, 0, acpi_pst_sysctl_freqs_bin, "IU",
1155                                 "available frequencies");
1156
1157                 SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
1158                                 SYSCTL_CHILDREN(dom->pd_sysctl_tree),
1159                                 OID_AUTO, "power",
1160                                 CTLTYPE_OPAQUE | CTLFLAG_RD,
1161                                 dom, 0, acpi_pst_sysctl_power, "IU",
1162                                 "power of available frequencies");
1163
1164                 SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
1165                                 SYSCTL_CHILDREN(dom->pd_sysctl_tree),
1166                                 OID_AUTO, "members",
1167                                 CTLTYPE_STRING | CTLFLAG_RD,
1168                                 dom, 0, acpi_pst_sysctl_members, "A",
1169                                 "member cpus");
1170
1171                 if (acpi_pst_md != NULL &&
1172                     acpi_pst_md->pmd_set_pstate != NULL) {
1173                         SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
1174                                         SYSCTL_CHILDREN(dom->pd_sysctl_tree),
1175                                         OID_AUTO, "select",
1176                                         CTLTYPE_UINT | CTLFLAG_RW,
1177                                         dom, 0, acpi_pst_sysctl_select,
1178                                         "IU", "select freq");
1179                 }
1180         }
1181
1182         if (has_domain && acpi_pst_md != NULL &&
1183             acpi_pst_md->pmd_set_pstate != NULL) {
1184                 SYSCTL_ADD_PROC(&cpu->glob_sysctl_ctx,
1185                                 SYSCTL_CHILDREN(cpu->glob_sysctl_tree),
1186                                 OID_AUTO, "px_global",
1187                                 CTLTYPE_UINT | CTLFLAG_RW,
1188                                 NULL, 0, acpi_pst_sysctl_global,
1189                                 "IU", "select freq for all domains");
1190                 SYSCTL_ADD_INT(&cpu->glob_sysctl_ctx,
1191                                SYSCTL_CHILDREN(cpu->glob_sysctl_tree),
1192                                OID_AUTO, "px_handle_notify", CTLFLAG_RW,
1193                                &acpi_pst_handle_notify, 0,
1194                                "handle type 0x80 notify");
1195
1196                 acpi_pst_global_set_pstate(acpi_pstate_start);
1197         }
1198 }
1199
1200 static int
1201 acpi_pst_sysctl_freqs(SYSCTL_HANDLER_ARGS)
1202 {
1203         int i, error, sstart, scount;
1204
1205         error = 0;
1206         sstart = acpi_pstate_start;
1207         scount = acpi_pstate_count;
1208         for (i = 0; i < acpi_npstates; ++i) {
1209                 if (error == 0 && i)
1210                         error = SYSCTL_OUT(req, " ", 1);
1211                 if (error == 0) {
1212                         const char *pat;
1213                         char buf[32];
1214
1215                         if (i < sstart || i >= scount)
1216                                 pat = "(%u)";
1217                         else
1218                                 pat = "%u";
1219
1220                         ksnprintf(buf, sizeof(buf), pat,
1221                                   acpi_pstates[i].st_freq);
1222                         error = SYSCTL_OUT(req, buf, strlen(buf));
1223                 }
1224         }
1225         return error;
1226 }
1227
1228 static int
1229 acpi_pst_sysctl_freqs_bin(SYSCTL_HANDLER_ARGS)
1230 {
1231         uint32_t freqs[ACPI_NPSTATE_MAX];
1232         int cnt, i, sstart, scount;
1233
1234         sstart = acpi_pstate_start;
1235         scount = acpi_pstate_count;
1236
1237         cnt = scount - sstart;
1238         for (i = 0; i < cnt; ++i)
1239                 freqs[i] = acpi_pstates[sstart + i].st_freq;
1240
1241         return sysctl_handle_opaque(oidp, freqs, cnt * sizeof(freqs[0]), req);
1242 }
1243
1244 static int
1245 acpi_pst_sysctl_power(SYSCTL_HANDLER_ARGS)
1246 {
1247         uint32_t power[ACPI_NPSTATE_MAX];
1248         int cnt, i, sstart, scount;
1249
1250         sstart = acpi_pstate_start;
1251         scount = acpi_pstate_count;
1252
1253         cnt = scount - sstart;
1254         for (i = 0; i < cnt; ++i)
1255                 power[i] = acpi_pstates[sstart + i].st_power;
1256
1257         return sysctl_handle_opaque(oidp, power, cnt * sizeof(power[0]), req);
1258 }
1259
1260 static int
1261 acpi_pst_sysctl_members(SYSCTL_HANDLER_ARGS)
1262 {
1263         struct acpi_pst_domain *dom = arg1;
1264         struct acpi_pst_softc *sc;
1265         int loop, error;
1266
1267         loop = error = 0;
1268         LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
1269                 char buf[32];
1270
1271                 if (error == 0 && loop)
1272                         error = SYSCTL_OUT(req, " ", 1);
1273                 if (error == 0) {
1274                         ksnprintf(buf, sizeof(buf), "cpu%d", sc->pst_cpuid);
1275                         error = SYSCTL_OUT(req, buf, strlen(buf));
1276                 }
1277
1278                 if (error == 0 && acpi_pst_md && acpi_pst_md->pmd_get_pstate) {
1279                         const struct acpi_pstate *pstate;
1280                         const char *str;
1281
1282                         pstate = acpi_pst_get_pstate(sc);
1283                         if (pstate == NULL) {
1284                                 str = "(*)";
1285                         } else {
1286                                 ksnprintf(buf, sizeof(buf), "(%d)",
1287                                           pstate->st_freq);
1288                                 str = buf;
1289                         }
1290                         error = SYSCTL_OUT(req, str, strlen(str));
1291                 }
1292                 ++loop;
1293         }
1294         return error;
1295 }
1296
1297 static int
1298 acpi_pst_sysctl_select(SYSCTL_HANDLER_ARGS)
1299 {
1300         struct acpi_pst_domain *dom = arg1;
1301         int error, i, freq;
1302
1303         KKASSERT(dom->pd_state >= 0 && dom->pd_state < acpi_npstates);
1304
1305         freq = acpi_pstates[dom->pd_state].st_freq;
1306
1307         error = sysctl_handle_int(oidp, &freq, 0, req);
1308         if (error || req->newptr == NULL)
1309                 return error;
1310
1311         i = acpi_pst_freq2index(freq);
1312         if (i < 0)
1313                 return EINVAL;
1314
1315         acpi_pst_domain_set_pstate(dom, i, NULL);
1316         return 0;
1317 }
1318
1319 static int
1320 acpi_pst_sysctl_global(SYSCTL_HANDLER_ARGS)
1321 {
1322         int error, i, freq;
1323
1324         KKASSERT(acpi_pst_global_state >= 0 &&
1325                  acpi_pst_global_state < acpi_npstates);
1326
1327         freq = acpi_pstates[acpi_pst_global_state].st_freq;
1328
1329         error = sysctl_handle_int(oidp, &freq, 0, req);
1330         if (error || req->newptr == NULL)
1331                 return error;
1332
1333         i = acpi_pst_freq2index(freq);
1334         if (i < 0)
1335                 return EINVAL;
1336
1337         acpi_pst_global_set_pstate(i);
1338
1339         return 0;
1340 }
1341
1342 static void
1343 acpi_pst_check_csr_handler(struct cpuhelper_msg *msg)
1344 {
1345         struct acpi_pst_chmsg *pmsg = (struct acpi_pst_chmsg *)msg;
1346         int error;
1347
1348         error = acpi_pst_md->pmd_check_csr(pmsg->ch_ctrl, pmsg->ch_status);
1349         cpuhelper_replymsg(msg, error);
1350 }
1351
1352 static int
1353 acpi_pst_check_csr(struct acpi_pst_softc *sc)
1354 {
1355         struct acpi_pst_chmsg msg;
1356
1357         if (acpi_pst_md == NULL)
1358                 return 0;
1359
1360         cpuhelper_initmsg(&msg.ch_msg, &curthread->td_msgport,
1361             acpi_pst_check_csr_handler, NULL, MSGF_PRIORITY);
1362         msg.ch_ctrl = &sc->pst_creg;
1363         msg.ch_status = &sc->pst_sreg;
1364
1365         return (cpuhelper_domsg(&msg.ch_msg, sc->pst_cpuid));
1366 }
1367
1368 static void
1369 acpi_pst_check_pstates_handler(struct cpuhelper_msg *msg)
1370 {
1371         int error;
1372
1373         error = acpi_pst_md->pmd_check_pstates(acpi_pstates, acpi_npstates);
1374         cpuhelper_replymsg(msg, error);
1375 }
1376
1377 static int
1378 acpi_pst_check_pstates(struct acpi_pst_softc *sc)
1379 {
1380         struct cpuhelper_msg msg;
1381
1382         if (acpi_pst_md == NULL)
1383                 return 0;
1384
1385         cpuhelper_initmsg(&msg, &curthread->td_msgport,
1386             acpi_pst_check_pstates_handler, NULL, MSGF_PRIORITY);
1387         return (cpuhelper_domsg(&msg, sc->pst_cpuid));
1388 }
1389
1390 static void
1391 acpi_pst_init_handler(struct cpuhelper_msg *msg)
1392 {
1393         struct acpi_pst_chmsg *pmsg = (struct acpi_pst_chmsg *)msg;
1394         int error;
1395
1396         error = acpi_pst_md->pmd_init(pmsg->ch_ctrl, pmsg->ch_status);
1397         cpuhelper_replymsg(msg, error);
1398 }
1399
1400 static int
1401 acpi_pst_init(struct acpi_pst_softc *sc)
1402 {
1403         struct acpi_pst_chmsg msg;
1404
1405         if (acpi_pst_md == NULL)
1406                 return 0;
1407
1408         cpuhelper_initmsg(&msg.ch_msg, &curthread->td_msgport,
1409             acpi_pst_init_handler, NULL, MSGF_PRIORITY);
1410         msg.ch_ctrl = &sc->pst_creg;
1411         msg.ch_status = &sc->pst_sreg;
1412
1413         return (cpuhelper_domsg(&msg.ch_msg, sc->pst_cpuid));
1414 }
1415
1416 static void
1417 acpi_pst_set_pstate_handler(struct cpuhelper_msg *msg)
1418 {
1419         struct acpi_pst_chmsg *pmsg = (struct acpi_pst_chmsg *)msg;
1420         int error;
1421
1422         error = acpi_pst_md->pmd_set_pstate(pmsg->ch_ctrl, pmsg->ch_status,
1423             msg->ch_cbarg);
1424         cpuhelper_replymsg(msg, error);
1425 }
1426
1427 static int
1428 acpi_pst_set_pstate(struct acpi_pst_softc *sc, const struct acpi_pstate *pstate)
1429 {
1430         struct acpi_pst_chmsg msg;
1431
1432         KKASSERT(acpi_pst_md != NULL);
1433
1434         cpuhelper_initmsg(&msg.ch_msg, &curthread->td_msgport,
1435             acpi_pst_set_pstate_handler, __DECONST(void *, pstate),
1436             MSGF_PRIORITY);
1437         msg.ch_ctrl = &sc->pst_creg;
1438         msg.ch_status = &sc->pst_sreg;
1439
1440         return (cpuhelper_domsg(&msg.ch_msg, sc->pst_cpuid));
1441 }
1442
1443 static void
1444 acpi_pst_get_pstate_handler(struct cpuhelper_msg *msg)
1445 {
1446         struct acpi_pst_chmsg *pmsg = (struct acpi_pst_chmsg *)msg;
1447         const struct acpi_pstate *pstate;
1448
1449         pstate = acpi_pst_md->pmd_get_pstate(pmsg->ch_status, acpi_pstates,
1450             acpi_npstates);
1451         msg->ch_cbarg = __DECONST(void *, pstate);
1452         cpuhelper_replymsg(msg, 0);
1453 }
1454
1455 static const struct acpi_pstate *
1456 acpi_pst_get_pstate(struct acpi_pst_softc *sc)
1457 {
1458         struct acpi_pst_chmsg msg;
1459
1460         if (acpi_pst_md == NULL)
1461                 return 0;
1462
1463         cpuhelper_initmsg(&msg.ch_msg, &curthread->td_msgport,
1464             acpi_pst_get_pstate_handler, NULL, MSGF_PRIORITY);
1465         msg.ch_status = &sc->pst_sreg;
1466
1467         cpuhelper_domsg(&msg.ch_msg, sc->pst_cpuid);
1468         return (msg.ch_msg.ch_cbarg);
1469 }
1470
1471 static int
1472 acpi_pst_alloc_resource(device_t dev, ACPI_OBJECT *obj, int idx,
1473                         struct acpi_pst_res *res)
1474 {
1475         struct acpi_pst_softc *sc = device_get_softc(dev);
1476         int error, type;
1477
1478         /* Save GAS */
1479         error = acpi_PkgRawGas(obj, idx, &res->pr_gas);
1480         if (error)
1481                 return error;
1482
1483         /* Allocate resource, if possible */
1484         res->pr_rid = sc->pst_parent->cpu_next_rid;
1485         acpi_bus_alloc_gas(dev, &type, &res->pr_rid, &res->pr_gas, &res->pr_res, 0);
1486         if (res->pr_res != NULL) {
1487                 sc->pst_parent->cpu_next_rid++;
1488                 res->pr_bt = rman_get_bustag(res->pr_res);
1489                 res->pr_bh = rman_get_bushandle(res->pr_res);
1490         } else {
1491                 res->pr_rid = 0;
1492         }
1493         return 0;
1494 }
1495
1496 static void
1497 acpi_pst_domain_check_nproc(device_t dev, struct acpi_pst_domain *dom)
1498 {
1499         struct acpi_pst_softc *pst;
1500         int i;
1501
1502         i = 0;
1503         LIST_FOREACH(pst, &dom->pd_pstlist, pst_link)
1504                 ++i;
1505         if (i == dom->pd_nproc) {
1506                 /*
1507                  * Some stupid BIOSes will set wrong "# of processors",
1508                  * e.g. 1 for CPU w/ hyperthreading; Be lenient here.
1509                  */
1510                 if (bootverbose) {
1511                         device_printf(dev, "domain%u already contains %d "
1512                             "P-States\n", dom->pd_dom, dom->pd_nproc);
1513                 }
1514                 dom->pd_nproc++;
1515         }
1516         KKASSERT(i < dom->pd_nproc);
1517 }
1518
1519 static int
1520 acpi_pst_eval_ppc(struct acpi_pst_softc *sc, int *sstart)
1521 {
1522         ACPI_BUFFER buf;
1523         ACPI_STATUS status;
1524         ACPI_OBJECT *obj;
1525
1526         buf.Pointer = NULL;
1527         buf.Length = ACPI_ALLOCATE_BUFFER;
1528         status = AcpiEvaluateObject(sc->pst_handle, "_PPC", NULL, &buf);
1529         if (!ACPI_FAILURE(status)) {
1530                 ACPI_OBJECT_LIST arglist;
1531                 ACPI_OBJECT arg[2];
1532
1533                 obj = (ACPI_OBJECT *)buf.Pointer;
1534                 if (obj->Type == ACPI_TYPE_INTEGER) {
1535                         if (obj->Integer.Value >= acpi_npstates) {
1536                                 /*
1537                                  * Some broken BIOSes can set this
1538                                  * to any values; let's simply start
1539                                  * from the first P-state, which
1540                                  * should be the highest frequency
1541                                  * available.
1542                                  */
1543                                 if (bootverbose) {
1544                                         device_printf(sc->pst_dev,
1545                                             "Invalid _PPC value %ju, "
1546                                             "npstates %d\n",
1547                                             (uintmax_t)obj->Integer.Value,
1548                                             acpi_npstates);
1549                                 }
1550                                 *sstart = 0;
1551                         } else {
1552                                 *sstart = obj->Integer.Value;
1553                                 if (bootverbose) {
1554                                         device_printf(sc->pst_dev, "_PPC %d\n",
1555                                             *sstart);
1556                                 }
1557                         }
1558                 } else {
1559                         device_printf(sc->pst_dev, "Invalid _PPC object\n");
1560                         AcpiOsFree(obj);
1561                         return ENXIO;
1562                 }
1563
1564                 /* Free _PPC */
1565                 AcpiOsFree(obj);
1566
1567                 /* _PPC has been successfully processed */
1568                 arglist.Pointer = arg;
1569                 arglist.Count = 2;
1570                 arg[0].Type = ACPI_TYPE_INTEGER;
1571                 arg[0].Integer.Value = 0x80;
1572                 arg[1].Type = ACPI_TYPE_INTEGER;
1573                 arg[1].Integer.Value = 0;
1574                 AcpiEvaluateObject(sc->pst_handle, "_OST", &arglist, NULL);
1575
1576                 return 0;
1577         }
1578         return ENOENT;
1579 }
1580
1581 static int
1582 acpi_pst_eval_pdl(struct acpi_pst_softc *sc, int *scount)
1583 {
1584         ACPI_BUFFER buf;
1585         ACPI_STATUS status;
1586         ACPI_OBJECT *obj;
1587
1588         buf.Pointer = NULL;
1589         buf.Length = ACPI_ALLOCATE_BUFFER;
1590         status = AcpiEvaluateObject(sc->pst_handle, "_PDL", NULL, &buf);
1591         if (!ACPI_FAILURE(status)) {
1592                 obj = (ACPI_OBJECT *)buf.Pointer;
1593                 if (obj->Type == ACPI_TYPE_INTEGER) {
1594                         if (obj->Integer.Value >= acpi_npstates) {
1595                                 device_printf(sc->pst_dev,
1596                                     "Invalid _PDL value\n");
1597                                 AcpiOsFree(obj);
1598                                 return ENXIO;
1599                         }
1600                         if (obj->Integer.Value >= acpi_pstate_start) {
1601                                 *scount = obj->Integer.Value + 1;
1602                                 if (bootverbose) {
1603                                         device_printf(sc->pst_dev, "_PDL %d\n",
1604                                             *scount);
1605                                 }
1606                         } else {
1607                                 /* Prefer _PPC as stated in ACPI 5.1 8.4.4.6 */
1608                                 device_printf(sc->pst_dev, "conflict _PDL %ju "
1609                                     "and _PPC %d, ignore\n",
1610                                     (uintmax_t)obj->Integer.Value,
1611                                     acpi_pstate_start);
1612                         }
1613                 } else {
1614                         device_printf(sc->pst_dev, "Invalid _PDL object\n");
1615                         AcpiOsFree(obj);
1616                         return ENXIO;
1617                 }
1618
1619                 /* Free _PDL */
1620                 AcpiOsFree(obj);
1621
1622                 return 0;
1623         }
1624         return ENOENT;
1625 }
1626
1627 /*
1628  * Notify is serialized by acpi task thread.
1629  */
1630 static void
1631 acpi_pst_notify(device_t dev)
1632 {
1633         struct acpi_pst_softc *sc = device_get_softc(dev);
1634         boolean_t fixup = FALSE;
1635
1636         if (!acpi_pst_handle_notify)
1637                 return;
1638
1639         /*
1640          * NOTE:
1641          * _PPC and _PDL evaluation order is critical.  _PDL
1642          * evaluation depends on _PPC evaluation.
1643          */
1644         if (sc->pst_flags & ACPI_PST_FLAG_PPC) {
1645                 int sstart = acpi_pstate_start;
1646
1647                 acpi_pst_eval_ppc(sc, &sstart);
1648                 if (acpi_pstate_start != sstart && sc->pst_cpuid == 0) {
1649                         acpi_pstate_start = sstart;
1650                         fixup = TRUE;
1651                 }
1652         }
1653         if (sc->pst_flags & ACPI_PST_FLAG_PDL) {
1654                 int scount = acpi_pstate_count;
1655
1656                 acpi_pst_eval_pdl(sc, &scount);
1657                 if (acpi_pstate_count != scount && sc->pst_cpuid == 0) {
1658                         acpi_pstate_count = scount;
1659                         fixup = TRUE;
1660                 }
1661         }
1662
1663         if (fixup && acpi_pst_md != NULL &&
1664             acpi_pst_md->pmd_set_pstate != NULL)
1665                 acpi_pst_global_fixup_pstate();
1666 }