ich777/CoreFreq
1
0
Fork 0
mirror of https://github.com/cyring/CoreFreq.git synced 2025-07-23 12:13:07 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
3a7c7033f2bc99fc80677ae18c9acae55bbd34e9
CoreFreq/x86_64/corefreqk.c

25461 lines
669 KiB
C
Raw Blame History

/*
* CoreFreq
* Copyright (C) 2015-2025 CYRIL COURTIAT
* Licenses: GPL2
*
* Time Capsule[12.24.2024]
* Cyril to Christiane Courtiat
* Love you Mum; rest in peace
*
* Upadacitinib [07.29.2024]
* Infliximab [11.17.2023]
* Vedolizumab [05.25.2023]
*
* CYRIL INGENIERIE[11.30.2022]
* Company closed down
*
* Time Capsule[02.02.2022]
* Cyril to Marcel Courtiat
* RIP Daddy; love you forever
*/
#include <linux/version.h>
#include <linux/module.h>
#include <linux/cpu.h>
#include <linux/pci.h>
#ifdef CONFIG_DMI
#include <linux/dmi.h>
#endif /* CONFIG_DMI */
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/cdev.h>
#include <linux/device.h>
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/percpu.h>
#include <linux/utsname.h>
#ifdef CONFIG_CPU_IDLE
#include <linux/cpuidle.h>
#endif /* CONFIG_CPU_IDLE */
#ifdef CONFIG_CPU_FREQ
#include <linux/cpufreq.h>
#endif /* CONFIG_CPU_FREQ */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
#include <linux/sched/signal.h>
#endif /* KERNEL_VERSION(4, 11, 0) */
#include <linux/clocksource.h>
#include <asm/msr.h>
#ifdef CONFIG_HAVE_NMI
#include <asm/nmi.h>
#endif
#ifdef CONFIG_XEN
#include <xen/xen.h>
#endif /* CONFIG_XEN */
#include <asm/mwait.h>
#ifdef CONFIG_AMD_NB
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 16, 0)
#include <asm/amd/nb.h>
#else
#include <asm/amd_nb.h>
#endif
#endif /* CONFIG_AMD_NB */
#ifdef CONFIG_ACPI
#include <linux/acpi.h>
#include <acpi/processor.h>
#endif
#ifdef CONFIG_ACPI_CPPC_LIB
#include <acpi/cppc_acpi.h>
#endif
#include "bitasm.h"
#include "amd_reg.h"
#include "intel_reg.h"
#include "coretypes.h"
#include "corefreq-api.h"
#include "corefreqk.h"
MODULE_AUTHOR ("CYRIL COURTIAT <labs[at]cyring[dot]fr>");
MODULE_DESCRIPTION ("CoreFreq Processor Driver");
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 12, 0)
MODULE_SUPPORTED_DEVICE ("Intel,AMD,HYGON");
#endif
MODULE_LICENSE ("GPL");
MODULE_VERSION (COREFREQ_VERSION);
static signed int ArchID = -1;
module_param(ArchID, int, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(ArchID, "Force an architecture (ID)");
static signed int AutoClock = 0b11;
module_param(AutoClock, int, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(AutoClock, "Estimate Clock Frequency 0:Spec; 1:Once; 2:Auto");
static unsigned int SleepInterval = 0;
module_param(SleepInterval, uint, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(SleepInterval, "Timer interval (ms)");
static unsigned int TickInterval = 0;
module_param(TickInterval, uint, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(TickInterval, "System requested interval (ms)");
static signed int Experimental = 0;
module_param(Experimental, int, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Experimental, "Enable features under development");
static signed int CPU_Count = -1;
module_param(CPU_Count, int, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(CPU_Count, "-1:Kernel(default); 0:Hardware; >0: User value");
static signed short Target_Ratio_Unlock = -1;
module_param(Target_Ratio_Unlock, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Target_Ratio_Unlock, "1:Target Ratio Unlock; 0:Lock");
static signed short Clock_Ratio_Unlock = -1;
module_param(Clock_Ratio_Unlock, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Clock_Ratio_Unlock, "1:MinRatio; 2:MaxRatio; 3:Both Unlock");
static signed short Turbo_Ratio_Unlock = -1;
module_param(Turbo_Ratio_Unlock, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Turbo_Ratio_Unlock, "1:Turbo Ratio Unlock; 0:Lock");
static signed short Uncore_Ratio_Unlock = -1;
module_param(Uncore_Ratio_Unlock, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Uncore_Ratio_Unlock, "1:Uncore Ratio Unlock; 0:Lock");
static signed int ServiceProcessor = -1; /* -1=ANY ; 0=BSP */
module_param(ServiceProcessor, int, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(ServiceProcessor, "Select a CPU to run services with");
static SERVICE_PROC DefaultSMT = RESET_SERVICE;
static unsigned short RDPMC_Enable = 0;
module_param(RDPMC_Enable, ushort, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(RDPMC_Enable, "Enable RDPMC bit in CR4 register");
static unsigned short NMI_Disable = 1;
module_param(NMI_Disable, ushort, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(NMI_Disable, "Disable the NMI Handler");
static int Override_SubCstate_Depth = 0;
static unsigned short Override_SubCstate[CPUIDLE_STATE_MAX] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0
};
module_param_array( Override_SubCstate,ushort, &Override_SubCstate_Depth, \
S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Override_SubCstate, "Override Sub C-States");
static signed short PkgCStateLimit = -1;
module_param(PkgCStateLimit, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(PkgCStateLimit, "Package C-State Limit");
static signed short IOMWAIT_Enable = -1;
module_param(IOMWAIT_Enable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(IOMWAIT_Enable, "I/O MWAIT Redirection Enable");
static signed short CStateIORedir = -1;
module_param(CStateIORedir, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(CStateIORedir, "Power Mgmt IO Redirection C-State");
static signed short Config_TDP_Level = -1;
module_param(Config_TDP_Level, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Config_TDP_Level, "Config TDP Control Level");
static unsigned int Custom_TDP_Count = 0;
static signed short Custom_TDP_Offset[PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE)] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0
};
module_param_array(Custom_TDP_Offset, short, &Custom_TDP_Count , \
S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Custom_TDP_Offset, "TDP Limit Offset (watt)");
static unsigned int PowerTimeWindow_Count = 0;
static signed short PowerTimeWindow[PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE)] = {
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1
};
module_param_array(PowerTimeWindow, short, &PowerTimeWindow_Count, \
S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(PowerTimeWindow, "Power Limit PL(n) Time Window");
static unsigned int TDP_Limiting_Count = 0;
static signed short Activate_TDP_Limit[PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE)] = {
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1
};
module_param_array(Activate_TDP_Limit, short, &TDP_Limiting_Count, \
S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Activate_TDP_Limit, "Activate TDP Limiting");
static unsigned int TDP_Clamping_Count = 0;
static signed short Activate_TDP_Clamp[PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE)] = {
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1
};
module_param_array(Activate_TDP_Clamp, short, &TDP_Clamping_Count, \
S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Activate_TDP_Clamp, "Activate TDP Clamping");
static unsigned short Custom_TDC_Offset = 0;
module_param(Custom_TDC_Offset, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Custom_TDC_Offset, "TDC Limit Offset (amp)");
static signed short Activate_TDC_Limit = -1;
module_param(Activate_TDC_Limit, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Activate_TDC_Limit, "Activate TDC Limiting");
static signed short L1_HW_PREFETCH_Disable = -1;
module_param(L1_HW_PREFETCH_Disable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(L1_HW_PREFETCH_Disable, "Disable L1 HW Prefetcher");
static signed short L1_HW_IP_PREFETCH_Disable = -1;
module_param(L1_HW_IP_PREFETCH_Disable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(L1_HW_IP_PREFETCH_Disable, "Disable L1 HW IP Prefetcher");
static signed short L1_NPP_PREFETCH_Disable = -1;
module_param(L1_NPP_PREFETCH_Disable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(L1_NPP_PREFETCH_Disable, "Disable L1 NPP Prefetcher");
static signed short L1_Scrubbing_Enable = -1;
module_param(L1_Scrubbing_Enable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(L1_Scrubbing_Enable, "Enable L1 Scrubbing");
static signed short L2_HW_PREFETCH_Disable = -1;
module_param(L2_HW_PREFETCH_Disable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(L2_HW_PREFETCH_Disable, "Disable L2 HW Prefetcher");
static signed short L2_HW_CL_PREFETCH_Disable = -1;
module_param(L2_HW_CL_PREFETCH_Disable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(L2_HW_CL_PREFETCH_Disable, "Disable L2 HW CL Prefetcher");
static signed short L2_AMP_PREFETCH_Disable = -1;
module_param(L2_AMP_PREFETCH_Disable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(L2_AMP_PREFETCH_Disable, "Adaptive Multipath Probability");
static signed short L2_NLP_PREFETCH_Disable = -1;
module_param(L2_NLP_PREFETCH_Disable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(L2_NLP_PREFETCH_Disable, "Disable L2 NLP Prefetcher");
static signed short L1_STRIDE_PREFETCH_Disable = -1;
module_param(L1_STRIDE_PREFETCH_Disable, short,S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(L1_STRIDE_PREFETCH_Disable, "Disable L1 Stride Prefetcher");
static signed short L1_REGION_PREFETCH_Disable = -1;
module_param(L1_REGION_PREFETCH_Disable, short,S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(L1_REGION_PREFETCH_Disable, "Disable L1 Region Prefetcher");
static signed short L1_BURST_PREFETCH_Disable = -1;
module_param(L1_BURST_PREFETCH_Disable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(L1_BURST_PREFETCH_Disable, "Disable L1 Burst Prefetcher");
static signed short L2_STREAM_PREFETCH_Disable = -1;
module_param(L2_STREAM_PREFETCH_Disable, short,S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(L2_STREAM_PREFETCH_Disable, "Disable L2 Stream Prefetcher");
static signed short L2_UPDOWN_PREFETCH_Disable = -1;
module_param(L2_UPDOWN_PREFETCH_Disable, short,S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(L2_UPDOWN_PREFETCH_Disable, "Disable L2 Up/Down Prefetcher");
static signed short LLC_Streamer_Disable = -1;
module_param(LLC_Streamer_Disable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(LLC_Streamer_Disable, "Disable LLC Streamer");
static signed short SpeedStep_Enable = -1;
module_param(SpeedStep_Enable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(SpeedStep_Enable, "Enable SpeedStep");
static signed short C1E_Enable = -1;
module_param(C1E_Enable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(C1E_Enable, "Enable SpeedStep C1E");
static unsigned int TurboBoost_Enable_Count = 1;
static signed short TurboBoost_Enable[2] = {-1, -1};
module_param_array(TurboBoost_Enable, short, &TurboBoost_Enable_Count, \
S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(TurboBoost_Enable, "Enable Turbo Boost");
static signed short C3A_Enable = -1;
module_param(C3A_Enable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(C3A_Enable, "Enable C3 Auto Demotion");
static signed short C1A_Enable = -1;
module_param(C1A_Enable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(C1A_Enable, "Enable C3 Auto Demotion");
static signed short C3U_Enable = -1;
module_param(C3U_Enable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(C3U_Enable, "Enable C3 UnDemotion");
static signed short C1U_Enable = -1;
module_param_named(C1U_Enable,C1U_Enable,short,S_IRUSR|S_IRGRP|S_IROTH);
module_param_named(C2U_Enable,C1U_Enable,short,S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(C1U_Enable, "Enable C1 UnDemotion");
MODULE_PARM_DESC(C2U_Enable, "Enable C2 UnDemotion");
static signed short CC6_Enable = -1;
module_param(CC6_Enable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(CC6_Enable, "Enable Core C6 State");
static signed short PC6_Enable = -1;
module_param(PC6_Enable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(PC6_Enable, "Enable Package C6 State");
static signed short ODCM_Enable = -1;
module_param(ODCM_Enable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(ODCM_Enable, "Enable On-Demand Clock Modulation");
static signed short ODCM_DutyCycle = -1;
module_param(ODCM_DutyCycle, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(ODCM_DutyCycle, "ODCM DutyCycle [0-7] | [0-14]");
static signed short PowerMGMT_Unlock = -1;
module_param(PowerMGMT_Unlock, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(PowerMGMT_Unlock, "Unlock Power Management");
static signed short PowerPolicy = -1;
module_param(PowerPolicy, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(PowerPolicy, "Power Policy Preference [0-15]");
static signed short Turbo_Activation_Ratio = -1;
module_param(Turbo_Activation_Ratio, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Turbo_Activation_Ratio, "Turbo Activation Ratio");
static signed int PState_FID = -1;
module_param(PState_FID, int, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(PState_FID, "P-State Frequency Id");
static signed int PState_VID = -1;
module_param(PState_VID, int, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(PState_VID, "P-State Voltage Id");
static enum RATIO_BOOST Ratio_Boost_Count = 0;
static signed int Ratio_Boost[BOOST(SIZE) - BOOST(18C)] = {
/* 18C */ -1,
/* 17C */ -1,
/* 16C */ -1,
/* 15C */ -1,
/* 14C */ -1,
/* 13C */ -1,
/* 12C */ -1,
/* 11C */ -1,
/* 10C */ -1,
/* 9C */ -1,
/* 8C */ -1,
/* 7C */ -1,
/* 6C */ -1,
/* 5C */ -1,
/* 4C */ -1,
/* 3C */ -1,
/* 2C */ -1,
/* 1C */ -1
};
module_param_array(Ratio_Boost, int, &Ratio_Boost_Count, \
S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Ratio_Boost, "Turbo Boost Frequency ratios");
static signed int Ratio_PPC = -1;
module_param(Ratio_PPC, int, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Ratio_PPC, "Target Performance ratio");
static signed short HWP_Enable = -1;
module_param(HWP_Enable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(HWP_Enable, "Hardware-Controlled Performance States");
static signed short HWP_EPP = -1;
module_param(HWP_EPP, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(HWP_EPP, "Energy Performance Preference");
static enum RATIO_BOOST Ratio_HWP_Count = 0;
static signed int Ratio_HWP[1 + (BOOST(HWP_TGT) - BOOST(HWP_MIN))] = {
/* HWP_MIN */ -1,
/* HWP_MAX */ -1,
/* HWP_TGT */ -1
};
module_param_array(Ratio_HWP, int, &Ratio_HWP_Count, \
S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Ratio_HWP, "Hardware-Controlled Performance ratios");
static signed short HDC_Enable = -1;
module_param(HDC_Enable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(HDC_Enable, "Hardware Duty Cycling");
static signed short EEO_Disable = -1;
module_param(EEO_Disable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(EEO_Disable, "Disable Energy Efficiency Optimization");
static signed short R2H_Disable = -1;
module_param(R2H_Disable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(R2H_Disable, "Disable Race to Halt");
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
static unsigned long Clear_Events = 0;
module_param(Clear_Events, ulong, S_IRUSR|S_IRGRP|S_IROTH);
#else
static unsigned long long Clear_Events = 0;
module_param(Clear_Events, ullong, S_IRUSR|S_IRGRP|S_IROTH);
#endif
MODULE_PARM_DESC(Clear_Events, "Clear Thermal and Power Events");
static unsigned int ThermalPoint_Count = 0;
static signed short ThermalPoint[THM_POINTS_DIM] = {
-1, -1, -1, -1, -1
};
module_param_array(ThermalPoint, short, &ThermalPoint_Count, \
S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(ThermalPoint, "Thermal Point");
static unsigned int PkgThermalPoint_Count = 0;
static signed short PkgThermalPoint[THM_POINTS_DIM] = {
-1, -1, -1, -1, -1
};
module_param_array(PkgThermalPoint, short, &PkgThermalPoint_Count, \
S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(PkgThermalPoint, "Package Thermal Point");
static signed short ThermalOffset = 0;
module_param(ThermalOffset, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(ThermalOffset, "Thermal Offset");
static int ThermalScope = -1;
module_param(ThermalScope, int, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(ThermalScope, "[0:None; 1:SMT; 2:Core; 3:Package]");
static int VoltageScope = -1;
module_param(VoltageScope, int, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(VoltageScope, "[0:None; 1:SMT; 2:Core; 3:Package]");
static int PowerScope = -1;
module_param(PowerScope, int, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(PowerScope, "[0:None; 1:SMT; 2:Core; 3:Package]");
static signed short Register_CPU_Idle = -1;
module_param(Register_CPU_Idle, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Register_CPU_Idle, "Register the Kernel cpuidle driver");
static signed short Register_CPU_Freq = -1;
module_param(Register_CPU_Freq, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Register_CPU_Freq, "Register the Kernel cpufreq driver");
static signed short Register_Governor = -1;
module_param(Register_Governor, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Register_Governor, "Register the Kernel governor");
static signed short Register_ClockSource = -1;
module_param(Register_ClockSource, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Register_ClockSource, "Register Clock Source driver");
static signed short Idle_Route = -1;
module_param(Idle_Route, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Idle_Route, "[0:Default; 1:I/O; 2:HALT; 3:MWAIT]");
static signed short Mech_IBRS = -1;
module_param(Mech_IBRS, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Mech_IBRS, "Mitigation Mechanism IBRS");
static signed short Mech_STIBP = -1;
module_param(Mech_STIBP, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Mech_STIBP, "Mitigation Mechanism STIBP");
static signed short Mech_SSBD = -1;
module_param(Mech_SSBD, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Mech_SSBD, "Mitigation Mechanism SSBD");
static signed short Mech_IBPB = -1;
module_param(Mech_IBPB, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Mech_IBPB, "Mitigation Mechanism IBPB");
static signed short Mech_SBPB = -1;
module_param(Mech_SBPB, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Mech_SBPB, "Mitigation Mechanism SBPB");
static signed short Mech_L1D_FLUSH = -1;
module_param(Mech_L1D_FLUSH, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Mech_L1D_FLUSH, "Mitigation Mechanism Cache L1D Flush");
static signed short Mech_PSFD = -1;
module_param(Mech_PSFD, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Mech_PSFD, "Mitigation Mechanism PSFD");
static signed short Mech_BTC_NOBR = -1;
module_param(Mech_BTC_NOBR, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Mech_BTC_NOBR, "Mitigation Mechanism BTC-NOBR");
static signed short Mech_XPROC_LEAK = -1;
module_param(Mech_XPROC_LEAK, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Mech_XPROC_LEAK, "Mitigation Mech. Cross Processor Leak");
static signed short Mech_AGENPICK = -1;
module_param(Mech_AGENPICK, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(Mech_AGENPICK, "Mitigation Mech. LsCfgDisAgenPick");
static signed short WDT_Enable = -1;
module_param(WDT_Enable, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(WDT_Enable, "Watchdog Hardware Timer");
static signed short HSMP_Attempt = -1;
module_param(HSMP_Attempt, short, S_IRUSR|S_IRGRP|S_IROTH);
MODULE_PARM_DESC(HSMP_Attempt, "Attempt the HSMP interface");
static struct {
signed int Major;
struct cdev *kcdev;
dev_t nmdev, mkdev;
struct class *clsdev;
#ifdef CONFIG_CPU_IDLE
struct cpuidle_device __percpu *IdleDevice;
struct cpuidle_driver IdleDriver;
#endif /* CONFIG_CPU_IDLE */
#ifdef CONFIG_CPU_FREQ
struct cpufreq_driver FreqDriver;
struct cpufreq_governor FreqGovernor;
#endif /* CONFIG_CPU_FREQ */
#ifdef CONFIG_PM_SLEEP
bool ResumeFromSuspend;
#endif /* CONFIG_PM_SLEEP */
unsigned int SubCstate[SUB_CSTATE_COUNT];
} CoreFreqK = {
#ifdef CONFIG_CPU_IDLE
.IdleDriver = {
.name = "corefreqk-idle",
.owner = THIS_MODULE
},
#endif /* CONFIG_CPU_IDLE */
#ifdef CONFIG_CPU_FREQ
.FreqDriver = {
.name = "corefreqk-perf",
.flags = CPUFREQ_CONST_LOOPS,
.exit = CoreFreqK_Policy_Exit,
/*MANDATORY*/ .init = CoreFreqK_Policy_Init,
/*MANDATORY*/ .verify = CoreFreqK_Policy_Verify,
/*MANDATORY*/ .setpolicy = CoreFreqK_SetPolicy,
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0)
.bios_limit= CoreFreqK_Bios_Limit,
.set_boost = CoreFreqK_SetBoost
#else
.bios_limit= CoreFreqK_Bios_Limit
#endif
},
.FreqGovernor = {
.name = "corefreq-policy",
.owner = THIS_MODULE,
.show_setspeed = CoreFreqK_Show_SetSpeed,
.store_setspeed = CoreFreqK_Store_SetSpeed
},
#endif /* CONFIG_CPU_FREQ */
#ifdef CONFIG_PM_SLEEP
.ResumeFromSuspend = false,
#endif /* CONFIG_PM_SLEEP */
};
static KPUBLIC *KPublic = NULL;
static KPRIVATE *KPrivate = NULL;
static ktime_t RearmTheTimer;
#define AT( _loc_ ) [ _loc_ ]
#define OF( _ptr_ , ...) -> _ptr_ __VA_ARGS__
#define RO( _ptr_ , ...) OF( _ptr_##_RO , __VA_ARGS__ )
#define RW( _ptr_ , ...) OF( _ptr_##_RW , __VA_ARGS__ )
#define ADDR( _head_ , _mbr_ ) ( _head_ _mbr_ )
#define PUBLIC(...) ADDR( KPublic , __VA_ARGS__ )
#define PRIVATE(...) ADDR( KPrivate, __VA_ARGS__ )
#define RESET_ARRAY(_array, _cnt, _val, ... ) \
({ \
unsigned int rst; \
for (rst = 0; rst < _cnt; rst++) { \
_array[rst] __VA_ARGS__ = _val; \
} \
})
static long CoreFreqK_Thermal_Scope(int scope)
{
if ((scope >= FORMULA_SCOPE_NONE) && (scope <= FORMULA_SCOPE_PKG))
{
PUBLIC(RO(Proc))->thermalFormula = \
(KIND_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula) << 8)|scope;
return RC_SUCCESS;
} else {
return -EINVAL;
}
}
static long CoreFreqK_Voltage_Scope(int scope)
{
if ((scope >= FORMULA_SCOPE_NONE) && (scope <= FORMULA_SCOPE_PKG))
{
PUBLIC(RO(Proc))->voltageFormula = \
(KIND_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula) << 8)|scope;
return RC_SUCCESS;
} else {
return -EINVAL;
}
}
static long CoreFreqK_Power_Scope(int scope)
{
if ((scope >= FORMULA_SCOPE_NONE) && (scope <= FORMULA_SCOPE_PKG))
{
PUBLIC(RO(Proc))->powerFormula = \
(KIND_OF_FORMULA(PUBLIC(RO(Proc))->powerFormula) << 8)|scope;
return RC_SUCCESS;
} else {
return -EINVAL;
}
}
static unsigned int FixMissingRatioAndFrequency(unsigned int r32, CLOCK *pClock)
{
unsigned long long r64 = r32;
if (PUBLIC(RO(Proc))->Features.Factory.Freq != 0)
{
if ((r32 == 0) && (pClock->Q > 0))
{ /* Fix missing ratio. */
r64=DIV_ROUND_CLOSEST(PUBLIC(RO(Proc))->Features.Factory.Freq, pClock->Q);
PUBLIC(RO(Core,AT(PUBLIC(RO(Proc))->Service.Core)))->Boost[BOOST(MAX)]=\
(unsigned int) r64;
}
}
else if (r32 > 0)
{ /* Fix the Factory frequency (unit: MHz) */
r64 = pClock->Hz * r32;
r64 = r64 / 1000000LLU;
PUBLIC(RO(Proc))->Features.Factory.Freq = (unsigned int) r64;
}
PUBLIC(RO(Proc))->Features.Factory.Clock.Q = pClock->Q;
PUBLIC(RO(Proc))->Features.Factory.Clock.R = pClock->R;
PUBLIC(RO(Proc))->Features.Factory.Clock.Hz = pClock->Hz;
if (PUBLIC(RO(Proc))->Features.Factory.Clock.Hz > 0)
{
r64 = PUBLIC(RO(Proc))->Features.Factory.Freq * 1000000LLU;
r64 = DIV_ROUND_CLOSEST(r64, PUBLIC(RO(Proc))->Features.Factory.Clock.Hz);
PUBLIC(RO(Proc))->Features.Factory.Ratio = (unsigned int) r64;
}
return (unsigned int) r64;
}
static unsigned long long
CoreFreqK_Read_CS_From_Invariant_TSC(struct clocksource *cs)
{
unsigned long long TSC __attribute__ ((aligned (8)));
UNUSED(cs);
RDTSCP64(TSC);
return TSC;
}
static unsigned long long
CoreFreqK_Read_CS_From_Variant_TSC(struct clocksource *cs)
{
unsigned long long TSC __attribute__ ((aligned (8)));
UNUSED(cs);
RDTSC64(TSC);
return TSC;
}
static struct clocksource CoreFreqK_CS = {
.name = "corefreq_tsc",
.rating = 300,
.mask = CLOCKSOURCE_MASK(64),
.flags = CLOCK_SOURCE_IS_CONTINUOUS
| CLOCK_SOURCE_VALID_FOR_HRES,
};
static long CoreFreqK_UnRegister_ClockSource(void)
{
long rc = -EINVAL;
if (PUBLIC(RO(Proc))->Registration.Driver.CS & REGISTRATION_ENABLE)
{
int rx = clocksource_unregister(&CoreFreqK_CS);
switch ( rx ) {
case 0:
PUBLIC(RO(Proc))->Registration.Driver.CS = REGISTRATION_DISABLE;
rc = RC_SUCCESS;
break;
default:
rc = (long) rx;
break;
}
}
return rc;
}
static long CoreFreqK_Register_ClockSource(unsigned int cpu)
{
long rc = -EINVAL;
if (Register_ClockSource == 1)
{
unsigned long long Freq_Hz;
unsigned int Freq_KHz;
if ((PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC == 1)
|| (PUBLIC(RO(Proc))->Features.ExtInfo.EDX.RDTSCP == 1))
{
CoreFreqK_CS.read = CoreFreqK_Read_CS_From_Invariant_TSC;
}
else
{
CoreFreqK_CS.read = CoreFreqK_Read_CS_From_Variant_TSC;
}
Freq_Hz = PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)]
* PUBLIC(RO(Core, AT(cpu)))->Clock.Hz;
Freq_KHz = Freq_Hz / 1000U;
if (Freq_KHz != 0)
{
int rx = clocksource_register_khz(&CoreFreqK_CS, Freq_KHz);
switch (rx) {
default:
fallthrough;
case -EBUSY:
PUBLIC(RO(Proc))->Registration.Driver.CS = REGISTRATION_DISABLE;
rc = (long) rx;
break;
case 0:
PUBLIC(RO(Proc))->Registration.Driver.CS = REGISTRATION_ENABLE;
rc = RC_SUCCESS;
pr_debug("%s: Freq_KHz[%u] Kernel CPU_KHZ[%u] TSC_KHZ[%u]\n" \
"LPJ[%lu] mask[%llx] mult[%u] shift[%u]\n",
CoreFreqK_CS.name, Freq_KHz, cpu_khz, tsc_khz, loops_per_jiffy,
CoreFreqK_CS.mask, CoreFreqK_CS.mult, CoreFreqK_CS.shift);
break;
}
}
} else {
PUBLIC(RO(Proc))->Registration.Driver.CS = REGISTRATION_DISABLE;
}
return rc;
}
static void VendorFromCPUID(char *pVendorID, unsigned int *pLargestFunc,
unsigned int *pCRC, enum HYPERVISOR *pHypervisor,
unsigned long leaf, unsigned long subLeaf)
{
static const struct {
char *vendorID;
size_t vendorLen;
enum CRC_MANUFACTURER mfrCRC;
enum HYPERVISOR hypervisor;
} mfrTbl[] = {
{VENDOR_INTEL ,__builtin_strlen(VENDOR_INTEL) ,CRC_INTEL , BARE_METAL},
{VENDOR_AMD ,__builtin_strlen(VENDOR_AMD) ,CRC_AMD , BARE_METAL},
{VENDOR_HYGON ,__builtin_strlen(VENDOR_HYGON) ,CRC_HYGON , BARE_METAL},
{VENDOR_KVM ,__builtin_strlen(VENDOR_KVM) ,CRC_KVM , HYPERV_KVM},
{VENDOR_VBOX ,__builtin_strlen(VENDOR_VBOX) ,CRC_VBOX , HYPERV_VBOX},
{VENDOR_KBOX ,__builtin_strlen(VENDOR_KBOX) ,CRC_KBOX , HYPERV_KBOX},
{VENDOR_VMWARE ,__builtin_strlen(VENDOR_VMWARE),CRC_VMWARE,HYPERV_VMWARE},
{VENDOR_HYPERV ,__builtin_strlen(VENDOR_HYPERV),CRC_HYPERV,HYPERV_HYPERV}
};
const unsigned int mfrSize = sizeof(mfrTbl) / sizeof(mfrTbl[0]);
unsigned int eax = 0x0, ebx = 0x0, ecx = 0x0, edx = 0x0; /*DWORD Only!*/
__asm__ volatile
(
"movq %4, %%rax \n\t"
"movq %5, %%rcx \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (eax),
"=r" (ebx),
"=r" (ecx),
"=r" (edx)
: "ir" (leaf),
"ir" (subLeaf)
: "%rax", "%rbx", "%rcx", "%rdx"
);
pVendorID[ 0] = ebx;
pVendorID[ 1] = (ebx >> 8);
pVendorID[ 2] = (ebx >> 16);
pVendorID[ 3] = (ebx >> 24);
pVendorID[ 4] = edx;
pVendorID[ 5] = (edx >> 8);
pVendorID[ 6] = (edx >> 16);
pVendorID[ 7] = (edx >> 24);
pVendorID[ 8] = ecx;
pVendorID[ 9] = (ecx >> 8);
pVendorID[10] = (ecx >> 16);
pVendorID[11] = (ecx >> 24);
pVendorID[12] = '\0';
(*pLargestFunc) = eax;
for (eax = 0; eax < mfrSize; eax++) {
if (!strncmp(pVendorID, mfrTbl[eax].vendorID, mfrTbl[eax].vendorLen))
{
(*pCRC) = mfrTbl[eax].mfrCRC;
(*pHypervisor) = mfrTbl[eax].hypervisor;
return;
}
}
}
static signed int SearchArchitectureID(void)
{
signed int id;
for (id = ARCHITECTURES - 1; id > 0; id--)
{ /* Search for an architecture signature. */
if ( (PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily \
== Arch[id].Signature.ExtFamily)
&& (PUBLIC(RO(Proc))->Features.Std.EAX.Family \
== Arch[id].Signature.Family)
&& ( ( (PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel \
== Arch[id].Signature.ExtModel)
&& (PUBLIC(RO(Proc))->Features.Std.EAX.Model \
== Arch[id].Signature.Model) )
|| (!Arch[id].Signature.ExtModel \
&& !Arch[id].Signature.Model) ) )
{
break;
}
}
return id;
}
static void BrandCleanup(char *pBrand, char inOrder[])
{
unsigned long ix, jx;
for (jx = 0; jx < BRAND_LENGTH; jx++) {
if (inOrder[jx] != 0x20) {
break;
}
}
for (ix = 0; jx < BRAND_LENGTH; jx++) {
if (!(inOrder[jx] == 0x20 && inOrder[jx + 1] == 0x20)) {
pBrand[ix++] = inOrder[jx];
}
}
}
static void BrandFromCPUID(char *buffer)
{
BRAND Brand;
unsigned long ix;
unsigned int jx , px = 0;
for (ix = 0; ix < 3; ix++)
{ __asm__ volatile
(
"movq %4, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (Brand.AX),
"=r" (Brand.BX),
"=r" (Brand.CX),
"=r" (Brand.DX)
: "r" (0x80000002LU + ix)
: "%rax", "%rbx", "%rcx", "%rdx"
);
for (jx = 0; jx < 4; jx++, px++) {
buffer[px ] = Brand.AX.Chr[jx];
buffer[px + 4] = Brand.BX.Chr[jx];
buffer[px + 8] = Brand.CX.Chr[jx];
buffer[px + 12] = Brand.DX.Chr[jx];
}
px += BRAND_PART;
}
}
static unsigned int Intel_Brand(char *pBrand, char buffer[])
{
unsigned int ix, frequency = 0, multiplier = 0;
BrandFromCPUID(buffer);
for (ix = 0; ix < (BRAND_LENGTH - 2); ix++)
{
if ((buffer[ix + 1] == 'H') && (buffer[ix + 2] == 'z')) {
switch (buffer[ix]) {
case 'M':
multiplier = 1;
break;
case 'G':
multiplier = 1000;
break;
case 'T':
multiplier = 1000000;
break;
}
break;
}
}
if (multiplier > 0)
{
if (buffer[ix - 3] == '.') {
frequency = (int) (buffer[ix-4] - '0') * multiplier;
frequency += (int) (buffer[ix-2] - '0') * (multiplier / 10);
frequency += (int) (buffer[ix-1] - '0') * (multiplier / 100);
} else {
frequency = (int) (buffer[ix-4] - '0') * 1000;
frequency += (int) (buffer[ix-3] - '0') * 100;
frequency += (int) (buffer[ix-2] - '0') * 10;
frequency += (int) (buffer[ix-1] - '0');
frequency *= frequency;
}
}
BrandCleanup(pBrand, buffer);
return frequency;
}
/* Retreive the Processor(BSP) features. */
static void Query_Features(void *pArg)
{ /* Must have x86 CPUID 0x0, 0x1, and Intel CPUID 0x4 */
INIT_ARG *iArg = (INIT_ARG *) pArg;
unsigned int eax = 0x0, ebx = 0x0, ecx = 0x0, edx = 0x0; /*DWORD Only!*/
enum HYPERVISOR hypervisor = HYPERV_NONE;
VendorFromCPUID(iArg->Features->Info.Vendor.ID,
&iArg->Features->Info.LargestStdFunc,
&iArg->Features->Info.Vendor.CRC,
&hypervisor,
0x0LU, 0x0LU);
if (hypervisor != BARE_METAL) {
iArg->rc = -ENXIO;
return;
}
__asm__ volatile
(
"movq $0x1, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (iArg->Features->Std.EAX),
"=r" (iArg->Features->Std.EBX),
"=r" (iArg->Features->Std.ECX),
"=r" (iArg->Features->Std.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
if (iArg->Features->Info.LargestStdFunc >= 0x5)
{
__asm__ volatile
(
"movq $0x5, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (iArg->Features->MWait.EAX),
"=r" (iArg->Features->MWait.EBX),
"=r" (iArg->Features->MWait.ECX),
"=r" (iArg->Features->MWait.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
switch (Override_SubCstate_Depth) {
case 8:
iArg->Features->MWait.EDX.SubCstate_MWAIT7 = Override_SubCstate[7];
fallthrough;
case 7:
iArg->Features->MWait.EDX.SubCstate_MWAIT6 = Override_SubCstate[6];
fallthrough;
case 6:
iArg->Features->MWait.EDX.SubCstate_MWAIT5 = Override_SubCstate[5];
fallthrough;
case 5:
iArg->Features->MWait.EDX.SubCstate_MWAIT4 = Override_SubCstate[4];
fallthrough;
case 4:
iArg->Features->MWait.EDX.SubCstate_MWAIT3 = Override_SubCstate[3];
fallthrough;
case 3:
iArg->Features->MWait.EDX.SubCstate_MWAIT2 = Override_SubCstate[2];
fallthrough;
case 2:
iArg->Features->MWait.EDX.SubCstate_MWAIT1 = Override_SubCstate[1];
fallthrough;
case 1:
iArg->Features->MWait.EDX.SubCstate_MWAIT0 = Override_SubCstate[0];
break;
};
CoreFreqK.SubCstate[0] = iArg->Features->MWait.EDX.SubCstate_MWAIT0;
CoreFreqK.SubCstate[1] = iArg->Features->MWait.EDX.SubCstate_MWAIT1;
/*C1E*/ CoreFreqK.SubCstate[2] = iArg->Features->MWait.EDX.SubCstate_MWAIT1;
CoreFreqK.SubCstate[3] = iArg->Features->MWait.EDX.SubCstate_MWAIT2;
CoreFreqK.SubCstate[4] = iArg->Features->MWait.EDX.SubCstate_MWAIT3;
CoreFreqK.SubCstate[5] = iArg->Features->MWait.EDX.SubCstate_MWAIT4;
CoreFreqK.SubCstate[6] = iArg->Features->MWait.EDX.SubCstate_MWAIT5;
CoreFreqK.SubCstate[7] = iArg->Features->MWait.EDX.SubCstate_MWAIT6;
CoreFreqK.SubCstate[8] = iArg->Features->MWait.EDX.SubCstate_MWAIT7;
}
if (iArg->Features->Info.LargestStdFunc >= 0x6)
{
__asm__ volatile
(
"movq $0x6, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (iArg->Features->Power.EAX),
"=r" (iArg->Features->Power.EBX),
"=r" (iArg->Features->Power.ECX),
"=r" (iArg->Features->Power.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
}
if (iArg->Features->Info.LargestStdFunc >= 0x7)
{
__asm__ volatile
(
"movq $0x7, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (iArg->Features->ExtFeature.EAX),
"=r" (iArg->Features->ExtFeature.EBX),
"=r" (iArg->Features->ExtFeature.ECX),
"=r" (iArg->Features->ExtFeature.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
if (iArg->Features->ExtFeature.EAX.MaxSubLeaf >= 1)
{
__asm__ volatile
(
"movq $0x7, %%rax \n\t"
"movq $0x1, %%rcx \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (iArg->Features->ExtFeature_Leaf1_EAX),
"=r" (ebx),
"=r" (ecx),
"=r" (iArg->Features->ExtFeature_Leaf1_EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
}
if (iArg->Features->ExtFeature.EAX.MaxSubLeaf >= 2)
{
__asm__ volatile
(
"movq $0x7, %%rax \n\t"
"movq $0x2, %%rcx \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (eax),
"=r" (ebx),
"=r" (ecx),
"=r" (iArg->Features->ExtFeature_Leaf2_EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
}
}
if (iArg->Features->Info.LargestStdFunc >= 0xd)
{
__asm__ volatile
(
"movq $0xd, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (iArg->Features->ExtState.EAX),
"=r" (iArg->Features->ExtState.EBX),
"=r" (iArg->Features->ExtState.ECX),
"=r" (iArg->Features->ExtState.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
__asm__ volatile
(
"movq $0xd, %%rax \n\t"
"movq $0x1, %%rcx \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (iArg->Features->ExtState_Leaf1.EAX),
"=r" (iArg->Features->ExtState_Leaf1.EBX),
"=r" (iArg->Features->ExtState_Leaf1.ECX),
"=r" (iArg->Features->ExtState_Leaf1.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
}
/* Must have 0x80000000,0x80000001,0x80000002,0x80000003,0x80000004 */
__asm__ volatile
(
"movq $0x80000000, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (iArg->Features->Info.LargestExtFunc),
"=r" (ebx),
"=r" (ecx),
"=r" (edx)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
__asm__ volatile
(
"movq $0x80000001, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (eax),
"=r" (ebx),
"=r" (iArg->Features->ExtInfo.ECX),
"=r" (iArg->Features->ExtInfo.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
if (iArg->Features->Info.LargestExtFunc >= 0x80000007)
{
__asm__ volatile
(
"movq $0x80000007, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (iArg->Features->AdvPower.EAX),
"=r" (iArg->Features->AdvPower.EBX),
"=r" (iArg->Features->AdvPower.ECX),
"=r" (iArg->Features->AdvPower.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
}
if (iArg->Features->Info.LargestExtFunc >= 0x80000008)
{
__asm__ volatile
(
"movq $0x80000008, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (iArg->Features->leaf80000008.EAX),
"=r" (iArg->Features->leaf80000008.EBX),
"=r" (iArg->Features->leaf80000008.ECX),
"=r" (iArg->Features->leaf80000008.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
}
/* Reset the performance features bits (present is zero) */
iArg->Features->PerfMon.EBX.CoreCycles = 1;
iArg->Features->PerfMon.EBX.InstrRetired = 1;
iArg->Features->PerfMon.EBX.RefCycles = 1;
iArg->Features->PerfMon.EBX.LLC_Ref = 1;
iArg->Features->PerfMon.EBX.LLC_Misses = 1;
iArg->Features->PerfMon.EBX.BranchRetired = 1;
iArg->Features->PerfMon.EBX.BranchMispred = 1;
iArg->Features->PerfMon.EBX.TopdownSlots = 1;
/* Per Vendor features */
if (iArg->Features->Info.Vendor.CRC == CRC_INTEL)
{
__asm__ volatile
(
"movq $0x4, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (eax),
"=r" (ebx),
"=r" (ecx),
"=r" (edx)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
iArg->SMT_Count = (eax >> 26) & 0x3f;
iArg->SMT_Count++;
if (iArg->Features->Info.LargestStdFunc >= 0xa)
{
__asm__ volatile
(
"movq $0xa, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (iArg->Features->PerfMon.EAX),
"=r" (iArg->Features->PerfMon.EBX),
"=r" (iArg->Features->PerfMon.ECX),
"=r" (iArg->Features->PerfMon.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
}
/* Extract the factory frequency from the brand string. */
iArg->Features->Factory.Freq = Intel_Brand( iArg->Features->Info.Brand,
iArg->Brand );
} else if ( (iArg->Features->Info.Vendor.CRC == CRC_AMD)
|| (iArg->Features->Info.Vendor.CRC == CRC_HYGON) )
{ /* AMD PM version: Use Intel bits as AMD placeholder. */
iArg->Features->PerfMon.EAX.Version = 1;
/* Specified as Core Performance 48 bits General Counters. */
iArg->Features->PerfMon.EAX.MonWidth = 48;
if (iArg->Features->ExtInfo.ECX.PerfCore) {
switch (iArg->Features->Std.EAX.ExtFamily) {
case 0x6:
case 0x8 ... 0xa:
/* PPR: six core performance event counters per thread */
iArg->Features->PerfMon.EAX.MonCtrs = 6;
break;
case 0x0 ... 0x5:
case 0x7:
iArg->Features->PerfMon.EAX.MonCtrs = 4;
break;
}
} else { /* CPUID 0F_00h */
iArg->Features->PerfMon.EAX.MonCtrs = 4;
}
/* Specified as Data Fabric or Northbridge Performance Events Counter */
if (iArg->Features->ExtInfo.ECX.PerfNB)
{ /* PPR: four Data Fabric performance events counters */
iArg->Features->Factory.PMC.NB = 4;
} else {
iArg->Features->Factory.PMC.NB = 0;
}
/* Specified as L3 Cache or L2I-ext Performance Events Counters */
if (iArg->Features->ExtInfo.ECX.PerfLLC) {
switch (iArg->Features->Std.EAX.ExtFamily) {
case 0x8 ... 0xB:
/* PPR: six performance events counters per L3 complex */
iArg->Features->Factory.PMC.LLC = 6;
break;
case 0x6:
if (iArg->Features->Std.EAX.ExtModel < 0x6) {
break;
}
fallthrough;
case 0x7:
iArg->Features->Factory.PMC.LLC = 4;
break;
default:
iArg->Features->Factory.PMC.LLC = 4;
break;
}
} else {
iArg->Features->Factory.PMC.LLC = 0;
}
/* Fix the Performance Counters. Use Intel bits as AMD placeholder */
iArg->Features->PerfMon.EDX.FixWidth = 64;
if ( iArg->Features->Power.ECX.HCF_Cap
| iArg->Features->AdvPower.EDX.EffFrqRO )
{
iArg->Features->PerfMon.EBX.CoreCycles = 0;
iArg->Features->PerfMon.EBX.RefCycles = 0;
iArg->Features->PerfMon.EDX.FixCtrs += 2;
}
if (iArg->Features->ExtInfo.ECX.PerfLLC)
{ /* PerfCtrExtLLC: Last Level Cache performance counter extensions */
iArg->Features->PerfMon.EBX.LLC_Ref = 0;
}
if (iArg->Features->Info.LargestExtFunc >= 0x80000008)
{
if (iArg->Features->Std.EAX.ExtFamily < 0xB) {
iArg->SMT_Count = iArg->Features->leaf80000008.ECX.NC + 1;
} else {
iArg->SMT_Count = iArg->Features->leaf80000008.ECX.F1Ah.NC + 1;
}
/* Add the Retired Instructions Perf Counter to the Fixed set */
if (iArg->Features->leaf80000008.EBX.IRPerf)
{
iArg->Features->PerfMon.EBX.InstrRetired = 0;
iArg->Features->PerfMon.EDX.FixCtrs++;
}
}
else if (iArg->Features->Std.EDX.HTT)
{
iArg->SMT_Count = iArg->Features->Std.EBX.Max_SMT_ID;
} else {
iArg->SMT_Count = 1;
}
if (iArg->Features->Info.LargestExtFunc >= 0x80000021)
{
__asm__ volatile
(
"movq $0x80000021, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (iArg->Features->ExtFeature2_EAX),
"=r" (ebx),
"=r" (ecx),
"=r" (edx)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
}
if (iArg->Features->Info.LargestExtFunc >= 0x80000022)
{
CPUID_0x80000022 leaf80000022 = {
.EAX = {0}, .EBX = {0}, .ECX = {0}, .EDX = {0}
};
__asm__ volatile
(
"movq $0x80000022, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (leaf80000022.EAX),
"=r" (leaf80000022.EBX),
"=r" (leaf80000022.ECX),
"=r" (leaf80000022.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
iArg->Features->PerfMon.EAX.Version += leaf80000022.EAX.PerfMonV2;
if (leaf80000022.EBX.NumPerfCtrCore > 0) {
iArg->Features->PerfMon.EAX.MonCtrs=leaf80000022.EBX.NumPerfCtrCore;
}
if (leaf80000022.EBX.NumPerfCtrNB > 0) {
iArg->Features->Factory.PMC.NB = leaf80000022.EBX.NumPerfCtrNB;
}
}
BrandFromCPUID(iArg->Brand);
BrandCleanup(iArg->Features->Info.Brand, iArg->Brand);
}
}
static void Compute_Interval(void)
{
if ( (SleepInterval >= LOOP_MIN_MS)
&& (SleepInterval <= LOOP_MAX_MS))
{
PUBLIC(RO(Proc))->SleepInterval = SleepInterval;
} else {
PUBLIC(RO(Proc))->SleepInterval = LOOP_DEF_MS;
}
/* Compute the tick steps . */
PUBLIC(RO(Proc))->tickReset = \
( (TickInterval >= PUBLIC(RO(Proc))->SleepInterval)
&& (TickInterval <= LOOP_MAX_MS) ) ?
TickInterval
: KMAX(TICK_DEF_MS, PUBLIC(RO(Proc))->SleepInterval);
PUBLIC(RO(Proc))->tickReset /= PUBLIC(RO(Proc))->SleepInterval;
PUBLIC(RO(Proc))->tickStep = PUBLIC(RO(Proc))->tickReset;
RearmTheTimer = ktime_set( 0, PUBLIC(RO(Proc))->SleepInterval
* 1000000LU );
}
#ifdef CONFIG_SMP
#define THIS_LPJ this_cpu_read(cpu_info.loops_per_jiffy)
#else
#define THIS_LPJ loops_per_jiffy
#endif
#define COMPUTE_LPJ(BCLK_Hz, COF) ( (BCLK_Hz * COF) / HZ )
#if defined(DELAY_TSC) && (DELAY_TSC == 1)
/* udelay() built with TSC implementation */
#define CLOCK_TSC( CYCLES, _TIMER, CTR ) \
({ \
__asm__ volatile \
( \
ASM_RD##_TIMER(r14) \
"addq %[cycles], %%rax" "\n\t" \
"movq %%rax , %%r15" "\n\t" \
"1:" "\n\t" \
ASM_RD##_TIMER(r13) \
"cmpq %%r15 , %%r13" "\n\t" \
"jc 1b" "\n\t" \
"movq %%r14 , %[ctr0]" "\n\t" \
"movq %%r13 , %[ctr1]" \
: [ctr0] "=m" (CTR[0]), \
[ctr1] "=m" (CTR[1]) \
: [cycles] "ir" (CYCLES) \
: "%rax", "%rbx", "%rcx", "%rdx", \
"%r13", "%r14", "%r15", \
"cc", "memory" \
); \
})
#define CLOCK2CYCLE(INTERVAL_NS) ((INTERVAL_NS * THIS_LPJ * HZ) / 1000000LLU)
#define CLOCK_DELAY(INTERVAL_NS, _TIMER, CTR) \
({ \
CLOCK_TSC( CLOCK2CYCLE(INTERVAL_NS), _TIMER, CTR ); \
})
#define CLOCK_OVERHEAD(_TIMER, CTR) CLOCK_TSC( 1LLU, _TIMER, CTR )
#else
#define CLOCK_DELAY(INTERVAL_NS, _TIMER, CTR) \
({ \
RD##_TIMER##64(CTR[0]); \
udelay(INTERVAL_NS); \
RD##_TIMER##64(CTR[1]); \
})
#define CLOCK_OVERHEAD(_TIMER, CTR) CLOCK_DELAY( 0LLU, _TIMER, CTR )
#endif
static void ComputeWithSerializedTSC(COMPUTE_ARG *pCompute)
{
unsigned int loop;
/* Writeback and Invalidate Caches. */
WBINVD();
/* Warm-up & Overhead */
for (loop = 0; loop < OCCURRENCES; loop++)
{
CLOCK_OVERHEAD(TSCP, pCompute->TSC[0][loop].V);
}
/* Estimation */
for (loop = 0; loop < OCCURRENCES; loop++)
{
CLOCK_DELAY(1000LLU, TSCP, pCompute->TSC[1][loop].V);
}
}
static void ComputeWithUnSerializedTSC(COMPUTE_ARG *pCompute)
{
unsigned int loop;
/* Writeback and Invalidate Caches. */
WBINVD();
/* Warm-up & Overhead */
for (loop = 0; loop < OCCURRENCES; loop++)
{
CLOCK_OVERHEAD(TSC, pCompute->TSC[0][loop].V);
}
/* Estimation */
for (loop = 0; loop < OCCURRENCES; loop++)
{
CLOCK_DELAY(1000LLU, TSC, pCompute->TSC[1][loop].V);
}
}
static void Compute_TSC(void *arg)
{
COMPUTE_ARG *pCompute = (COMPUTE_ARG *) arg;
unsigned long long D[2][OCCURRENCES];
unsigned int ratio = (unsigned int) pCompute->Clock.Q;
unsigned int loop = 0, what = 0, best[2] = {0, 0}, top[2] = {0, 0};
/*
TSC[0] stores the overhead
TSC[1] stores the estimation
*/
/* Is the TSC invariant or can serialize ? */
if ((PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC == 1)
|| (PUBLIC(RO(Proc))->Features.ExtInfo.EDX.RDTSCP == 1))
{
ComputeWithSerializedTSC(pCompute);
} else {
ComputeWithUnSerializedTSC(pCompute);
}
/* Select the best clock. */
for (loop = 0; loop < OCCURRENCES; loop++) {
for (what = 0; what < 2; what++) {
D[what][loop] = pCompute->TSC[what][loop].V[1]
- pCompute->TSC[what][loop].V[0];
}
}
for (loop = 0; loop < OCCURRENCES; loop++) {
unsigned int inner = 0, count[2] = {0, 0};
for (inner = loop; inner < OCCURRENCES; inner++) {
for (what = 0; what < 2; what++) {
if (D[what][loop] == D[what][inner])
count[what]++;
}
}
for (what = 0; what < 2; what++) {
if ((count[what] > top[what])
|| ((count[what] == top[what])
&& (D[what][loop] < D[what][best[what]]))) {
top[what] = count[what];
best[what] = loop;
}
}
}
/* Substract the overhead . */
D[1][best[1]] -= D[0][best[0]];
/* Compute the Base Clock . */
REL_BCLK(pCompute->Clock, ratio, D[1][best[1]], 1LLU);
}
static CLOCK Compute_Clock(unsigned int cpu, COMPUTE_ARG *pCompute)
{
/* Synchronously call the Base Clock estimation on a pinned CPU.
* 1/ Preemption is disabled by smp_call_function_single() > get_cpu()
* 2/ IRQ are suspended by generic_exec_single(func) > local_irq_save()
* 3/ Function 'func' is executed
* 4/ IRQ are resumed > local_irq_restore()
* 5/ Preemption is enabled > put_cpu()
*/
smp_call_function_single(cpu, Compute_TSC, pCompute, 1);
return pCompute->Clock;
}
static void ClockToHz(CLOCK *clock)
{
clock->Hz = clock->Q * 1000000L;
clock->Hz += clock->R * PRECISION;
}
/* [Genuine Intel] */
static CLOCK BaseClock_GenuineIntel(unsigned int ratio)
{
CLOCK clock = {.Q = 100, .R = 0, .Hz = 100000000L};
if ((PUBLIC(RO(Proc))->Features.Factory.Freq > 0) && (ratio > 0))
{
clock.Hz = (PUBLIC(RO(Proc))->Features.Factory.Freq * 1000000L)
/ ratio;
clock.Q = PUBLIC(RO(Proc))->Features.Factory.Freq / ratio;
clock.R = (PUBLIC(RO(Proc))->Features.Factory.Freq % ratio)
* PRECISION;
}
return clock;
};
/* [Authentic AMD] */
static CLOCK BaseClock_AuthenticAMD(unsigned int ratio)
{ /* For AMD Families 0Fh, 10h up to 16h */
CLOCK clock = {.Q = 100, .R = 0, .Hz = 100000000L};
UNUSED(ratio);
return clock;
};
/* [Core] */
static CLOCK BaseClock_Core(unsigned int ratio)
{
CLOCK clock = {.Q = 100, .R = 0};
FSB_FREQ FSB = {.value = 0};
RDMSR(FSB, MSR_FSB_FREQ);
switch (FSB.Bus_Speed) {
case 0b101: {
clock.Q = 100;
clock.R = 0;
};
break;
case 0b001: {
clock.Q = 133;
clock.R = 3333;
}
break;
case 0b011: {
clock.Q = 166;
clock.R = 6666;
}
break;
}
ClockToHz(&clock);
clock.R *= ratio;
return clock;
};
/* [Core2] */
static CLOCK BaseClock_Core2(unsigned int ratio)
{
CLOCK clock = {.Q = 100, .R = 0};
FSB_FREQ FSB = {.value = 0};
RDMSR(FSB, MSR_FSB_FREQ);
switch (FSB.Bus_Speed) {
case 0b101: {
clock.Q = 100;
clock.R = 0;
}
break;
case 0b001: {
clock.Q = 133;
clock.R = 3333;
}
break;
case 0b011: {
clock.Q = 166;
clock.R = 6666;
}
break;
case 0b010: {
clock.Q = 200;
clock.R = 0;
}
break;
case 0b000: {
clock.Q = 266;
clock.R = 6666;
}
break;
case 0b100: {
clock.Q = 333;
clock.R = 3333;
}
break;
case 0b110: {
clock.Q = 400;
clock.R = 0;
}
break;
}
ClockToHz(&clock);
clock.R *= ratio;
return clock;
};
/* [Atom] */
static CLOCK BaseClock_Atom(unsigned int ratio)
{
CLOCK clock = {.Q = 83, .R = 0};
FSB_FREQ FSB = {.value = 0};
RDMSR(FSB, MSR_FSB_FREQ);
switch (FSB.Bus_Speed) {
case 0b111: {
clock.Q = 83;
clock.R = 2000;
}
break;
case 0b101: {
clock.Q = 99;
clock.R = 8400;
}
break;
case 0b001: {
clock.Q = 133;
clock.R = 2000;
}
break;
case 0b011: {
clock.Q = 166;
clock.R = 4000;
}
break;
default: {
clock.Q = 83;
clock.R = 2000;
}
break;
}
ClockToHz(&clock);
clock.R *= ratio;
return clock;
};
/* [Airmont] */
static CLOCK BaseClock_Airmont(unsigned int ratio)
{
CLOCK clock = {.Q = 87, .R = 5};
FSB_FREQ FSB = {.value = 0};
RDMSR(FSB, MSR_FSB_FREQ);
switch (FSB.Airmont.Bus_Speed) {
case 0b0000: {
clock.Q = 83;
clock.R = 3333;
}
break;
case 0b0001: {
clock.Q = 100;
clock.R = 0000;
}
break;
case 0b0010: {
clock.Q = 133;
clock.R = 3333;
}
break;
case 0b0011: {
clock.Q = 116;
clock.R = 6666;
}
break;
case 0b0100: {
clock.Q = 80;
clock.R = 0000;
}
break;
case 0b0101: {
clock.Q = 93;
clock.R = 3333;
}
break;
case 0b0110: {
clock.Q = 90;
clock.R = 0000;
}
break;
case 0b0111: {
clock.Q = 88;
clock.R = 9000;
}
break;
case 0b1000:
break;
}
ClockToHz(&clock);
clock.R *= ratio;
return clock;
};
/* [Silvermont] */
static CLOCK BaseClock_Silvermont(unsigned int ratio)
{
CLOCK clock = {.Q = 83, .R = 3};
FSB_FREQ FSB = {.value = 0};
RDMSR(FSB, MSR_FSB_FREQ);
switch (FSB.Bus_Speed)
{
case 0b100: {
clock.Q = 80;
clock.R = 0;
}
break;
case 0b000: {
clock.Q = 83;
clock.R = 3000;
}
break;
case 0b001: {
clock.Q = 100;
clock.R = 0;
}
break;
case 0b010: {
clock.Q = 133;
clock.R = 3333;
}
break;
case 0b011: {
clock.Q = 116;
clock.R = 7000;
}
break;
}
ClockToHz(&clock);
clock.R *= ratio;
return clock;
};
/* [Nehalem] */
static CLOCK BaseClock_Nehalem(unsigned int ratio)
{
CLOCK clock = {.Q = 133, .R = 3333};
ClockToHz(&clock);
clock.R *= ratio;
return clock;
};
/* [Westmere] */
static CLOCK BaseClock_Westmere(unsigned int ratio)
{
CLOCK clock = {.Q = 133, .R = 3333};
ClockToHz(&clock);
clock.R *= ratio;
return clock;
};
/* [SandyBridge] */
static CLOCK BaseClock_SandyBridge(unsigned int ratio)
{
CLOCK clock = {.Q = 100, .R = 0};
ClockToHz(&clock);
clock.R *= ratio;
return clock;
};
/* [IvyBridge] */
static CLOCK BaseClock_IvyBridge(unsigned int ratio)
{
CLOCK clock = {.Q = 100, .R = 0};
ClockToHz(&clock);
clock.R *= ratio;
return clock;
};
/* [Haswell] */
static CLOCK BaseClock_Haswell(unsigned int ratio)
{
CLOCK clock = {.Q = 100, .R = 0};
ClockToHz(&clock);
clock.R *= ratio;
return clock;
};
/* [Skylake] */
static CLOCK BaseClock_Skylake(unsigned int ratio)
{
CLOCK clock = {.Q = 100, .R = 0};
if (PUBLIC(RO(Proc))->Features.Info.LargestStdFunc >= 0x16) {
unsigned int eax = 0x0, ebx = 0x0, edx = 0x0, fsb = 0;
__asm__ volatile
(
"movq $0x16, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (eax),
"=r" (ebx),
"=r" (fsb),
"=r" (edx)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
if (fsb > 0)
clock.Q = fsb;
else
clock.Q = 100;
}
ClockToHz(&clock);
clock.R *= ratio;
return clock;
};
static CLOCK BaseClock_AMD_Family_17h(unsigned int ratio)
{ /* Source: AMD PPR Family 17h Chap. 1.4/ Table 11: REFCLK = 100 MHz */
CLOCK clock = {.Q = 100, .R = 0, .Hz = 100000000L};
UNUSED(ratio);
return clock;
};
static void Define_CPUID(CORE_RO *Core, const CPUID_STRUCT CpuIDforVendor[])
{ /* Per vendor, define a CPUID dump table to query . */
enum CPUID_ENUM i;
for (i = 0; i < CPUID_MAX_FUNC; i++) {
Core->CpuID[i].func = CpuIDforVendor[i].func;
Core->CpuID[i].sub = CpuIDforVendor[i].sub;
}
}
static void Cache_Topology(CORE_RO *Core)
{
unsigned long level = 0x0;
if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL) {
for (level = 0; level < CACHE_MAX_LEVEL; level++) {
__asm__ volatile
(
"movq $0x4, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"movq %4, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (Core->T.Cache[level].AX),
"=r" (Core->T.Cache[level].BX),
"=r" (Core->T.Cache[level].Set),
"=r" (Core->T.Cache[level].DX)
: "r" (level)
: "%rax", "%rbx", "%rcx", "%rdx"
);
if (!Core->T.Cache[level].Type)
break;
}
}
else if ( (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
||(PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON) )
{
struct CACHE_INFO CacheInfo; /* Employ same Intel algorithm. */
if (PUBLIC(RO(Proc))->Features.Info.LargestExtFunc >= 0x80000005)
{
Core->T.Cache[0].Level = 1;
Core->T.Cache[0].Type = 2; /* Inst. */
Core->T.Cache[1].Level = 1;
Core->T.Cache[1].Type = 1; /* Data */
/* Fn8000_0005 L1 Data and Inst. caches. */
__asm__ volatile
(
"movq $0x80000005, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (CacheInfo.AX),
"=r" (CacheInfo.BX),
"=r" (CacheInfo.CX),
"=r" (CacheInfo.DX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
/* L1 Inst. */
Core->T.Cache[0].Way = CacheInfo.CPUID_0x80000005_L1I.Assoc;
Core->T.Cache[0].Size = CacheInfo.CPUID_0x80000005_L1I.Size;
/* L1 Data */
Core->T.Cache[1].Way = CacheInfo.CPUID_0x80000005_L1D.Assoc;
Core->T.Cache[1].Size = CacheInfo.CPUID_0x80000005_L1D.Size;
}
if (PUBLIC(RO(Proc))->Features.Info.LargestExtFunc >= 0x80000006)
{
Core->T.Cache[2].Level = 2;
Core->T.Cache[2].Type = 3; /* Unified! */
Core->T.Cache[3].Level = 3;
Core->T.Cache[3].Type = 3;
/* Fn8000_0006 L2 and L3 caches. */
__asm__ volatile
(
"movq $0x80000006, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (CacheInfo.AX),
"=r" (CacheInfo.BX),
"=r" (CacheInfo.CX),
"=r" (CacheInfo.DX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
/* L2 */
Core->T.Cache[2].Way = CacheInfo.CPUID_0x80000006_L2.Assoc;
Core->T.Cache[2].Size = CacheInfo.CPUID_0x80000006_L2.Size;
/* L3 */
Core->T.Cache[3].Way = CacheInfo.CPUID_0x80000006_L3.Assoc;
Core->T.Cache[3].Size = CacheInfo.CPUID_0x80000006_L3.Size;
}
}
}
static unsigned int L3_SubCache_AMD_Piledriver(unsigned int bits)
{ /* Return the AMD Piledriver L3 Sub-Cache size in unit of 512 KB */
switch (bits) {
case 0xc:
return 4;
case 0xd:
case 0xe:
return 2;
default:
return 0;
}
}
static void Map_AMD_Topology(void *arg)
{
if (arg != NULL)
{
CORE_RO *Core = (CORE_RO *) arg;
struct CPUID_0x00000001_EBX leaf1_ebx = {0};
CPUID_0x80000008 leaf80000008 = {
.EAX = {0}, .EBX = {0}, .ECX = {{0}}, .EDX = {0}
};
bool CPU_Complex = true;
Cache_Topology(Core);
RDMSR(Core->T.Base, MSR_IA32_APICBASE);
__asm__ volatile
(
"movq $0x1, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%ebx, %0"
: "=r" (leaf1_ebx)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
__asm__ volatile
(
"movq $0x80000008, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%ecx, %0"
: "=r" (leaf80000008.ECX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
switch (PUBLIC(RO(Proc))->ArchID) {
default:
case AMD_Family_0Fh: /* Legacy processor. */
Core->T.ApicID = leaf1_ebx.Init_APIC_ID;
Core->T.CoreID = leaf1_ebx.Init_APIC_ID;
Core->T.PackageID = 0;
break;
case AMD_Family_10h:
{
CPUID_0x80000001 leaf80000001 = {
.EAX = {0}, .EBX = {{0}}, .ECX = {{0}}, .EDX = {{0}}
};
__asm__ volatile
(
"movq $0x80000001, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (leaf80000001.EAX),
"=r" (leaf80000001.EBX),
"=r" (leaf80000001.ECX),
"=r" (leaf80000001.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
Core->T.Cluster.Node = leaf80000001.ECX.NodeId;
}
fallthrough;
case AMD_Family_11h:
case AMD_Family_12h:
case AMD_Family_14h:
Core->T.ApicID = leaf1_ebx.Init_APIC_ID;
Core->T.CoreID = leaf1_ebx.Init_APIC_ID;
Core->T.PackageID = leaf1_ebx.Init_APIC_ID
>> leaf80000008.ECX.ApicIdCoreIdSize;
break;
case AMD_Family_15h:
if ((PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel == 0x0)
&& (PUBLIC(RO(Proc))->Features.Std.EAX.Model >= 0x0)
&& (PUBLIC(RO(Proc))->Features.Std.EAX.Model <= 0xf))
{
L3_CACHE_PARAMETER L3;
RDPCI(L3, PCI_AMD_L3_CACHE_PARAMETER);
Core->T.Cache[3].Size+=L3_SubCache_AMD_Piledriver(L3.SubCacheSize0);
Core->T.Cache[3].Size+=L3_SubCache_AMD_Piledriver(L3.SubCacheSize1);
Core->T.Cache[3].Size+=L3_SubCache_AMD_Piledriver(L3.SubCacheSize2);
Core->T.Cache[3].Size+=L3_SubCache_AMD_Piledriver(L3.SubCacheSize3);
}
fallthrough;
case AMD_Family_16h:
Core->T.ApicID = leaf1_ebx.Init_APIC_ID;
Core->T.PackageID = leaf1_ebx.Init_APIC_ID
>> leaf80000008.ECX.ApicIdCoreIdSize;
Core->T.CoreID = leaf1_ebx.Init_APIC_ID
- (Core->T.PackageID
<< leaf80000008.ECX.ApicIdCoreIdSize);
if (PUBLIC(RO(Proc))->Features.ExtInfo.ECX.ExtApicId == 1)
{
CPUID_0x8000001e leaf8000001e;
__asm__ volatile
(
"movq $0x8000001e, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (leaf8000001e.EAX),
"=r" (leaf8000001e.EBX),
"=r" (leaf8000001e.ECX),
"=r" (leaf8000001e.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
/* TODO(Case of leaf8000001e.EAX.ExtApicId) */
Core->T.Cluster.Node= leaf8000001e.ECX.NodeId;
Core->T.Cluster.CMP = leaf8000001e.EBX.CompUnitId;
}
break;
/* Zen APU */
case AMD_Zen_APU:
case AMD_ZenPlus_APU:
case AMD_Zen_Dali:
case AMD_Zen2_Renoir:
case AMD_Zen2_LCN:
case AMD_Zen2_Ariel:
case AMD_Zen2_Jupiter:
case AMD_Zen2_Galileo:
case AMD_Zen2_MDN:
case AMD_Zen3_CZN:
case AMD_Zen3Plus_RMB:
case AMD_Zen4_PHX:
case AMD_Zen4_PHXR:
case AMD_Zen4_PHX2:
case AMD_Zen4_HWK:
case AMD_Zen5_STX:
case AMD_Zen5_KRK:
case AMD_Zen5_STXH:
CPU_Complex = false;
fallthrough;
/* Zen CPU Complex */
case AMD_Zen:
case AMD_ZenPlus:
case AMD_EPYC_Rome_CPK:
case AMD_Zen2_MTS:
case AMD_Zen3_VMR:
case AMD_EPYC_Milan:
case AMD_Zen3_Chagall:
case AMD_Zen3_Badami:
case AMD_Zen4_Genoa:
case AMD_Zen4_RPL:
case AMD_Zen4_Bergamo:
case AMD_Zen4_STP:
case AMD_Zen5_Eldora:
case AMD_Zen5_Turin:
case AMD_Zen5_Turin_Dense:
case AMD_Family_17h:
case Hygon_Family_18h:
case AMD_Family_19h:
case AMD_Family_1Ah:
if (PUBLIC(RO(Proc))->Features.ExtInfo.ECX.ExtApicId == 1)
{
struct CACHE_INFO CacheInfo = {
.AX = 0, .BX = 0, .CX = 0, .DX = 0, .Size = 0
};
CPUID_0x8000001e leaf8000001e = {
.EAX = {0}, .EBX = {{0}}, .ECX = {0}, .EDX = {0}
};
/* Fn8000_001D Cache Properties. */
unsigned long idx, level[CACHE_MAX_LEVEL] = {1, 0, 2, 3};
for (idx = 0; idx < CACHE_MAX_LEVEL; idx++ ) {
__asm__ volatile
(
"movq $0x8000001d, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"movq %4, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (CacheInfo.AX),
"=r" (CacheInfo.BX),
"=r" (CacheInfo.CX),
"=r" (CacheInfo.DX)
: "r" (idx)
: "%rax", "%rbx", "%rcx", "%rdx"
);
Core->T.Cache[level[idx]].WrBack = CacheInfo.WrBack;
Core->T.Cache[level[idx]].Inclus = CacheInfo.Inclus;
}
/* Fn8000_001E {ExtApic, Core, Node} Identifiers. */
__asm__ volatile
(
"movq $0x8000001e, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (leaf8000001e.EAX),
"=r" (leaf8000001e.EBX),
"=r" (leaf8000001e.ECX),
"=r" (leaf8000001e.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
Core->T.ApicID = leaf8000001e.EAX.ExtApicId;
Core->T.CoreID = leaf8000001e.EBX.CoreId;
Core->T.PackageID = leaf8000001e.ECX.NodeId;
if (leaf8000001e.EBX.ThreadsPerCore > 0)
{ /* SMT is enabled . */
Core->T.ThreadID = leaf8000001e.EAX.ExtApicId & 1;
}
else
{ /* SMT is disabled. */
Core->T.ThreadID = 0;
}
Core->T.Cluster.Node = leaf8000001e.ECX.NodeId;
if (PUBLIC(RO(Proc))->Features.Info.LargestExtFunc >= 0x80000026)
{
CPUID_0x80000026 leaf80000026 = {
.EAX = {0}, .EBX = {0}, .ECX = {0}, .EDX = {0}
};
__asm__ volatile
(
"movq $0x80000026, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (leaf80000026.EAX),
"=r" (leaf80000026.EBX),
"=r" (leaf80000026.ECX),
"=r" (leaf80000026.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
if ( ((PUBLIC(RO(Proc))->ArchID == AMD_Zen5_Turin)
|| (PUBLIC(RO(Proc))->ArchID == AMD_Zen5_Turin_Dense))
&& (leaf80000008.ECX.F1Ah.NC > 0xff) )
{
Core->T.Cluster.CCD=(leaf80000026.EDX.Extended_APIC_ID & 0xf00);
Core->T.Cluster.CCD= Core->T.Cluster.CCD >> 8;
}
else
{
Core->T.Cluster.CCD=(leaf80000026.EDX.Extended_APIC_ID & 0xf0);
if (leaf80000008.ECX.NC >= 0x7f) {
Core->T.Cluster.CCD= Core->T.Cluster.CCD >> 6;
} else if (leaf80000008.ECX.NC >= 0x3f) {
Core->T.Cluster.CCD= Core->T.Cluster.CCD >> 5;
} else {
Core->T.Cluster.CCD= Core->T.Cluster.CCD >> 4;
}
}
} else {
Core->T.Cluster.CCD = (Core->T.ApicID & 0xf0) >> 4;
}
if (CPU_Complex == true ) {
if (leaf80000008.ECX.NC >= 0x7f) {
Core->T.Cluster.CCX = Core->T.CoreID >> 4;
} else if (leaf80000008.ECX.NC >= 0x3f) {
Core->T.Cluster.CCX = Core->T.CoreID >> 3;
} else {
Core->T.Cluster.CCX = Core->T.CoreID >> 2;
}
}
} else { /* Fallback algorithm. */
Core->T.ApicID = leaf1_ebx.Init_APIC_ID;
Core->T.PackageID = leaf1_ebx.Init_APIC_ID
>> leaf80000008.ECX.ApicIdCoreIdSize;
Core->T.CoreID = leaf1_ebx.Init_APIC_ID
- (Core->T.PackageID
<< leaf80000008.ECX.ApicIdCoreIdSize);
Core->T.ThreadID = 0;
}
break;
}
}
}
/*
Enumerate the topology of Processors, Cores and Threads
Remark: Early single-core processors are not processed.
Sources: Intel Software Developer's Manual vol 3A Chap. 8.9 /
Intel whitepaper: Detecting Hyper-Threading Technology /
*/
static unsigned short FindMaskWidth(unsigned short maxCount)
{
unsigned short maskWidth = 0, count = (maxCount - 1);
if (BITBSR(count, maskWidth) == 0)
maskWidth++;
return maskWidth;
}
static void Map_Intel_Topology(void *arg)
{
if (arg != NULL)
{
CORE_RO *Core = (CORE_RO *) arg;
unsigned short SMT_Mask_Width, CORE_Mask_Width,
SMT_Select_Mask, CORE_Select_Mask, PKG_Select_Mask;
struct CPUID_0x00000001_EBX leaf1_ebx;
struct
{ /* CPUID 4 */
unsigned int
Type : 5-0,
Level : 8-5,
Init : 9-8,
Assoc : 10-9,
Unused : 14-10,
Cache_SMT_ID : 26-14,
Max_Core_ID : 32-26;
} leaf4_eax;
RDMSR(Core->T.Base, MSR_IA32_APICBASE);
__asm__ volatile
(
"movq $0x1, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%ebx, %0"
: "=r" (leaf1_ebx)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
if (PUBLIC(RO(Proc))->Features.Std.EDX.HTT) {
SMT_Mask_Width = leaf1_ebx.Max_SMT_ID;
__asm__ volatile
(
"movq $0x4, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0"
: "=r" (leaf4_eax)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
CORE_Mask_Width = leaf4_eax.Max_Core_ID + 1;
} else {
SMT_Mask_Width = 0;
CORE_Mask_Width = 1;
}
if (CORE_Mask_Width != 0) {
SMT_Mask_Width = FindMaskWidth(SMT_Mask_Width) / CORE_Mask_Width;
}
SMT_Select_Mask = ~((-1) << SMT_Mask_Width);
CORE_Select_Mask = (~((-1) << (CORE_Mask_Width + SMT_Mask_Width)))
^ SMT_Select_Mask;
PKG_Select_Mask = (-1) << (CORE_Mask_Width + SMT_Mask_Width);
Core->T.ThreadID = leaf1_ebx.Init_APIC_ID & SMT_Select_Mask;
Core->T.CoreID = (leaf1_ebx.Init_APIC_ID & CORE_Select_Mask)
>> SMT_Mask_Width;
Core->T.PackageID = (leaf1_ebx.Init_APIC_ID & PKG_Select_Mask)
>> (CORE_Mask_Width + SMT_Mask_Width);
Core->T.ApicID = leaf1_ebx.Init_APIC_ID;
Cache_Topology(Core);
}
}
static void Map_Intel_Extended_Topology(void *arg)
{
if (arg != NULL) {
CORE_RO *Core = (CORE_RO *) arg;
long InputLevel = 0;
int NoMoreLevels = 0,
SMT_Mask_Width = 0, SMT_Select_Mask = 0,
CorePlus_Mask_Width = 0, CoreOnly_Select_Mask = 0,
Package_Select_Mask = 0;
CPUID_TOPOLOGY_LEAF ExtTopology = {
.AX.Register = 0,
.BX.Register = 0,
.CX.Register = 0,
.DX.Register = 0
};
RDMSR(Core->T.Base, MSR_IA32_APICBASE);
do {
__asm__ volatile
(
"movq $0xb, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"movq %4, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (ExtTopology.AX),
"=r" (ExtTopology.BX),
"=r" (ExtTopology.CX),
"=r" (ExtTopology.DX)
: "r" (InputLevel)
: "%rax", "%rbx", "%rcx", "%rdx"
);
/* Exit from the loop if the BX register equals 0 or
if the requested level exceeds the level of a Core. */
if (!ExtTopology.BX.Register || (InputLevel > LEVEL_CORE))
NoMoreLevels = 1;
else {
switch (ExtTopology.CX.Type) {
case LEVEL_THREAD: {
SMT_Mask_Width = ExtTopology.AX.SHRbits;
SMT_Select_Mask = ~((-1) << SMT_Mask_Width);
Core->T.ThreadID = ExtTopology.DX.x2ApicID
& SMT_Select_Mask;
}
break;
case LEVEL_CORE: {
CorePlus_Mask_Width = ExtTopology.AX.SHRbits;
CoreOnly_Select_Mask = (~((-1) << CorePlus_Mask_Width))
^ SMT_Select_Mask;
Core->T.CoreID = (ExtTopology.DX.x2ApicID
& CoreOnly_Select_Mask)
>> SMT_Mask_Width;
Package_Select_Mask = (-1) << CorePlus_Mask_Width;
Core->T.PackageID = (ExtTopology.DX.x2ApicID
& Package_Select_Mask)
>> CorePlus_Mask_Width;
}
break;
}
}
InputLevel++;
} while (!NoMoreLevels);
Core->T.ApicID = ExtTopology.DX.x2ApicID;
if ((PUBLIC(RO(Proc))->Features.Info.LargestStdFunc >= 0xa)
&& PUBLIC(RO(Proc))->Features.ExtFeature.EDX.Hybrid)
{
__asm__ volatile
(
"movq $0x1a, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0"
: "=r" (Core->T.Cluster.ID)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
}
Cache_Topology(Core);
}
}
static int Core_Topology(unsigned int cpu)
{
int rc = smp_call_function_single(cpu,
( (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
||(PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON) ) ?
Map_AMD_Topology
: (PUBLIC(RO(Proc))->Features.Info.LargestStdFunc >= 0xb) ?
Map_Intel_Extended_Topology : Map_Intel_Topology,
PUBLIC(RO(Core, AT(cpu))), 1); /* Synchronous call. */
if ( !rc
&& (PUBLIC(RO(Proc))->Features.HTT_Enable == 0)
&& (PUBLIC(RO(Core, AT(cpu)))->T.ThreadID > 0) )
{
PUBLIC(RO(Proc))->Features.HTT_Enable = 1;
}
return rc;
}
static unsigned int Proc_Topology(void)
{
unsigned int cpu, CountEnabledCPU = 0;
for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++) {
PUBLIC(RO(Core, AT(cpu)))->T.Base.value = 0;
PUBLIC(RO(Core, AT(cpu)))->T.ApicID = -1;
PUBLIC(RO(Core, AT(cpu)))->T.CoreID = -1;
PUBLIC(RO(Core, AT(cpu)))->T.ThreadID = -1;
PUBLIC(RO(Core, AT(cpu)))->T.PackageID = -1;
PUBLIC(RO(Core, AT(cpu)))->T.Cluster.ID = 0;
BITSET(LOCKLESS, PUBLIC(RO(Core, AT(cpu)))->OffLine, HW);
BITSET(LOCKLESS, PUBLIC(RO(Core, AT(cpu)))->OffLine, OS);
if (cpu_present(cpu)) { /* CPU state probed by the OS. */
if (Core_Topology(cpu) == 0) {
/* CPU state based on the hardware. */
if (PUBLIC(RO(Core, AT(cpu)))->T.ApicID >= 0)
{
BITCLR(LOCKLESS, PUBLIC(RO(Core, AT(cpu)))->OffLine,HW);
CountEnabledCPU++;
}
BITCLR(LOCKLESS, PUBLIC(RO(Core, AT(cpu)))->OffLine, OS);
}
}
}
return CountEnabledCPU;
}
#define HyperThreading_Technology() \
( \
PUBLIC(RO(Proc))->CPU.OnLine = Proc_Topology() \
)
static void Package_Init_Reset(void)
{
PUBLIC(RO(Proc))->Features.TgtRatio_Unlock = 1;
PUBLIC(RO(Proc))->Features.ClkRatio_Unlock = 0;
PUBLIC(RO(Proc))->Features.TDP_Unlock = 0;
PUBLIC(RO(Proc))->Features.Turbo_Unlock = 0;
PUBLIC(RO(Proc))->Features.TurboActiv_Lock = 1;
PUBLIC(RO(Proc))->Features.TDP_Cfg_Lock = 1;
PUBLIC(RO(Proc))->Features.Uncore_Unlock = 0;
}
static void Default_Unlock_Reset(void)
{
switch (Target_Ratio_Unlock) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PUBLIC(RO(Proc))->Features.TgtRatio_Unlock = Target_Ratio_Unlock;
break;
}
switch (Clock_Ratio_Unlock) {
case COREFREQ_TOGGLE_OFF:
case 0b01:
case 0b10:
case 0b11:
PUBLIC(RO(Proc))->Features.ClkRatio_Unlock = Clock_Ratio_Unlock;
break;
}
switch (Turbo_Ratio_Unlock) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PUBLIC(RO(Proc))->Features.Turbo_Unlock = Turbo_Ratio_Unlock;
break;
}
switch (Uncore_Ratio_Unlock) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PUBLIC(RO(Proc))->Features.Uncore_Unlock = Uncore_Ratio_Unlock;
break;
}
switch (HSMP_Attempt) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PUBLIC(RO(Proc))->Features.HSMP_Capable = HSMP_Attempt;
break;
}
}
static void OverrideCodeNameString(PROCESSOR_SPECIFIC *pSpecific)
{
StrCopy(PUBLIC(RO(Proc))->Architecture,
Arch[
PUBLIC(RO(Proc))->ArchID
].Architecture[pSpecific->CodeNameIdx], CODENAME_LEN);
}
static void OverrideUnlockCapability(PROCESSOR_SPECIFIC *pSpecific)
{
if (pSpecific->Latch & LATCH_TGT_RATIO_UNLOCK) {
PUBLIC(RO(Proc))->Features.TgtRatio_Unlock=pSpecific->TgtRatioUnlocked;
}
if (pSpecific->Latch & LATCH_CLK_RATIO_UNLOCK) {
PUBLIC(RO(Proc))->Features.ClkRatio_Unlock=pSpecific->ClkRatioUnlocked;
}
if (pSpecific->Latch & LATCH_TURBO_UNLOCK) {
PUBLIC(RO(Proc))->Features.Turbo_Unlock = pSpecific->TurboUnlocked;
}
if (pSpecific->Latch & LATCH_UNCORE_UNLOCK) {
PUBLIC(RO(Proc))->Features.Uncore_Unlock = pSpecific->UncoreUnlocked;
}
if (pSpecific->Latch & LATCH_HSMP_CAPABLE) {
PUBLIC(RO(Proc))->Features.HSMP_Capable = pSpecific->HSMP_Capable;
}
}
static PROCESSOR_SPECIFIC *LookupProcessor(void)
{
PROCESSOR_SPECIFIC *pSpecific;
for (pSpecific = Arch[PUBLIC(RO(Proc))->ArchID].Specific;
(pSpecific != NULL) && (pSpecific->Brand != NULL);
pSpecific++)
{
char **brands, *brand;
for (brands = pSpecific->Brand, brand = *brands;
brand != NULL;
brands++, brand = *brands)
{
if (strstr(PUBLIC(RO(Proc))->Features.Info.Brand, brand)) {
return pSpecific;
}
}
}
return NULL;
}
static void Intel_FlexRatio(void)
{
static struct {
struct SIGNATURE Arch;
unsigned short grantFlex : 1-0,
experimental : 2-1,
freeToUse : 8-2,
bitsLayout : 16-8;
} list[] = {
{_Core_Yonah, 0, 1, 0, 1},
{_Core_Conroe, 0, 1, 0, 1},
{_Core_Kentsfield, 0, 1, 0, 1},
{_Core_Conroe_616, 0, 1, 0, 1},
{_Core_Penryn, 1, 1, 0, 1}, /* 06_17 */
{_Core_Dunnington, 0, 1, 0, 1},
{_Atom_Bonnell, 0, 1, 0, 0}, /* 06_1C */
{_Atom_Silvermont, 0, 1, 0, 0}, /* 06_26 */
{_Atom_Lincroft, 0, 1, 0, 0}, /* 06_27 */
{_Atom_Clover_Trail, 0, 1, 0, 0}, /* 06_35 */
{_Atom_Saltwell, 0, 1, 0, 0}, /* 06_36 */
{_Silvermont_Bay_Trail, 0, 1, 0, 0}, /* 06_37 */
{_Atom_Avoton, 0, 1, 0, 0}, /* 06_4D */
{_Atom_Airmont, 0, 1, 0, 0}, /* 06_4C */
{_Atom_Goldmont, 1, 1, 0, 0}, /* 06_5C */
{_Atom_Sofia, 1, 1, 0, 0}, /* 06_5D */
{_Atom_Merrifield, 1, 1, 0, 0}, /* 06_4A */
{_Atom_Moorefield, 1, 1, 0, 0}, /* 06_5A */
{_Nehalem_Bloomfield, 0, 0, 0, 1}, /* 06_1A */
{_Nehalem_Lynnfield, 1, 1, 0, 1}, /* 06_1E */
{_Nehalem_MB, 1, 1, 0, 1}, /* 06_1F */
{_Nehalem_EX, 1, 1, 0, 1}, /* 06_2E */
{_Westmere, 1, 1, 0, 1}, /* 06_25 */
{_Westmere_EP, 1, 0, 0, 1}, /* 06_2C : R/W */
{_Westmere_EX, 1, 1, 0, 1}, /* 06_2F */
{_SandyBridge, 1, 1, 0, 0}, /* 06_2A */
{_SandyBridge_EP, 1, 1, 0, 0}, /* 06_2D */
{_IvyBridge, 1, 0, 0, 0}, /* 06_3A */
{_IvyBridge_EP, 1, 1, 0, 0}, /* 06_3E */
{_Haswell_DT, 1, 1, 0, 0}, /* 06_3C */
{_Haswell_EP, 1, 1, 0, 0}, /* 06_3F */
{_Haswell_ULT, 1, 1, 0, 0}, /* 06_45 */
{_Haswell_ULX, 1, 1, 0, 0}, /* 06_46 */
{_Broadwell, 1, 1, 0, 0}, /* 06_3D */
{_Broadwell_D, 1, 1, 0, 0}, /* 06_56 */
{_Broadwell_H, 1, 1, 0, 0}, /* 06_47 */
{_Broadwell_EP, 1, 1, 0, 0}, /* 06_4F */
{_Skylake_UY, 1, 1, 0, 0}, /* 06_4E */
{_Skylake_S, 1, 1, 0, 0}, /* 06_5E */
{_Skylake_X, 1, 1, 0, 0}, /* 06_55 */
{_Xeon_Phi, 0, 1, 0, 0}, /* 06_57 */
{_Kabylake, 1, 1, 0, 0}, /* 06_9E */
{_Kabylake_UY, 1, 1, 0, 0}, /* 06_8E */
{_Cannonlake_U, 1, 1, 0, 0}, /* 06_66 */
{_Cannonlake_H, 1, 1, 0, 0},
{_Geminilake, 1, 1, 0, 0}, /* 06_7A */
{_Icelake_UY, 1, 1, 0, 0}, /* 06_7E */
{_Icelake_X, 1, 1, 0, 0},
{_Icelake_D, 1, 1, 0, 0},
{_Sunny_Cove, 1, 1, 0, 0},
{_Tigerlake, 1, 1, 0, 0},
{_Tigerlake_U, 1, 0, 0, 0}, /* 06_8C : R/W */
{_Cometlake, 1, 1, 0, 0},
{_Cometlake_UY, 1, 1, 0, 0},
{_Atom_Denverton, 1, 1, 0, 0},
{_Tremont_Jacobsville, 1, 1, 0, 0},
{_Tremont_Lakefield, 1, 1, 0, 0},
{_Tremont_Elkhartlake, 1, 1, 0, 0},
{_Tremont_Jasperlake, 1, 1, 0, 0},
{_Sapphire_Rapids, 1, 1, 0, 0},
{_Emerald_Rapids, 1, 1, 0, 0},
{_Granite_Rapids_X, 1, 1, 0, 0},
{_Granite_Rapids_D, 1, 1, 0, 0},
{_Sierra_Forest, 1, 1, 0, 0},
{_Grand_Ridge, 1, 1, 0, 0},
{_Rocketlake, 1, 1, 0, 0},
{_Rocketlake_U, 1, 1, 0, 0},
{_Alderlake_S, 1, 0, 0, 0}, /* 06_97 */
{_Alderlake_H, 1, 0, 0, 0}, /* 06_9A */
{_Alderlake_N, 1, 1, 0, 0},
{_Meteorlake_M, 1, 1, 0, 0},
{_Meteorlake_N, 1, 1, 0, 0},
{_Meteorlake_S, 1, 1, 0, 0},
{_Raptorlake, 1, 0, 0, 0}, /* 06_B7 */
{_Raptorlake_P, 1, 1, 0, 0},
{_Raptorlake_S, 1, 1, 0, 0},
{_Lunarlake, 1, 1, 0, 0}, /* 06_BD */
{_Arrowlake, 1, 1, 0, 0}, /* 06_C6 */
{_Arrowlake_H, 1, 1, 0, 0}, /* 06_C5 */
{_Arrowlake_U, 1, 1, 0, 0}, /* 06_B5 */
{_Pantherlake, 1, 1, 0, 0}, /* 06_CC */
{_Clearwater_Forest, 1, 1, 0, 0}, /* 06_DD */
{_Bartlettlake_S, 1, 1, 0, 0} /* 06_D7 */
};
const unsigned int ids = sizeof(list) / sizeof(list[0]);
unsigned int id;
for (id = 0; id < ids; id++) {
if ((list[id].Arch.ExtFamily == PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily)
&& (list[id].Arch.Family == PUBLIC(RO(Proc))->Features.Std.EAX.Family)
&& (list[id].Arch.ExtModel == PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel)
&& (list[id].Arch.Model == PUBLIC(RO(Proc))->Features.Std.EAX.Model))
{
if (list[id].grantFlex) {
if (!list[id].experimental
|| (list[id].experimental
&& PUBLIC(RO(Proc))->Registration.Experimental))
{
FLEX_RATIO flexReg = {.value = 0};
RDMSR(flexReg, MSR_FLEX_RATIO);
switch (list[id].bitsLayout) {
default:
case 0:
PUBLIC(RO(Proc))->Features.OC_Enable = flexReg.OC_ENABLED;
PUBLIC(RO(Proc))->Features.Factory.Bins = flexReg.OC_BINS;
PUBLIC(RO(Proc))->Features.OC_Lock = flexReg.OC_LOCK;
break;
case 1:
PUBLIC(RO(Proc))->Features.OC_Enable = flexReg.OC_ENABLED;
PUBLIC(RO(Proc))->Features.Factory.Bins=flexReg.CLOCK_FLEX_MAX;
break;
}
PUBLIC(RO(Proc))->Features.Factory.Overclock = \
ABS_FREQ_MHz( signed int,
PUBLIC(RO(Proc))->Features.Factory.Bins,
PUBLIC(RO(Proc))->Features.Factory.Clock );
}
}
break;
}
}
}
static int Intel_MaxBusRatio(PLATFORM_ID *PfID)
{
struct SIGNATURE allowList[] = {
_Core_Conroe, /* 06_0F */
_Core_Penryn, /* 06_17 */
_Atom_Bonnell, /* 06_1C */
_Atom_Silvermont, /* 06_26 */
_Atom_Lincroft, /* 06_27 */
_Atom_Clover_Trail, /* 06_35 */
_Atom_Saltwell, /* 06_36 */
_Silvermont_Bay_Trail, /* 06_37 */
_Atom_Bonnell, /* 06_1C */
_Atom_Airmont /* 06_4C */
};
const int ids = sizeof(allowList) / sizeof(allowList[0]);
int id;
for (id = 0; id < ids; id++) {
if ((allowList[id].ExtFamily \
== PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily)
&& (allowList[id].Family \
== PUBLIC(RO(Proc))->Features.Std.EAX.Family)
&& (allowList[id].ExtModel \
== PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel)
&& (allowList[id].Model \
== PUBLIC(RO(Proc))->Features.Std.EAX.Model))
{
RDMSR((*PfID), MSR_IA32_PLATFORM_ID);
return 0;
}
}
return -1;
}
static void Intel_Core_Platform_Info(unsigned int cpu)
{
PLATFORM_ID PfID = {.value = 0};
PLATFORM_INFO PfInfo = {.value = 0};
PERF_STATUS PerfStatus = {.value = 0};
unsigned int ratio0 = 10, ratio1 = 10; /*Arbitrary values*/
RDMSR(PfInfo, MSR_PLATFORM_INFO);
if (PfInfo.value != 0) {
ratio0 = PfInfo.MaxNonTurboRatio;
}
RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
if (PerfStatus.value != 0) { /* Chap. 18.18.3.4 */
if (PerfStatus.CORE.XE_Enable) {
ratio1 = PerfStatus.CORE.MaxBusRatio;
} else {
if (Intel_MaxBusRatio(&PfID) == 0) {
if (PfID.value != 0)
{
ratio1 = PfID.MaxBusRatio;
}
}
}
} else {
if (Intel_MaxBusRatio(&PfID) == 0) {
if (PfID.value != 0)
{
ratio1 = PfID.MaxBusRatio;
}
}
}
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = KMIN(ratio0, ratio1);
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = KMAX(ratio0, ratio1);
}
static void Compute_Intel_Core_Burst(unsigned int cpu)
{
PERF_STATUS PerfStatus = {.value = 0};
RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
if (PRIVATE(OF(Specific)) != NULL)
{
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(1C)] = \
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)]
+ PRIVATE(OF(Specific))->Boost[0]
+ PRIVATE(OF(Specific))->Boost[1];
if (PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(1C)] > 0) {
PUBLIC(RO(Proc))->Features.SpecTurboRatio = 1;
}
} else { /* Is Processor half ratio or Burst capable ? */
if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA)
{
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(1C)] = \
PerfStatus.value != 0 ? PerfStatus.CORE.MaxBusRatio
: 2 + PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)];
PUBLIC(RO(Proc))->Features.SpecTurboRatio = 1;
} else {
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(1C)] = 0;
PUBLIC(RO(Proc))->Features.SpecTurboRatio = 0;
}
}
}
static PLATFORM_INFO Intel_Platform_Info(unsigned int cpu)
{
PLATFORM_INFO PfInfo = {.value = 0};
RDMSR(PfInfo, MSR_PLATFORM_INFO);
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = PfInfo.MinimumRatio;
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = PfInfo.MaxNonTurboRatio;
return PfInfo;
}
typedef union {
unsigned long long value;
} TURBO_RATIO_CFG_MSR;
typedef union {
TURBO_RATIO_CFG_MSR MSR;
TURBO_RATIO_CONFIG0 Cfg0;
TURBO_RATIO_CONFIG1 Cfg1;
TURBO_RATIO_CONFIG2 Cfg2;
} TURBO_CONFIG;
typedef struct {
CLOCK_ARG *pClockMod;
TURBO_CONFIG Config;
long rc;
} CLOCK_TURBO_ARG;
static void Assign_8C_Boost(unsigned int *pBoost, TURBO_CONFIG *pConfig)
{
pBoost[BOOST(8C)] = pConfig->Cfg0.MaxRatio_8C;
pBoost[BOOST(7C)] = pConfig->Cfg0.MaxRatio_7C;
pBoost[BOOST(6C)] = pConfig->Cfg0.MaxRatio_6C;
pBoost[BOOST(5C)] = pConfig->Cfg0.MaxRatio_5C;
pBoost[BOOST(4C)] = pConfig->Cfg0.MaxRatio_4C;
pBoost[BOOST(3C)] = pConfig->Cfg0.MaxRatio_3C;
pBoost[BOOST(2C)] = pConfig->Cfg0.MaxRatio_2C;
pBoost[BOOST(1C)] = pConfig->Cfg0.MaxRatio_1C;
}
static void Assign_15C_Boost(unsigned int *pBoost, TURBO_CONFIG *pConfig)
{
pBoost[BOOST(15C)] = pConfig->Cfg1.IVB_EP.MaxRatio_15C;
pBoost[BOOST(14C)] = pConfig->Cfg1.IVB_EP.MaxRatio_14C;
pBoost[BOOST(13C)] = pConfig->Cfg1.IVB_EP.MaxRatio_13C;
pBoost[BOOST(12C)] = pConfig->Cfg1.IVB_EP.MaxRatio_12C;
pBoost[BOOST(11C)] = pConfig->Cfg1.IVB_EP.MaxRatio_11C;
pBoost[BOOST(10C)] = pConfig->Cfg1.IVB_EP.MaxRatio_10C;
pBoost[BOOST(9C) ] = pConfig->Cfg1.IVB_EP.MaxRatio_9C;
}
static void Assign_16C_Boost(unsigned int *pBoost, TURBO_CONFIG *pConfig)
{
pBoost[BOOST(16C)] = pConfig->Cfg1.HSW_EP.MaxRatio_16C;
pBoost[BOOST(15C)] = pConfig->Cfg1.HSW_EP.MaxRatio_15C;
pBoost[BOOST(14C)] = pConfig->Cfg1.HSW_EP.MaxRatio_14C;
pBoost[BOOST(13C)] = pConfig->Cfg1.HSW_EP.MaxRatio_13C;
pBoost[BOOST(12C)] = pConfig->Cfg1.HSW_EP.MaxRatio_12C;
pBoost[BOOST(11C)] = pConfig->Cfg1.HSW_EP.MaxRatio_11C;
pBoost[BOOST(10C)] = pConfig->Cfg1.HSW_EP.MaxRatio_10C;
pBoost[BOOST(9C) ] = pConfig->Cfg1.HSW_EP.MaxRatio_9C;
}
static void Assign_18C_Boost(unsigned int *pBoost, TURBO_CONFIG *pConfig)
{
pBoost[BOOST(18C)] = pConfig->Cfg2.MaxRatio_18C;
pBoost[BOOST(17C)] = pConfig->Cfg2.MaxRatio_17C;
}
static void Assign_SKL_X_Boost(unsigned int *pBoost, TURBO_CONFIG *pConfig)
{
pBoost[BOOST(16C)] = pConfig->Cfg1.SKL_X.NUMCORE_7;
pBoost[BOOST(15C)] = pConfig->Cfg1.SKL_X.NUMCORE_6;
pBoost[BOOST(14C)] = pConfig->Cfg1.SKL_X.NUMCORE_5;
pBoost[BOOST(13C)] = pConfig->Cfg1.SKL_X.NUMCORE_4;
pBoost[BOOST(12C)] = pConfig->Cfg1.SKL_X.NUMCORE_3;
pBoost[BOOST(11C)] = pConfig->Cfg1.SKL_X.NUMCORE_2;
pBoost[BOOST(10C)] = pConfig->Cfg1.SKL_X.NUMCORE_1;
pBoost[BOOST(9C) ] = pConfig->Cfg1.SKL_X.NUMCORE_0;
}
static long For_All_Turbo_Clock(CLOCK_ARG *pClockMod, void (*ConfigFunc)(void*))
{
long rc = RC_SUCCESS;
unsigned int cpu = PUBLIC(RO(Proc))->CPU.Count;
do {
cpu--; /* From last AP to BSP */
if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS)
&& ((pClockMod->cpu == -1) || (pClockMod->cpu == cpu)))
{
CLOCK_TURBO_ARG ClockTurbo = {
.pClockMod = pClockMod,
.Config = {.MSR = {.value = 0}},
.rc = RC_SUCCESS
};
smp_call_function_single(cpu, ConfigFunc, &ClockTurbo, 1);
rc = ClockTurbo.rc;
}
} while ((cpu != 0) && (rc >= RC_SUCCESS)) ;
return rc;
}
static void Intel_Turbo_Cfg8C_PerCore(void *arg)
{
CLOCK_TURBO_ARG *pClockCfg8C = (CLOCK_TURBO_ARG *) arg;
unsigned int registerAddress = MSR_TURBO_RATIO_LIMIT;
if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.Hybrid == 1)
{
CORE_RO *Core;
unsigned int cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
if (Core->T.Cluster.Hybrid.CoreType == Hybrid_Atom) {
registerAddress = MSR_SECONDARY_TURBO_RATIO_LIMIT;
}
} else if (PUBLIC(RO(Proc))->ArchID == Atom_Airmont) {
registerAddress = MSR_ATOM_CORE_TURBO_RATIOS;
}
RDMSR(pClockCfg8C->Config.Cfg0, registerAddress);
if (pClockCfg8C->pClockMod != NULL) /* Read-Only function called ? */
{
pClockCfg8C->rc = RC_SUCCESS;
if (PUBLIC(RO(Proc))->Features.Turbo_Unlock)
{
unsigned short WrRd8C = 0;
switch (pClockCfg8C->pClockMod->NC) {
case 1:
if (pClockCfg8C->pClockMod->cpu == -1) {
pClockCfg8C->Config.Cfg0.MaxRatio_1C = pClockCfg8C->pClockMod->Ratio;
} else {
pClockCfg8C->Config.Cfg0.MaxRatio_1C += pClockCfg8C->pClockMod->Offset;
}
WrRd8C = 1;
break;
case 2:
if (pClockCfg8C->pClockMod->cpu == -1) {
pClockCfg8C->Config.Cfg0.MaxRatio_2C = pClockCfg8C->pClockMod->Ratio;
} else {
pClockCfg8C->Config.Cfg0.MaxRatio_2C += pClockCfg8C->pClockMod->Offset;
}
WrRd8C = 1;
break;
case 3:
if (pClockCfg8C->pClockMod->cpu == -1) {
pClockCfg8C->Config.Cfg0.MaxRatio_3C = pClockCfg8C->pClockMod->Ratio;
} else {
pClockCfg8C->Config.Cfg0.MaxRatio_3C += pClockCfg8C->pClockMod->Offset;
}
WrRd8C = 1;
break;
case 4:
if (pClockCfg8C->pClockMod->cpu == -1) {
pClockCfg8C->Config.Cfg0.MaxRatio_4C = pClockCfg8C->pClockMod->Ratio;
} else {
pClockCfg8C->Config.Cfg0.MaxRatio_4C += pClockCfg8C->pClockMod->Offset;
}
WrRd8C = 1;
break;
case 5:
if (pClockCfg8C->pClockMod->cpu == -1) {
pClockCfg8C->Config.Cfg0.MaxRatio_5C = pClockCfg8C->pClockMod->Ratio;
} else {
pClockCfg8C->Config.Cfg0.MaxRatio_5C += pClockCfg8C->pClockMod->Offset;
}
WrRd8C = 1;
break;
case 6:
if (pClockCfg8C->pClockMod->cpu == -1) {
pClockCfg8C->Config.Cfg0.MaxRatio_6C = pClockCfg8C->pClockMod->Ratio;
} else {
pClockCfg8C->Config.Cfg0.MaxRatio_6C += pClockCfg8C->pClockMod->Offset;
}
WrRd8C = 1;
break;
case 7:
if (pClockCfg8C->pClockMod->cpu == -1) {
pClockCfg8C->Config.Cfg0.MaxRatio_7C = pClockCfg8C->pClockMod->Ratio;
} else {
pClockCfg8C->Config.Cfg0.MaxRatio_7C += pClockCfg8C->pClockMod->Offset;
}
WrRd8C = 1;
break;
case 8:
if (pClockCfg8C->pClockMod->cpu == -1) {
pClockCfg8C->Config.Cfg0.MaxRatio_8C = pClockCfg8C->pClockMod->Ratio;
} else {
pClockCfg8C->Config.Cfg0.MaxRatio_8C += pClockCfg8C->pClockMod->Offset;
}
WrRd8C = 1;
break;
default:
WrRd8C = 0;
break;
}
if (WrRd8C) {
WRMSR(pClockCfg8C->Config.Cfg0, registerAddress);
RDMSR(pClockCfg8C->Config.Cfg0, registerAddress);
pClockCfg8C->rc = RC_OK_COMPUTE;
}
} else {
pClockCfg8C->rc = -RC_TURBO_PREREQ;
}
}
}
static long Intel_Turbo_Config8C(CLOCK_ARG *pClockMod)
{
long rc = For_All_Turbo_Clock(pClockMod, Intel_Turbo_Cfg8C_PerCore);
return rc;
}
static void Intel_Turbo_Cfg15C_PerCore(void *arg)
{
CLOCK_TURBO_ARG *pClockCfg15C = (CLOCK_TURBO_ARG *) arg;
RDMSR(pClockCfg15C->Config.Cfg1, MSR_TURBO_RATIO_LIMIT1);
if (pClockCfg15C->pClockMod != NULL)
{
pClockCfg15C->rc = RC_SUCCESS;
if (PUBLIC(RO(Proc))->Features.Turbo_Unlock)
{
unsigned short WrRd15C = 0;
switch (pClockCfg15C->pClockMod->NC) {
case 9:
if (pClockCfg15C->pClockMod->cpu == -1) {
pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_9C = \
pClockCfg15C->pClockMod->Ratio;
} else {
pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_9C += \
pClockCfg15C->pClockMod->Offset;
}
WrRd15C = 1;
break;
case 10:
if (pClockCfg15C->pClockMod->cpu == -1) {
pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_10C = \
pClockCfg15C->pClockMod->Ratio;
} else {
pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_10C += \
pClockCfg15C->pClockMod->Offset;
}
WrRd15C = 1;
break;
case 11:
if (pClockCfg15C->pClockMod->cpu == -1) {
pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_11C = \
pClockCfg15C->pClockMod->Ratio;
} else {
pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_11C += \
pClockCfg15C->pClockMod->Offset;
}
WrRd15C = 1;
break;
case 12:
if (pClockCfg15C->pClockMod->cpu == -1) {
pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_12C = \
pClockCfg15C->pClockMod->Ratio;
} else {
pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_12C += \
pClockCfg15C->pClockMod->Offset;
}
WrRd15C = 1;
break;
case 13:
if (pClockCfg15C->pClockMod->cpu == -1) {
pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_13C = \
pClockCfg15C->pClockMod->Ratio;
} else {
pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_13C += \
pClockCfg15C->pClockMod->Offset;
}
WrRd15C = 1;
break;
case 14:
if (pClockCfg15C->pClockMod->cpu == -1) {
pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_14C = \
pClockCfg15C->pClockMod->Ratio;
} else {
pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_14C += \
pClockCfg15C->pClockMod->Offset;
}
WrRd15C = 1;
break;
case 15:
if (pClockCfg15C->pClockMod->cpu == -1) {
pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_15C = \
pClockCfg15C->pClockMod->Ratio;
} else {
pClockCfg15C->Config.Cfg1.IVB_EP.MaxRatio_15C += \
pClockCfg15C->pClockMod->Offset;
}
WrRd15C = 1;
break;
default:
WrRd15C = 0;
break;
}
if (WrRd15C) {
WRMSR(pClockCfg15C->Config.Cfg1, MSR_TURBO_RATIO_LIMIT1);
RDMSR(pClockCfg15C->Config.Cfg1, MSR_TURBO_RATIO_LIMIT1);
pClockCfg15C->rc = RC_OK_COMPUTE;
}
} else {
pClockCfg15C->rc = -RC_TURBO_PREREQ;
}
}
}
static long Intel_Turbo_Config15C(CLOCK_ARG *pClockMod)
{
long rc = For_All_Turbo_Clock(pClockMod, Intel_Turbo_Cfg15C_PerCore);
return rc;
}
static void Intel_Turbo_Cfg16C_PerCore(void *arg)
{
CLOCK_TURBO_ARG *pClockCfg16C = (CLOCK_TURBO_ARG *) arg;
RDMSR(pClockCfg16C->Config.Cfg1, MSR_TURBO_RATIO_LIMIT1);
if (pClockCfg16C->pClockMod != NULL)
{
pClockCfg16C->rc = RC_SUCCESS;
if (PUBLIC(RO(Proc))->Features.Turbo_Unlock)
{
unsigned short WrRd16C = 0;
switch (pClockCfg16C->pClockMod->NC) {
case 9:
if (pClockCfg16C->pClockMod->cpu == -1) {
pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_9C = \
pClockCfg16C->pClockMod->Ratio;
} else {
pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_9C += \
pClockCfg16C->pClockMod->Offset;
}
WrRd16C = 1;
break;
case 10:
if (pClockCfg16C->pClockMod->cpu == -1) {
pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_10C = \
pClockCfg16C->pClockMod->Ratio;
} else {
pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_10C += \
pClockCfg16C->pClockMod->Offset;
}
WrRd16C = 1;
break;
case 11:
if (pClockCfg16C->pClockMod->cpu == -1) {
pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_11C = \
pClockCfg16C->pClockMod->Ratio;
} else {
pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_11C += \
pClockCfg16C->pClockMod->Offset;
}
WrRd16C = 1;
break;
case 12:
if (pClockCfg16C->pClockMod->cpu == -1) {
pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_12C = \
pClockCfg16C->pClockMod->Ratio;
} else {
pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_12C += \
pClockCfg16C->pClockMod->Offset;
}
WrRd16C = 1;
break;
case 13:
if (pClockCfg16C->pClockMod->cpu == -1) {
pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_13C = \
pClockCfg16C->pClockMod->Ratio;
} else {
pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_13C += \
pClockCfg16C->pClockMod->Offset;
}
WrRd16C = 1;
break;
case 14:
if (pClockCfg16C->pClockMod->cpu == -1) {
pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_14C = \
pClockCfg16C->pClockMod->Ratio;
} else {
pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_14C += \
pClockCfg16C->pClockMod->Offset;
}
WrRd16C = 1;
break;
case 15:
if (pClockCfg16C->pClockMod->cpu == -1) {
pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_15C = \
pClockCfg16C->pClockMod->Ratio;
} else {
pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_15C += \
pClockCfg16C->pClockMod->Offset;
}
WrRd16C = 1;
break;
case 16:
if (pClockCfg16C->pClockMod->cpu == -1) {
pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_16C = \
pClockCfg16C->pClockMod->Ratio;
} else {
pClockCfg16C->Config.Cfg1.HSW_EP.MaxRatio_16C += \
pClockCfg16C->pClockMod->Offset;
}
WrRd16C = 1;
break;
default:
WrRd16C = 0;
break;
}
if (WrRd16C) {
WRMSR(pClockCfg16C->Config.Cfg1, MSR_TURBO_RATIO_LIMIT1);
RDMSR(pClockCfg16C->Config.Cfg1, MSR_TURBO_RATIO_LIMIT1);
pClockCfg16C->rc = RC_OK_COMPUTE;
}
} else {
pClockCfg16C->rc = -RC_TURBO_PREREQ;
}
}
}
static long Intel_Turbo_Config16C(CLOCK_ARG *pClockMod)
{
long rc = For_All_Turbo_Clock(pClockMod, Intel_Turbo_Cfg16C_PerCore);
return rc;
}
static void Intel_Turbo_Cfg18C_PerCore(void *arg)
{
CLOCK_TURBO_ARG *pClockCfg18C = (CLOCK_TURBO_ARG *) arg;
RDMSR(pClockCfg18C->Config.Cfg2, MSR_TURBO_RATIO_LIMIT2);
if (pClockCfg18C->pClockMod != NULL)
{
pClockCfg18C->rc = RC_SUCCESS;
if (PUBLIC(RO(Proc))->Features.Turbo_Unlock)
{
unsigned short WrRd18C = 0;
switch (pClockCfg18C->pClockMod->NC) {
case 17:
if (pClockCfg18C->pClockMod->cpu == -1) {
pClockCfg18C->Config.Cfg2.MaxRatio_17C = \
pClockCfg18C->pClockMod->Ratio;
} else {
pClockCfg18C->Config.Cfg2.MaxRatio_17C += \
pClockCfg18C->pClockMod->Offset;
}
WrRd18C = 1;
break;
case 18:
if (pClockCfg18C->pClockMod->cpu == -1) {
pClockCfg18C->Config.Cfg2.MaxRatio_18C = \
pClockCfg18C->pClockMod->Ratio;
} else {
pClockCfg18C->Config.Cfg2.MaxRatio_18C += \
pClockCfg18C->pClockMod->Offset;
}
WrRd18C = 1;
break;
default:
WrRd18C = 0;
break;
}
if (WrRd18C) {
WRMSR(pClockCfg18C->Config.Cfg2, MSR_TURBO_RATIO_LIMIT2);
RDMSR(pClockCfg18C->Config.Cfg2, MSR_TURBO_RATIO_LIMIT2);
pClockCfg18C->rc = RC_OK_COMPUTE;
}
} else {
pClockCfg18C->rc = -RC_TURBO_PREREQ;
}
}
}
static long Intel_Turbo_Config18C(CLOCK_ARG *pClockMod)
{
long rc = For_All_Turbo_Clock(pClockMod, Intel_Turbo_Cfg18C_PerCore);
return rc;
}
static void Intel_Turbo_Cfg_SKL_X_PerCore(void *arg)
{
CLOCK_TURBO_ARG *pClockCfg16C = (CLOCK_TURBO_ARG *) arg;
RDMSR(pClockCfg16C->Config.Cfg1, MSR_TURBO_RATIO_LIMIT1);
if (pClockCfg16C->pClockMod != NULL)
{
pClockCfg16C->rc = RC_SUCCESS;
if (PUBLIC(RO(Proc))->Features.Turbo_Unlock)
{
unsigned short WrRd16C = 0;
switch (pClockCfg16C->pClockMod->NC) {
case 9:
if (pClockCfg16C->pClockMod->cpu == -1) {
pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_0 = \
pClockCfg16C->pClockMod->Ratio;
} else {
pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_0 += \
pClockCfg16C->pClockMod->Offset;
}
WrRd16C = 1;
break;
case 10:
if (pClockCfg16C->pClockMod->cpu == -1) {
pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_1 = \
pClockCfg16C->pClockMod->Ratio;
} else {
pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_1 += \
pClockCfg16C->pClockMod->Offset;
}
WrRd16C = 1;
break;
case 11:
if (pClockCfg16C->pClockMod->cpu == -1) {
pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_2 = \
pClockCfg16C->pClockMod->Ratio;
} else {
pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_2 += \
pClockCfg16C->pClockMod->Offset;
}
WrRd16C = 1;
break;
case 12:
if (pClockCfg16C->pClockMod->cpu == -1) {
pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_3 = \
pClockCfg16C->pClockMod->Ratio;
} else {
pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_3 += \
pClockCfg16C->pClockMod->Offset;
}
WrRd16C = 1;
break;
case 13:
if (pClockCfg16C->pClockMod->cpu == -1) {
pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_4 = \
pClockCfg16C->pClockMod->Ratio;
} else {
pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_4 += \
pClockCfg16C->pClockMod->Offset;
}
WrRd16C = 1;
break;
case 14:
if (pClockCfg16C->pClockMod->cpu == -1) {
pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_5 = \
pClockCfg16C->pClockMod->Ratio;
} else {
pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_5 += \
pClockCfg16C->pClockMod->Offset;
}
WrRd16C = 1;
break;
case 15:
if (pClockCfg16C->pClockMod->cpu == -1) {
pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_6 = \
pClockCfg16C->pClockMod->Ratio;
} else {
pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_6 += \
pClockCfg16C->pClockMod->Offset;
}
WrRd16C = 1;
break;
case 16:
if (pClockCfg16C->pClockMod->cpu == -1) {
pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_7 = \
pClockCfg16C->pClockMod->Ratio;
} else {
pClockCfg16C->Config.Cfg1.SKL_X.NUMCORE_7 += \
pClockCfg16C->pClockMod->Offset;
}
WrRd16C = 1;
break;
default:
WrRd16C = 0;
break;
}
if (WrRd16C) {
WRMSR(pClockCfg16C->Config.Cfg1, MSR_TURBO_RATIO_LIMIT1);
RDMSR(pClockCfg16C->Config.Cfg1, MSR_TURBO_RATIO_LIMIT1);
pClockCfg16C->rc = RC_OK_COMPUTE;
}
} else {
pClockCfg16C->rc = -RC_TURBO_PREREQ;
}
}
}
static long Skylake_X_Turbo_Config16C(CLOCK_ARG *pClockMod)
{
long rc = For_All_Turbo_Clock(pClockMod, Intel_Turbo_Cfg_SKL_X_PerCore);
return rc;
}
static long TurboClock_IvyBridge_EP(CLOCK_ARG *pClockMod)
{
long rc = Intel_Turbo_Config8C(pClockMod);
if (rc >= RC_SUCCESS)
{
rc = Intel_Turbo_Config15C(pClockMod);
if (rc >= RC_SUCCESS)
{
TURBO_RATIO_CONFIG1 Cfg1;
RDMSR(Cfg1, MSR_TURBO_RATIO_LIMIT1);
Cfg1.IVB_EP.Semaphore = 1;
WRMSR(Cfg1, MSR_TURBO_RATIO_LIMIT1);
}
}
return rc;
}
static long TurboClock_Haswell_EP(CLOCK_ARG *pClockMod)
{
long rc = Intel_Turbo_Config8C(pClockMod);
if (rc >= RC_SUCCESS) {
rc = Intel_Turbo_Config16C(pClockMod);
}
if (rc >= RC_SUCCESS) {
rc = Intel_Turbo_Config18C(pClockMod);
if (rc >= RC_SUCCESS)
{
TURBO_RATIO_CONFIG2 Cfg2;
RDMSR(Cfg2, MSR_TURBO_RATIO_LIMIT2);
Cfg2.Semaphore = 1;
WRMSR(Cfg2, MSR_TURBO_RATIO_LIMIT2);
}
}
return rc;
}
static long TurboClock_Broadwell_EP(CLOCK_ARG *pClockMod)
{
long rc = Intel_Turbo_Config8C(pClockMod);
if (rc >= RC_SUCCESS) {
rc = Intel_Turbo_Config16C(pClockMod);
}
if (rc >= RC_SUCCESS) {
rc = Intel_Turbo_Config18C(pClockMod);
if (rc >= RC_SUCCESS)
{
TURBO_RATIO_CONFIG3 Cfg3;
RDMSR(Cfg3, MSR_TURBO_RATIO_LIMIT3);
Cfg3.Semaphore = 1;
WRMSR(Cfg3, MSR_TURBO_RATIO_LIMIT3);
}
}
return rc;
}
static long TurboClock_Skylake_X(CLOCK_ARG *pClockMod)
{
long rc = Intel_Turbo_Config8C(pClockMod);
if (rc >= RC_SUCCESS) {
rc = Skylake_X_Turbo_Config16C(pClockMod);
}
return rc;
}
static void PerCore_Intel_HWP_Notification(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
if ((arg != NULL) && PUBLIC(RO(Proc))->Features.Power.EAX.HWP_Int) {
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->HWP_Mask, Core->Bind);
/* HWP Notifications are fully disabled. */
Core->PowerThermal.HWP_Interrupt.value = 0;
WRMSR(Core->PowerThermal.HWP_Interrupt, MSR_HWP_INTERRUPT);
RDMSR(Core->PowerThermal.HWP_Interrupt, MSR_HWP_INTERRUPT);
if (Core->PowerThermal.HWP_Interrupt.value == 0) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->HWP, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->HWP, Core->Bind);
}
}
}
static void PerCore_Intel_HWP_Ignition(void *arg)
{
if ((arg != NULL) && PUBLIC(RO(Proc))->Features.Power.EAX.HWP_EPP) {
CORE_RO *Core = (CORE_RO *) arg;
RDMSR(Core->PowerThermal.HWP_Capabilities, MSR_IA32_HWP_CAPABILITIES);
RDMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
Core->PowerThermal.HWP_Request.Minimum_Perf =
Core->PowerThermal.HWP_Capabilities.Lowest;
Core->PowerThermal.HWP_Request.Maximum_Perf =
Core->PowerThermal.HWP_Capabilities.Highest;
Core->PowerThermal.HWP_Request.Desired_Perf =
Core->PowerThermal.HWP_Capabilities.Guaranteed;
if ((HWP_EPP >= 0) && (HWP_EPP <= 0xff)) {
Core->PowerThermal.HWP_Request.Energy_Pref = HWP_EPP;
}
WRMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
RDMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
}
}
static void Intel_Hardware_Performance(void)
{
PM_ENABLE PM_Enable = {.value = 0};
HDC_CONTROL HDC_Control = {.value = 0};
if (PUBLIC(RO(Proc))->Features.Power.EAX.HWP_Reg)
{
/* MSR_IA32_PM_ENABLE is a Package register. */
RDMSR(PM_Enable, MSR_IA32_PM_ENABLE);
/* Is the HWP requested and its current state is off ? */
if ((HWP_Enable == COREFREQ_TOGGLE_ON) && (PM_Enable.HWP_Enable == 0))
{
unsigned int cpu;
/* From last AP to BSP */
cpu = PUBLIC(RO(Proc))->CPU.Count;
do {
cpu--;
if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS)) {
/* Synchronous call. */
smp_call_function_single(cpu,
PerCore_Intel_HWP_Notification,
&PUBLIC(RO(Core, AT(cpu))), 1);
}
} while (cpu != 0) ;
if (BITCMP_CC(BUS_LOCK, \
PUBLIC(RW(Proc))->HWP, PUBLIC(RO(Proc))->HWP_Mask) )
{
/* Enable the Hardware-controlled Performance States. */
PM_Enable.HWP_Enable = 1;
WRMSR(PM_Enable, MSR_IA32_PM_ENABLE);
RDMSR(PM_Enable, MSR_IA32_PM_ENABLE);
if (PM_Enable.HWP_Enable)
{
cpu = PUBLIC(RO(Proc))->CPU.Count;
do {
cpu--;
if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS)) {
/* Asynchronous call. */
smp_call_function_single(cpu,
PerCore_Intel_HWP_Ignition,
&PUBLIC(RO(Core, AT(cpu))), 0);
}
} while (cpu != 0) ;
}
}
}
}
PUBLIC(RO(Proc))->Features.HWP_Enable = PM_Enable.HWP_Enable;
/* Hardware Duty Cycling */
if (PUBLIC(RO(Proc))->Features.Power.EAX.HDC_Reg)
{
RDMSR(HDC_Control, MSR_IA32_PKG_HDC_CTL);
switch (HDC_Enable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
HDC_Control.HDC_Enable = HDC_Enable;
WRMSR(HDC_Control, MSR_IA32_PKG_HDC_CTL);
RDMSR(HDC_Control, MSR_IA32_PKG_HDC_CTL);
break;
}
}
PUBLIC(RO(Proc))->Features.HDC_Enable = HDC_Control.HDC_Enable;
}
static void Skylake_PowerControl(void)
{
unsigned short WrRdMSR = 0;
POWER_CONTROL PowerCtrl = {.value = 0};
RDMSR(PowerCtrl, MSR_IA32_POWER_CTL);
switch (EEO_Disable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PowerCtrl.EEO_Disable = EEO_Disable;
WrRdMSR = 1;
break;
}
switch (R2H_Disable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PowerCtrl.R2H_Disable = R2H_Disable;
WrRdMSR = 1;
break;
}
if (WrRdMSR) {
WRMSR(PowerCtrl, MSR_IA32_POWER_CTL);
RDMSR(PowerCtrl, MSR_IA32_POWER_CTL);
}
PUBLIC(RO(Proc))->Features.EEO_Enable = !PowerCtrl.EEO_Disable;
PUBLIC(RO(Proc))->Features.R2H_Enable = !PowerCtrl.R2H_Disable;
}
static void SandyBridge_Uncore_Ratio(unsigned int cpu)
{
PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MIN)] = \
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)];
PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MAX)] = \
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)];
}
static long Haswell_Uncore_Ratio(CLOCK_ARG *pClockMod)
{
long rc = RC_SUCCESS;
UNCORE_RATIO_LIMIT UncoreRatio = {.value = 0};
RDMSR(UncoreRatio, MSR_HSW_UNCORE_RATIO_LIMIT);
if (pClockMod != NULL) { /* Called as Read-Only ? */
if (PUBLIC(RO(Proc))->Features.Uncore_Unlock)
{
unsigned short WrRdMSR;
switch (pClockMod->NC) {
case CLOCK_MOD_MAX:
if (pClockMod->cpu == -1) {
UncoreRatio.MaxRatio = pClockMod->Ratio;
} else {
UncoreRatio.MaxRatio += pClockMod->Offset;
}
WrRdMSR = 1;
break;
case CLOCK_MOD_MIN:
if (pClockMod->cpu == -1) {
UncoreRatio.MinRatio = pClockMod->Ratio;
} else {
UncoreRatio.MinRatio += pClockMod->Offset;
}
WrRdMSR = 1;
break;
default:
WrRdMSR = 0;
rc = -RC_UNIMPLEMENTED;
break;
}
if (WrRdMSR) {
WRMSR(UncoreRatio, MSR_HSW_UNCORE_RATIO_LIMIT);
RDMSR(UncoreRatio, MSR_HSW_UNCORE_RATIO_LIMIT);
rc = RC_OK_COMPUTE;
}
} else {
rc = -RC_UNCORE_PREREQ;
}
}
PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MIN)]=UncoreRatio.MinRatio;
PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MAX)]=UncoreRatio.MaxRatio;
return rc;
}
static void Nehalem_PowerLimit(void)
{
unsigned short WrRdMSR = 0;
NEHALEM_POWER_LIMIT PowerLimit = {.value = 0};
RDMSR(PowerLimit, MSR_TURBO_POWER_CURRENT_LIMIT);
if (Custom_TDP_Count > 0) {
if (Custom_TDP_Offset[PWR_DOMAIN(PKG)] != 0)
{ /* Register is an 8 watt multiplier */
signed short TDP_Limit = PowerLimit.TDP_Limit >> 3;
TDP_Limit += Custom_TDP_Offset[PWR_DOMAIN(PKG)];
if (TDP_Limit >= 0)
{
PowerLimit.TDP_Limit = TDP_Limit << 3;
WrRdMSR = 1;
}
}
}
if (TDP_Limiting_Count > 0) {
switch (Activate_TDP_Limit[PWR_DOMAIN(PKG)]) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PowerLimit.TDP_Override = Activate_TDP_Limit[PWR_DOMAIN(PKG)];
WrRdMSR = 1;
break;
}
}
if (Custom_TDC_Offset != 0) {
signed short TDC_Limit = PowerLimit.TDC_Limit >> 3;
TDC_Limit += Custom_TDC_Offset;
if (TDC_Limit > 0)
{
PowerLimit.TDC_Limit = TDC_Limit << 3;
WrRdMSR = 1;
}
}
switch (Activate_TDC_Limit) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PowerLimit.TDC_Override = Activate_TDC_Limit;
WrRdMSR = 1;
break;
}
if (WrRdMSR) {
if (PUBLIC(RO(Proc))->Features.TDP_Unlock) {
WRMSR(PowerLimit, MSR_TURBO_POWER_CURRENT_LIMIT);
RDMSR(PowerLimit, MSR_TURBO_POWER_CURRENT_LIMIT);
}
}
PUBLIC(RO(Proc))->PowerThermal.Domain[PWR_DOMAIN(PKG)].Unlock = \
PUBLIC(RO(Proc))->Features.TDP_Unlock;
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Domain_Limit1 = PowerLimit.TDP_Limit;
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Enable_Limit1 = PowerLimit.TDP_Override;
/* TDP: 1/(2 << (3-1)) = 1/8 watt */
PUBLIC(RO(Proc))->PowerThermal.Unit.PU = 3;
/* TDC: 1/8 amp */
PUBLIC(RO(Proc))->PowerThermal.TDC = PowerLimit.TDC_Limit >> 3;
PUBLIC(RO(Proc))->PowerThermal.Enable_Limit.TDC=PowerLimit.TDC_Override;
}
static void Intel_PowerInterface(void)
{
RDMSR(PUBLIC(RO(Proc))->PowerThermal.Unit, MSR_RAPL_POWER_UNIT);
RDMSR(PUBLIC(RO(Proc))->PowerThermal.PowerInfo, MSR_PKG_POWER_INFO);
}
static void Intel_DomainPowerLimit( unsigned int MSR_DOMAIN_POWER_LIMIT,
unsigned long long PowerLimitLockMask,
enum PWR_DOMAIN pw )
{
const unsigned int lt = PWR_LIMIT_SIZE * pw, rt = 1 + lt;
unsigned short WrRdMSR = 0;
DOMAIN_POWER_LIMIT PowerLimit = {.value = 0};
RDMSR(PowerLimit, MSR_DOMAIN_POWER_LIMIT);
PUBLIC(RO(Proc))->PowerThermal.Domain[pw].Unlock = \
!((PowerLimit.value & PowerLimitLockMask) == PowerLimitLockMask);
if (PUBLIC(RO(Proc))->PowerThermal.Unit.PU > 0)
{
unsigned short pwrUnits = PUBLIC(RO(Proc))->PowerThermal.Unit.PU - 1;
pwrUnits = 2 << pwrUnits;
if (Custom_TDP_Count > lt) {
if (Custom_TDP_Offset[lt] != 0)
{
signed short TDP_Limit = PowerLimit.Domain_Limit1 / pwrUnits;
TDP_Limit = TDP_Limit + Custom_TDP_Offset[lt];
if (TDP_Limit >= 0) {
PowerLimit.Domain_Limit1 = pwrUnits * TDP_Limit;
WrRdMSR = 1;
}
}
}
if (PowerLimitLockMask == PKG_POWER_LIMIT_LOCK_MASK) {
if (Custom_TDP_Count > rt) {
if (Custom_TDP_Offset[rt] != 0)
{
signed short TDP_Limit = PowerLimit.Domain_Limit2 / pwrUnits;
TDP_Limit = TDP_Limit + Custom_TDP_Offset[rt];
if (TDP_Limit >= 0) {
PowerLimit.Domain_Limit2 = pwrUnits * TDP_Limit;
WrRdMSR = 1;
}
}
}
}
switch (pw) {
case PWR_DOMAIN(PKG):
case PWR_DOMAIN(CORES):
case PWR_DOMAIN(UNCORE):
case PWR_DOMAIN(RAM):
case PWR_DOMAIN(PLATFORM):
if (PowerTimeWindow_Count > lt) {
if ((PowerTimeWindow[lt] >= 0)
&& (PowerTimeWindow[lt] <= 0b1111111))
{
PowerLimit.TimeWindow1 = PowerTimeWindow[lt];
WrRdMSR = 1;
}
}
break;
case PWR_DOMAIN(SIZE):
break;
}
switch (pw) {
case PWR_DOMAIN(PLATFORM):
if (!PUBLIC(RO(Proc))->Registration.Experimental) {
break;
}
fallthrough;
case PWR_DOMAIN(PKG):
if (PowerTimeWindow_Count > rt) {
if ((PowerTimeWindow[rt] >= 0)
&& (PowerTimeWindow[rt] <= 0b1111111))
{
PowerLimit.TimeWindow2 = PowerTimeWindow[rt];
WrRdMSR = 1;
}
}
break;
case PWR_DOMAIN(CORES):
case PWR_DOMAIN(UNCORE):
case PWR_DOMAIN(RAM):
case PWR_DOMAIN(SIZE):
break;
}
}
if (TDP_Limiting_Count > lt) {
switch (Activate_TDP_Limit[lt]) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PowerLimit.Enable_Limit1 = Activate_TDP_Limit[lt];
WrRdMSR = 1;
break;
}
}
switch (pw) {
case PWR_DOMAIN(RAM):
if (!PUBLIC(RO(Proc))->Registration.Experimental) {
break;
}
fallthrough;
default:
if (TDP_Clamping_Count > lt) {
switch (Activate_TDP_Clamp[lt]) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
if (PUBLIC(RO(Proc))->Features.Power.EAX.PLN) {
PowerLimit.Clamping1 = Activate_TDP_Clamp[lt];
WrRdMSR = 1;
}
break;
}
}
break;
}
if (PowerLimitLockMask == PKG_POWER_LIMIT_LOCK_MASK) {
if (TDP_Limiting_Count > rt) {
switch (Activate_TDP_Limit[rt]) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PowerLimit.Enable_Limit2 = Activate_TDP_Limit[rt];
WrRdMSR = 1;
break;
}
}
switch (pw) {
case PWR_DOMAIN(RAM):
if (!PUBLIC(RO(Proc))->Registration.Experimental) {
break;
}
fallthrough;
default:
if (TDP_Clamping_Count > rt) {
switch (Activate_TDP_Clamp[rt]) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PowerLimit.Clamping2 = Activate_TDP_Clamp[rt];
WrRdMSR = 1;
break;
}
}
break;
}
}
if (WrRdMSR && PUBLIC(RO(Proc))->PowerThermal.Domain[pw].Unlock) {
WRMSR(PowerLimit, MSR_DOMAIN_POWER_LIMIT);
}
RDMSR(PowerLimit, MSR_DOMAIN_POWER_LIMIT);
PUBLIC(RO(Proc))->PowerThermal.Domain[pw].PowerLimit = PowerLimit;
}
static void Intel_Pkg_CST_IRTL(const unsigned int MSR, PKGCST_IRTL *PCST)
{
RDMSR((*PCST), MSR);
}
static long Intel_ThermalOffset(bool programmableTj)
{
long rc = -EINVAL;
if (ThermalOffset != 0) {
if (programmableTj)
{
TJMAX TjMax = {.value = 0};
RDMSR(TjMax, MSR_IA32_TEMPERATURE_TARGET);
switch (PUBLIC(RO(Proc))->ArchID) {
case Atom_Goldmont:
case Xeon_Phi: /* TODO(06_85h) */ {
const short offset = TjMax.Atom.Offset + ThermalOffset;
if ((offset >= 0) && (offset <= 0b111111)) {
TjMax.Atom.Offset = offset;
WRMSR(TjMax, MSR_IA32_TEMPERATURE_TARGET);
RDMSR(TjMax, MSR_IA32_TEMPERATURE_TARGET);
rc = TjMax.Atom.Offset == offset ? RC_OK_COMPUTE
: -RC_UNIMPLEMENTED;
}
}
break;
case IvyBridge_EP:
case Broadwell_EP:
case Broadwell_D:
case Skylake_X: {
const short offset = TjMax.EP.Offset + ThermalOffset;
if ((offset >= 0) && (offset <= 0b1111)) {
TjMax.EP.Offset = offset;
WRMSR(TjMax, MSR_IA32_TEMPERATURE_TARGET);
RDMSR(TjMax, MSR_IA32_TEMPERATURE_TARGET);
rc = TjMax.EP.Offset == offset ? RC_OK_COMPUTE
: -RC_UNIMPLEMENTED;
}
}
break;
default: {
const short offset = TjMax.Offset + ThermalOffset;
if ((offset >= 0) && (offset <= 0b1111)) {
TjMax.Offset = offset;
WRMSR(TjMax, MSR_IA32_TEMPERATURE_TARGET);
RDMSR(TjMax, MSR_IA32_TEMPERATURE_TARGET);
rc = TjMax.Offset == offset ? RC_OK_COMPUTE
: -RC_UNIMPLEMENTED;
}
}
break;
case Core_Yonah ... Core_Dunnington:
rc = -RC_UNIMPLEMENTED;
break;
}
} else {
rc = -RC_UNIMPLEMENTED;
}
} else {
rc = RC_SUCCESS;
}
return rc;
}
static void Intel_Processor_PIN(bool capable)
{
if (capable) {
INTEL_PPIN_CTL PPinCtl = {.value = 0};
RDMSR(PPinCtl, MSR_PPIN_CTL);
if (PPinCtl.Enable) {
INTEL_PPIN_NUM PPinNum = {.value = 0};
RDMSR(PPinNum, MSR_PPIN);
PUBLIC(RO(Proc))->Features.Factory.PPIN = PPinNum.value;
}
}
}
static void AMD_Processor_PIN(bool capable)
{
if (capable) {
AMD_PPIN_CTL PPinCtl = {.value = 0};
RDMSR(PPinCtl, MSR_AMD_PPIN_CTL);
if (PPinCtl.Enable) {
AMD_PPIN_NUM PPinNum = {.value = 0};
RDMSR(PPinNum, MSR_AMD_PPIN);
PUBLIC(RO(Proc))->Features.Factory.PPIN = PPinNum.value;
}
}
}
static long AMD_F17h_CPPC(void)
{
AMD_CPPC_ENABLE CPPC_Enable = {.value = 0};
if (PUBLIC(RO(Proc))->Features.leaf80000008.EBX.CPPC)
{
RDMSR(CPPC_Enable, MSR_AMD_CPPC_ENABLE);
if ((HWP_Enable == COREFREQ_TOGGLE_ON) && (CPPC_Enable.CPPC_Enable == 0))
{
CPPC_Enable.CPPC_Enable = 1;
WRMSR(CPPC_Enable, MSR_AMD_CPPC_ENABLE);
RDMSR(CPPC_Enable, MSR_AMD_CPPC_ENABLE);
}
PUBLIC(RO(Proc))->Features.HWP_Enable = CPPC_Enable.CPPC_Enable;
return 0;
}
return -ENODEV;
}
static void Compute_ACPI_CPPC_Bounds(unsigned int cpu)
{
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
if (Core->PowerThermal.ACPI_CPPC.Highest > \
PUBLIC(RO(Proc))->PowerThermal.ACPI_CPPC.Maximum)
{
PUBLIC(RO(Proc))->PowerThermal.ACPI_CPPC.Maximum = \
Core->PowerThermal.ACPI_CPPC.Highest;
}
if (Core->PowerThermal.ACPI_CPPC.Highest < \
PUBLIC(RO(Proc))->PowerThermal.ACPI_CPPC.Minimum)
{
PUBLIC(RO(Proc))->PowerThermal.ACPI_CPPC.Minimum = \
Core->PowerThermal.ACPI_CPPC.Highest;
}
}
static signed int Disable_ACPI_CPPC(unsigned int cpu, void *arg)
{
#if defined(CONFIG_ACPI_CPPC_LIB) \
&& LINUX_VERSION_CODE >= KERNEL_VERSION(5, 17, 0)
signed int rc = cppc_set_enable((signed int) cpu, false);
#else
signed int rc = -ENODEV;
#endif /* CONFIG_ACPI_CPPC_LIB */
UNUSED(arg);
if (rc != 0) {
pr_debug("CoreFreq: cppc_set_enable(cpu=%u, false) error %d\n",
cpu, rc);
}
Compute_ACPI_CPPC_Bounds(cpu);
return rc;
}
static signed int Enable_ACPI_CPPC(unsigned int cpu, void *arg)
{
#if defined(CONFIG_ACPI_CPPC_LIB) \
&& LINUX_VERSION_CODE >= KERNEL_VERSION(5, 17, 0)
signed int rc = cppc_set_enable((signed int) cpu, true);
#else
signed int rc = -ENODEV;
#endif /* CONFIG_ACPI_CPPC_LIB */
UNUSED(arg);
if (rc != 0) {
pr_debug("CoreFreq: cppc_set_enable(cpu=%u, true) error %d\n",
cpu, rc);
}
Compute_ACPI_CPPC_Bounds(cpu);
return rc;
}
static signed int Get_ACPI_CPPC_Registers(unsigned int cpu, void *arg)
{
#ifdef CONFIG_ACPI_CPPC_LIB
struct cppc_perf_fb_ctrs CPPC_Perf;
struct cppc_perf_caps CPPC_Caps;
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
signed int rc = 0;
UNUSED(arg);
if ((rc = cppc_get_perf_ctrs(Core->Bind, &CPPC_Perf)) == 0) {
if ((rc = cppc_get_perf_caps(Core->Bind, &CPPC_Caps)) != 0)
pr_debug("CoreFreq: cppc_get_perf_caps(cpu=%u) error %d\n",
Core->Bind, rc);
} else {
pr_debug("CoreFreq: cppc_get_perf_ctrs(cpu=%u) error %d\n",
Core->Bind, rc);
}
if (rc == 0) {
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0)
unsigned long long desired_perf = 0;
#endif
Core->PowerThermal.ACPI_CPPC = (struct ACPI_CPPC_STRUCT) {
.Highest = CPPC_Caps.highest_perf,
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0)
.Guaranteed = CPPC_Caps.guaranteed_perf == 0 ?
CPPC_Caps.nominal_perf
: CPPC_Caps.guaranteed_perf,
#else
.Guaranteed = CPPC_Caps.nominal_perf,
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0)
.Efficient = CPPC_Caps.nominal_freq,
.Lowest = CPPC_Caps.lowest_freq,
.Minimum = CPPC_Caps.lowest_freq,
#else
.Efficient = CPPC_Caps.nominal_perf,
.Lowest = CPPC_Caps.lowest_perf,
.Minimum = CPPC_Caps.lowest_perf,
#endif
.Maximum = CPPC_Caps.highest_perf,
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0)
.Desired = CPPC_Perf.reference_perf,
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)
.Desired = CPPC_Caps.reference_perf,
#else
.Desired = 0,
#endif
.Energy = 0
};
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0)
#if (defined(CONFIG_SCHED_BORE) || defined(CONFIG_CACHY)) \
&& (LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0))
rc = cppc_get_desired_perf(Core->Bind, &desired_perf);
rc = rc == -EINVAL ? 0 : rc;
#else
rc = cppc_get_desired_perf(Core->Bind, &desired_perf);
#endif
if (rc == 0) {
Core->PowerThermal.ACPI_CPPC.Desired = desired_perf;
} else {
pr_debug("CoreFreq: cppc_get_desired_perf(cpu=%u) error %d\n",
Core->Bind, rc);
}
#endif
}
return rc;
#else
return -ENODEV;
#endif /* CONFIG_ACPI_CPPC_LIB */
}
static signed int Get_EPP_ACPI_CPPC(unsigned int cpu)
{
signed int rc = -ENODEV;
#if defined(CONFIG_ACPI_CPPC_LIB) \
&& LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0)
u64 epp_perf;
if ((rc = cppc_get_epp_perf((signed int) cpu, &epp_perf)) == 0)
{
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
Core->PowerThermal.ACPI_CPPC.Energy = epp_perf;
} else {
pr_debug("CoreFreq: cppc_get_epp_perf(cpu=%u) error %d\n", cpu, rc);
}
#endif
return rc;
}
static signed int Put_EPP_ACPI_CPPC(unsigned int cpu, signed short epp)
{
signed int rc = -ENODEV;
#if defined(CONFIG_ACPI_CPPC_LIB) \
&& LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0)
struct cppc_perf_ctrls perf_ctrls = {
.max_perf = 0,
.min_perf = 0,
.desired_perf = 0,
.energy_perf = epp
};
if ((rc = cppc_set_epp_perf((signed int) cpu, &perf_ctrls, true)) < 0) {
pr_debug("CoreFreq: cppc_set_epp_perf(cpu=%u) error %d\n", cpu, rc);
}
#endif
return rc;
}
static signed int Set_EPP_ACPI_CPPC(unsigned int cpu, void *arg)
{
signed int rc = 0;
UNUSED(arg);
if ((HWP_EPP >= 0) && (HWP_EPP <= 0xff)) {
if ((rc = Put_EPP_ACPI_CPPC(cpu, HWP_EPP)) == 0) {
rc = Get_EPP_ACPI_CPPC(cpu);
}
}
return rc;
}
static signed int Read_ACPI_CPPC_Registers(unsigned int cpu, void *arg)
{
signed int rc = Get_ACPI_CPPC_Registers(cpu, arg);
Compute_ACPI_CPPC_Bounds(cpu);
if (Get_EPP_ACPI_CPPC(cpu) == 0) {
PUBLIC(RO(Proc))->Features.OSPM_EPP = 1;
}
Set_EPP_ACPI_CPPC(cpu, arg);
return rc;
}
static signed int Read_ACPI_PCT_Registers(unsigned int cpu)
{
#if defined(CONFIG_ACPI)
signed int rc = 0;
struct acpi_processor *pr = per_cpu(processors, cpu);
if (pr == NULL) {
PUBLIC(RO(Proc))->Features.ACPI_PCT_CAP = 0;
PUBLIC(RO(Proc))->Features.ACPI_PCT = 0;
rc = -ENODEV;
} else {
struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
acpi_status status=acpi_evaluate_object(pr->handle,"_PCT",NULL,&buffer);
if (ACPI_FAILURE(status)) {
PUBLIC(RO(Proc))->Features.ACPI_PCT_CAP = 0;
PUBLIC(RO(Proc))->Features.ACPI_PCT = 0;
rc = -ENODEV;
} else {
union acpi_object *pct = (union acpi_object *) buffer.pointer;
if ((pct == NULL) || (pct->type != ACPI_TYPE_PACKAGE)
|| (pct->package.count < 2))
{
PUBLIC(RO(Proc))->Features.ACPI_PCT_CAP = 0;
PUBLIC(RO(Proc))->Features.ACPI_PCT = 0;
rc = -EFAULT;
} else {
PUBLIC(RO(Proc))->Features.ACPI_PCT_CAP = 1;
PUBLIC(RO(Proc))->Features.ACPI_PCT = 1;
}
}
kfree(buffer.pointer);
}
return rc;
#else
PUBLIC(RO(Proc))->Features.ACPI_PCT_CAP = 0;
PUBLIC(RO(Proc))->Features.ACPI_PCT = 0;
return -ENODEV;
#endif
}
static signed int Read_ACPI_PSS_Registers(unsigned int cpu)
{
#if defined(CONFIG_ACPI)
signed int rc = 0;
struct acpi_processor *pr = per_cpu(processors, cpu);
if (pr == NULL) {
PUBLIC(RO(Proc))->Features.ACPI_PSS_CAP = 0;
PUBLIC(RO(Proc))->Features.ACPI_PSS = 0;
rc = -ENODEV;
} else {
struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
acpi_status status=acpi_evaluate_object(pr->handle,"_PSS",NULL,&buffer);
if (ACPI_FAILURE(status)) {
PUBLIC(RO(Proc))->Features.ACPI_PSS_CAP = 0;
PUBLIC(RO(Proc))->Features.ACPI_PSS = 0;
rc = -ENODEV;
} else {
union acpi_object *pss = (union acpi_object *) buffer.pointer;
if ((pss == NULL) || (pss->type != ACPI_TYPE_PACKAGE)
|| (pss->package.count == 0))
{
PUBLIC(RO(Proc))->Features.ACPI_PSS_CAP = 0;
PUBLIC(RO(Proc))->Features.ACPI_PSS = 0;
rc = -EFAULT;
} else {
PUBLIC(RO(Proc))->Features.ACPI_PSS_CAP = 1;
PUBLIC(RO(Proc))->Features.ACPI_PSS = pss->package.count & 0xf;
}
}
kfree(buffer.pointer);
}
return rc;
#else
PUBLIC(RO(Proc))->Features.ACPI_PSS_CAP = 0;
PUBLIC(RO(Proc))->Features.ACPI_PSS = 0;
return -ENODEV;
#endif
}
static signed int Read_ACPI_PPC_Registers(unsigned int cpu)
{
#if defined(CONFIG_ACPI)
signed int rc = 0;
struct acpi_processor *pr = per_cpu(processors, cpu);
if (pr == NULL) {
PUBLIC(RO(Proc))->Features.ACPI_PPC_CAP = 0;
PUBLIC(RO(Proc))->Features.ACPI_PPC = 0;
rc = -ENODEV;
} else {
struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
acpi_status status=acpi_evaluate_object(pr->handle,"_PPC",NULL,&buffer);
if (ACPI_FAILURE(status)) {
PUBLIC(RO(Proc))->Features.ACPI_PPC_CAP = 0;
PUBLIC(RO(Proc))->Features.ACPI_PPC = 0;
rc = -ENODEV;
} else {
union acpi_object *ppc = (union acpi_object *) buffer.pointer;
if ((ppc == NULL) || (ppc->type != ACPI_TYPE_INTEGER))
{
PUBLIC(RO(Proc))->Features.ACPI_PPC_CAP = 0;
PUBLIC(RO(Proc))->Features.ACPI_PPC = 0;
rc = -EFAULT;
} else {
union acpi_object *obj = buffer.pointer;
if ((obj != NULL) && (obj->type == ACPI_TYPE_INTEGER)) {
PUBLIC(RO(Proc))->Features.ACPI_PPC_CAP = 1;
PUBLIC(RO(Proc))->Features.ACPI_PPC = obj->integer.value & 0xf;
} else {
PUBLIC(RO(Proc))->Features.ACPI_PPC_CAP = 0;
PUBLIC(RO(Proc))->Features.ACPI_PPC = 0;
}
}
}
kfree(buffer.pointer);
}
return rc;
#else
PUBLIC(RO(Proc))->Features.ACPI_PPC_CAP = 0;
PUBLIC(RO(Proc))->Features.ACPI_PPC = 0;
return -ENODEV;
#endif
}
static signed int Read_ACPI_CST_Registers(unsigned int cpu)
{
#if defined(CONFIG_ACPI)
signed int rc = 0;
struct acpi_processor *pr = per_cpu(processors, cpu);
if (pr == NULL) {
PUBLIC(RO(Proc))->Features.ACPI_CST_CAP = 0;
PUBLIC(RO(Proc))->Features.ACPI_CST = 0;
rc = -ENODEV;
} else {
struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
acpi_status status=acpi_evaluate_object(pr->handle,"_CST",NULL,&buffer);
if (ACPI_FAILURE(status)) {
PUBLIC(RO(Proc))->Features.ACPI_CST_CAP = 0;
PUBLIC(RO(Proc))->Features.ACPI_CST = 0;
rc = -ENODEV;
} else {
union acpi_object *cst = (union acpi_object *) buffer.pointer;
if ((cst == NULL) || (cst->type != ACPI_TYPE_PACKAGE)
|| (cst->package.count == 0))
{
PUBLIC(RO(Proc))->Features.ACPI_CST_CAP = 0;
PUBLIC(RO(Proc))->Features.ACPI_CST = 0;
rc = -EFAULT;
} else {
PUBLIC(RO(Proc))->Features.ACPI_CST_CAP = 1;
PUBLIC(RO(Proc))->Features.ACPI_CST = \
cst->package.elements[0].integer.value & 0xf;
}
}
kfree(buffer.pointer);
}
return rc;
#else
PUBLIC(RO(Proc))->Features.ACPI_CST_CAP = 0;
PUBLIC(RO(Proc))->Features.ACPI_CST = 0;
return -ENODEV;
#endif
}
static void For_All_ACPI_CPPC( signed int (*CPPC_Func)(unsigned int, void*),
void *arg )
{
#if defined(CONFIG_ACPI_CPPC_LIB) \
&& LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0)
signed int rc = acpi_cpc_valid() == false;
#elif defined(CONFIG_ACPI)
signed int rc = acpi_disabled;
#else
signed int rc = false;
#endif
unsigned int cpu;
PUBLIC(RO(Proc))->Features.OSPM_CPC = !rc;
PUBLIC(RO(Proc))->PowerThermal.ACPI_CPPC.Minimum = 255U;
PUBLIC(RO(Proc))->PowerThermal.ACPI_CPPC.Maximum = 1U;
for (cpu = 0; (cpu < PUBLIC(RO(Proc))->CPU.Count) && (rc == 0); cpu++)
{
if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS)) {
rc = CPPC_Func(cpu, arg);
}
}
PUBLIC(RO(Proc))->Features.ACPI_CPPC = (rc == 0);
}
static void Query_Same_Platform_Features(unsigned int cpu)
{
PLATFORM_INFO PfInfo;
PfInfo = Intel_Platform_Info(cpu);
PUBLIC(RO(Proc))->Features.TDP_Unlock = PfInfo.ProgrammableTDP;
PUBLIC(RO(Proc))->Features.TDP_Levels = PfInfo.ConfigTDPlevels;
PUBLIC(RO(Proc))->Features.Turbo_Unlock = PfInfo.ProgrammableTurbo;
if ((PRIVATE(OF(Specific)) = LookupProcessor()) != NULL) {
OverrideCodeNameString(PRIVATE(OF(Specific)));
OverrideUnlockCapability(PRIVATE(OF(Specific)));
}
Default_Unlock_Reset();
Intel_Processor_PIN(PfInfo.PPIN_CAP);
Intel_ThermalOffset(PfInfo.ProgrammableTj);
PUBLIC(RO(Proc))->Features.SpecTurboRatio = 0;
}
static void Nehalem_Platform_Info(unsigned int cpu)
{
Query_Same_Platform_Features(cpu);
PUBLIC(RO(Proc))->Features.SpecTurboRatio += 8; /* 8C */
}
static void IvyBridge_EP_Platform_Info(unsigned int cpu)
{
const unsigned int NC = \
PUBLIC(RO(Proc))->CPU.Count >> PUBLIC(RO(Proc))->Features.HTT_Enable;
Query_Same_Platform_Features(cpu);
PUBLIC(RO(Proc))->Features.SpecTurboRatio += 8; /* 8C */
if (NC > 8) {
PUBLIC(RO(Proc))->Features.SpecTurboRatio += 7; /* 15C */
}
}
static void Haswell_EP_Platform_Info(unsigned int cpu)
{
const unsigned int NC = \
PUBLIC(RO(Proc))->CPU.Count >> PUBLIC(RO(Proc))->Features.HTT_Enable;
Query_Same_Platform_Features(cpu);
PUBLIC(RO(Proc))->Features.SpecTurboRatio += 8; /* 8C */
if (NC > 8) {
PUBLIC(RO(Proc))->Features.SpecTurboRatio += 8; /* 16C */
}
if (NC > 16) {
PUBLIC(RO(Proc))->Features.SpecTurboRatio += 2; /* 18C */
}
}
static void Skylake_X_Platform_Info(unsigned int cpu)
{
const unsigned int NC = \
PUBLIC(RO(Proc))->CPU.Count >> PUBLIC(RO(Proc))->Features.HTT_Enable;
switch (PUBLIC(RO(Proc))->Features.Std.EAX.Stepping) {
case 11:
case 10:
StrCopy(PUBLIC(RO(Proc))->Architecture,
Arch[PUBLIC(RO(Proc))->ArchID].Architecture[CN_COOPERLAKE_X],
CODENAME_LEN);
break;
case 7:
StrCopy(PUBLIC(RO(Proc))->Architecture,
Arch[PUBLIC(RO(Proc))->ArchID].Architecture[CN_CASCADELAKE_X],
CODENAME_LEN);
break;
case 4:
StrCopy(PUBLIC(RO(Proc))->Architecture,
Arch[PUBLIC(RO(Proc))->ArchID].Architecture[CN_SKYLAKE_X],
CODENAME_LEN);
break;
}
Query_Same_Platform_Features(cpu);
PUBLIC(RO(Proc))->Features.SpecTurboRatio += 8; /* 8C */
if (NC > 8) {
PUBLIC(RO(Proc))->Features.SpecTurboRatio += 8; /* 16C */
}
}
static void Probe_AMD_DataFabric(void)
{
#ifdef CONFIG_AMD_NB
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
if (PRIVATE(OF(Zen)).Device.DF == NULL)
{
struct pci_dev *dev;
dev = pci_get_domain_bus_and_slot(0x0,0x0,PCI_DEVFN(0x18, 0x0));
if (dev != NULL)
{
PRIVATE(OF(Zen)).Device.DF = dev;
}
}
#endif
#endif /* CONFIG_AMD_NB */
}
typedef void (*ROUTER)(void __iomem *mchmap, unsigned short mc);
static PCI_CALLBACK Router( struct pci_dev *dev, unsigned int offset,
unsigned int bsize, unsigned long long wsize,
ROUTER route, unsigned short mc )
{
void __iomem *mchmap;
union {
unsigned long long addr;
struct {
unsigned int low;
unsigned int high;
};
} mchbar;
unsigned long long wmask = BITCPL(wsize);
unsigned char mchbarEnable = 0;
switch (bsize) {
case 32:
pci_read_config_dword(dev, offset , &mchbar.low);
mchbar.high = 0;
break;
case 64:
pci_read_config_dword(dev, offset , &mchbar.low);
pci_read_config_dword(dev, offset + 4, &mchbar.high);
break;
}
mchbarEnable = BITVAL(mchbar.addr, 0);
if (mchbarEnable) {
mchbar.addr &= wmask;
mchbar.addr += wsize * mc;
mchmap = ioremap(mchbar.addr, wsize);
if (mchmap != NULL) {
route(mchmap, mc);
iounmap(mchmap);
return 0;
} else
return (PCI_CALLBACK) -ENOMEM;
} else
return (PCI_CALLBACK) -ENOMEM;
}
#if defined(ARCH_PMC)
PCI_CALLBACK GetMemoryBAR(int M, int B, int D, int F, unsigned int offset,
unsigned int bsize, unsigned long long wsize,
unsigned short range,
struct pci_dev **device, void __iomem **memmap)
{
if ((*device) == NULL) {
if (((*device) = pci_get_domain_bus_and_slot(M, B, PCI_DEVFN(D,F))) != NULL)
{
union {
unsigned long long addr;
struct {
unsigned int low;
unsigned int high;
};
} membar;
unsigned long long wmask = BITCPL(wsize);
unsigned char membarEnable = 0;
switch (bsize) {
case 32:
pci_read_config_dword((*device), offset , &membar.low);
membar.high = 0;
break;
case 64:
pci_read_config_dword((*device), offset , &membar.low);
pci_read_config_dword((*device), offset + 4, &membar.high);
break;
}
membarEnable = BITVAL(membar.addr, 0);
if (membarEnable) {
membar.addr &= wmask;
membar.addr += wsize * range;
if (((*memmap) = ioremap(membar.addr, wsize)) != NULL) {
return 0;
} else
return (PCI_CALLBACK) -ENOMEM;
} else
return (PCI_CALLBACK) -ENOMEM;
}
return (PCI_CALLBACK) -EINVAL;
}
return (PCI_CALLBACK) -EEXIST;
}
void PutMemoryBAR(struct pci_dev **device, void __iomem **memmap)
{
if ((*memmap) != NULL) {
iounmap((*memmap));
(*memmap) = NULL;
}
if ((*device) != NULL)
{
pci_dev_put((*device));
(*device) = NULL;
}
}
#endif /* ARCH_PMC */
static void Query_P945(void __iomem *mchmap, unsigned short mc)
{ /* Source: Mobile Intel 945 Express Chipset Family. */
unsigned short cha;
PUBLIC(RO(Proc))->Uncore.Bus.ClkCfg.value = readl(mchmap + 0xc00);
PUBLIC(RO(Proc))->Uncore.MC[mc].P945.DCC.value = readl(mchmap + 0x200);
switch (PUBLIC(RO(Proc))->Uncore.MC[mc].P945.DCC.DAMC) {
case 0b00:
case 0b11:
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 1;
break;
case 0b01:
case 0b10:
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 2;
break;
}
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 1;
for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
{
unsigned short rank, rankCount;
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P945.DRT0.value = \
readl(mchmap + 0x110 + 0x80 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P945.DRT1.value = \
readw(mchmap + 0x114 + 0x80 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P945.DRT2.value = \
readl(mchmap + 0x118 + 0x80 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P945.BANK.value = \
readw(mchmap + 0x10e + 0x80 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P945.WIDTH.value = \
readw(mchmap + 0x40c + 0x80 * cha);
if (cha == 0) {
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P945.WIDTH.value &= \
0b11111111;
rankCount = 4;
} else {
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P945.WIDTH.value &= 0b1111;
rankCount = 2;
}
for (rank = 0; rank < rankCount; rank++)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P945.DRB[rank].value = \
readb(mchmap + 0x100 + rank + 0x80 * cha);
}
}
}
static void Query_P955(void __iomem *mchmap, unsigned short mc)
{ /* Source: Intel 82955X Memory Controller Hub (MCH) */
unsigned short cha;
PUBLIC(RO(Proc))->Uncore.Bus.ClkCfg.value = readl(mchmap + 0xc00);
PUBLIC(RO(Proc))->Uncore.MC[mc].P955.DCC.value = readl(mchmap + 0x200);
switch (PUBLIC(RO(Proc))->Uncore.MC[mc].P955.DCC.DAMC) {
case 0b00:
case 0b11:
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 1;
break;
case 0b01:
case 0b10:
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 2;
break;
}
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 1;
for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
{
unsigned short rank;
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P955.DRT1.value = \
readw(mchmap + 0x114 + 0x80 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P955.BANK.value = \
readw(mchmap + 0x10e + 0x80 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P955.WIDTH.value = \
readw(mchmap + 0x40c + 0x80 * cha);
for (rank = 0; rank < 4; rank++)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P955.DRB[rank].value = \
readb(mchmap + 0x100 + rank + 0x80 * cha);
}
}
}
static void Query_P965(void __iomem *mchmap, unsigned short mc)
{
unsigned short cha;
PUBLIC(RO(Proc))->Uncore.Bus.ClkCfg.value = readl(mchmap + 0xc00);
PUBLIC(RO(Proc))->Uncore.MC[mc].P965.CKE0.value = readl(mchmap + 0x260);
PUBLIC(RO(Proc))->Uncore.MC[mc].P965.CKE1.value = readl(mchmap + 0x660);
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
(PUBLIC(RO(Proc))->Uncore.MC[mc].P965.CKE0.RankPop0 != 0)
+ (PUBLIC(RO(Proc))->Uncore.MC[mc].P965.CKE1.RankPop0 != 0);
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = \
PUBLIC(RO(Proc))->Uncore.MC[mc].P965.CKE0.SingleDimmPop ? 1 : 2;
if (PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount > 1) {
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount += \
PUBLIC(RO(Proc))->Uncore.MC[mc].P965.CKE1.SingleDimmPop ? 1 : 2;
}
for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P965.DRT0.value = \
readl(mchmap + 0x29c + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P965.DRT1.value = \
readw(mchmap + 0x250 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P965.DRT2.value = \
readl(mchmap + 0x252 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P965.DRT3.value = \
readw(mchmap + 0x256 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P965.DRT4.value = \
readl(mchmap + 0x258 + 0x400 * cha);
}
}
static void Query_G965(void __iomem *mchmap, unsigned short mc)
{ /* Source: Mobile Intel 965 Express Chipset Family. */
unsigned short cha, slot;
PUBLIC(RO(Proc))->Uncore.Bus.ClkCfg.value = readl(mchmap + 0xc00);
PUBLIC(RO(Proc))->Uncore.MC[mc].G965.DRB0.value = readl(mchmap+0x1200);
PUBLIC(RO(Proc))->Uncore.MC[mc].G965.DRB1.value = readl(mchmap+0x1300);
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
(PUBLIC(RO(Proc))->Uncore.MC[mc].G965.DRB0.Rank1Addr != 0)
+ (PUBLIC(RO(Proc))->Uncore.MC[mc].G965.DRB1.Rank1Addr != 0);
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = \
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount > 1 ? 1:2;
for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].G965.DRT0.value = \
readl(mchmap + 0x1210 + 0x100 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].G965.DRT1.value = \
readl(mchmap + 0x1214 + 0x100 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].G965.DRT2.value = \
readl(mchmap + 0x1218 + 0x100 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].G965.DRT3.value = \
readl(mchmap + 0x121c + 0x100 * cha);
for (slot = 0; slot < PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount; slot++)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].DIMM[slot].DRA.value = \
readl(mchmap + 0x1208 + 0x100 * cha);
}
}
}
static void Query_P35(void __iomem *mchmap, unsigned short mc)
{ /* Source: Intel 3 Series Express Chipset Family. */
unsigned short cha;
PUBLIC(RO(Proc))->Uncore.Bus.ClkCfg.value = readl(mchmap + 0xc00);
PUBLIC(RO(Proc))->Uncore.MC[mc].P35.CKE0.value = readl(mchmap + 0x260);
PUBLIC(RO(Proc))->Uncore.MC[mc].P35.CKE1.value = readl(mchmap + 0x660);
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
(PUBLIC(RO(Proc))->Uncore.MC[mc].P35.CKE0.RankPop0 != 0)
+ (PUBLIC(RO(Proc))->Uncore.MC[mc].P35.CKE1.RankPop0 != 0);
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = \
PUBLIC(RO(Proc))->Uncore.MC[mc].P35.CKE0.SingleDimmPop ? 1 : 2;
if (PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount > 1) {
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount += \
PUBLIC(RO(Proc))->Uncore.MC[mc].P35.CKE1.SingleDimmPop ? 1 : 2;
}
for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
{
unsigned short rank;
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P35.DRT0.value = \
readw(mchmap + 0x265 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P35.DRT1.value = \
readw(mchmap + 0x250 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P35.DRT2.value = \
readl(mchmap + 0x252 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P35.DRT3.value = \
readw(mchmap + 0x256 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P35.DRT4.value = \
readl(mchmap + 0x258 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P35.DRT5.value = \
readw(mchmap + 0x25d + 0x400 * cha);
for (rank = 0; rank < 4; rank++)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P35.DRB[rank].value = \
readw(mchmap + 0x200 + (2 * rank) + 0x400 * cha);
}
for (rank = 0; rank < 2; rank++)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].P35.DRA[rank].value = \
readw(mchmap + 0x208 + (2 * rank) + 0x400 * cha);
}
}
}
static kernel_ulong_t Query_NHM_Timing( struct pci_dev *pdev,
unsigned short mc,
unsigned short cha )
{ /*Source:Micron Technical Note DDR3 Power-Up, Initialization, & Reset*/
struct pci_dev *dev = pci_get_domain_bus_and_slot(
pci_domain_nr(pdev->bus),
pdev->bus->number,
PCI_DEVFN(4, 0)
);
if (dev != NULL)
{
pci_read_config_dword(dev, 0x58,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].NHM.DIMM_Init.value);
pci_read_config_dword(dev, 0x70,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].NHM.MR0_1.value);
pci_read_config_dword(dev, 0x74,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].NHM.MR2_3.value);
pci_read_config_dword(dev ,0x80,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].NHM.Rank_A.value);
pci_read_config_dword(dev ,0x84,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].NHM.Rank_B.value);
pci_read_config_dword(dev ,0x88,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].NHM.Bank.value);
pci_read_config_dword(dev ,0x8c,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].NHM.Refresh.value);
pci_read_config_dword(dev ,0x90,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].NHM.CKE_Timing.value);
pci_read_config_dword(dev, 0xb8,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].NHM.Sched.value);
pci_dev_put(dev);
return 0;
} else
return -ENODEV;
}
static kernel_ulong_t Query_NHM_DIMM( struct pci_dev *pdev,
unsigned short mc,
unsigned short cha )
{
struct pci_dev *dev = pci_get_domain_bus_and_slot(
pci_domain_nr(pdev->bus),
pdev->bus->number,
PCI_DEVFN(4, 1)
);
if (dev != NULL)
{
unsigned short slot;
for (slot = 0; slot < PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount; slot++)
{
pci_read_config_dword(dev, 0x48 + 4 * slot,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].DIMM[slot].DOD.value);
}
pci_dev_put(dev);
return 0;
} else {
return -ENODEV;
}
}
static void Query_NHM_MaxDIMMs(struct pci_dev *dev, unsigned short mc)
{
pci_read_config_dword(dev, 0x64,
&PUBLIC(RO(Proc))->Uncore.MC[mc].MaxDIMMs.NHM.DOD.value);
switch (PUBLIC(RO(Proc))->Uncore.MC[mc].MaxDIMMs.NHM.DOD.MAXNUMDIMMS) {
case 0b00:
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 1;
break;
case 0b01:
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 2;
break;
case 0b10:
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 3;
break;
default:
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 0;
break;
}
}
static kernel_ulong_t Query_NHM_IMC(struct pci_dev *dev, unsigned short mc)
{
kernel_ulong_t rc = 0;
unsigned short cha;
Query_NHM_MaxDIMMs(dev, mc);
pci_read_config_dword(dev, 0x48,
&PUBLIC(RO(Proc))->Uncore.MC[mc].NHM.CONTROL.value);
pci_read_config_dword(dev, 0x4c,
&PUBLIC(RO(Proc))->Uncore.MC[mc].NHM.STATUS.value);
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
(PUBLIC(RO(Proc))->Uncore.MC[mc].NHM.CONTROL.CHANNEL0_ACTIVE != 0)
+ (PUBLIC(RO(Proc))->Uncore.MC[mc].NHM.CONTROL.CHANNEL1_ACTIVE != 0)
+ (PUBLIC(RO(Proc))->Uncore.MC[mc].NHM.CONTROL.CHANNEL2_ACTIVE != 0);
for (cha = 0;
(cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount) && !rc; cha++)
{
rc = Query_NHM_Timing(dev, mc, cha) & Query_NHM_DIMM(dev, mc, cha);
}
return rc;
}
static kernel_ulong_t Query_Lynnfield_IMC(struct pci_dev *dev,unsigned short mc)
{
kernel_ulong_t rc = 0;
unsigned short cha;
Query_NHM_MaxDIMMs(dev, mc);
pci_read_config_dword(dev, 0x48,
&PUBLIC(RO(Proc))->Uncore.MC[mc].NHM.CONTROL.value);
pci_read_config_dword(dev, 0x4c,
&PUBLIC(RO(Proc))->Uncore.MC[mc].NHM.STATUS.value);
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
(PUBLIC(RO(Proc))->Uncore.MC[mc].NHM.CONTROL.CHANNEL0_ACTIVE != 0)
+ (PUBLIC(RO(Proc))->Uncore.MC[mc].NHM.CONTROL.CHANNEL1_ACTIVE != 0);
for (cha = 0;
(cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount) && !rc; cha++)
{
rc = Query_NHM_Timing(dev, mc, cha) & Query_NHM_DIMM(dev, mc, cha);
}
return rc;
}
static void BIOS_DDR(void __iomem *mchmap)
{
PUBLIC(RO(Proc))->Uncore.Bus.BIOS_DDR.value = readl(mchmap + 0x5e00);
}
static void Query_SNB_IMC(void __iomem *mchmap, unsigned short mc)
{ /* Sources: 2nd & 3rd Generation Intel Core Processor Family
Intel Xeon Processor E3-1200 Family */
unsigned short cha, dimmCount[2];
PUBLIC(RO(Proc))->Uncore.Bus.ClkCfg.value = readl(mchmap + 0xc00);
PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MADCH.value = readl(mchmap + 0x5000);
PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD0.value = readl(mchmap + 0x5004);
PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD1.value = readl(mchmap + 0x5008);
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 0;
dimmCount[0]=(PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD0.Dimm_A_Size > 0)
+(PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD0.Dimm_B_Size > 0);
dimmCount[1]=(PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD1.Dimm_A_Size > 0)
+(PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD1.Dimm_B_Size > 0);
for (cha = 0; cha < 2; cha++)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount += (dimmCount[cha] > 0);
}
for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB.DBP.value = \
readl(mchmap + 0x4000 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB.RAP.value = \
readl(mchmap + 0x4004 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB.RWP.value = \
readl(mchmap + 0x4008 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB.OTP.value = \
readl(mchmap + 0x400c + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB.PDWN.value = \
readl(mchmap + 0x40b0 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB.RFP.value = \
readl(mchmap + 0x4294 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB.RFTP.value = \
readl(mchmap + 0x4298 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB.SRFTP.value = \
readl(mchmap + 0x42a4 + 0x400 * cha);
}
/* Is Dual DIMM Per Channel Disable ? */
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = \
(PUBLIC(RO(Proc))->Uncore.Bus.SNB_Cap.DDPCD == 1) ?
1 : PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount;
BIOS_DDR(mchmap);
}
static void Query_Turbo_TDP_Config(void __iomem *mchmap)
{
TURBO_ACTIVATION TurboActivation = {.value = 0};
CONFIG_TDP_NOMINAL NominalTDP = {.value = 0};
CONFIG_TDP_CONTROL ControlTDP = {.value = 0};
CONFIG_TDP_LEVEL ConfigTDP[2] = {{.value = 0}, {.value = 0}};
unsigned int cpu, local = get_cpu(); /* TODO(preempt_disable) */
NominalTDP.value = readl(mchmap + 0x5f3c);
PUBLIC(RO(Core, AT(local)))->Boost[BOOST(TDP)] = NominalTDP.Ratio;
ConfigTDP[0].value = readq(mchmap + 0x5f40);
PUBLIC(RO(Core, AT(local)))->Boost[BOOST(TDP1)] = ConfigTDP[0].Ratio;
ConfigTDP[1].value = readq(mchmap + 0x5f48);
PUBLIC(RO(Core, AT(local)))->Boost[BOOST(TDP2)] = ConfigTDP[1].Ratio;
ControlTDP.value = readl(mchmap + 0x5f50);
PUBLIC(RO(Proc))->Features.TDP_Cfg_Lock = ControlTDP.Lock;
PUBLIC(RO(Proc))->Features.TDP_Cfg_Level = ControlTDP.Level;
TurboActivation.value = readl(mchmap + 0x5f54);
PUBLIC(RO(Core, AT(local)))->Boost[BOOST(ACT)]=TurboActivation.MaxRatio;
PUBLIC(RO(Proc))->Features.TurboActiv_Lock = TurboActivation.Ratio_Lock;
put_cpu(); /* TODO(preempt_enable) */
switch (PUBLIC(RO(Proc))->Features.TDP_Cfg_Level) {
case 2:
PUBLIC(RO(Proc))->PowerThermal.PowerInfo.ThermalSpecPower = \
ConfigTDP[1].PkgPower;
PUBLIC(RO(Proc))->PowerThermal.PowerInfo.MinimumPower = \
ConfigTDP[1].MinPower;
PUBLIC(RO(Proc))->PowerThermal.PowerInfo.MaximumPower = \
ConfigTDP[1].MaxPower;
break;
case 1:
PUBLIC(RO(Proc))->PowerThermal.PowerInfo.ThermalSpecPower = \
ConfigTDP[0].PkgPower;
PUBLIC(RO(Proc))->PowerThermal.PowerInfo.MinimumPower = \
ConfigTDP[0].MinPower;
PUBLIC(RO(Proc))->PowerThermal.PowerInfo.MaximumPower = \
ConfigTDP[0].MaxPower;
break;
case 0:
/*TODO(Unknown CSR for Nominal {Pkg, Min, Max} Power settings ?) */
break;
}
PUBLIC(RO(Proc))->Features.TDP_Levels = 3;
for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++) {
if (cpu != local)
{
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(TDP)] = \
PUBLIC(RO(Core, AT(local)))->Boost[BOOST(TDP)];
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(TDP1)] = \
PUBLIC(RO(Core, AT(local)))->Boost[BOOST(TDP1)];
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(TDP2)] = \
PUBLIC(RO(Core, AT(local)))->Boost[BOOST(TDP2)];
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(ACT)] = \
PUBLIC(RO(Core, AT(local)))->Boost[BOOST(ACT)];
}
}
}
static void Query_HSW_IMC(void __iomem *mchmap, unsigned short mc)
{ /* Source: Desktop 4th & 5th Generation Intel Core Processor Family */
unsigned short cha, dimmCount[2];
PUBLIC(RO(Proc))->Uncore.Bus.ClkCfg.value = readl(mchmap + 0xc00);
PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MADCH.value= readl(mchmap + 0x5000);
PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD0.value = readl(mchmap + 0x5004);
PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD1.value = readl(mchmap + 0x5008);
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 0;
dimmCount[0]=(PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD0.Dimm_A_Size > 0)
+(PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD0.Dimm_B_Size > 0);
dimmCount[1]=(PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD1.Dimm_A_Size > 0)
+(PUBLIC(RO(Proc))->Uncore.MC[mc].SNB.MAD1.Dimm_B_Size > 0);
for (cha = 0; cha < 2; cha++)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount += (dimmCount[cha] > 0);
}
for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
{
/*TODO( Unsolved: What is the channel #1 'X' factor of Haswell registers ? )*/
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].HSW.REG4C00.value = \
readl(mchmap + 0x4c00);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].HSW.Timing.value = \
readl(mchmap + 0x4c04);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].HSW.Rank_A.value = \
readl(mchmap + 0x4c08);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].HSW.Rank_B.value = \
readl(mchmap + 0x4c0c);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].HSW.Rank.value = \
readl(mchmap + 0x4c14);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].HSW.Refresh.value = \
readl(mchmap + 0x4e98);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].HSW.PDWN.value = \
readl(mchmap + 0x4cb0);
}
/* Is Dual DIMM Per Channel Disable ? */
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = \
(PUBLIC(RO(Proc))->Uncore.Bus.SNB_Cap.DDPCD == 1) ?
1 : PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount;
if (mc == 0) {
Query_Turbo_TDP_Config(mchmap);
}
}
static void Query_HSW_CLK(void __iomem *mchmap, unsigned short mc)
{
BIOS_DDR(mchmap);
Query_HSW_IMC(mchmap, mc);
}
static void SKL_SA(void __iomem *mchmap)
{
PUBLIC(RO(Proc))->Uncore.Bus.SKL_SA_Pll.value = readl(mchmap + 0x5918);
}
static void Query_SKL_IMC(void __iomem *mchmap, unsigned short mc)
{ /* Source: 6th & 7th Generation Intel Processor for S-Platforms Vol 2*/
unsigned short cha;
/* Intra channel configuration */
PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADCH.value = readl(mchmap+0x5000);
if (PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADCH.CH_L_MAP)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADC0.value = readl(mchmap+0x5008);
PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADC1.value = readl(mchmap+0x5004);
} else {
PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADC0.value = readl(mchmap+0x5004);
PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADC1.value = readl(mchmap+0x5008);
}
/* DIMM parameters */
PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADD0.value = readl(mchmap+0x500c);
PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADD1.value = readl(mchmap+0x5010);
/* Sum up any present DIMM per channel. */
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
((PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADD0.Dimm_L_Size != 0)
|| (PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADD0.Dimm_S_Size != 0))
+ ((PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADD1.Dimm_L_Size != 0)
|| (PUBLIC(RO(Proc))->Uncore.MC[mc].SKL.MADD1.Dimm_S_Size != 0));
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 2;
for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SKL.Timing.value = \
readl(mchmap + 0x4000 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SKL.ACT.value = \
readl(mchmap + 0x4004 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SKL.RDRD.value = \
readl(mchmap + 0x400c + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SKL.RDWR.value = \
readl(mchmap + 0x4010 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SKL.WRRD.value = \
readl(mchmap + 0x4014 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SKL.WRWR.value = \
readl(mchmap + 0x4018 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SKL.Sched.value = \
readl(mchmap + 0x401c + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SKL.ODT.value = \
readl(mchmap + 0x4070 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SKL.Refresh.value = \
readl(mchmap + 0x423c + 0x400 * cha);
}
if (mc == 0) {
Query_Turbo_TDP_Config(mchmap);
SKL_SA(mchmap);
}
}
static void RKL_SA(void __iomem *mchmap)
{
PUBLIC(RO(Proc))->Uncore.Bus.ADL_SA_Pll.value = readq(mchmap+0x5918);
PUBLIC(RO(Proc))->PowerThermal.VID.SOC = \
PUBLIC(RO(Proc))->Uncore.Bus.ADL_SA_Pll.SA_VOLTAGE;
}
static void Query_RKL_IMC(void __iomem *mchmap, unsigned short mc)
{ /* Source: 11th Generation Intel Core Processor Datasheet Vol 2 */
unsigned short cha;
/* Intra channel configuration */
PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADCH.value = readl(mchmap+0x5000);
if (PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADCH.CH_L_MAP)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADC0.value = readl(mchmap+0x5008);
PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADC1.value = readl(mchmap+0x5004);
} else {
PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADC0.value = readl(mchmap+0x5004);
PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADC1.value = readl(mchmap+0x5008);
}
/* DIMM parameters */
PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADD0.value = readl(mchmap+0x500c);
PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADD1.value = readl(mchmap+0x5010);
/* Sum up any present DIMM per channel. */
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
((PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADD0.Dimm_L_Size != 0)
|| (PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADD0.Dimm_S_Size != 0))
+ ((PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADD1.Dimm_L_Size != 0)
|| (PUBLIC(RO(Proc))->Uncore.MC[mc].RKL.MADD1.Dimm_S_Size != 0));
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 2;
for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.Timing.value = \
readl(mchmap + 0x4000 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.ACT.value = \
readl(mchmap + 0x4004 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.RDRD.value = \
readl(mchmap + 0x400c + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.RDWR.value = \
readl(mchmap + 0x4010 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.WRRD.value = \
readl(mchmap + 0x4014 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.WRWR.value = \
readl(mchmap + 0x4018 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.Sched.value = \
readq(mchmap + 0x4088 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.PWDEN.value = \
readq(mchmap + 0x4050 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.ODT.value = \
readq(mchmap + 0x4070 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.Refresh.value = \
readl(mchmap + 0x423c + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].RKL.SRExit.value = \
readl(mchmap + 0x42c4 + 0x400 * cha);
}
if (mc == 0) {
Query_Turbo_TDP_Config(mchmap);
BIOS_DDR(mchmap);
RKL_SA(mchmap);
}
}
#define TGL_SA RKL_SA
static void Query_TGL_IMC(void __iomem *mchmap, unsigned short mc)
{ /* Source: 11th Generation Intel Core Processor Datasheet Vol 2 */
unsigned short cha;
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 0;
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 0;
/* Intra channel configuration */
PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADCH.value = readl(mchmap+0x5000);
if (PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADCH.value == 0xffffffff)
{
goto EXIT_TGL_IMC;
}
if (PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADCH.CH_L_MAP)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADC0.value = readl(mchmap+0x5008);
PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADC1.value = readl(mchmap+0x5004);
} else {
PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADC0.value = readl(mchmap+0x5004);
PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADC1.value = readl(mchmap+0x5008);
}
if ( (PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADC0.value == 0xffffffff)
|| (PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADC1.value == 0xffffffff) )
{
goto EXIT_TGL_IMC;
}
/* DIMM parameters */
PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADD0.value = readl(mchmap+0x500c);
PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADD1.value = readl(mchmap+0x5010);
if ( (PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADD0.value == 0xffffffff)
|| (PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADD1.value == 0xffffffff) )
{
goto EXIT_TGL_IMC;
}
/* Sum up any present DIMM per channel. */
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
((PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADD0.Dimm_L_Size != 0)
|| (PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADD0.Dimm_S_Size != 0))
+ ((PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADD1.Dimm_L_Size != 0)
|| (PUBLIC(RO(Proc))->Uncore.MC[mc].TGL.MADD1.Dimm_S_Size != 0));
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 2;
for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.Timing.value = \
readq(mchmap + 0x4000 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.ACT.value = \
readl(mchmap + 0x4008 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.RDRD.value = \
readl(mchmap + 0x400c + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.RDWR.value = \
readl(mchmap + 0x4010 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.WRRD.value = \
readl(mchmap + 0x4014 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.WRWR.value = \
readl(mchmap + 0x4018 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.Sched.value = \
readq(mchmap + 0x4088 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.PWDEN.value = \
readq(mchmap + 0x4050 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.ODT.value = \
readq(mchmap + 0x4070 + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.Refresh.value = \
readl(mchmap + 0x423c + 0x400 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].TGL.SRExit.value = \
readq(mchmap + 0x42c0 + 0x400 * cha);
}
if (mc == 0) {
Query_Turbo_TDP_Config(mchmap);
BIOS_DDR(mchmap);
TGL_SA(mchmap);
}
EXIT_TGL_IMC:
EMPTY_STMT();
}
#define ADL_SA TGL_SA
static void Query_ADL_IMC(void __iomem *mchmap, unsigned short mc)
{ /* Source: 12th Generation Intel Core Processor Datasheet Vol 2 */
unsigned short cha, virtualCount;
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 0;
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 0;
/* Intra channel configuration */
PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADCH.value = readl(mchmap+0xd800);
if (PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADCH.value == 0xffffffff)
{
goto EXIT_ADL_IMC;
}
if (PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADCH.CH_L_MAP)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADC0.value = readl(mchmap+0xd808);
PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADC1.value = readl(mchmap+0xd804);
} else {
PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADC0.value = readl(mchmap+0xd804);
PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADC1.value = readl(mchmap+0xd808);
}
if ( (PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADC0.value == 0xffffffff)
|| (PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADC1.value == 0xffffffff) )
{
goto EXIT_ADL_IMC;
}
/* DIMM parameters */
PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADD0.value = readl(mchmap+0xd80c);
PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADD1.value = readl(mchmap+0xd810);
if ( (PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADD0.value == 0xffffffff)
|| (PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADD1.value == 0xffffffff) )
{
goto EXIT_ADL_IMC;
}
/* Check for 2 DIMMs Per Channel is enabled */
if (PUBLIC(RO(Proc))->Uncore.Bus.ADL_Cap_A.DDPCD == 0)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 2;
switch (PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADCH.DDR_TYPE) {
case 0b00: /* DDR4 */
virtualCount = 1;
break;
case 0b11: /* LPDDR4 */
case 0b01: /* DDR5 */
case 0b10: /* LPDDR5 */
default:
virtualCount = 2;
break;
}
} else {
/* Guessing activated channel from the populated DIMM. */
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
((PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADD0.Dimm_L_Size != 0)
|| (PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADD0.Dimm_S_Size != 0))
+ ((PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADD1.Dimm_L_Size != 0)
|| (PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADD1.Dimm_S_Size != 0));
virtualCount = PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount;
}
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 1;
for (cha = 0 ; cha < virtualCount; cha++)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].ADL.Timing.value = \
readq(mchmap + 0xe000 + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].ADL.ACT.value = \
readl(mchmap + 0xe008 + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].ADL.RDRD.value = \
readl(mchmap + 0xe00c + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].ADL.RDWR.value = \
readl(mchmap + 0xe010 + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].ADL.WRRD.value = \
readl(mchmap + 0xe014 + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].ADL.WRWR.value = \
readl(mchmap + 0xe018 + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].ADL.Sched.value = \
readq(mchmap + 0xe088 + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].ADL.PWDEN.value = \
readq(mchmap + 0xe050 + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].ADL.ODT.value = \
readq(mchmap + 0xe070 + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].ADL.Refresh.value = \
readl(mchmap + 0xe43c + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].ADL.SRExit.value = \
readq(mchmap + 0xe4c0 + 0x800 * cha);
}
if (mc == 0) {
Query_Turbo_TDP_Config(mchmap);
BIOS_DDR(mchmap);
ADL_SA(mchmap);
}
EXIT_ADL_IMC:
EMPTY_STMT();
}
#define MTL_SA ADL_SA
static void Query_MTL_IMC(void __iomem *mchmap, unsigned short mc)
{ /* Source: 13th and 14th Gen. Ultra series 1 and 2. Datasheet Vol 2 */
unsigned short cha, virtualCount;
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 0;
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 0;
/* Intra channel configuration */
PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADCH.value = readl(mchmap+0xd800);
if (PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADCH.value == 0xffffffff)
{
goto EXIT_MTL_IMC;
}
if (PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADCH.CH_L_MAP)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADC0.value = readl(mchmap+0xd808);
PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADC1.value = readl(mchmap+0xd804);
} else {
PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADC0.value = readl(mchmap+0xd804);
PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADC1.value = readl(mchmap+0xd808);
}
if ( (PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADC0.value == 0xffffffff)
|| (PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADC1.value == 0xffffffff) )
{
goto EXIT_MTL_IMC;
}
/* DIMM parameters */
PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADD0.value = readl(mchmap+0xd80c);
PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADD1.value = readl(mchmap+0xd810);
if ( (PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADD0.value == 0xffffffff)
|| (PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADD1.value == 0xffffffff) )
{
goto EXIT_MTL_IMC;
}
/* Check for 2 DIMMs Per Channel is enabled */
if (PUBLIC(RO(Proc))->Uncore.Bus.MTL_Cap_A.DDPCD == 0)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 2;
switch (PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADCH.DDR_TYPE) {
case 0b00: /* DDR4 */
virtualCount = 1;
break;
case 0b11: /* LPDDR4 */
case 0b01: /* DDR5 */
case 0b10: /* LPDDR5 */
default:
virtualCount = 2;
break;
}
} else {
/* Guessing activated channel from the populated DIMM. */
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
((PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADD0.Dimm_L_Size != 0)
|| (PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADD0.Dimm_S_Size != 0))
+ ((PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADD1.Dimm_L_Size != 0)
|| (PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADD1.Dimm_S_Size != 0));
virtualCount = PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount;
}
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 1;
for (cha = 0 ; cha < virtualCount; cha++)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].MTL.Timing.value = \
readq(mchmap + 0xe000 + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].MTL.ACT.value = \
readq(mchmap + 0xe138 + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].MTL.RDRD.value = \
readl(mchmap + 0xe00c + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].MTL.RDWR.value = \
readl(mchmap + 0xe010 + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].MTL.WRRD.value = \
readl(mchmap + 0xe014 + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].MTL.WRWR.value = \
readl(mchmap + 0xe018 + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].MTL.Sched.value = \
readq(mchmap + 0xe088 + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].MTL.PWDEN.value = \
readq(mchmap + 0xe050 + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].MTL.CAS.value = \
readq(mchmap + 0xe070 + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].MTL.Refresh.value = \
readq(mchmap + 0xe4a0 + 0x800 * cha);
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].MTL.SRExit.value = \
readq(mchmap + 0xe4c0 + 0x800 * cha);
}
EXIT_MTL_IMC:
EMPTY_STMT();
}
static void Query_MTL_Package_IMC(void __iomem *mchmap, unsigned short mc)
{
Query_Turbo_TDP_Config(mchmap);
MTL_SA(mchmap);
/* Source: Intel Core Ultra 200S and 200HX Series Proc. CFG & MEM Registers */
PUBLIC(RO(Proc))->Uncore.Bus.MTL_BCLK.value = readq(mchmap + 0x5f60);
PUBLIC(RO(Proc))->Uncore.Bus.MTL_CR_MEM.value = readl(mchmap + 0x13d00);
PUBLIC(RO(Proc))->Uncore.Bus.MTL_CR_BIOS.value= readl(mchmap + 0x13d08);
}
static void Query_GLK_IMC(void __iomem *mchmap, unsigned short mc)
{ /* Source: Intel Pentium Silver and Intel Celeron Processors Vol 2 */
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = \
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = 0;
}
static PCI_CALLBACK P945(struct pci_dev *dev)
{
PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
return Router(dev, 0x44, 32, 0x4000, Query_P945, 0);
}
static PCI_CALLBACK P955(struct pci_dev *dev)
{
PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
return Router(dev, 0x44, 32, 0x4000, Query_P955, 0);
}
static PCI_CALLBACK P965(struct pci_dev *dev)
{
PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
return Router(dev, 0x48, 64, 0x4000, Query_P965, 0);
}
static PCI_CALLBACK G965(struct pci_dev *dev)
{
PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
return Router(dev, 0x48, 64, 0x4000, Query_G965, 0);
}
static PCI_CALLBACK P35(struct pci_dev *dev)
{
PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
return Router(dev, 0x48, 64, 0x4000, Query_P35, 0);
}
#define WDT_Technology(TCO1_CNT, Halt_Get_Method, Halt_Set_Method) \
({ \
Halt_Get_Method \
\
switch (WDT_Enable) { \
case COREFREQ_TOGGLE_OFF: \
case COREFREQ_TOGGLE_ON: \
TCO1_CNT.TCO_TMR_HALT = !WDT_Enable; \
Halt_Set_Method \
Halt_Get_Method \
break; \
} \
if (TCO1_CNT.TCO_TMR_HALT) { \
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->WDT, \
PUBLIC(RO(Proc))->Service.Core); \
} else { \
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->WDT, \
PUBLIC(RO(Proc))->Service.Core); \
} \
})
static PCI_CALLBACK ICH_TCO(struct pci_dev *dev)
{
kernel_ulong_t rc;
if ((rc = pci_request_regions(dev, DRV_DEVNAME)) == 0)
{
Intel_TCO1_CNT TCO1_CNT = {.value = 0};
WDT_Technology( TCO1_CNT,
{ pci_read_config_word(dev, 0x40 + 8, &TCO1_CNT.value); },
{ pci_write_config_word(dev, 0x40 + 8, TCO1_CNT.value); } );
pci_release_regions(dev);
}
return (PCI_CALLBACK) rc;
}
static PCI_CALLBACK TCOBASE(struct pci_dev *dev)
{
kernel_ulong_t rc = -ENODEV;
struct device *parent = &dev->dev;
Intel_TCOCTL CTL = {.value = 0};
Intel_TCOBASE TCO = {.value = 0};
pci_read_config_dword(dev, 0x50, &TCO.value);
pci_read_config_dword(dev, 0x54, &CTL.value);
if (TCO.IOS && CTL.BASE_EN) {
if (!devm_request_region(parent, TCO.TCOBA, 32, DRV_DEVNAME))
{
Intel_TCO1_CNT TCO1_CNT = {.value = 0};
WDT_Technology( TCO1_CNT,
{ TCO1_CNT.value = inw(TCO.TCOBA + 8); },
{ outw(TCO1_CNT.value, TCO.TCOBA + 8); } );
devm_request_region(parent, TCO.TCOBA, 32, DRV_DEVNAME);
rc = RC_SUCCESS;
}
}
return (PCI_CALLBACK) rc;
}
#undef WDT_Technology
static PCI_CALLBACK SoC_MCR(struct pci_dev *dev)
{/* DRP */
PCI_MCR MsgCtrlReg = {
.MBZ = 0, .Bytes = 0, .Offset = 0x0, .Port = 0x1, .OpCode = 0x10
};
pci_write_config_dword(dev, 0xd0, MsgCtrlReg.value);
pci_read_config_dword(dev, 0xd4,
&PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DRP.value);
/* DTR0 */
MsgCtrlReg.Offset = 0x1;
pci_write_config_dword(dev, 0xd0, MsgCtrlReg.value);
pci_read_config_dword(dev, 0xd4,
&PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DTR0.value);
/* DTR1 */
MsgCtrlReg.Offset = 0x2;
pci_write_config_dword(dev, 0xd0, MsgCtrlReg.value);
pci_read_config_dword(dev, 0xd4,
&PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DTR1.value);
/* DTR2 */
MsgCtrlReg.Offset = 0x3;
pci_write_config_dword(dev, 0xd0, MsgCtrlReg.value);
pci_read_config_dword(dev, 0xd4,
&PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DTR2.value);
/* DTR3 */
MsgCtrlReg.Offset = 0x4;
pci_write_config_dword(dev, 0xd0, MsgCtrlReg.value);
pci_read_config_dword(dev, 0xd4,
&PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DTR3.value);
/* DRFC */
MsgCtrlReg.Offset = 0x8;
pci_write_config_dword(dev, 0xd0, MsgCtrlReg.value);
pci_read_config_dword(dev, 0xd4,
&PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DRFC.value);
/* DRMC */
MsgCtrlReg.Offset = 0xb;
pci_write_config_dword(dev, 0xd0, MsgCtrlReg.value);
pci_read_config_dword(dev, 0xd4,
&PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DRMC.value);
/* BIOS_CFG */
MsgCtrlReg.Port = 0x4;
MsgCtrlReg.Offset = 0x6;
pci_write_config_dword(dev, 0xd0, MsgCtrlReg.value);
pci_read_config_dword(dev, 0xd4,
&PUBLIC(RO(Proc))->Uncore.MC[0].SLM.BIOS_CFG.value);
return (PCI_CALLBACK) 0;
}
static PCI_CALLBACK SoC_SLM(struct pci_dev *dev)
{
PCI_CALLBACK rc = SoC_MCR(dev);
if ((PCI_CALLBACK) 0 == rc)
{
PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
PUBLIC(RO(Proc))->Uncore.MC[0].ChannelCount = 1
+ ( PUBLIC(RO(Proc))->Uncore.MC[0].SLM.BIOS_CFG.EFF_DUAL_CH_EN
| !PUBLIC(RO(Proc))->Uncore.MC[0].SLM.BIOS_CFG.DUAL_CH_DIS );
PUBLIC(RO(Proc))->Uncore.MC[0].SlotCount = (
PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DRP.RKEN0
| PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DRP.RKEN1
) + (
PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DRP.RKEN2
| PUBLIC(RO(Proc))->Uncore.MC[0].SLM.DRP.RKEN3
);
}
return rc;
}
static PCI_CALLBACK SoC_AMT(struct pci_dev *dev)
{
PCI_CALLBACK rc = SoC_MCR(dev);
if ((PCI_CALLBACK) 0 == rc)
{
PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
PUBLIC(RO(Proc))->Uncore.MC[0].ChannelCount = 1
+ ( PUBLIC(RO(Proc))->Uncore.MC[0].SLM.BIOS_CFG.EFF_DUAL_CH_EN
| !PUBLIC(RO(Proc))->Uncore.MC[0].SLM.BIOS_CFG.DUAL_CH_DIS );
PUBLIC(RO(Proc))->Uncore.MC[0].SlotCount = 1;
}
return rc;
}
static PCI_CALLBACK Lynnfield_IMC(struct pci_dev *dev)
{ /* Clarksfield; Lynnfield */
kernel_ulong_t rc = 0;
unsigned short mc;
PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
for (mc = 0; (mc < PUBLIC(RO(Proc))->Uncore.CtrlCount) && !rc; mc++) {
rc = Query_Lynnfield_IMC(dev, 0);
}
return (PCI_CALLBACK) rc;
}
static PCI_CALLBACK Jasper_Forest_IMC(struct pci_dev *pdev)
{
kernel_ulong_t rc = 0;
unsigned int cpu = 0, SocketID = 0, bus_number = pdev->bus->number;
PUBLIC(RO(Proc))->Uncore.CtrlCount = 0;
while ((cpu < PUBLIC(RO(Proc))->CPU.Count) && !rc)
{
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
if (Core->T.PackageID == SocketID)
{
struct pci_dev *dev = pci_get_domain_bus_and_slot(
pci_domain_nr(pdev->bus),
bus_number,
PCI_DEVFN(3, 0) );
if (dev != NULL)
{
const unsigned short mc = ((unsigned short) SocketID)
% MC_MAX_CTRL;
rc = Query_NHM_IMC(dev, mc);
pci_dev_put(dev);
}
PUBLIC(RO(Proc))->Uncore.CtrlCount++;
if (bus_number < 0xff) {
bus_number++;
}
SocketID++;
}
cpu++;
}
return (PCI_CALLBACK) rc;
}
static PCI_CALLBACK Nehalem_IMC(struct pci_dev *dev)
{ /* Arrandale; Beckton; Bloomfield; Clarkdale; Eagleton; Gainestown; Gulftown*/
kernel_ulong_t rc = 0;
unsigned short mc;
PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
for (mc = 0; (mc < PUBLIC(RO(Proc))->Uncore.CtrlCount) && !rc; mc++) {
rc = Query_NHM_IMC(dev, mc);
}
return (PCI_CALLBACK) rc;
}
static PCI_CALLBACK NHM_IMC_TR(struct pci_dev *dev)
{
pci_read_config_dword(dev, 0x50,
&PUBLIC(RO(Proc))->Uncore.Bus.DimmClock.value);
return (PCI_CALLBACK) 0;
}
static PCI_CALLBACK NHM_NON_CORE(struct pci_dev *dev)
{
NHM_CURRENT_UCLK_RATIO UncoreClock = {.value = 0};
pci_read_config_dword(dev, 0xc0, &UncoreClock.value);
PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MAX)]=UncoreClock.UCLK;
PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MIN)]=UncoreClock.MinRatio;
return (PCI_CALLBACK) 0;
}
static PCI_CALLBACK X58_QPI(struct pci_dev *dev)
{
pci_read_config_dword(dev, 0xd0,
&PUBLIC(RO(Proc))->Uncore.Bus.QuickPath.value);
Intel_FlexRatio();
return (PCI_CALLBACK) 0;
}
static PCI_CALLBACK X58_VTD(struct pci_dev *dev)
{
kernel_ulong_t rc = 0;
unsigned int VTBAR = 0;
pci_read_config_dword(dev, 0x180, &VTBAR);
if (BITVAL(VTBAR, 0) == 1)
{
void __iomem *VT_d_MMIO;
const unsigned int VT_d_Bar = VTBAR & 0xffffe000;
VT_d_MMIO = ioremap(VT_d_Bar, 0x1000);
if (VT_d_MMIO != NULL)
{
PUBLIC(RO(Proc))->Uncore.Bus.IOMMU_Ver.value = readl(VT_d_MMIO + 0x0);
PUBLIC(RO(Proc))->Uncore.Bus.IOMMU_Cap.value = readq(VT_d_MMIO + 0x8);
iounmap(VT_d_MMIO);
}
PUBLIC(RO(Proc))->Uncore.Bus.QuickPath.X58.VT_d = 0;
} else {
PUBLIC(RO(Proc))->Uncore.Bus.QuickPath.X58.VT_d = 1;
}
return (PCI_CALLBACK) rc;
}
static PCI_CALLBACK SNB_IMC(struct pci_dev *dev)
{
pci_read_config_dword(dev, 0xe4,
&PUBLIC(RO(Proc))->Uncore.Bus.SNB_Cap.value);
PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
return Router(dev, 0x48, 64, 0x8000, Query_SNB_IMC, 0);
}
static PCI_CALLBACK IVB_IMC(struct pci_dev *dev)
{
pci_read_config_dword(dev, 0xe4,
&PUBLIC(RO(Proc))->Uncore.Bus.SNB_Cap.value);
pci_read_config_dword(dev, 0xe8,
&PUBLIC(RO(Proc))->Uncore.Bus.IVB_Cap.value);
Intel_FlexRatio();
PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
return Router(dev, 0x48, 64, 0x8000, Query_SNB_IMC, 0);
}
static PCI_CALLBACK SNB_EP_CAP(struct pci_dev *dev)
{
pci_read_config_dword(dev, 0x84,
&PUBLIC(RO(Proc))->Uncore.Bus.SNB_EP_Cap0.value);
pci_read_config_dword(dev, 0x88,
&PUBLIC(RO(Proc))->Uncore.Bus.SNB_EP_Cap1.value);
pci_read_config_dword(dev, 0x8c,
&PUBLIC(RO(Proc))->Uncore.Bus.SNB_EP_Cap2.value);
pci_read_config_dword(dev, 0x90,
&PUBLIC(RO(Proc))->Uncore.Bus.SNB_EP_Cap3.value);
pci_read_config_dword(dev, 0x94,
&PUBLIC(RO(Proc))->Uncore.Bus.SNB_EP_Cap4.value);
return (PCI_CALLBACK) 0;
}
static kernel_ulong_t SNB_EP_CTRL(struct pci_dev *dev, unsigned short mc)
{
pci_read_config_dword(dev, 0x7c,
&PUBLIC(RO(Proc))->Uncore.MC[mc].SNB_EP.TECH.value);
pci_read_config_dword(dev, 0x80,
&PUBLIC(RO(Proc))->Uncore.MC[mc].SNB_EP.TAD.value);
return 0;
}
static kernel_ulong_t SNB_EP_IMC(struct pci_dev *dev , unsigned short mc,
unsigned short cha)
{
pci_read_config_dword(dev, 0x200,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB_EP.DBP.value);
pci_read_config_dword(dev, 0x204,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB_EP.RAP.value);
pci_read_config_dword(dev, 0x208,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB_EP.RWP.value);
pci_read_config_dword(dev, 0x20c,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB_EP.OTP.value);
pci_read_config_dword(dev, 0x210,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB_EP.RFP.value);
pci_read_config_dword(dev, 0x214,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB_EP.RFTP.value);
pci_read_config_dword(dev, 0x218,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].SNB_EP.SRFTP.value);
return 0;
}
static kernel_ulong_t SNB_EP_TAD(struct pci_dev *dev, unsigned short mc,
unsigned short cha)
{
unsigned short slotCount;
pci_read_config_dword(dev, 0x80,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].DIMM[0].MTR.value);
pci_read_config_dword(dev, 0x84,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].DIMM[1].MTR.value);
pci_read_config_dword(dev, 0x88,
&PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].DIMM[2].MTR.value);
slotCount = \
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].DIMM[0].MTR.DIMM_POP
+ PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].DIMM[1].MTR.DIMM_POP
+ PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].DIMM[2].MTR.DIMM_POP;
if (slotCount > PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount) {
PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount = slotCount;
}
return 0;
}
static PCI_CALLBACK SNB_EP_QPI(struct pci_dev *dev)
{
QPI_FREQUENCY QuickPath = {.value = 0};
pci_read_config_dword(dev, 0xd4, &QuickPath.value);
if (QuickPath.EP.QPIFREQSEL > \
PUBLIC(RO(Proc))->Uncore.Bus.QuickPath.EP.QPIFREQSEL)
{
PUBLIC(RO(Proc))->Uncore.Bus.QuickPath = QuickPath;
}
return (PCI_CALLBACK) 0;
}
static PCI_CALLBACK SNB_EP_HB(struct pci_dev *pdev)
{
struct pci_dev *dev;
kernel_ulong_t rc = 0;
CPUBUSNO BUSNO[MC_MAX_CTRL] = {{.value = 0}};
const unsigned int HostBridge = pdev->bus->number;
const unsigned int domain = pci_domain_nr(pdev->bus);
unsigned short mc;
PUBLIC(RO(Proc))->Uncore.CtrlCount = 0;
dev = pci_get_domain_bus_and_slot(domain, HostBridge, PCI_DEVFN(5, 0));
if (dev != NULL)
{
pci_read_config_dword(dev, 0x108, &BUSNO[0].value);
pci_dev_put(dev);
}
if (BUSNO[0].CFG.Valid)
{
PUBLIC(RO(Proc))->Uncore.CtrlCount++;
dev = pci_get_domain_bus_and_slot(domain, BUSNO[0].CFG.UNC + 1,
PCI_DEVFN(5, 0));
if (dev != NULL)
{
pci_read_config_dword(dev, 0x108, &BUSNO[1].value);
pci_dev_put(dev);
if (BUSNO[1].CFG.Valid) {
PUBLIC(RO(Proc))->Uncore.CtrlCount++;
}
}
}
for (mc = 0; mc < PUBLIC(RO(Proc))->Uncore.CtrlCount; mc++)
{
const unsigned int TAD = mc < 2 ? 15 : 29;
const unsigned int TCR = mc < 2 ? 16 : 30;
const unsigned int fun[4] = {0, 1, 4, 5};
const unsigned int QPI[3] = {8, 9, 24};
unsigned short cha, link;
dev = pci_get_domain_bus_and_slot(domain, BUSNO[mc].CFG.UNC,
PCI_DEVFN(15, 0));
if (dev != NULL)
{
rc = SNB_EP_CTRL(dev, mc);
pci_dev_put(dev);
}
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 0;
for (cha = 0; cha < 4; cha++)
{
dev = pci_get_domain_bus_and_slot(domain, BUSNO[mc].CFG.UNC,
PCI_DEVFN(TCR, fun[cha]));
if (dev != NULL)
{
rc = SNB_EP_IMC(dev, mc, cha);
pci_dev_put(dev);
dev = pci_get_domain_bus_and_slot(domain, BUSNO[mc].CFG.UNC,
PCI_DEVFN(TAD, 2 + cha));
if (dev != NULL)
{
rc = SNB_EP_TAD(dev, mc, cha);
pci_dev_put(dev);
}
PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount = 1 + cha;
}
}
for (link = 0; link < 3; link++)
{
dev = pci_get_domain_bus_and_slot(domain, BUSNO[mc].CFG.UNC,
PCI_DEVFN(QPI[link], 0));
if (dev != NULL)
{
rc = (kernel_ulong_t) SNB_EP_QPI(dev);
pci_dev_put(dev);
}
}
dev = pci_get_domain_bus_and_slot(domain, BUSNO[mc].CFG.UNC,
PCI_DEVFN(10, 3));
if (dev != NULL)
{
rc = (kernel_ulong_t) SNB_EP_CAP(dev);
pci_dev_put(dev);
}
}
return (PCI_CALLBACK) rc;
}
static PCI_CALLBACK HSW_HOST(struct pci_dev *dev, ROUTER Query)
{
pci_read_config_dword(dev, 0xe4,
&PUBLIC(RO(Proc))->Uncore.Bus.SNB_Cap.value);
pci_read_config_dword(dev, 0xe8,
&PUBLIC(RO(Proc))->Uncore.Bus.IVB_Cap.value);
Intel_FlexRatio();
PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
return Router(dev, 0x48, 64, 0x8000, Query, 0);
}
static PCI_CALLBACK HSW_IMC(struct pci_dev *dev)
{
return HSW_HOST(dev, Query_HSW_IMC);
}
static PCI_CALLBACK HSW_CLK(struct pci_dev *dev)
{
return HSW_HOST(dev, Query_HSW_CLK);
}
static kernel_ulong_t HSW_EP_CTRL(struct pci_dev *dev, unsigned short mc)
{
pci_read_config_dword(dev, 0x7c,
&PUBLIC(RO(Proc))->Uncore.MC[mc].HSW_EP.TECH.value);
pci_read_config_dword(dev, 0x80,
&PUBLIC(RO(Proc))->Uncore.MC[mc].HSW_EP.TAD.value);
return 0;
}
static PCI_CALLBACK HSW_EP_CTRL0(struct pci_dev *dev)
{
if (PUBLIC(RO(Proc))->Uncore.CtrlCount < 1) {
PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
}
HSW_EP_CTRL(dev, 0);
return (PCI_CALLBACK) 0;
}
static PCI_CALLBACK HSW_EP_CTRL1(struct pci_dev *dev)
{
if (PUBLIC(RO(Proc))->Uncore.CtrlCount < 2) {
PUBLIC(RO(Proc))->Uncore.CtrlCount = 2;
}
HSW_EP_CTRL(dev, 1);
return (PCI_CALLBACK) 0;
}
#define HSW_EP_IMC SNB_EP_IMC
static PCI_CALLBACK HSW_EP_IMC_CTRL0_CHA0(struct pci_dev *dev)
{
const unsigned short
channelMap = PUBLIC(RO(Proc))->Uncore.MC[0].HSW_EP.TECH.CHN_DISABLE;
if (BITVAL(channelMap, 0) == 0) {
if (PUBLIC(RO(Proc))->Uncore.MC[0].ChannelCount < 1) {
PUBLIC(RO(Proc))->Uncore.MC[0].ChannelCount = 1;
}
return (PCI_CALLBACK) HSW_EP_IMC(dev, 0, 0);
} else {
return (PCI_CALLBACK) -ENODEV;
}
}
static PCI_CALLBACK HSW_EP_IMC_CTRL0_CHA1(struct pci_dev *dev)
{
const unsigned short
channelMap = PUBLIC(RO(Proc))->Uncore.MC[0].HSW_EP.TECH.CHN_DISABLE;
if (BITVAL(channelMap, 1) == 0) {
if (PUBLIC(RO(Proc))->Uncore.MC[0].ChannelCount < 2) {
PUBLIC(RO(Proc))->Uncore.MC[0].ChannelCount = 2;
}
return (PCI_CALLBACK) HSW_EP_IMC(dev, 0, 1);
} else {
return (PCI_CALLBACK) -ENODEV;
}
}
static PCI_CALLBACK HSW_EP_IMC_CTRL0_CHA2(struct pci_dev *dev)
{
const unsigned short
channelMap = PUBLIC(RO(Proc))->Uncore.MC[0].HSW_EP.TECH.CHN_DISABLE;
if (BITVAL(channelMap, 2) == 0) {
if (PUBLIC(RO(Proc))->Uncore.MC[0].ChannelCount < 3) {
PUBLIC(RO(Proc))->Uncore.MC[0].ChannelCount = 3;
}
return (PCI_CALLBACK) HSW_EP_IMC(dev, 0, 2);
} else {
return (PCI_CALLBACK) -ENODEV;
}
}
static PCI_CALLBACK HSW_EP_IMC_CTRL0_CHA3(struct pci_dev *dev)
{
const unsigned short
channelMap = PUBLIC(RO(Proc))->Uncore.MC[0].HSW_EP.TECH.CHN_DISABLE;
if (BITVAL(channelMap, 3) == 0) {
if (PUBLIC(RO(Proc))->Uncore.MC[0].ChannelCount < 4) {
PUBLIC(RO(Proc))->Uncore.MC[0].ChannelCount = 4;
}
return (PCI_CALLBACK) HSW_EP_IMC(dev, 0, 3);
} else {
return (PCI_CALLBACK) -ENODEV;
}
}
static PCI_CALLBACK HSW_EP_IMC_CTRL1_CHA0(struct pci_dev *dev)
{
const unsigned short
channelMap = PUBLIC(RO(Proc))->Uncore.MC[1].HSW_EP.TECH.CHN_DISABLE;
if (BITVAL(channelMap, 0) == 0) {
if (PUBLIC(RO(Proc))->Uncore.MC[1].ChannelCount < 1) {
PUBLIC(RO(Proc))->Uncore.MC[1].ChannelCount = 1;
}
return (PCI_CALLBACK) HSW_EP_IMC(dev, 1, 0);
} else {
return (PCI_CALLBACK) -ENODEV;
}
}
static PCI_CALLBACK HSW_EP_IMC_CTRL1_CHA1(struct pci_dev *dev)
{
const unsigned short
channelMap = PUBLIC(RO(Proc))->Uncore.MC[1].HSW_EP.TECH.CHN_DISABLE;
if (BITVAL(channelMap, 1) == 0) {
if (PUBLIC(RO(Proc))->Uncore.MC[1].ChannelCount < 2) {
PUBLIC(RO(Proc))->Uncore.MC[1].ChannelCount = 2;
}
return (PCI_CALLBACK) HSW_EP_IMC(dev, 1, 1);
} else {
return (PCI_CALLBACK) -ENODEV;
}
}
static PCI_CALLBACK HSW_EP_IMC_CTRL1_CHA2(struct pci_dev *dev)
{
const unsigned short
channelMap = PUBLIC(RO(Proc))->Uncore.MC[1].HSW_EP.TECH.CHN_DISABLE;
if (BITVAL(channelMap, 2) == 0) {
if (PUBLIC(RO(Proc))->Uncore.MC[1].ChannelCount < 3) {
PUBLIC(RO(Proc))->Uncore.MC[1].ChannelCount = 3;
}
return (PCI_CALLBACK) HSW_EP_IMC(dev, 1, 2);
} else {
return (PCI_CALLBACK) -ENODEV;
}
}
static PCI_CALLBACK HSW_EP_IMC_CTRL1_CHA3(struct pci_dev *dev)
{
const unsigned short
channelMap = PUBLIC(RO(Proc))->Uncore.MC[1].HSW_EP.TECH.CHN_DISABLE;
if (BITVAL(channelMap, 3) == 0) {
if (PUBLIC(RO(Proc))->Uncore.MC[1].ChannelCount < 4) {
PUBLIC(RO(Proc))->Uncore.MC[1].ChannelCount = 4;
}
return (PCI_CALLBACK) HSW_EP_IMC(dev, 1, 3);
} else {
return (PCI_CALLBACK) -ENODEV;
}
}
#define HSW_EP_TAD SNB_EP_TAD
static PCI_CALLBACK HSW_EP_TAD_CTRL0_CHA0(struct pci_dev *dev)
{
return (PCI_CALLBACK) HSW_EP_TAD(dev, 0, 0);
}
static PCI_CALLBACK HSW_EP_TAD_CTRL0_CHA1(struct pci_dev *dev)
{
return (PCI_CALLBACK) HSW_EP_TAD(dev, 0, 1);
}
static PCI_CALLBACK HSW_EP_TAD_CTRL0_CHA2(struct pci_dev *dev)
{
return (PCI_CALLBACK) HSW_EP_TAD(dev, 0, 2);
}
static PCI_CALLBACK HSW_EP_TAD_CTRL0_CHA3(struct pci_dev *dev)
{
return (PCI_CALLBACK) HSW_EP_TAD(dev, 0, 3);
}
static PCI_CALLBACK HSW_EP_TAD_CTRL1_CHA0(struct pci_dev *dev)
{
return (PCI_CALLBACK) HSW_EP_TAD(dev, 1, 0);
}
static PCI_CALLBACK HSW_EP_TAD_CTRL1_CHA1(struct pci_dev *dev)
{
return (PCI_CALLBACK) HSW_EP_TAD(dev, 1, 1);
}
static PCI_CALLBACK HSW_EP_TAD_CTRL1_CHA2(struct pci_dev *dev)
{
return (PCI_CALLBACK) HSW_EP_TAD(dev, 1, 2);
}
static PCI_CALLBACK HSW_EP_TAD_CTRL1_CHA3(struct pci_dev *dev)
{
return (PCI_CALLBACK) HSW_EP_TAD(dev, 1, 3);
}
static void SoC_SKL_VTD(void)
{
if (PUBLIC(RO(Proc))->Uncore.Bus.SKL_Cap_A.VT_d == 0)
{
if (!request_mem_region(0xfed90000, 0x1000, DRV_DEVNAME)) {
pr_warn("CoreFreq: SoC_SKL_VTD: request_mem_region 0xfed90000\n");
} else {
void __iomem *VT_d_MMIO;
const unsigned int VTBAR = 0xfed90000;
VT_d_MMIO = ioremap(VTBAR, 0x1000);
if (VT_d_MMIO != NULL)
{
PUBLIC(RO(Proc))->Uncore.Bus.IOMMU_Ver.value = readl(VT_d_MMIO + 0x0);
PUBLIC(RO(Proc))->Uncore.Bus.IOMMU_Cap.value = readq(VT_d_MMIO + 0x8);
iounmap(VT_d_MMIO);
}
release_mem_region(0xfed90000, 0x1000);
}
}
}
static PCI_CALLBACK SKL_HOST( struct pci_dev *dev,
ROUTER Query,
unsigned long long wsize,
unsigned short mc )
{
pci_read_config_dword(dev, 0xe4,
&PUBLIC(RO(Proc))->Uncore.Bus.SKL_Cap_A.value);
pci_read_config_dword(dev, 0xe8,
&PUBLIC(RO(Proc))->Uncore.Bus.SKL_Cap_B.value);
pci_read_config_dword(dev, 0xec,
&PUBLIC(RO(Proc))->Uncore.Bus.SKL_Cap_C.value);
Intel_FlexRatio();
SoC_SKL_VTD();
return Router(dev, 0x48, 64, wsize, Query, mc);
}
static PCI_CALLBACK SKL_IMC(struct pci_dev *dev)
{
PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
return SKL_HOST(dev, Query_SKL_IMC, 0x8000, 0);
}
static PCI_CALLBACK CML_PCH(struct pci_dev *dev)
{
UNUSED(dev);
return (PCI_CALLBACK) 0;
}
static PCI_CALLBACK RKL_IMC(struct pci_dev *dev)
{ /* Same address offsets as used in Skylake */
PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
return SKL_HOST(dev, Query_RKL_IMC, 0x8000, 0);
}
static PCI_CALLBACK TGL_IMC(struct pci_dev *dev)
{ /* Source: drivers/edac/igen6_edac.c */
PCI_CALLBACK rc = 0;
unsigned short mc;
PUBLIC(RO(Proc))->Uncore.CtrlCount = 0;
/* Controller #0 is not necessary activated but enabled. */
for (mc = 0; mc < MC_MAX_CTRL; mc++)
{
rc = SKL_HOST(dev, Query_TGL_IMC, 0x10000, mc);
if ( ((PCI_CALLBACK) 0 == rc)
&& (PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount > 0) ) {
PUBLIC(RO(Proc))->Uncore.CtrlCount = mc + 1;
}
}
pci_read_config_dword(dev, 0xf0,
&PUBLIC(RO(Proc))->Uncore.Bus.TGL_Cap_E.value);
return rc;
}
static PCI_CALLBACK ADL_HOST( struct pci_dev *dev,
ROUTER Query,
unsigned long long wsize,
unsigned short mc )
{
pci_read_config_dword(dev, 0xe4,
&PUBLIC(RO(Proc))->Uncore.Bus.ADL_Cap_A.value);
pci_read_config_dword(dev, 0xe8,
&PUBLIC(RO(Proc))->Uncore.Bus.ADL_Cap_B.value);
pci_read_config_dword(dev, 0xec,
&PUBLIC(RO(Proc))->Uncore.Bus.ADL_Cap_C.value);
pci_read_config_dword(dev, 0xf0,
&PUBLIC(RO(Proc))->Uncore.Bus.ADL_Cap_E.value);
Intel_FlexRatio();
SoC_SKL_VTD();
return Router(dev, 0x48, 64, wsize, Query, mc);
}
static PCI_CALLBACK ADL_IMC(struct pci_dev *dev)
{ /* Source: 12th Generation Intel Core Processors datasheet, vol 2 */
PCI_CALLBACK rc = 0;
unsigned short mc, cha;
PUBLIC(RO(Proc))->Uncore.CtrlCount = 0;
/* MCHBAR matches bits 41 to 17 ; two MC x 64KB memory space */
for (mc = 0; mc < MC_MAX_CTRL; mc++)
{
rc = ADL_HOST(dev, Query_ADL_IMC, 0x10000, mc);
if ( (PCI_CALLBACK) 0 == rc)
{
if (PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADCH.value != 0xffffffff) {
switch (PUBLIC(RO(Proc))->Uncore.MC[mc].ADL.MADCH.DDR_TYPE) {
case 0b01: /* DDR5 */
case 0b10: /* LPDDR5 */
for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].ADL.Sched.ReservedBits1=0;
}
break;
}
}
if (PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount > 0)
{
PUBLIC(RO(Proc))->Uncore.CtrlCount = mc + 1;
}
}
}
return rc;
}
static PCI_CALLBACK MTL_HOST( struct pci_dev *dev,
ROUTER Query,
unsigned long long wsize,
unsigned short mc )
{
pci_read_config_dword(dev, 0xe4,
&PUBLIC(RO(Proc))->Uncore.Bus.MTL_Cap_A.value);
pci_read_config_dword(dev, 0xe8,
&PUBLIC(RO(Proc))->Uncore.Bus.MTL_Cap_B.value);
pci_read_config_dword(dev, 0xec,
&PUBLIC(RO(Proc))->Uncore.Bus.MTL_Cap_C.value);
pci_read_config_dword(dev, 0xf0,
&PUBLIC(RO(Proc))->Uncore.Bus.MTL_Cap_E.value);
Intel_FlexRatio();
SoC_SKL_VTD();
return Router(dev, 0x48, 64, wsize, Query, mc);
}
static PCI_CALLBACK MTL_IMC(struct pci_dev *dev)
{/* Source: D0:F0 Host Bridge and DRAM Controller - MCHBAR (part 5) Registers */
PCI_CALLBACK rc = 0;
unsigned short mc, cha;
PUBLIC(RO(Proc))->Uncore.CtrlCount = 0;
/* MCHBAR matches bits 41 to 17 ; two MC x 64KB memory space */
for (mc = 0; mc < MC_MAX_CTRL; mc++)
{
rc = MTL_HOST(dev, Query_MTL_IMC, 0x10000, mc);
if ( (PCI_CALLBACK) 0 == rc)
{
if (PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADCH.value != 0xffffffff) {
switch (PUBLIC(RO(Proc))->Uncore.MC[mc].MTL.MADCH.DDR_TYPE) {
case 0b01: /* DDR5 */
case 0b10: /* LPDDR5 */
for (cha = 0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount; cha++)
{
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].MTL.Sched.ReservedBits1=0;
}
break;
}
}
if (PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount > 0)
{
PUBLIC(RO(Proc))->Uncore.CtrlCount = mc + 1;
}
}
}
rc = MTL_HOST(dev, Query_MTL_Package_IMC, 0x14000, 0);
return rc;
}
static PCI_CALLBACK GLK_IMC(struct pci_dev *dev)
{
PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
pci_read_config_dword(dev, 0xe4,
&PUBLIC(RO(Proc))->Uncore.Bus.GLK_Cap_A.value);
pci_read_config_dword(dev, 0xe8,
&PUBLIC(RO(Proc))->Uncore.Bus.GLK_Cap_B.value);
Intel_FlexRatio();
SoC_SKL_VTD();
return Router(dev, 0x48, 64, 0x8000, Query_GLK_IMC, 0);
}
static PCI_CALLBACK AMD_0Fh_MCH(struct pci_dev *dev)
{ /* Source: BKDG for AMD NPT Family 0Fh Processors. */
unsigned short cha, slot, chip;
/* As defined by specifications. */
PUBLIC(RO(Proc))->Uncore.CtrlCount = 1;
/* DRAM Configuration low register. */
pci_read_config_dword(dev, 0x90,
&PUBLIC(RO(Proc))->Uncore.MC[0].AMD0Fh.DCRL.value);
/* DRAM Configuration high register. */
pci_read_config_dword(dev, 0x94,
&PUBLIC(RO(Proc))->Uncore.MC[0].AMD0Fh.DCRH.value);
/* One channel if 64 bits / two channels if 128 bits width. */
PUBLIC(RO(Proc))->Uncore.MC[0].ChannelCount = \
PUBLIC(RO(Proc))->Uncore.MC[0].AMD0Fh.DCRL.Width128 + 1;
/* DIMM Geometry. */
for (chip = 0; chip < 8; chip++)
{
cha = chip >> 2;
slot = chip % 4;
pci_read_config_dword(dev, 0x40 + 4 * chip,
&PUBLIC(RO(Proc))->Uncore.MC[0].Channel[cha].DIMM[slot].MBA.value);
PUBLIC(RO(Proc))->Uncore.MC[0].SlotCount += \
PUBLIC(RO(Proc))->Uncore.MC[0].Channel[cha].DIMM[slot].MBA.CSEnable;
}
/* DIMM Size. */
pci_read_config_dword(dev, 0x80,
&PUBLIC(RO(Proc))->Uncore.MC[0].MaxDIMMs.AMD0Fh.CS.value);
/* DRAM Timings. */
pci_read_config_dword(dev, 0x88,
&PUBLIC(RO(Proc))->Uncore.MC[0].Channel[0].AMD0Fh.DTRL.value);
/* Assume same timings for both channels. */
PUBLIC(RO(Proc))->Uncore.MC[0].Channel[1].AMD0Fh.DTRL.value = \
PUBLIC(RO(Proc))->Uncore.MC[0].Channel[0].AMD0Fh.DTRL.value;
return (PCI_CALLBACK) 0;
}
static PCI_CALLBACK AMD_0Fh_HTT(struct pci_dev *dev)
{
unsigned int link;
pci_read_config_dword(dev, 0x60,
&PUBLIC(RO(Proc))->Uncore.Bus.NodeID.value);
switch ( PCI_SLOT(dev->devfn) ) {
case 24:
PUBLIC(RO(Core, AT(0)))->T.Cluster.Node = \
PUBLIC(RO(Proc))->Uncore.Bus.NodeID.Node;
break;
case 25:
if ( ( (1 + PUBLIC(RO(Proc))->Uncore.Bus.NodeID.CPUCnt)
>=PUBLIC(RO(Proc))->CPU.Count )
&& (PUBLIC(RO(Proc))->CPU.Count >= 2) )
{
PUBLIC(RO(Core, AT(1)))->T.Cluster.Node = \
PUBLIC(RO(Proc))->Uncore.Bus.NodeID.Node;
}
break;
}
pci_read_config_dword(dev, 0x64,
&PUBLIC(RO(Proc))->Uncore.Bus.UnitID.value);
for (link = 0; link < 3; link++)
{
pci_read_config_dword(dev, 0x88 + 0x20 * link,
&PUBLIC(RO(Proc))->Uncore.Bus.LDTi_Freq[link].value);
};
return (PCI_CALLBACK) 0;
}
static PCI_CALLBACK AMD_Zen_IOMMU(struct pci_dev *dev)
{
/* Sources:
* AMD I/O Virtualization Technology (IOMMU) Specification Jan. 2020
* coreboot/src/soc/amd/picasso/agesa_acpi.c
*/
AMD_IOMMU_CAP_BAR IOMMU_Cap_Bar;
unsigned char IOMMU_UnLock = 0;
PUBLIC(RO(Proc))->Uncore.Bus.IOMMU_CR.value = 0x0;
pci_read_config_dword( dev, 0x40,
&PUBLIC(RO(Proc))->Uncore.Bus.IOMMU_HDR.value );
pci_read_config_dword(dev, 0x44, &IOMMU_Cap_Bar.low);
pci_read_config_dword(dev, 0x48, &IOMMU_Cap_Bar.high);
IOMMU_UnLock = IOMMU_Cap_Bar.addr & 0x1;
IOMMU_Cap_Bar.addr = IOMMU_Cap_Bar.addr & 0xffffc000;
if ((IOMMU_Cap_Bar.addr != 0x0) && IOMMU_UnLock)
{
void __iomem *IOMMU_MMIO;
const size_t bsize = PUBLIC(RO(Proc))->Uncore.Bus.IOMMU_HDR.EFRSup ?
0x80000 : 0x4000;
if ((IOMMU_MMIO = ioremap(IOMMU_Cap_Bar.addr, bsize)) != NULL)
{
PUBLIC(RO(Proc))->Uncore.Bus.IOMMU_CR.value = readq(IOMMU_MMIO + 0x18);
iounmap(IOMMU_MMIO);
}
else
{
return (PCI_CALLBACK) -ENOMEM;
}
}
return (PCI_CALLBACK) 0;
}
static void AMD_Zen_UMC_Aggregate(unsigned short mc, unsigned short cha,
bool *Got_Mem_Clock, bool *Got_Div_Clock)
{
if ((*Got_Mem_Clock) == false)
{
enum UNCORE_BOOST Mem_Clock = 0;
switch ( PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h\
.CONFIG.BurstLength )
{
case 0x0: /* BL2 */
case 0x1: /* BL4 */
case 0x2: /* BL8 */
if (PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.MISC.DDR4.MEMCLK)
{
Mem_Clock = PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h\
.MISC.DDR4.MEMCLK;
(*Got_Mem_Clock) = true;
}
break;
case 0x3: /* BL16 */
if (PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.MISC.DDR5.MEMCLK)
{
Mem_Clock = PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h\
.MISC.DDR5.MEMCLK;
Mem_Clock = DIV_ROUND_CLOSEST(Mem_Clock, 100U);
(*Got_Mem_Clock) = true;
}
break;
}
if ((*Got_Mem_Clock) == true) {
PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MAX)] = \
PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MIN)] = Mem_Clock;
}
}
if ((*Got_Div_Clock) == false)
{
enum UNCORE_BOOST Div_Clock = 1;
if (PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DbgMisc.UMC_Ready)
{
switch( PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h\
.CONFIG.BurstLength )
{
case 0x0: /* BL2 */
case 0x1: /* BL4 */
case 0x2: /* BL8 */
Div_Clock = PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h\
.DbgMisc.UCLK_Divisor == 0 ? 2 : 1;
Div_Clock = Div_Clock * 3;
(*Got_Div_Clock) = true;
break;
case 0x3: /* BL16 */
Div_Clock = PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h\
.DbgMisc.UCLK_Divisor == 0 ? 1 : 2;
(*Got_Div_Clock) = true;
break;
}
}
if ((*Got_Div_Clock) == true) {
PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MIN)] = \
DIV_ROUND_CLOSEST(
PUBLIC(RO(Proc))->Uncore.Boost[UNCORE_BOOST(MIN)],
Div_Clock
);
}
}
}
static void AMD_Zen_UMC(struct pci_dev *dev,
const unsigned int UMC_BAR,
const unsigned int CS_MASK[2][2],
const unsigned int UMC_DAC,
const unsigned int UMC_DIMM_CFG,
unsigned short mc, unsigned short cha,
bool *Got_Mem_Clock, bool *Got_Div_Clock)
{
unsigned int CHIP_BAR[2][2] = {
[0] = {
[0] = UMC_BAR + CS_MASK[0][0],
[1] = UMC_BAR + CS_MASK[0][1]
},
[1] = {
[0] = UMC_BAR + CS_MASK[1][0],
[1] = UMC_BAR + CS_MASK[1][1]
}
};
unsigned short chip, ranks = 0;
for (chip = 0; chip < MC_MAX_DIMM; chip++)
{
const unsigned short slot = chip & 1;
unsigned short sec;
Core_AMD_SMN_Read(
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].DIMM[slot].AMD17h.DAC,
UMC_BAR + UMC_DAC + (slot << 2), dev
);
Core_AMD_SMN_Read(
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].DIMM[slot].AMD17h.CFG,
UMC_BAR + UMC_DIMM_CFG + (slot << 2), dev
);
for (sec = 0; sec < 2; sec++)
{
unsigned int addr[2];
addr[1] = CHIP_BAR[sec][1] + ((chip >> 1) << 2);
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha]\
.AMD17h.CHIP[chip][sec].Mask,
addr[1], dev );
addr[0] = CHIP_BAR[sec][0] + ((chip - (chip > 2)) << 2);
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha]\
.AMD17h.CHIP[chip][sec].Chip,
addr[0], dev );
ranks += BITVAL(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha]\
.AMD17h.CHIP[chip][sec].Chip.value, 0);
}
}
for (chip = 0; chip < MC_MAX_DIMM; chip++) {
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].DIMM[chip].AMD17h.Ranks = \
ranks;
}
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.CONFIG,
UMC_BAR + 0x100, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.SPAZ,
UMC_BAR + 0x12c, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.ENCR,
UMC_BAR + 0x144, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.MISC,
UMC_BAR + 0x200, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR1,
UMC_BAR + 0x204, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR2,
UMC_BAR + 0x208, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR3,
UMC_BAR + 0x20c, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR4,
UMC_BAR + 0x210, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR5,
UMC_BAR + 0x214, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR6,
UMC_BAR + 0x218, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR7,
UMC_BAR + 0x21c, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR8,
UMC_BAR + 0x220, dev);
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR9,
UMC_BAR + 0x224, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR10,
UMC_BAR + 0x228, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR12,
UMC_BAR + 0x230, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR13,
UMC_BAR + 0x234, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR14,
UMC_BAR + 0x238, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR20,
UMC_BAR + 0x250, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR21,
UMC_BAR + 0x254, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR22,
UMC_BAR + 0x258, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTRFC,
UMC_BAR + 0x260, dev );
switch ( PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha]\
.AMD17h.CONFIG.BurstLength ) {
case 0x3: /* BL16 */
{
unsigned int offset; /* DWORD */
for (offset = 0x2c0; offset < 0x2d0; offset = offset + 0x4)
{
Core_AMD_SMN_Read(
PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.RFCSB,
UMC_BAR + offset, dev );
if (PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.RFCSB.value)
{
break;
}
}
}
break;
}
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DTR35,
UMC_BAR + 0x28c, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.BGS,
UMC_BAR + 0x58, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.BGS_ALT,
UMC_BAR + 0xd0, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.ECC.Lo,
UMC_BAR + 0xdf0, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.ECC.Hi,
UMC_BAR + 0xdf4, dev );
Core_AMD_SMN_Read(PUBLIC(RO(Proc))->Uncore.MC[mc].Channel[cha].AMD17h.DbgMisc,
UMC_BAR + 0xd6c, dev );
AMD_Zen_UMC_Aggregate(mc, cha, Got_Mem_Clock, Got_Div_Clock);
}
static PCI_CALLBACK AMD_17h_DataFabric( struct pci_dev *pdev,
const unsigned int CS_MASK[2][2],
const unsigned int UMC_DAC,
const unsigned int UMC_DIMM_CFG,
const unsigned short umc_max,
const unsigned short cha_max,
const unsigned int devfn[] )
{
struct pci_dev *dev;
unsigned short umc;
bool Got_Mem_Clock = false, Got_Div_Clock = false;
PUBLIC(RO(Proc))->Uncore.CtrlCount = 0;
for (umc = 0; umc < umc_max; umc++)
{
const unsigned int domain = pci_domain_nr(pdev->bus);
unsigned short cha = 0, cnt;
dev = pci_get_domain_bus_and_slot(domain, 0x0, devfn[umc]);
if (dev != NULL)
{
PUBLIC(RO(Proc))->Uncore.CtrlCount++;
PUBLIC(RO(Proc))->Uncore.MC[umc].SlotCount = 2;
for (cnt = 0; cnt < cha_max; cnt++)
{
AMD_17_UMC_SDP_CTRL SDP_CTRL = {.value = 0};
Core_AMD_SMN_Read(SDP_CTRL, SMU_AMD_UMC_BASE_CHA_F17H(cnt)+0x104, dev);
if ((SDP_CTRL.value != 0xffffffff) && (SDP_CTRL.SdpInit))
{
AMD_Zen_UMC( dev,
SMU_AMD_UMC_BASE_CHA_F17H(cnt),
CS_MASK,
UMC_DAC, UMC_DIMM_CFG,
umc, cha,
&Got_Mem_Clock, &Got_Div_Clock );
cha++;
}
}
PUBLIC(RO(Proc))->Uncore.MC[umc].ChannelCount = cha;
pci_dev_put(dev);
} else {
pr_warn("CoreFreq: AMD_17h_DataFabric()" \
" Break UMC(%hu) probing @ PCI(0x%x:0x0:0x%x.0x%x)\n",
umc, domain, PCI_SLOT(devfn[umc]), PCI_FUNC(devfn[umc]));
break;
}
}
return (PCI_CALLBACK) 0;
}
static void AMD_UMC_Normalize_Channels(void)
{
unsigned short umc;
for (umc = 0; umc < PUBLIC(RO(Proc))->Uncore.CtrlCount; umc++)
{ /* If UMC is quad channels (in 2 x 32-bits) then unpopulate odd channels*/
if (PUBLIC(RO(Proc))->Uncore.MC[umc].ChannelCount >= 4) {
unsigned short cha;
for (cha=0; cha < PUBLIC(RO(Proc))->Uncore.MC[umc].ChannelCount; cha++)
{
if (cha & 1) {
unsigned short slot;
for(slot=0;slot < PUBLIC(RO(Proc))->Uncore.MC[umc].SlotCount;slot++)
{
const unsigned short chipselect_pair = slot << 1;
BITCLR(LOCKLESS, PUBLIC(RO(Proc))->Uncore.MC[umc].Channel[cha]\
.AMD17h.CHIP[chipselect_pair][0].Chip.value, 0);
}
}
}
}
}
}
static PCI_CALLBACK AMD_DataFabric_Zeppelin(struct pci_dev *pdev)
{
if (strncmp(PUBLIC(RO(Proc))->Architecture,
Arch[PUBLIC(RO(Proc))->ArchID].Architecture[CN_WHITEHAVEN],
CODENAME_LEN) == 0)
{ /* Two controllers */
return AMD_17h_DataFabric( pdev,
(const unsigned int[2][2]) {
{ 0x0, 0x20},
{0x10, 0x28}
},
0x30, 0x80,
2, MC_MAX_CHA,
(const unsigned int[]) {PCI_DEVFN(0x18, 0x0),
PCI_DEVFN(0x19, 0x0)} );
}
else if (strncmp(PUBLIC(RO(Proc))->Architecture,
Arch[PUBLIC(RO(Proc))->ArchID].Architecture[CN_NAPLES],
CODENAME_LEN) == 0)
{ /* Four controllers */
return AMD_17h_DataFabric( pdev,
(const unsigned int[2][2]) {
{ 0x0, 0x20},
{0x10, 0x28}
},
0x30, 0x80,
4, MC_MAX_CHA,
(const unsigned int[]) {PCI_DEVFN(0x18, 0x0),
PCI_DEVFN(0x19, 0x0),
PCI_DEVFN(0x1a, 0x0),
PCI_DEVFN(0x1b, 0x0)} );
}
else /* CN_SNOWY_OWL, CN_SUMMIT_RIDGE */
{ /* One controller */
return AMD_17h_DataFabric( pdev,
(const unsigned int[2][2]) {
{ 0x0, 0x20},
{0x10, 0x28}
},
0x30, 0x80,
1, MC_MAX_CHA,
(const unsigned int[]) {PCI_DEVFN(0x18, 0x0)} );
}
}
static PCI_CALLBACK AMD_DataFabric_Raven(struct pci_dev *pdev)
{
return AMD_17h_DataFabric( pdev,
(const unsigned int[2][2]) {
{ 0x0, 0x20},
{0x10, 0x28}
},
0x30, 0x80,
1, MC_MAX_CHA,
(const unsigned int[]) {PCI_DEVFN(0x18, 0x0)} );
}
static PCI_CALLBACK AMD_DataFabric_Matisse(struct pci_dev *pdev)
{
return AMD_17h_DataFabric( pdev,
(const unsigned int[2][2]) {
{ 0x0, 0x20},
{0x10, 0x28}
},
0x30, 0x80,
1, MC_MAX_CHA,
(const unsigned int[]) {PCI_DEVFN(0x18, 0x0)} );
}
static PCI_CALLBACK AMD_DataFabric_Starship(struct pci_dev *pdev)
{
return AMD_17h_DataFabric( pdev,
(const unsigned int[2][2]) {
{ 0x0, 0x20},
{0x10, 0x28}
},
0x30, 0x80,
1, MC_MAX_CHA,
(const unsigned int[]) {PCI_DEVFN(0x18, 0x0)} );
}
static PCI_CALLBACK AMD_DataFabric_Renoir(struct pci_dev *pdev)
{
return AMD_17h_DataFabric( pdev,
(const unsigned int[2][2]) {
{ 0x0, 0x20},
{0x10, 0x28}
},
0x30, 0x80,
1, MC_MAX_CHA,
(const unsigned int[]) {PCI_DEVFN(0x18, 0x0)} );
}
static PCI_CALLBACK AMD_DataFabric_Ariel(struct pci_dev *pdev)
{
return AMD_17h_DataFabric( pdev,
(const unsigned int[2][2]) {
{ 0x0, 0x20},
{0x10, 0x28}
},
0x30, 0x80,
1, MC_MAX_CHA,
(const unsigned int[]) {PCI_DEVFN(0x18, 0x0)} );
}
static PCI_CALLBACK AMD_DataFabric_Raven2(struct pci_dev *pdev)
{
return AMD_17h_DataFabric( pdev,
(const unsigned int[2][2]) {
{ 0x0, 0x20},
{0x10, 0x28}
},
0x30, 0x80,
1, MC_MAX_CHA,
(const unsigned int[]) {PCI_DEVFN(0x18, 0x0)} );
}
static PCI_CALLBACK AMD_DataFabric_Fireflight(struct pci_dev *pdev)
{
return AMD_17h_DataFabric( pdev,
(const unsigned int[2][2]) {
{ 0x0, 0x20},
{0x10, 0x28}
},
0x30, 0x80,
1, MC_MAX_CHA,
(const unsigned int[]) {PCI_DEVFN(0x18, 0x0)} );
}
static PCI_CALLBACK AMD_DataFabric_Arden(struct pci_dev *pdev)
{
return AMD_17h_DataFabric( pdev,
(const unsigned int[2][2]) {
{ 0x0, 0x20},
{0x10, 0x28}
},
0x30, 0x80,
1, MC_MAX_CHA,
(const unsigned int[]) {PCI_DEVFN(0x18, 0x0)} );
}
static PCI_CALLBACK AMD_DataFabric_VanGogh(struct pci_dev *pdev)
{
return AMD_17h_DataFabric( pdev,
(const unsigned int[2][2]) {
{ 0x0, 0x20},
{0x10, 0x28}
},
0x30, 0x80,
1, MC_MAX_CHA,
(const unsigned int[]) {PCI_DEVFN(0x18, 0x0)} );
}
static PCI_CALLBACK AMD_DataFabric_Vermeer(struct pci_dev *pdev)
{
return AMD_17h_DataFabric( pdev,
(const unsigned int[2][2]) {
{ 0x0, 0x20},
{0x10, 0x28}
},
0x30, 0x80,
1, MC_MAX_CHA,
(const unsigned int[]) {PCI_DEVFN(0x18, 0x0)} );
}
static PCI_CALLBACK AMD_DataFabric_Cezanne(struct pci_dev *pdev)
{
return AMD_17h_DataFabric( pdev,
(const unsigned int[2][2]) {
{ 0x0, 0x20},
{0x10, 0x28}
},
0x30, 0x80,
1, MC_MAX_CHA,
(const unsigned int[]) {PCI_DEVFN(0x18, 0x0)} );
}
static PCI_CALLBACK AMD_DataFabric_Rembrandt(struct pci_dev *pdev)
{
PCI_CALLBACK ret = AMD_17h_DataFabric( pdev,
(const unsigned int[2][2]) {
{ 0x0, 0x20},
{0x10, 0x30}
},
0x40, 0x90,
1, MC_MAX_CHA,
(const unsigned int[]) {PCI_DEVFN(0x18, 0x0)} );
if ((PCI_CALLBACK) 0 == ret) {
AMD_UMC_Normalize_Channels();
}
return ret;
}
static PCI_CALLBACK AMD_DataFabric_Raphael(struct pci_dev *pdev)
{
return AMD_17h_DataFabric( pdev,
(const unsigned int[2][2]) {
{ 0x0, 0x20},
{0x10, 0x30}
},
0x44, 0x90,
1, MC_MAX_CHA,
(const unsigned int[]) {PCI_DEVFN(0x18, 0x0)} );
}
static PCI_CALLBACK AMD_DataFabric_Genoa(struct pci_dev *pdev)
{
return AMD_17h_DataFabric( pdev,
(const unsigned int[2][2]) {
{ 0x0, 0x20},
{0x10, 0x30}
},
0x40, 0x90,
4, MC_MAX_CHA,
(const unsigned int[]) {PCI_DEVFN(0x18, 0x0),
PCI_DEVFN(0x19, 0x0),
PCI_DEVFN(0x1a, 0x0),
PCI_DEVFN(0x1b, 0x0)} );
}
static PCI_CALLBACK AMD_DataFabric_Phoenix(struct pci_dev *pdev)
{
PCI_CALLBACK ret = AMD_17h_DataFabric( pdev,
(const unsigned int[2][2]) {
{ 0x0, 0x20},
{0x10, 0x30}
},
0x40, 0x98,
1, MC_MAX_CHA,
(const unsigned int[]) {PCI_DEVFN(0x18, 0x0)} );
if ((PCI_CALLBACK) 0 == ret) {
AMD_UMC_Normalize_Channels();
}
return ret;
}
static void CoreFreqK_ResetChip(struct pci_dev *dev)
{
UNUSED(dev);
memset( PUBLIC(RO(Proc))->Uncore.Chip, 0,
CHIP_MAX_PCI*sizeof(struct CHIP_ST) );
}
static void CoreFreqK_AppendChip(struct pci_dev *dev)
{
unsigned int idx;
for (idx = 0; idx < CHIP_MAX_PCI; idx++)
{
if (PUBLIC(RO(Proc))->Uncore.Chip[idx].VID == 0)
{
PUBLIC(RO(Proc))->Uncore.Chip[idx].VID = dev->vendor;
PUBLIC(RO(Proc))->Uncore.Chip[idx].DID = dev->device;
break;
}
}
}
static int CoreFreqK_ProbePCI( struct pci_device_id PCI_ids[],
void (*PreProbe)(struct pci_dev*),
void (*PostProbe)(struct pci_dev*) )
{
struct pci_device_id *id = PCI_ids;
struct pci_dev *dev = NULL;
int rc = -ENODEV;
if (PreProbe != NULL) {
PreProbe(dev);
}
while (id->vendor || id->subvendor || id->class_mask)
{
dev = pci_get_device(id->vendor, id->device, NULL);
if (dev != NULL) {
if (!pci_enable_device(dev))
{
PCI_CALLBACK Callback = (PCI_CALLBACK) id->driver_data;
if ((rc = (int) Callback(dev)) == 0)
{
if (PostProbe != NULL) {
PostProbe(dev);
}
}
pci_disable_device(dev);
}
pci_dev_put(dev);
}
id++;
}
return rc;
}
static void Query_Same_Genuine_Features(void)
{
if ((PRIVATE(OF(Specific)) = LookupProcessor()) != NULL)
{
OverrideCodeNameString(PRIVATE(OF(Specific)));
OverrideUnlockCapability(PRIVATE(OF(Specific)));
}
Default_Unlock_Reset();
if (PUBLIC(RO(Proc))->Features.Turbo_Unlock)
{
PUBLIC(RO(Proc))->Features.SpecTurboRatio = 1;
} else {
PUBLIC(RO(Proc))->Features.SpecTurboRatio = 0;
}
}
static void Query_GenuineIntel(unsigned int cpu)
{
Query_Same_Genuine_Features();
Intel_Core_Platform_Info(cpu);
HyperThreading_Technology();
}
static void Query_Core2(unsigned int cpu)
{
Query_Same_Genuine_Features();
Intel_Core_Platform_Info(cpu);
HyperThreading_Technology();
}
static void Query_Atom_Bonnell(unsigned int cpu)
{
Query_Same_Genuine_Features();
Intel_Core_Platform_Info(cpu);
HyperThreading_Technology();
}
static void Query_Silvermont(unsigned int cpu)
{ /* Tables 2-6, 2-7, 2-8(BT), 2-9(BT) */
/* Query the Min and Max frequency ratios */
PLATFORM_INFO PfInfo = {.value = 0};
RDMSR(PfInfo, MSR_PLATFORM_INFO);
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = PfInfo.MinimumRatio;
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = PfInfo.MaxNonTurboRatio;
/* Assume a count of Boost ratios equals to the CPU count */
PUBLIC(RO(Proc))->Features.SpecTurboRatio=PUBLIC(RO(Proc))->CPU.Count;
/* But can be overridden following the specifications */
if ((PRIVATE(OF(Specific)) = LookupProcessor()) != NULL)
{
OverrideCodeNameString(PRIVATE(OF(Specific)));
OverrideUnlockCapability(PRIVATE(OF(Specific)));
}
Default_Unlock_Reset();
HyperThreading_Technology();
/* The architecture is gifted of Power and Energy registers */
RDMSR(PUBLIC(RO(Proc))->PowerThermal.Unit, MSR_RAPL_POWER_UNIT);
}
static void Query_Goldmont(unsigned int cpu) /* Tables 2-6, 2-12 */
{
PLATFORM_INFO PfInfo = {.value = 0};
RDMSR(PfInfo, MSR_PLATFORM_INFO);
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = PfInfo.MinimumRatio;
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = PfInfo.MaxNonTurboRatio;
PUBLIC(RO(Proc))->Features.SpecTurboRatio=PUBLIC(RO(Proc))->CPU.Count;
if ((PRIVATE(OF(Specific)) = LookupProcessor()) != NULL)
{
OverrideCodeNameString(PRIVATE(OF(Specific)));
OverrideUnlockCapability(PRIVATE(OF(Specific)));
}
Default_Unlock_Reset();
HyperThreading_Technology();
Intel_PowerInterface();
if (cpu == PUBLIC(RO(Proc))->Service.Core) {
Read_ACPI_PCT_Registers(cpu);
Read_ACPI_PSS_Registers(cpu);
Read_ACPI_PPC_Registers(cpu);
Read_ACPI_CST_Registers(cpu);
}
}
static void Query_Airmont(unsigned int cpu) /* Tables 2-6, 2-7, 2-8, 2-11 */
{
Query_Silvermont(cpu);
}
static void Query_Nehalem(unsigned int cpu) /* Table 2-15 */
{
Nehalem_Platform_Info(cpu);
HyperThreading_Technology();
if (cpu == PUBLIC(RO(Proc))->Service.Core) {
Read_ACPI_PCT_Registers(cpu);
Read_ACPI_PSS_Registers(cpu);
Read_ACPI_PPC_Registers(cpu);
Read_ACPI_CST_Registers(cpu);
}
}
static void Query_Nehalem_EX(unsigned int cpu) /* Tables 2-15, 2-17 */
{
Query_Same_Genuine_Features();
Intel_Core_Platform_Info(cpu);
HyperThreading_Technology();
}
static void Query_Avoton(unsigned int cpu) /* Table 2-10 */
{
Query_Silvermont(cpu);
RDMSR(PUBLIC(RO(Proc))->PowerThermal.PowerInfo, MSR_AVN_PKG_POWER_INFO);
}
static void Query_SandyBridge(unsigned int cpu)
{
Nehalem_Platform_Info(cpu);
HyperThreading_Technology();
SandyBridge_Uncore_Ratio(cpu);
Intel_PowerInterface();
if (cpu == PUBLIC(RO(Proc))->Service.Core) {
Read_ACPI_PCT_Registers(cpu);
Read_ACPI_PSS_Registers(cpu);
Read_ACPI_PPC_Registers(cpu);
Read_ACPI_CST_Registers(cpu);
}
}
static void Query_SandyBridge_EP(unsigned int cpu)
{
Query_SandyBridge(cpu);
}
static void Query_IvyBridge(unsigned int cpu)
{
Nehalem_Platform_Info(cpu);
HyperThreading_Technology();
SandyBridge_Uncore_Ratio(cpu);
Intel_PowerInterface();
if (cpu == PUBLIC(RO(Proc))->Service.Core) {
Read_ACPI_PCT_Registers(cpu);
Read_ACPI_PSS_Registers(cpu);
Read_ACPI_PPC_Registers(cpu);
Read_ACPI_CST_Registers(cpu);
}
}
static void Query_IvyBridge_EP(unsigned int cpu)
{
IvyBridge_EP_Platform_Info(cpu);
HyperThreading_Technology();
SandyBridge_Uncore_Ratio(cpu);
Intel_PowerInterface();
if (cpu == PUBLIC(RO(Proc))->Service.Core) {
Read_ACPI_PCT_Registers(cpu);
Read_ACPI_PSS_Registers(cpu);
Read_ACPI_PPC_Registers(cpu);
Read_ACPI_CST_Registers(cpu);
}
}
static void Query_Haswell(unsigned int cpu)
{
if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA)
{
PUBLIC(RO(Proc))->Features.Uncore_Unlock = 1;
}
Nehalem_Platform_Info(cpu);
HyperThreading_Technology();
SandyBridge_Uncore_Ratio(cpu);
Intel_PowerInterface();
if (cpu == PUBLIC(RO(Proc))->Service.Core) {
Read_ACPI_PCT_Registers(cpu);
Read_ACPI_PSS_Registers(cpu);
Read_ACPI_PPC_Registers(cpu);
Read_ACPI_CST_Registers(cpu);
}
}
static void Query_Haswell_EP(unsigned int cpu)
{
if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA)
{
PUBLIC(RO(Proc))->Features.Uncore_Unlock = 1;
}
Haswell_EP_Platform_Info(cpu);
HyperThreading_Technology();
Haswell_Uncore_Ratio(NULL);
Intel_PowerInterface();
if (cpu == PUBLIC(RO(Proc))->Service.Core) {
Read_ACPI_PCT_Registers(cpu);
Read_ACPI_PSS_Registers(cpu);
Read_ACPI_PPC_Registers(cpu);
Read_ACPI_CST_Registers(cpu);
}
}
static void Query_Haswell_ULT(unsigned int cpu)
{
Query_IvyBridge(cpu);
}
static void Query_Haswell_ULX(unsigned int cpu)
{
Query_IvyBridge(cpu);
}
static void Query_Broadwell(unsigned int cpu)
{
if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA)
{
PUBLIC(RO(Proc))->Features.Uncore_Unlock = 1;
}
Nehalem_Platform_Info(cpu);
HyperThreading_Technology();
Haswell_Uncore_Ratio(NULL);
Intel_PowerInterface();
Intel_Hardware_Performance();
if (cpu == PUBLIC(RO(Proc))->Service.Core) {
Read_ACPI_PCT_Registers(cpu);
Read_ACPI_PSS_Registers(cpu);
Read_ACPI_PPC_Registers(cpu);
Read_ACPI_CST_Registers(cpu);
}
}
static void Query_Broadwell_EP(unsigned int cpu)
{
Query_Haswell_EP(cpu);
Intel_Hardware_Performance();
}
static void Query_Skylake(unsigned int cpu)
{
Query_Broadwell(cpu);
PUBLIC(RO(Proc))->Features.EEO_Capable = 1;
PUBLIC(RO(Proc))->Features.R2H_Capable = 1;
Skylake_PowerControl();
}
static void Query_Kaby_Lake(unsigned int cpu)
{
Power_ACCU_Skylake = Power_ACCU_SKL_PLATFORM;
Query_Skylake(cpu);
}
static void Query_Skylake_X(unsigned int cpu)
{
if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA)
{
PUBLIC(RO(Proc))->Features.Uncore_Unlock = 1;
}
Skylake_X_Platform_Info(cpu);
HyperThreading_Technology();
Haswell_Uncore_Ratio(NULL);
Intel_PowerInterface();
Intel_Hardware_Performance();
}
static unsigned short Compute_AuthenticAMD_Boost(unsigned int cpu)
{
unsigned short SpecTurboRatio = 0;
/* Lowest frequency according to BKDG */
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = 8;
if (PUBLIC(RO(Proc))->Features.AdvPower.EDX.HwPstate == 1)
{
COFVID CofVid = {.value = 0};
switch (Arch[PUBLIC(RO(Proc))->ArchID].Signature.ExtFamily) {
case 0x1:
case 0x6:
case 0x7:
RDMSR(CofVid, MSR_AMD_COFVID_STATUS);
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)]=CofVid.Arch_COF.MaxCpuCof;
break;
case 0x2:
RDMSR(CofVid, MSR_AMD_COFVID_STATUS);
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = \
CofVid.Arch_Pll.MainPllOpFidMax;
if (CofVid.Arch_Pll.MainPllOpFidMax > 0)
{
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] += 0x8;
}
break;
case 0x3:
case 0x5:
RDMSR(CofVid, MSR_AMD_COFVID_STATUS);
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = \
CofVid.Arch_Pll.MainPllOpFidMax;
if (CofVid.Arch_Pll.MainPllOpFidMax > 0)
{
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] += 0x10;
}
break;
}
} else {
/* TODO(Get PLL for non HwPstate processor) */
}
if (PRIVATE(OF(Specific)) != NULL)
{
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(1C)] = \
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)]
+ PRIVATE(OF(Specific))->Boost[0];
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(2C)] = \
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(1C)]
+ PRIVATE(OF(Specific))->Boost[1];
SpecTurboRatio = 2;
} else {
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(1C)] = \
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)];
SpecTurboRatio = 0;
}
return SpecTurboRatio;
}
static void Query_VirtualMachine(unsigned int cpu)
{
Query_Same_Genuine_Features();
/* Reset Max ratio to call the clock estimation in Controller_Init() */
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = 0;
if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL)
{
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = 10;
}
else if ( (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
|| (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON) )
{ /* Lowest frequency according to BKDG */
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = 8;
if (PRIVATE(OF(Specific)) != NULL) {
/* Save the thermal parameters if specified */
PUBLIC(RO(Proc))->PowerThermal.Param = PRIVATE(OF(Specific))->Param;
} else {
PUBLIC(RO(Proc))->PowerThermal.Param.Target = 0;
}
}
HyperThreading_Technology();
}
static void Query_AuthenticAMD(unsigned int cpu)
{ /* Fallback algorithm for unspecified AMD architectures. */
PRIVATE(OF(Specific)) = LookupProcessor();
if (PRIVATE(OF(Specific)) != NULL) {
/* Save thermal parameters for later use in the Daemon */
PUBLIC(RO(Proc))->PowerThermal.Param = PRIVATE(OF(Specific))->Param;
/* Override Processor CodeName & Locking capabilities */
OverrideCodeNameString(PRIVATE(OF(Specific)));
OverrideUnlockCapability(PRIVATE(OF(Specific)));
} else {
PUBLIC(RO(Proc))->PowerThermal.Param.Target = 0;
}
Default_Unlock_Reset();
PUBLIC(RO(Proc))->Features.SpecTurboRatio = \
Compute_AuthenticAMD_Boost(cpu);
HyperThreading_Technology();
}
static unsigned short Compute_AMD_Family_0Fh_Boost(unsigned int cpu)
{ /* Source: BKDG for AMD NPT Family 0Fh: Chap. 13.8 */
unsigned short SpecTurboRatio = 0;
if (PUBLIC(RO(Proc))->Features.AdvPower.EDX.FID == 1)
{ /* Processor supports FID changes . */
FIDVID_STATUS FidVidStatus = {.value = 0};
RDMSR(FidVidStatus, MSR_K7_FID_VID_STATUS);
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = \
VCO[FidVidStatus.StartFID].MCF;
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = 8 + FidVidStatus.MaxFID;
if (FidVidStatus.StartFID < 0b1000)
{
unsigned int t;
for (t = 0; t < 5; t++)
{
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(SIZE)-5+t] = \
VCO[FidVidStatus.StartFID].PCF[t];
}
SpecTurboRatio = 5;
} else {
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(1C)] = \
8 + FidVidStatus.MaxFID;
SpecTurboRatio = 1;
}
} else {
RDMSR(PUBLIC(RO(Core, AT(cpu)))->SystemRegister.HWCR, MSR_K7_HWCR);
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = 8
+ PUBLIC(RO(Core, AT(cpu)))->SystemRegister.HWCR.Family_0Fh.StartFID;
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = \
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)];
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(1C) ] = \
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)];
SpecTurboRatio = 1;
}
return SpecTurboRatio;
}
static void Query_AMD_Family_0Fh(unsigned int cpu)
{
PRIVATE(OF(Specific)) = LookupProcessor();
if (PRIVATE(OF(Specific)) != NULL)
{
PUBLIC(RO(Proc))->PowerThermal.Param = PRIVATE(OF(Specific))->Param;
OverrideCodeNameString(PRIVATE(OF(Specific)));
OverrideUnlockCapability(PRIVATE(OF(Specific)));
} else {
PUBLIC(RO(Proc))->PowerThermal.Param.Target = 0;
}
Default_Unlock_Reset();
PUBLIC(RO(Proc))->Features.SpecTurboRatio = \
Compute_AMD_Family_0Fh_Boost(cpu);
HyperThreading_Technology();
}
static void Compute_AMD_Family_10h_Boost(unsigned int cpu)
{
unsigned int pstate, sort[5] = {
BOOST(1C), BOOST(MAX), BOOST(2C), BOOST(3C), BOOST(MIN)
};
for (pstate = 0; pstate <= 4; pstate++)
{
unsigned int fid, did;
PSTATEDEF PstateDef = {.value = 0};
RDMSR(PstateDef, (MSR_AMD_PSTATE_DEF_BASE + pstate));
fid = PstateDef.Family_10h.CpuFid + 0x10;
did = 1 << PstateDef.Family_10h.CpuDid;
PUBLIC(RO(Core, AT(cpu)))->Boost[sort[pstate]] = fid / did;
}
}
static void Query_AMD_Family_10h(unsigned int cpu)
{
PRIVATE(OF(Specific)) = LookupProcessor();
if (PRIVATE(OF(Specific)) != NULL)
{
PUBLIC(RO(Proc))->PowerThermal.Param = PRIVATE(OF(Specific))->Param;
OverrideCodeNameString(PRIVATE(OF(Specific)));
OverrideUnlockCapability(PRIVATE(OF(Specific)));
} else {
PUBLIC(RO(Proc))->PowerThermal.Param.Target = 0;
}
Default_Unlock_Reset();
Compute_AMD_Family_10h_Boost(cpu);
PUBLIC(RO(Proc))->Features.SpecTurboRatio = 3;
HyperThreading_Technology();
}
static void Compute_AMD_Family_11h_Boost(unsigned int cpu)
{
unsigned int pstate, sort[8] = {
BOOST(1C), BOOST(MAX), BOOST(2C), BOOST(3C),
BOOST(4C), BOOST(5C) , BOOST(6C), BOOST(MIN)
};
for (pstate = 0; pstate <= 7; pstate++)
{
unsigned int fid, did;
PSTATEDEF PstateDef = {.value = 0};
RDMSR(PstateDef, (MSR_AMD_PSTATE_DEF_BASE + pstate));
fid = PstateDef.Family_10h.CpuFid + 0x8;
did = 1 << PstateDef.Family_10h.CpuDid;
PUBLIC(RO(Core, AT(cpu)))->Boost[sort[pstate]] = fid / did;
}
}
static void Query_AMD_Family_11h(unsigned int cpu)
{
PRIVATE(OF(Specific)) = LookupProcessor();
if (PRIVATE(OF(Specific)) != NULL)
{
PUBLIC(RO(Proc))->PowerThermal.Param = PRIVATE(OF(Specific))->Param;
OverrideCodeNameString(PRIVATE(OF(Specific)));
OverrideUnlockCapability(PRIVATE(OF(Specific)));
} else {
PUBLIC(RO(Proc))->PowerThermal.Param.Target = 0;
}
Default_Unlock_Reset();
Compute_AMD_Family_11h_Boost(cpu);
PUBLIC(RO(Proc))->Features.SpecTurboRatio = 6;
HyperThreading_Technology();
}
static void Compute_AMD_Family_12h_Boost(unsigned int cpu)
{
unsigned int pstate, sort[8] = {
BOOST(1C), BOOST(MAX), BOOST(2C), BOOST(3C),
BOOST(4C), BOOST(5C) , BOOST(6C), BOOST(MIN)
};
for (pstate = 0; pstate <= 7; pstate++)
{
unsigned int fid, did;
PSTATEDEF PstateDef = {.value = 0};
RDMSR(PstateDef, (MSR_AMD_PSTATE_DEF_BASE + pstate));
fid = PstateDef.Family_12h.CpuFid + 0x10;
did = PstateDef.Family_12h.CpuDid;
PUBLIC(RO(Core, AT(cpu)))->Boost[sort[pstate]] = fid / did;
}
}
static void Query_AMD_Family_12h(unsigned int cpu)
{
PRIVATE(OF(Specific)) = LookupProcessor();
if (PRIVATE(OF(Specific)) != NULL)
{
PUBLIC(RO(Proc))->PowerThermal.Param = PRIVATE(OF(Specific))->Param;
OverrideCodeNameString(PRIVATE(OF(Specific)));
OverrideUnlockCapability(PRIVATE(OF(Specific)));
} else {
PUBLIC(RO(Proc))->PowerThermal.Param.Target = 0;
}
Default_Unlock_Reset();
Compute_AMD_Family_12h_Boost(cpu);
PUBLIC(RO(Proc))->Features.SpecTurboRatio = 6;
HyperThreading_Technology();
}
static void Compute_AMD_Family_14h_Boost(unsigned int cpu)
{
COFVID CofVid = {.value = 0};
unsigned int MaxFreq = 100, ClockDiv;
unsigned int pstate, sort[8] = {
BOOST(1C), BOOST(MAX), BOOST(2C), BOOST(3C),
BOOST(4C), BOOST(5C) , BOOST(6C), BOOST(MIN)
};
RDMSR(CofVid, MSR_AMD_COFVID_STATUS);
if (CofVid.Arch_Pll.MainPllOpFidMax > 0)
MaxFreq *= (CofVid.Arch_Pll.MainPllOpFidMax + 0x10);
for (pstate = 0; pstate <= 7; pstate++)
{
PSTATEDEF PstateDef = {.value = 0};
RDMSR(PstateDef, (MSR_AMD_PSTATE_DEF_BASE + pstate));
ClockDiv = (PstateDef.Family_14h.CpuDidMSD + 1) * 4;
ClockDiv += PstateDef.Family_14h.CpuDidLSD;
PUBLIC(RO(Core, AT(cpu)))->Boost[sort[pstate]] = \
(MaxFreq * 4) / ClockDiv;
} /* Frequency @ MainPllOpFidMax (MHz) */
}
static void Query_AMD_Family_14h(unsigned int cpu)
{
PRIVATE(OF(Specific)) = LookupProcessor();
if (PRIVATE(OF(Specific)) != NULL)
{
PUBLIC(RO(Proc))->PowerThermal.Param = PRIVATE(OF(Specific))->Param;
OverrideCodeNameString(PRIVATE(OF(Specific)));
OverrideUnlockCapability(PRIVATE(OF(Specific)));
} else {
PUBLIC(RO(Proc))->PowerThermal.Param.Target = 0;
}
Default_Unlock_Reset();
Compute_AMD_Family_14h_Boost(cpu);
PUBLIC(RO(Proc))->Features.SpecTurboRatio = 6;
HyperThreading_Technology();
}
static unsigned int AMD_F15h_CoreCOF(unsigned int FID, unsigned int DID)
{/* CoreCOF (MHz) = 100 * (CpuFid[5:0] + 10h) / (2 ^ CpuDid) */
unsigned int COF = (FID + 0x10) / (1 << DID);
return COF;
}
#define AMD_F15h_CoreFID(COF, DID) \
({ \
unsigned int FID = (COF * (1 << DID)) - 0x10; \
\
FID; \
})
static void Compute_AMD_Family_15h_Boost(unsigned int cpu)
{
if (PUBLIC(RO(Proc))->Features.AdvPower.EDX.HwPstate)
{
unsigned int pstate, sort[8] = {
BOOST(1C), BOOST(MAX), BOOST(2C), BOOST(3C),
BOOST(4C), BOOST(5C) , BOOST(6C), BOOST(MIN)
};
for (pstate = 0; pstate <= 7; pstate++)
{
PSTATEDEF PstateDef = {.value = 0};
unsigned int COF;
RDMSR(PstateDef, (MSR_AMD_PSTATE_DEF_BASE + pstate));
COF = AMD_F15h_CoreCOF( PstateDef.Family_15h.CpuFid,
PstateDef.Family_15h.CpuDid );
PUBLIC(RO(Core, AT(cpu)))->Boost[sort[pstate]] = COF;
}
}
}
static void Query_AMD_Family_15h(unsigned int cpu)
{
Compute_AMD_Family_15h_Boost(cpu);
PUBLIC(RO(Proc))->Features.SpecTurboRatio = 6;
HyperThreading_Technology();
/* Find micro-architecture based on the CPUID model. Bulldozer initialized */
PRIVATE(OF(Specific)) = LookupProcessor();
switch (PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel) {
case 0x0:
if ( (PUBLIC(RO(Proc))->Features.Std.EAX.Model >= 0x0)
&& (PUBLIC(RO(Proc))->Features.Std.EAX.Model <= 0xf) )
{
StrCopy(PUBLIC(RO(Proc))->Architecture,
Arch[PUBLIC(RO(Proc))->ArchID].Architecture[CN_PILEDRIVER],
CODENAME_LEN);
}
break;
case 0x1:
if ( (PUBLIC(RO(Proc))->Features.Std.EAX.Model >= 0x0)
&& (PUBLIC(RO(Proc))->Features.Std.EAX.Model <= 0xf) )
{
StrCopy(PUBLIC(RO(Proc))->Architecture,
Arch[PUBLIC(RO(Proc))->ArchID].Architecture[CN_PILEDRIVER],
CODENAME_LEN);
}
break;
case 0x3:
if ( (PUBLIC(RO(Proc))->Features.Std.EAX.Model >= 0x0)
&& (PUBLIC(RO(Proc))->Features.Std.EAX.Model <= 0xf) )
{
StrCopy(PUBLIC(RO(Proc))->Architecture,
Arch[PUBLIC(RO(Proc))->ArchID].Architecture[CN_STEAMROLLER],
CODENAME_LEN);
}
break;
case 0x6:
case 0x7:
if ( (PUBLIC(RO(Proc))->Features.Std.EAX.Model >= 0x0)
&& (PUBLIC(RO(Proc))->Features.Std.EAX.Model <= 0xf) )
{
StrCopy(PUBLIC(RO(Proc))->Architecture,
Arch[PUBLIC(RO(Proc))->ArchID].Architecture[CN_EXCAVATOR],
CODENAME_LEN);
/* One thermal sensor through the SMU interface */
PUBLIC(RO(Proc))->thermalFormula = \
CoreFreqK_Thermal_Scope(FORMULA_SCOPE_PKG);
}
break;
}
if (PRIVATE(OF(Specific)) != NULL)
{
PUBLIC(RO(Proc))->PowerThermal.Param = PRIVATE(OF(Specific))->Param;
OverrideCodeNameString(PRIVATE(OF(Specific)));
OverrideUnlockCapability(PRIVATE(OF(Specific)));
} else {
PUBLIC(RO(Proc))->PowerThermal.Param.Target = 0;
}
Default_Unlock_Reset();
}
static COF_ST AMD_Zen_CoreCOF(PSTATEDEF PStateDef)
{
unsigned long remainder;
COF_ST COF;
switch (PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily) {
case 0xB:
/* Source: PPR Family 1Ah Model 02h.
* Arch: Granite Ridge, Strix Point, Turin, Dense
* CoreCOF = Core::X86::Msr::PStateDef[CpuFid[11:0]] * 5 MHz
*/
COF.Q = (PStateDef.Family_1Ah.CpuFid >> 1) / 10;
remainder = (PRECISION * PStateDef.Family_1Ah.CpuFid) >> 1;
remainder = remainder - (UNIT_KHz(1) * COF.Q);
COF.R = (unsigned short) remainder;
break;
default:
/* Source: PPR for AMD Family 17h Model 01h, Revision B1 Processors
* CoreCOF = (PStateDef[CpuFid[7:0]] / PStateDef[CpuDfsId]) * 200
*/
if (PStateDef.Family_17h.CpuDfsId != 0)
{
COF.Q = (PStateDef.Family_17h.CpuFid << 1)
/ PStateDef.Family_17h.CpuDfsId;
remainder = (UNIT_KHz(1) * (PStateDef.Family_17h.CpuFid
- ((COF.Q * PStateDef.Family_17h.CpuDfsId) >> 1))) >> 2;
COF.R = (unsigned short) remainder;
} else {
COF.Q = PStateDef.Family_17h.CpuFid >> 2;
COF.R = 0;
}
break;
}
return COF;
}
static unsigned short AMD_Zen_Compute_FID_DID( unsigned int COF,
unsigned int *FID,
unsigned int *DID )
{
unsigned int tmp = (*FID);
if ((*DID) != 0) {
tmp = (COF * (*DID)) >> 1;
} else {
tmp = COF << 2;
}
if (tmp < (1 << 8)) {
(*FID) = tmp;
return 0;
} else {
return 1;
}
}
static unsigned short AMD_Zen5_Compute_FID( unsigned int COF,
unsigned int *FID )
{
unsigned int tmp = (COF << 1) * 10;
if ((tmp >= 0x10) && (tmp <= 0xfff)) {
(*FID) = tmp;
return 0;
} else {
return 1;
}
}
static unsigned short AMD_Zen_CoreFID( unsigned int COF,
unsigned int *FID,
unsigned int *DID )
{
unsigned short ret;
switch (PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily) {
case 0xB:
ret = AMD_Zen5_Compute_FID(COF, FID);
break;
default:
ret = AMD_Zen_Compute_FID_DID(COF, FID, DID);
while ((ret != 0) && ((*DID) > 0))
{
(*DID) = (*DID) >> 1;
ret = AMD_Zen_Compute_FID_DID(COF, FID, DID);
}
break;
}
return ret;
}
static bool Compute_AMD_Zen_Boost(unsigned int cpu)
{
COF_ST COF = {.Q = 0, .R = 0};
unsigned int pstate, sort[8] = { /* P[0..7]-States */
BOOST(MAX), BOOST(1C), BOOST(2C), BOOST(3C),
BOOST(4C) , BOOST(5C), BOOST(6C), BOOST(7C)
};
PSTATEDEF PstateDef = {.value = 0};
PSTATECTRL PstateCtrl = {.value = 0};
for (pstate = BOOST(MIN); pstate < BOOST(SIZE); pstate++) {
PUBLIC(RO(Core, AT(cpu)))->Boost[pstate] = 0;
}
if (PUBLIC(RO(Proc))->Features.AdvPower.EDX.HwPstate)
{
PSTATELIMIT PstateLimit = {.value = 0};
RDMSR(PstateLimit, MSR_AMD_PSTATE_CURRENT_LIMIT);
RDMSR(PstateDef, (MSR_AMD_PSTATE_DEF_BASE
+ PstateLimit.Family_17h.PstateMaxVal));
COF = AMD_Zen_CoreCOF(PstateDef);
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = COF.Q;
} else {
/*Core & L3 frequencies < 400MHz are not supported by the architecture*/
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)] = 4;
}
/* Loop over all frequency ids. */
for (pstate = 0; pstate <= 7; pstate++)
{
RDMSR(PstateDef, (MSR_AMD_PSTATE_DEF_BASE + pstate));
/* Handle only valid P-States. */
if (PstateDef.Family_17h.PstateEn)
{
COF = AMD_Zen_CoreCOF(PstateDef);
PUBLIC(RO(Core, AT(cpu)))->Boost[sort[pstate]] = COF.Q;
}
}
PUBLIC(RO(Proc))->Features.SpecTurboRatio = pstate;
/* Read the Target P-State */
RDMSR(PstateCtrl, MSR_AMD_PERF_CTL);
RDMSR(PstateDef, MSR_AMD_PSTATE_DEF_BASE + PstateCtrl.PstateCmd);
COF = AMD_Zen_CoreCOF(PstateDef);
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(TGT)] = COF.Q;
/* If CPB is enabled then add Boost + XFR to the P0 ratio. */
RDMSR(PUBLIC(RO(Core, AT(cpu)))->SystemRegister.HWCR, MSR_K7_HWCR);
if (PUBLIC(RO(Core, AT(cpu)))->SystemRegister.HWCR.Family_17h.CpbDis == 0)
{
AMD_17_ZEN2_COF XtraCOF = {.value = 0};
switch (PUBLIC(RO(Proc))->ArchID) {
case AMD_Zen5_STXH:
case AMD_Zen5_KRK:
case AMD_Zen5_Eldora:
case AMD_Zen5_STX:
case AMD_Zen4_HWK:
case AMD_Zen4_PHX2:
case AMD_Zen4_PHXR:
case AMD_Zen4_PHX:
case AMD_Zen4_RPL:
case AMD_Zen3Plus_RMB:
case AMD_Zen3_VMR:
case AMD_Zen2_MTS:
Core_AMD_SMN_Read(XtraCOF,
SMU_AMD_F17H_MATISSE_COF,
PRIVATE(OF(Zen)).Device.DF);
break;
case AMD_Zen5_Turin:
case AMD_Zen5_Turin_Dense:
case AMD_Zen4_Bergamo:
case AMD_EPYC_Rome_CPK:
Core_AMD_SMN_Read(XtraCOF,
SMU_AMD_F17H_ZEN2_MCM_COF,
PRIVATE(OF(Zen)).Device.DF);
break;
case AMD_Zen4_Genoa:
break;
}
if (XtraCOF.value != 0)
{
unsigned int CPB = XtraCOF.BoostRatio >> 2,
XFR = !!(XtraCOF.BoostRatio & 0b11);
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(CPB)] = CPB;
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(XFR)] = CPB + XFR;
}
if (PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(CPB)] == 0)
{
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(CPB)] = \
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)];
if (PRIVATE(OF(Specific)) != NULL)
{
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(CPB)] += \
PRIVATE(OF(Specific))->Boost[0];
}
}
if (PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(XFR)] == 0)
{
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(XFR)] = \
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)];
if (PRIVATE(OF(Specific)) != NULL)
{
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(XFR)] += \
PRIVATE(OF(Specific))->Boost[0];
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(XFR)] += \
PRIVATE(OF(Specific))->Boost[1];
}
}
return true; /* Have CPB: inform to add the Xtra Boost ratios */
} else {
return false; /* CPB is disabled */
}
}
typedef struct {
CLOCK_ARG *pClockMod;
long rc;
unsigned long long PstateAddr;
unsigned int BoostIndex;
} CLOCK_ZEN_ARG;
static void TargetClock_AMD_Zen_PerCore(void *arg)
{
CLOCK_ZEN_ARG *pClockZen = (CLOCK_ZEN_ARG *) arg;
unsigned int cpu = smp_processor_id();
COF_ST COF = {.Q = 0, .R = 0};
unsigned int pstate,
target = (pClockZen->pClockMod->cpu == -1) ?
pClockZen->pClockMod->Ratio
:
PUBLIC(RO(Core, AT(cpu)))->Boost[pClockZen->BoostIndex]
+ pClockZen->pClockMod->Offset;
unsigned short RdWrMSR = 0;
if (target == 0) {
pstate = 0; /* AUTO Frequency is requested by User */
RdWrMSR = 1;
} else {
/* Look-up for the first enabled P-State with the same target frequency */
for (pstate = 0; pstate <= 7; pstate++)
{
PSTATEDEF PstateDef = {.value = 0};
RDMSR(PstateDef, pClockZen->PstateAddr + pstate);
if (PstateDef.Family_17h.PstateEn)
{
COF = AMD_Zen_CoreCOF(PstateDef);
if (COF.Q == target) {
RdWrMSR = 1;
break;
}
}
}
}
if (RdWrMSR == 1)
{
PSTATECTRL PstateCtrl = {.value = 0};
/* Command a new target P-state */
RDMSR(PstateCtrl, MSR_AMD_PERF_CTL);
PstateCtrl.PstateCmd = pstate;
WRMSR(PstateCtrl, MSR_AMD_PERF_CTL);
pClockZen->rc = RC_OK_COMPUTE;
} else {
pClockZen->rc = -RC_PSTATE_NOT_FOUND;
}
}
static void TurboClock_AMD_Zen_PerCore(void *arg)
{
CLOCK_ZEN_ARG *pClockZen = (CLOCK_ZEN_ARG *) arg;
PSTATEDEF PstateDef = {.value = 0};
COF_ST COF = {.Q = 0, .R = 0};
unsigned int FID, DID = 0;
const unsigned int smp = pClockZen->pClockMod->cpu == -1 ?
smp_processor_id() : pClockZen->pClockMod->cpu;
/* Make sure the Core Performance Boost is disabled. */
RDMSR(PUBLIC(RO(Core, AT(smp)))->SystemRegister.HWCR, MSR_K7_HWCR);
if (PUBLIC(RO(Core, AT(smp)))->SystemRegister.HWCR.Family_17h.CpbDis)
{
/* Apply if and only if the P-State is enabled ? */
RDMSR(PstateDef, pClockZen->PstateAddr);
if (PstateDef.Family_17h.PstateEn)
{
if (pClockZen->pClockMod->cpu == -1) { /* Request an absolute COF */
COF.Q = pClockZen->pClockMod->Ratio;
} else { /* Compute the requested COF based on a frequency offset */
COF = AMD_Zen_CoreCOF(PstateDef);
COF.Q = COF.Q + pClockZen->pClockMod->Offset;
}
switch (PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily) {
case 0xB:
FID = PstateDef.Family_1Ah.CpuFid;
break;
default:
FID = PstateDef.Family_17h.CpuFid;
DID = PstateDef.Family_17h.CpuDfsId;
break;
}
/* Attempt to write the P-State MSR with new FID and DID */
if (AMD_Zen_CoreFID(COF.Q, &FID, &DID) == 0)
{
switch (PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily) {
case 0xB:
PstateDef.Family_1Ah.CpuFid = FID;
break;
default:
PstateDef.Family_17h.CpuFid = FID;
PstateDef.Family_17h.CpuDfsId = DID;
break;
}
WRMSR(PstateDef, pClockZen->PstateAddr);
pClockZen->rc = RC_OK_COMPUTE;
} else {
pClockZen->rc = -RC_PSTATE_NOT_FOUND;
}
} else {
pClockZen->rc = -ENODEV;
}
} else {
pClockZen->rc = -RC_TURBO_PREREQ;
}
}
static void BaseClock_AMD_Zen_PerCore(void *arg)
{
CLOCK_ZEN_ARG *pClockZen = (CLOCK_ZEN_ARG *) arg;
PSTATEDEF PstateDef = {.value = 0};
COMPUTE_ARG Compute = { .TSC = { NULL, NULL } };
Compute.TSC[0] = kmalloc(STRUCT_SIZE, GFP_KERNEL);
if (Compute.TSC[0] == NULL) {
goto OutOfMemory;
}
Compute.TSC[1] = kmalloc(STRUCT_SIZE, GFP_KERNEL);
if (Compute.TSC[1] == NULL) {
goto OutOfMemory;
}
TurboClock_AMD_Zen_PerCore(arg);
if (pClockZen->rc == RC_OK_COMPUTE)
{
const unsigned int cpu = smp_processor_id();
/* Calibration Phase One */
RDMSR(PstateDef, pClockZen->PstateAddr);
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = \
AMD_Zen_CoreCOF(PstateDef).Q;
cpu_data(cpu).loops_per_jiffy = \
COMPUTE_LPJ( PUBLIC(RO(Core, AT(cpu)))->Clock.Hz,
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] );
/* Calibration Phase Two */
Compute.Clock.Q = PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)];
Compute.Clock.R = 0;
Compute.Clock.Hz = 0;
Compute_TSC(&Compute);
PUBLIC(RO(Core, AT(cpu)))->Clock = Compute.Clock;
/* Calibration Phase Three */
RDMSR(PstateDef, pClockZen->PstateAddr);
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = \
AMD_Zen_CoreCOF(PstateDef).Q;
cpu_data(cpu).loops_per_jiffy = \
COMPUTE_LPJ( PUBLIC(RO(Core, AT(cpu)))->Clock.Hz,
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] );
}
OutOfMemory:
if (Compute.TSC[1] != NULL) {
kfree(Compute.TSC[1]);
} else {
pClockZen->rc = -ENOMEM;
}
if (Compute.TSC[0] != NULL) {
kfree(Compute.TSC[0]);
} else {
pClockZen->rc = -ENOMEM;
}
}
static unsigned short CPPC_AMD_Zen_ScaleRatio( CORE_RO *Core,
unsigned int scale,
unsigned short hint,
unsigned short CPB )
{
unsigned int max_1C;
unsigned short scaled;
if (CPB) {
max_1C = Core->Boost[BOOST(CPB)] * hint;
} else {
max_1C = Core->Boost[BOOST(MAX)] * hint;
}
if (scale > 0) {
scaled = DIV_ROUND_CLOSEST(max_1C, scale);
scaled = scaled & 0xff;
} else {
scaled = 0;
}
return scaled;
}
static unsigned int CPPC_AMD_Zen_ScaleHint( CORE_RO *Core,
unsigned int scale,
signed int ratio,
unsigned short CPB )
{
enum RATIO_BOOST boost;
unsigned int flag = 1 << 31;
if (CPB) {
boost = BOOST(CPB);
} else {
boost = BOOST(MAX);
}
if ((ratio >= 0) && Core->Boost[boost] && (ratio <= Core->Boost[boost]))
{
unsigned short hint;
hint = DIV_ROUND_CLOSEST(scale * ratio, Core->Boost[boost]);
flag = flag ^ (1 << 31);
flag = flag | hint;
}
return flag;
}
static signed int Put_ACPI_CPPC_Registers(unsigned int cpu, void *arg)
{
#if defined (CONFIG_ACPI_CPPC_LIB) \
&& LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)
CLOCK_ZEN_ARG *pClockZen = (CLOCK_ZEN_ARG *) arg;
struct cppc_perf_fb_ctrs CPPC_Perf;
struct cppc_perf_caps CPPC_Caps;
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
RDMSR(Core->SystemRegister.HWCR, MSR_K7_HWCR);
if ((cppc_get_perf_ctrs(Core->Bind, &CPPC_Perf) == 0)
&& (cppc_get_perf_caps(Core->Bind, &CPPC_Caps) == 0))
{
unsigned short WrRdCPPC = 0;
struct cppc_perf_ctrls perf_ctrls = {
.max_perf = CPPC_Caps.highest_perf,
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0)
.min_perf = CPPC_AMD_Zen_ScaleHint(
Core,
Core->PowerThermal.ACPI_CPPC.Highest,
CPPC_Caps.lowest_freq / PRECISION,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
),
#else
.min_perf = CPPC_Caps.lowest_perf,
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0)
.desired_perf = CPPC_Perf.reference_perf
#else
.desired_perf = CPPC_Caps.reference_perf
#endif
};
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0)
unsigned long long desired_perf = 0;
if (cppc_get_desired_perf(Core->Bind, &desired_perf) == 0) {
perf_ctrls.desired_perf = desired_perf;
} else {
pr_debug("CoreFreq: cppc_get_desired_perf(cpu=%u) error\n",
Core->Bind);
}
#endif
switch (pClockZen->pClockMod->NC) {
case CLOCK_MOD_HWP_MIN:
pClockZen->rc = -RC_UNIMPLEMENTED;
break;
case CLOCK_MOD_HWP_MAX:
pClockZen->rc = -RC_UNIMPLEMENTED;
break;
case CLOCK_MOD_HWP_TGT:
{
unsigned int hint;
if (pClockZen->pClockMod->cpu == -1) {
hint = CPPC_AMD_Zen_ScaleHint(
Core,
Core->PowerThermal.ACPI_CPPC.Highest,
pClockZen->pClockMod->Ratio,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
perf_ctrls.desired_perf = hint & 0xff;
WrRdCPPC = 1;
} else if (pClockZen->pClockMod->cpu == cpu) {
hint = CPPC_AMD_Zen_ScaleHint(
Core,
Core->PowerThermal.ACPI_CPPC.Highest,
Core->Boost[BOOST(HWP_TGT)]
+ pClockZen->pClockMod->Offset,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
perf_ctrls.desired_perf = hint & 0xff;
WrRdCPPC = 1;
}
}
break;
default:
pClockZen->rc = -RC_UNIMPLEMENTED;
break;
}
if (WrRdCPPC == 1)
{
cppc_set_perf(cpu, &perf_ctrls);
if (cppc_get_perf_caps(cpu, &CPPC_Caps) == 0)
{
perf_ctrls.max_perf = CPPC_Caps.highest_perf;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0)
perf_ctrls.min_perf = \
CPPC_AMD_Zen_ScaleHint(
Core,
Core->PowerThermal.ACPI_CPPC.Highest,
CPPC_Caps.lowest_freq / PRECISION,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
Core->PowerThermal.ACPI_CPPC.Minimum = CPPC_Caps.lowest_freq;
#else
perf_ctrls.min_perf = CPPC_Caps.lowest_perf;
Core->PowerThermal.ACPI_CPPC.Minimum = \
CPPC_AMD_Zen_ScaleRatio(
Core,
255U,
CPPC_Caps.lowest_perf,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
#endif
Core->PowerThermal.ACPI_CPPC.Maximum = CPPC_Caps.highest_perf;
} else {
pr_debug("CoreFreq: cppc_get_perf_caps(cpu=%u) error\n", cpu);
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0)
if (cppc_get_desired_perf(cpu, &desired_perf) == 0) {
perf_ctrls.desired_perf = desired_perf;
Core->PowerThermal.ACPI_CPPC.Desired = desired_perf;
} else {
pr_debug("CoreFreq: cppc_get_desired_perf(cpu=%u) error\n",cpu);
}
#endif
pClockZen->rc = RC_OK_COMPUTE;
}
} else {
pr_debug("CoreFreq: cppc_get_perf_*(cpu=%u) error\n", Core->Bind);
}
return 0;
#else
return -ENODEV;
#endif /* CONFIG_ACPI_CPPC_LIB */
}
static signed int Write_ACPI_CPPC_Registers(unsigned int cpu, void *arg)
{
signed int rc = Put_ACPI_CPPC_Registers(cpu, arg);
Compute_ACPI_CPPC_Bounds(cpu);
return rc;
}
static void CPPC_AMD_Zen_PerCore(void *arg)
{
CLOCK_ZEN_ARG *pClockZen = (CLOCK_ZEN_ARG *) arg;
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
AMD_CPPC_REQUEST CPPC_Req = {.value = 0};
unsigned short WrRdCPPC = 0;
RDMSR(Core->SystemRegister.HWCR, MSR_K7_HWCR);
RDMSR(CPPC_Req, MSR_AMD_CPPC_REQ);
switch (pClockZen->pClockMod->NC) {
case CLOCK_MOD_HWP_MIN:
{
unsigned int hint;
if (pClockZen->pClockMod->cpu == -1) {
hint = CPPC_AMD_Zen_ScaleHint(
Core, 255U,
pClockZen->pClockMod->Ratio,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
} else {
hint = CPPC_AMD_Zen_ScaleHint(
Core, 255U,
Core->Boost[BOOST(HWP_MIN)]
+ pClockZen->pClockMod->Offset,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
}
if ((hint & (1 << 31)) == (1 << 31))
{
pClockZen->rc = -ERANGE;
} else {
CPPC_Req.Minimum_Perf = hint & 0xff;
WrRdCPPC = 1;
}
}
break;
case CLOCK_MOD_HWP_MAX:
{
unsigned int hint;
if (pClockZen->pClockMod->cpu == -1) {
hint = CPPC_AMD_Zen_ScaleHint(
Core, 255U,
pClockZen->pClockMod->Ratio,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
} else {
hint = CPPC_AMD_Zen_ScaleHint(
Core, 255U,
Core->Boost[BOOST(HWP_MAX)]
+ pClockZen->pClockMod->Offset,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
}
if ((hint & (1 << 31)) == (1 << 31))
{
pClockZen->rc = -ERANGE;
} else {
CPPC_Req.Maximum_Perf = hint & 0xff;
WrRdCPPC = 1;
}
}
break;
case CLOCK_MOD_HWP_TGT:
{
unsigned int hint;
if (pClockZen->pClockMod->cpu == -1) {
hint = CPPC_AMD_Zen_ScaleHint(
Core, 255U,
pClockZen->pClockMod->Ratio,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
} else {
hint = CPPC_AMD_Zen_ScaleHint(
Core, 255U,
Core->Boost[BOOST(HWP_TGT)]
+ pClockZen->pClockMod->Offset,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
}
if ((hint & (1 << 31)) == (1 << 31))
{
pClockZen->rc = -ERANGE;
} else {
CPPC_Req.Desired_Perf = hint & 0xff;
WrRdCPPC = 1;
}
}
break;
default:
pClockZen->rc = -RC_UNIMPLEMENTED;
break;
}
if (WrRdCPPC == 1)
{
WRMSR(CPPC_Req, MSR_AMD_CPPC_REQ);
RDMSR(CPPC_Req, MSR_AMD_CPPC_REQ);
Core->PowerThermal.HWP_Request.Minimum_Perf = \
CPPC_AMD_Zen_ScaleRatio(
Core, 255U, CPPC_Req.Minimum_Perf,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
Core->PowerThermal.HWP_Request.Maximum_Perf = \
CPPC_AMD_Zen_ScaleRatio(
Core, 255U, CPPC_Req.Maximum_Perf,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
Core->PowerThermal.HWP_Request.Desired_Perf = \
CPPC_AMD_Zen_ScaleRatio(
Core, 255U, CPPC_Req.Desired_Perf,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
Core->Boost[BOOST(HWP_MIN)]=Core->PowerThermal.HWP_Request.Minimum_Perf;
Core->Boost[BOOST(HWP_MAX)]=Core->PowerThermal.HWP_Request.Maximum_Perf;
Core->Boost[BOOST(HWP_TGT)]=Core->PowerThermal.HWP_Request.Desired_Perf;
pClockZen->rc = RC_OK_COMPUTE;
}
}
static long For_All_AMD_Zen_Clock( CLOCK_ZEN_ARG *pClockZen,
void (*PerCore)(void *) )
{
long rc = RC_SUCCESS;
unsigned int cpu = PUBLIC(RO(Proc))->CPU.Count;
do {
cpu--; /* From last AP to BSP */
if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS)
&& ((pClockZen->pClockMod->cpu == -1)||(pClockZen->pClockMod->cpu == cpu)))
{
smp_call_function_single(cpu, PerCore, pClockZen, 1);
rc = pClockZen->rc;
}
} while ((cpu != 0) && (rc >= RC_SUCCESS)) ;
return rc;
}
static long ClockSource_OC_Granted(void)
{
long rc = -RC_CLOCKSOURCE;
char *clockName = kmalloc(PAGE_SIZE, GFP_KERNEL);
if (clockName != NULL)
{
struct file *clockFile=filp_open(CLOCKSOURCE_PATH"/current_clocksource",
O_RDWR, (umode_t) 0);
if (clockFile != NULL)
{
loff_t pos = 0;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
ssize_t len = kernel_read(clockFile, clockName, PAGE_SIZE - 1, &pos);
#else
int len = kernel_read(clockFile, pos, clockName, PAGE_SIZE - 1);
#endif
if (len > 0)
{
const struct {
char *name;
size_t len;
} allowList[] = {
{ "hpet", __builtin_strlen("hpet") },
{ "acpi_pm", __builtin_strlen("acpi_pm") },
{ "jiffies", __builtin_strlen("jiffies") }
};
const size_t dim = sizeof(allowList) / sizeof(allowList[0]);
size_t idx;
for (idx = 0; idx < dim; idx++) {
if (0 == strncmp(allowList[idx].name, clockName,
allowList[idx].len))
{
rc = RC_SUCCESS;
break;
}
}
} else {
rc = len == 0 ? -EAGAIN : len;
}
filp_close(clockFile, NULL);
} else {
rc = -ENOENT;
}
kfree(clockName);
} else {
rc = -ENOMEM;
}
return rc;
}
static long For_All_AMD_Zen_BaseClock( CLOCK_ZEN_ARG *pClockZen,
void (*PerCore)(void*) )
{
long rc;
unsigned int cpu = PUBLIC(RO(Proc))->Service.Core;
if ((rc = ClockSource_OC_Granted()) == RC_SUCCESS)
{
rc = For_All_AMD_Zen_Clock(pClockZen, PerCore);
if (rc == RC_OK_COMPUTE)
{
PUBLIC(RO(Proc))->Features.Factory.Clock = (CLOCK) {.Q=0, .R=0, .Hz=0};
PUBLIC(RO(Proc))->Features.Factory.Ratio = 0;
PUBLIC(RO(Proc))->Features.Factory.Freq = 0;
FixMissingRatioAndFrequency(PUBLIC(RO(Core,AT(cpu)))->Boost[BOOST(MAX)],
&PUBLIC(RO(Core, AT(cpu)))->Clock);
loops_per_jiffy = cpu_data(cpu).loops_per_jiffy;
cpu_khz = tsc_khz = (unsigned int) ((loops_per_jiffy * HZ) / 1000LU);
}
}
return rc;
}
static long TurboClock_AMD_Zen(CLOCK_ARG *pClockMod)
{
if (pClockMod != NULL) {
if ((pClockMod->NC >= 1) && (pClockMod->NC <= 7))
{
CLOCK_ZEN_ARG ClockZen = { /* P[1..7]-States allowed */
.pClockMod = pClockMod,
.PstateAddr = MSR_AMD_PSTATE_DEF_BASE + pClockMod->NC,
.BoostIndex = BOOST(SIZE) - pClockMod->NC,
.rc = RC_SUCCESS
};
return For_All_AMD_Zen_Clock(&ClockZen, TurboClock_AMD_Zen_PerCore);
} else {
return -RC_UNIMPLEMENTED;
}
} else {
return -EINVAL;
}
}
static long ClockMod_AMD_Zen(CLOCK_ARG *pClockMod)
{
if (pClockMod != NULL) {
switch (pClockMod->NC) {
case CLOCK_MOD_MAX:
{
CLOCK_ZEN_ARG ClockZen = { /* P[0]:Max non-boosted P-State */
.pClockMod = pClockMod,
.PstateAddr = MSR_AMD_PSTATE_DEF_BASE,
.BoostIndex = BOOST(MAX),
.rc = RC_SUCCESS
};
return(For_All_AMD_Zen_BaseClock(&ClockZen, BaseClock_AMD_Zen_PerCore));
}
case CLOCK_MOD_TGT:
{
CLOCK_ZEN_ARG ClockZen = { /* Target non-boosted P-State*/
.pClockMod = pClockMod,
.PstateAddr = MSR_AMD_PSTATE_DEF_BASE,
.BoostIndex = BOOST(TGT),
.rc = RC_SUCCESS
};
return For_All_AMD_Zen_Clock(&ClockZen, TargetClock_AMD_Zen_PerCore);
}
case CLOCK_MOD_HWP_MIN:
case CLOCK_MOD_HWP_MAX:
case CLOCK_MOD_HWP_TGT:
{
CLOCK_ZEN_ARG ClockZen = {
.pClockMod = pClockMod,
.rc = RC_SUCCESS
};
if (PUBLIC(RO(Proc))->Features.HWP_Enable == 1) {
return For_All_AMD_Zen_Clock(&ClockZen, CPPC_AMD_Zen_PerCore);
}
else if (PUBLIC(RO(Proc))->Features.ACPI_CPPC == 1)
{
For_All_ACPI_CPPC(Write_ACPI_CPPC_Registers, &ClockZen);
return ClockZen.rc;
} else {
return -RC_UNIMPLEMENTED;
}
}
default:
return -RC_UNIMPLEMENTED;
}
} else {
return -EINVAL;
}
}
static void Query_AMD_F17h_Power_Limits(PROC_RO *Pkg)
{ /* Package Power Tracking */
Core_AMD_SMN_Read( Pkg->PowerThermal.Zen.PWR,
SMU_AMD_F17H_ZEN2_MCM_PWR,
PRIVATE(OF(Zen)).Device.DF );
/* Junction Temperature */
if (Pkg->PowerThermal.Zen.PWR.TjMax > 0) {
Pkg->PowerThermal.Param.Offset[THERMAL_TARGET] = \
Pkg->PowerThermal.Zen.PWR.TjMax;
}
/* Thermal Design Power */
Core_AMD_SMN_Read( Pkg->PowerThermal.Zen.TDP,
SMU_AMD_F17H_ZEN2_MCM_TDP,
PRIVATE(OF(Zen)).Device.DF );
/* Electric Design Current */
Core_AMD_SMN_Read( Pkg->PowerThermal.Zen.EDC,
SMU_AMD_F17H_ZEN2_MCM_EDC,
PRIVATE(OF(Zen)).Device.DF );
}
static unsigned int Query_AMD_HSMP_Interface(void)
{
HSMP_ARG arg[8];
unsigned int rx = HSMP_UNSPECIFIED;
if (PUBLIC(RO(Proc))->Features.HSMP_Capable)
{ /* Mark the SMU as Enable if the reachability test is successful */
RESET_ARRAY(arg, 8, 0, .value);
arg[0].value = 0x2;
rx = AMD_HSMP_Exec(HSMP_TEST_MSG, arg);
if ((rx == HSMP_RESULT_OK) && (arg[0].value == (0x2 + 0x1)))
{
PUBLIC(RO(Proc))->Features.HSMP_Enable = 1;
}
else if (IS_HSMP_OOO(rx))
{
PUBLIC(RO(Proc))->Features.HSMP_Enable = 0;
}
} else {
PUBLIC(RO(Proc))->Features.HSMP_Enable = 0;
}
PUBLIC(RO(Proc))->Features.Factory.SMU.Version = 0;
if (PUBLIC(RO(Proc))->Features.HSMP_Enable)
{
RESET_ARRAY(arg, 8, 0, .value);
rx = AMD_HSMP_Exec(HSMP_RD_SMU_VER, arg);
if (rx == HSMP_RESULT_OK)
{
PUBLIC(RO(Proc))->Features.Factory.SMU.Version = arg[0].value;
}
else if (IS_HSMP_OOO(rx))
{
PUBLIC(RO(Proc))->Features.HSMP_Enable = 0;
}
}
PUBLIC(RO(Proc))->Features.Factory.SMU.Interface = 0;
if (PUBLIC(RO(Proc))->Features.HSMP_Enable)
{
RESET_ARRAY(arg, 8, 0, .value);
rx = AMD_HSMP_Exec(HSMP_RD_VERSION, arg);
if (rx == HSMP_RESULT_OK)
{
PUBLIC(RO(Proc))->Features.Factory.SMU.Interface = arg[0].value;
}
else if (IS_HSMP_OOO(rx))
{
PUBLIC(RO(Proc))->Features.HSMP_Enable = 0;
}
}
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Enable_Limit1 = \
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Clamping1 = \
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].Unlock = \
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Domain_Limit1 = 0;
if (PUBLIC(RO(Proc))->Features.HSMP_Enable)
{
RESET_ARRAY(arg, 8, 0, .value);
rx = AMD_HSMP_Exec(HSMP_RD_PKG_PL1, arg);
if (rx == HSMP_RESULT_OK)
{
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Domain_Limit1 = arg[0].value / 1000;
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Enable_Limit1 = \
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Clamping1 = \
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].Unlock = \
/* Assumed PL1 is enabled, clamped, unlocked if value exists */
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Domain_Limit1 > 0 ? 1 : 0;
}
else if (IS_HSMP_OOO(rx))
{
PUBLIC(RO(Proc))->Features.HSMP_Enable = 0;
}
}
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Enable_Limit2 = \
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Clamping2 = \
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Domain_Limit2 = 0;
if (PUBLIC(RO(Proc))->Features.HSMP_Enable)
{
RESET_ARRAY(arg, 8, 0, .value);
rx = AMD_HSMP_Exec(HSMP_RD_MAX_PPT, arg);
if (rx == HSMP_RESULT_OK)
{
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Domain_Limit2 = arg[0].value / 1000;
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Enable_Limit2 = \
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Clamping2 = \
/* Assumed PL2 is enabled, clamped if value exists */
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Domain_Limit2 > 0 ? 1 : 0;
}
else if (IS_HSMP_OOO(rx))
{
PUBLIC(RO(Proc))->Features.HSMP_Enable = 0;
}
}
PUBLIC(RO(Proc))->Counter[0].FCLK = \
PUBLIC(RO(Proc))->Counter[1].FCLK = \
PUBLIC(RO(Proc))->Delta.FCLK = 0;
if (PUBLIC(RO(Proc))->Features.HSMP_Enable)
{
RESET_ARRAY(arg, 8, 0, .value);
rx = AMD_HSMP_Exec(HSMP_RD_DF_MCLK, arg);
if (rx == HSMP_RESULT_OK)
{
PUBLIC(RO(Proc))->Counter[0].FCLK = \
PUBLIC(RO(Proc))->Counter[1].FCLK = \
PUBLIC(RO(Proc))->Delta.FCLK = UNIT_MHz(arg[0].value);
}
else if (IS_HSMP_OOO(rx))
{
PUBLIC(RO(Proc))->Features.HSMP_Enable = 0;
}
}
return rx;
}
static void Query_AMD_Family_17h(unsigned int cpu)
{
PRIVATE(OF(Specific)) = LookupProcessor();
if (PRIVATE(OF(Specific)) != NULL)
{
/* Save thermal parameters for later use in the Daemon */
PUBLIC(RO(Proc))->PowerThermal.Param = PRIVATE(OF(Specific))->Param;
/* Override Processor CodeName & Locking capabilities */
OverrideCodeNameString(PRIVATE(OF(Specific)));
OverrideUnlockCapability(PRIVATE(OF(Specific)));
} else {
PUBLIC(RO(Proc))->PowerThermal.Param.Target = 0;
}
Default_Unlock_Reset();
if (Compute_AMD_Zen_Boost(cpu) == true)
{ /* Count the Xtra Boost ratios */
PUBLIC(RO(Proc))->Features.XtraCOF = 2;
}
else { /* Disabled CPB: Hide ratios */
PUBLIC(RO(Proc))->Features.XtraCOF = 0;
}
/* Apply same register bit fields as Intel RAPL_POWER_UNIT */
RDMSR(PUBLIC(RO(Proc))->PowerThermal.Unit, MSR_AMD_RAPL_POWER_UNIT);
HyperThreading_Technology();
AMD_Processor_PIN(PUBLIC(RO(Proc))->Features.leaf80000008.EBX.PPIN);
Query_AMD_HSMP_Interface();
}
static void Exit_AMD_F17h(void)
{
#ifdef CONFIG_AMD_NB
if (PRIVATE(OF(Zen)).Device.DF != NULL)
{
pci_dev_put(PRIVATE(OF(Zen)).Device.DF);
PRIVATE(OF(Zen)).Device.DF = NULL;
}
#endif /* CONFIG_AMD_NB */
}
static void Query_AMD_F17h_PerSocket(unsigned int cpu)
{
Core_AMD_Family_17h_Temp = CTL_AMD_Family_17h_Temp;
Probe_AMD_DataFabric();
Query_AMD_Family_17h(cpu);
if (cpu == PUBLIC(RO(Proc))->Service.Core) {
Query_AMD_F17h_Power_Limits(PUBLIC(RO(Proc)));
if (AMD_F17h_CPPC() == -ENODEV) {
For_All_ACPI_CPPC(Read_ACPI_CPPC_Registers, NULL);
}
Read_ACPI_PCT_Registers(cpu);
Read_ACPI_PSS_Registers(cpu);
Read_ACPI_PPC_Registers(cpu);
Read_ACPI_CST_Registers(cpu);
}
}
static void Query_AMD_F17h_PerCluster(unsigned int cpu)
{
Core_AMD_Family_17h_Temp = CCD_AMD_Family_17h_Zen2_Temp;
Probe_AMD_DataFabric();
Query_AMD_Family_17h(cpu);
if (cpu == PUBLIC(RO(Proc))->Service.Core) {
Query_AMD_F17h_Power_Limits(PUBLIC(RO(Proc)));
if (AMD_F17h_CPPC() == -ENODEV) {
For_All_ACPI_CPPC(Read_ACPI_CPPC_Registers, NULL);
}
Read_ACPI_PCT_Registers(cpu);
Read_ACPI_PSS_Registers(cpu);
Read_ACPI_PPC_Registers(cpu);
Read_ACPI_CST_Registers(cpu);
}
}
static void Query_AMD_F19h_11h_PerCluster(unsigned int cpu)
{
Core_AMD_Family_17h_Temp = CCD_AMD_Family_19h_Genoa_Temp;
Pkg_AMD_Family_17h_Temp = Pkg_AMD_Family_19h_Genoa_Temp;
Probe_AMD_DataFabric();
Query_AMD_Family_17h(cpu);
if (cpu == PUBLIC(RO(Proc))->Service.Core) {
Query_AMD_F17h_Power_Limits(PUBLIC(RO(Proc)));
if (AMD_F17h_CPPC() == -ENODEV) {
For_All_ACPI_CPPC(Read_ACPI_CPPC_Registers, NULL);
}
Read_ACPI_PCT_Registers(cpu);
Read_ACPI_PSS_Registers(cpu);
Read_ACPI_PPC_Registers(cpu);
Read_ACPI_CST_Registers(cpu);
}
}
static void Query_AMD_F19h_61h_PerCluster(unsigned int cpu)
{
Core_AMD_Family_17h_Temp = CCD_AMD_Family_19h_Zen4_Temp;
Probe_AMD_DataFabric();
Query_AMD_Family_17h(cpu);
if (cpu == PUBLIC(RO(Proc))->Service.Core) {
Query_AMD_F17h_Power_Limits(PUBLIC(RO(Proc)));
if (AMD_F17h_CPPC() == -ENODEV) {
For_All_ACPI_CPPC(Read_ACPI_CPPC_Registers, NULL);
}
Read_ACPI_PCT_Registers(cpu);
Read_ACPI_PSS_Registers(cpu);
Read_ACPI_PPC_Registers(cpu);
Read_ACPI_CST_Registers(cpu);
}
}
static void Query_Hygon_F18h(unsigned int cpu)
{
switch (PUBLIC(RO(Proc))->Features.Std.EAX.Model) {
case 0x0:
switch (PUBLIC(RO(Proc))->Features.Std.EAX.Stepping) {
case 0x2:
StrCopy(PUBLIC(RO(Proc))->Architecture,
Arch_Hygon_Family_18h[CN_DHYANA_V1],
CODENAME_LEN);
break;
}
break;
case 0x1:
switch (PUBLIC(RO(Proc))->Features.Std.EAX.Stepping) {
case 0x1:
StrCopy(PUBLIC(RO(Proc))->Architecture,
Arch_Hygon_Family_18h[CN_DHYANA_V2],
CODENAME_LEN);
break;
}
break;
}
Query_AMD_F17h_PerSocket(cpu);
}
static void Dump_CPUID(CORE_RO *Core)
{
unsigned int i;
__asm__ volatile
(
"xorq %%rax, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (Core->Query.StdFunc.LargestStdFunc),
"=r" (Core->Query.StdFunc.BX),
"=r" (Core->Query.StdFunc.CX),
"=r" (Core->Query.StdFunc.DX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
__asm__ volatile
(
"movq $0x80000000, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (Core->Query.ExtFunc.LargestExtFunc),
"=r" (Core->Query.ExtFunc.EBX),
"=r" (Core->Query.ExtFunc.ECX),
"=r" (Core->Query.ExtFunc.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
/* Intel: CPUID INSTRUCTION
If a value is entered for CPUID.EAX is invalid for a particular processor,
the data for the highest basic information leaf is returned.
*/
for (i = 0; i < CPUID_MAX_FUNC; i++)
{ __asm__ volatile
(
"xorq %%rax, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"mov %4, %%eax \n\t"
"mov %5, %%ecx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (Core->CpuID[i].reg[0]),
"=r" (Core->CpuID[i].reg[1]),
"=r" (Core->CpuID[i].reg[2]),
"=r" (Core->CpuID[i].reg[3])
: "r" (Core->CpuID[i].func),
"r" (Core->CpuID[i].sub)
: "%rax", "%rbx", "%rcx", "%rdx"
);
}
}
static void AMD_F17h_DCU_Technology(CORE_RO *Core) /* Per Core */
{
AMD_DC_CFG DC_Cfg1 = {.value = 0};
AMD_CU_CFG3 CU_Cfg3 = {.value = 0};
AMD_IC_CFG IC_Cfg = {.value = 0};
RDMSR(DC_Cfg1, MSR_AMD_DC_CFG);
switch (L1_HW_PREFETCH_Disable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
DC_Cfg1.L1_HW_Prefetch = L1_HW_PREFETCH_Disable;
WRMSR(DC_Cfg1, MSR_AMD_DC_CFG);
RDMSR(DC_Cfg1, MSR_AMD_DC_CFG);
break;
}
RDMSR(CU_Cfg3, MSR_AMD_CU_CFG3);
switch (L2_HW_PREFETCH_Disable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
CU_Cfg3.L2_HW_Prefetch = !L2_HW_PREFETCH_Disable;
WRMSR(CU_Cfg3, MSR_AMD_CU_CFG3);
RDMSR(CU_Cfg3, MSR_AMD_CU_CFG3);
break;
}
RDMSR(IC_Cfg, MSR_AMD_IC_CFG);
switch (L1_HW_IP_PREFETCH_Disable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
IC_Cfg.HW_IP_Prefetch = L1_HW_IP_PREFETCH_Disable;
WRMSR(IC_Cfg, MSR_AMD_IC_CFG);
RDMSR(IC_Cfg, MSR_AMD_IC_CFG);
break;
}
if (DC_Cfg1.L1_HW_Prefetch)
{
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_HW_Prefetch, Core->Bind);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_HW_Prefetch, Core->Bind);
}
if (CU_Cfg3.L2_HW_Prefetch)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_HW_Prefetch, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_HW_Prefetch, Core->Bind);
}
if (IC_Cfg.HW_IP_Prefetch == 1) {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_HW_IP_Prefetch, Core->Bind);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_HW_IP_Prefetch, Core->Bind);
}
if (PUBLIC(RO(Proc))->Features.ExtFeature2_EAX.PrefetchCtl_MSR == 1)
{
int ToggleFeature = 0;
AMD_PREFETCH_CONTROL PrefetchCtl = {.value = 0};
RDMSR(PrefetchCtl, MSR_AMD_PREFETCH_CTRL);
switch (L1_STRIDE_PREFETCH_Disable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PrefetchCtl.L1Stride = L1_STRIDE_PREFETCH_Disable;
ToggleFeature = 1;
break;
}
switch (L1_REGION_PREFETCH_Disable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PrefetchCtl.L1Region = L1_REGION_PREFETCH_Disable;
ToggleFeature = 1;
break;
}
switch (L1_BURST_PREFETCH_Disable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PrefetchCtl.L1Stream = L1_BURST_PREFETCH_Disable;
ToggleFeature = 1;
break;
}
switch (L2_STREAM_PREFETCH_Disable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PrefetchCtl.L2Stream = L2_STREAM_PREFETCH_Disable;
ToggleFeature = 1;
break;
}
switch (L2_UPDOWN_PREFETCH_Disable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PrefetchCtl.UpDown = L2_UPDOWN_PREFETCH_Disable;
ToggleFeature = 1;
break;
}
if (ToggleFeature == 1) {
WRMSR(PrefetchCtl, MSR_AMD_PREFETCH_CTRL);
RDMSR(PrefetchCtl, MSR_AMD_PREFETCH_CTRL);
}
if (PrefetchCtl.L1Stride)
{
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_Stride_Pf, Core->Bind);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_Stride_Pf, Core->Bind);
}
if (PrefetchCtl.L1Region)
{
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_Region_Pf, Core->Bind);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_Region_Pf, Core->Bind);
}
if (PrefetchCtl.L1Stream)
{
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_Burst_Pf, Core->Bind);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_Burst_Pf, Core->Bind);
}
if (PrefetchCtl.L2Stream)
{
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_Stream_HW_Pf, Core->Bind);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_Stream_HW_Pf, Core->Bind);
}
if (PrefetchCtl.UpDown)
{
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_UpDown_Pf, Core->Bind);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_UpDown_Pf, Core->Bind);
}
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->DCU_Mask, Core->Bind);
}
static void Intel_DCU_Technology(CORE_RO *Core) /*Per Core */
{ /* Avoton[06_4D], GDM[06_5C], NHM[06_1A, 06_1E, 06_1F, 06_2E], SNB+, Phi */
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1))
{
int ToggleFeature = 0;
MISC_FEATURE_CONTROL MiscFeatCtrl = {.value = 0};
RDMSR(MiscFeatCtrl, MSR_MISC_FEATURE_CONTROL);
switch (L2_HW_PREFETCH_Disable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
MiscFeatCtrl.L2_HW_Prefetch = L2_HW_PREFETCH_Disable;
ToggleFeature = 1;
break;
}
switch (L2_HW_CL_PREFETCH_Disable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
MiscFeatCtrl.L2_HW_CL_Prefetch = L2_HW_CL_PREFETCH_Disable;
ToggleFeature = 1;
break;
}
switch (L1_HW_PREFETCH_Disable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
MiscFeatCtrl.L1_HW_Prefetch = L1_HW_PREFETCH_Disable;
ToggleFeature = 1;
break;
}
switch (L1_HW_IP_PREFETCH_Disable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
MiscFeatCtrl.L1_HW_IP_Prefetch = L1_HW_IP_PREFETCH_Disable;
ToggleFeature = 1;
break;
}
if (ToggleFeature == 1)
{
WRMSR(MiscFeatCtrl, MSR_MISC_FEATURE_CONTROL);
RDMSR(MiscFeatCtrl, MSR_MISC_FEATURE_CONTROL);
}
if (MiscFeatCtrl.L2_HW_Prefetch == 1) {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_HW_Prefetch, Core->Bind);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_HW_Prefetch, Core->Bind);
}
if (MiscFeatCtrl.L2_HW_CL_Prefetch == 1) {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_HW_CL_Prefetch, Core->Bind);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_HW_CL_Prefetch, Core->Bind);
}
if (MiscFeatCtrl.L1_HW_Prefetch == 1) {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_HW_Prefetch, Core->Bind);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_HW_Prefetch, Core->Bind);
}
if (MiscFeatCtrl.L1_HW_IP_Prefetch == 1) {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_HW_IP_Prefetch, Core->Bind);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_HW_IP_Prefetch, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->DCU_Mask, Core->Bind);
switch (Core->T.Cluster.Hybrid.CoreType) {
case Hybrid_Atom:
{
ATOM_L2_PREFETCH_0X1321 MLC_Ctrl = {.value = 0};
ATOM_L2_PREFETCH_0X1320 LLC_Ctrl = {.value = 0};
RDMSR(MLC_Ctrl, MSR_ATOM_L2_PREFETCH_0X1321);
RDMSR(LLC_Ctrl, MSR_ATOM_L2_PREFETCH_0X1320);
switch (L2_NLP_PREFETCH_Disable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
MLC_Ctrl.L2_DISABLE_NEXT_LINE_PREFETCH = L2_NLP_PREFETCH_Disable;
WRMSR(MLC_Ctrl, MSR_ATOM_L2_PREFETCH_0X1321);
RDMSR(MLC_Ctrl, MSR_ATOM_L2_PREFETCH_0X1321);
break;
}
if (MLC_Ctrl.L2_DISABLE_NEXT_LINE_PREFETCH == 1) {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_NLP_Prefetch, Core->Bind);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_NLP_Prefetch, Core->Bind);
}
switch (LLC_Streamer_Disable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
LLC_Ctrl.LLC_STREAM_DISABLE = LLC_Streamer_Disable;
WRMSR(LLC_Ctrl, MSR_ATOM_L2_PREFETCH_0X1320);
RDMSR(LLC_Ctrl, MSR_ATOM_L2_PREFETCH_0X1320);
break;
}
if (LLC_Ctrl.LLC_STREAM_DISABLE == 1) {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->LLC_Streamer, Core->Bind);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->LLC_Streamer, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->ECORE_Mask, Core->Bind);
}
break;
case Hybrid_Core:
/* Invalid MSR_ATOM_L2_PREFETCH registers with P-Core */
break;
case Hybrid_RSVD1:
case Hybrid_RSVD2:
default:
break;
}
}
}
static void Intel_Core_MicroArchControl(CORE_RO *Core)
{
CORE_UARCH_CTL Core_Uarch_Ctl = {.value = 0};
RDMSR(Core_Uarch_Ctl, MSR_CORE_UARCH_CTL);
switch (L1_Scrubbing_Enable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Core_Uarch_Ctl.L1_Scrubbing_En = L1_Scrubbing_Enable;
WRMSR(Core_Uarch_Ctl, MSR_CORE_UARCH_CTL);
RDMSR(Core_Uarch_Ctl, MSR_CORE_UARCH_CTL);
break;
}
if (Core_Uarch_Ctl.L1_Scrubbing_En == 1) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_Scrubbing, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_Scrubbing, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->L1_Scrub_Mask, Core->Bind);
}
static void Intel_Core_MicroArchitecture(CORE_RO *Core) /* Per P-Core */
{ /* 06_7D, 06_7E, 06_8C, 06_8D, 06_97, 06_9A, 06_B7, 06_BA, 06_BF, MTL, ARL */
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1))
{
switch (Core->T.Cluster.Hybrid.CoreType) {
case Hybrid_Atom:
/* No MSR_CORE_UARCH_CTL(0x541) register with E-Core */
break;
case Hybrid_Core:
{
MISC_FEATURE_CONTROL MiscFeatCtrl = {.value = 0};
RDMSR(MiscFeatCtrl, MSR_MISC_FEATURE_CONTROL);
switch (L2_AMP_PREFETCH_Disable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
MiscFeatCtrl.L2_AMP_Prefetch = L2_AMP_PREFETCH_Disable;
WRMSR(MiscFeatCtrl, MSR_MISC_FEATURE_CONTROL);
RDMSR(MiscFeatCtrl, MSR_MISC_FEATURE_CONTROL);
break;
}
if (MiscFeatCtrl.L2_AMP_Prefetch == 1) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_AMP_Prefetch, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_AMP_Prefetch, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->L2_AMP_Mask, Core->Bind);
Intel_Core_MicroArchControl(Core);
}
break;
case Hybrid_RSVD1:
case Hybrid_RSVD2:
default:
/* Unspecified MSR_CORE_UARCH_CTL(0x541) register with IDs */
break;
}
}
}
static void Intel_Ultra7_MicroArchitecture(CORE_RO *Core)
{ /* 06_AA, 06_AB, 06_AC, 06_C6 */
MISC_FEATURE_CONTROL MiscFeatCtrl = {.value = 0};
RDMSR(MiscFeatCtrl, MSR_MISC_FEATURE_CONTROL);
switch (L1_NPP_PREFETCH_Disable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
MiscFeatCtrl.L1_NPP_Prefetch = L1_NPP_PREFETCH_Disable;
WRMSR(MiscFeatCtrl, MSR_MISC_FEATURE_CONTROL);
RDMSR(MiscFeatCtrl, MSR_MISC_FEATURE_CONTROL);
break;
}
if (MiscFeatCtrl.L1_NPP_Prefetch == 1) {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_NPP_Prefetch, Core->Bind);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_NPP_Prefetch, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->DCU_Mask, Core->Bind);
}
static void SpeedStep_Technology(CORE_RO *Core) /*Per Package*/
{
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (PUBLIC(RO(Proc))->Features.Std.ECX.EIST == 1)
{
MISC_PROC_FEATURES MiscFeatures = {.value = 0};
RDMSR(MiscFeatures, MSR_IA32_MISC_ENABLE);
switch (SpeedStep_Enable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
MiscFeatures.EIST = SpeedStep_Enable;
WRMSR(MiscFeatures, MSR_IA32_MISC_ENABLE);
RDMSR(MiscFeatures, MSR_IA32_MISC_ENABLE);
break;
}
if (MiscFeatures.EIST)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->SpeedStep, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SpeedStep, Core->Bind);
}
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SpeedStep, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->SpeedStep_Mask, Core->Bind);
}
}
static void Intel_Turbo_Config(CORE_RO *Core, void (*ConfigFunc)(void*),
void (*AssignFunc)(unsigned int*, TURBO_CONFIG*))
{ /*Per Package*/
CLOCK_TURBO_ARG ClockTurbo = {
.pClockMod = NULL, /* Read-Only Operation */
.Config = {.MSR = {.value = 0}},
.rc = RC_SUCCESS
};
ConfigFunc(&ClockTurbo);
AssignFunc(Core->Boost, &ClockTurbo.Config);
}
typedef void (*SET_TARGET)(CORE_RO*, unsigned int);
static void Set_Core2_Target(CORE_RO *Core, unsigned int ratio)
{
Core->PowerThermal.PerfControl.CORE.TargetRatio = ratio;
}
static void Set_Nehalem_Target(CORE_RO *Core, unsigned int ratio)
{
Core->PowerThermal.PerfControl.NHM.TargetRatio = ratio & 0xff;
}
static void Set_SandyBridge_Target(CORE_RO *Core, unsigned int ratio)
{
Core->PowerThermal.PerfControl.SNB.TargetRatio = ratio;
}
#define GET_CORE2_TARGET(Core) \
( \
Core->PowerThermal.PerfControl.CORE.TargetRatio \
)
typedef unsigned int (*GET_TARGET)(CORE_RO*);
static unsigned int Get_Core2_Target(CORE_RO *Core)
{
return GET_CORE2_TARGET(Core);
}
#define GET_NEHALEM_TARGET(Core) \
( \
Core->PowerThermal.PerfControl.NHM.TargetRatio & 0xff \
)
static unsigned int Get_Nehalem_Target(CORE_RO *Core)
{
return GET_NEHALEM_TARGET(Core);
}
#define GET_SANDYBRIDGE_TARGET(Core) \
( \
Core->PowerThermal.PerfControl.SNB.TargetRatio \
)
static unsigned int Get_SandyBridge_Target(CORE_RO *Core)
{
return GET_SANDYBRIDGE_TARGET(Core);
}
typedef int (*CMP_TARGET)(CORE_RO*, unsigned int);
static int Cmp_Core2_Target(CORE_RO *Core, unsigned int ratio)
{
return (Core->PowerThermal.PerfControl.CORE.TargetRatio == ratio);
}
static int Cmp_Nehalem_Target(CORE_RO *Core, unsigned int ratio)
{
return (Core->PowerThermal.PerfControl.NHM.TargetRatio == ratio);
}
static int Cmp_SandyBridge_Target(CORE_RO *Core, unsigned int ratio)
{
return (Core->PowerThermal.PerfControl.SNB.TargetRatio > ratio);
}
static int Cmp_Skylake_Target(CORE_RO *Core, unsigned int ratio)
{
return (Core->PowerThermal.PerfControl.SNB.TargetRatio >= ratio);
}
static bool IsPerformanceControlCapable(void)
{
struct SIGNATURE denyList[] = {
_Silvermont_Bay_Trail, /* 06_37 */
_Atom_Merrifield, /* 06_4A */
_Atom_Avoton, /* 06_4D */
_Atom_Moorefield, /* 06_5A */
_Atom_Sofia /* 06_5D */
};
const int ids = sizeof(denyList) / sizeof(denyList[0]);
int id;
for (id = 0; id < ids; id++) {
if((denyList[id].ExtFamily==PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily)
&& (denyList[id].Family == PUBLIC(RO(Proc))->Features.Std.EAX.Family)
&& (denyList[id].ExtModel == PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel)
&& (denyList[id].Model == PUBLIC(RO(Proc))->Features.Std.EAX.Model))
{
return (PUBLIC(RO(Proc))->Registration.Driver.CPUfreq == 1);
}
}
return true;
}
static bool WritePerformanceControl(PERF_CONTROL *pPerfControl)
{
bool isCapable = IsPerformanceControlCapable();
if (isCapable == true)
{
WRMSR((*pPerfControl), MSR_IA32_PERF_CTL);
}
return isCapable;
}
static void TurboBoost_Technology(CORE_RO *Core,SET_TARGET SetTarget,
GET_TARGET GetTarget,
CMP_TARGET CmpTarget,
unsigned int TurboRatio,
unsigned int ValidRatio)
{ /* Per SMT */
int ToggleFeature;
MISC_PROC_FEATURES MiscFeatures = {.value = 0};
RDMSR(MiscFeatures, MSR_IA32_MISC_ENABLE);
UNUSED(GetTarget);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->TurboBoost_Mask, Core->Bind);
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
if ( (MiscFeatures.Turbo_IDA == 0)
&& (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA) )
{
switch (TurboBoost_Enable[0]) {
case COREFREQ_TOGGLE_OFF: /* Restore the nominal P-state */
Core->PowerThermal.PerfControl.Turbo_IDA = 1;
SetTarget(Core, Core->Boost[BOOST(MAX)]);
ToggleFeature = 1;
break;
case COREFREQ_TOGGLE_ON: /* Request the Turbo P-state */
Core->PowerThermal.PerfControl.Turbo_IDA = 0;
SetTarget(Core, TurboRatio);
ToggleFeature = 1;
break;
default:
ToggleFeature = 0;
break;
}
if (ToggleFeature == 1)
{
WritePerformanceControl(&Core->PowerThermal.PerfControl);
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
}
if (Core->PowerThermal.PerfControl.Turbo_IDA == 0)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
}
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
}
if (PUBLIC(RO(Proc))->Features.HWP_Enable)
{
RDMSR(Core->PowerThermal.HWP_Capabilities, MSR_IA32_HWP_CAPABILITIES);
RDMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
if (PUBLIC(RO(Proc))->Features.Power.EAX.HWP_EPP)
{ /* EPP mode */
if ((HWP_EPP >= 0) && (HWP_EPP <= 0xff))
{
Core->PowerThermal.HWP_Request.Energy_Pref = HWP_EPP;
WRMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
RDMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
}
} else { /* EPB fallback mode */
if (PUBLIC(RO(Proc))->Registration.Driver.CPUfreq
& (REGISTRATION_ENABLE | REGISTRATION_FULLCTRL) )
{
/* Turbo is a function of the Target P-state */
if (!CmpTarget(Core, ValidRatio))
{
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
}
}
}
Core->Boost[BOOST(HWP_MIN)]=Core->PowerThermal.HWP_Request.Minimum_Perf;
Core->Boost[BOOST(HWP_MAX)]=Core->PowerThermal.HWP_Request.Maximum_Perf;
Core->Boost[BOOST(HWP_TGT)]=Core->PowerThermal.HWP_Request.Desired_Perf;
} else { /* EPB mode */
if (PUBLIC(RO(Proc))->Registration.Driver.CPUfreq
& (REGISTRATION_ENABLE | REGISTRATION_FULLCTRL) )
{
if (!CmpTarget(Core, ValidRatio))
{
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
}
}
}
}
static void DynamicAcceleration(CORE_RO *Core) /* Unique */
{
if (IsPerformanceControlCapable() == true)
{
if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA)
{
TurboBoost_Technology( Core,
Set_Core2_Target,
Get_Core2_Target,
Cmp_Core2_Target,
1 + Core->Boost[BOOST(MAX)],
1 + Core->Boost[BOOST(MAX)] );
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->TurboBoost_Mask, Core->Bind);
}
} else {
int ToggleFeature;
MISC_PROC_FEATURES MiscFeatures = {.value = 0};
RDMSR(MiscFeatures, MSR_IA32_MISC_ENABLE);
switch (TurboBoost_Enable[0]) {
case COREFREQ_TOGGLE_OFF:
MiscFeatures.Turbo_IDA = 1;
ToggleFeature = 1;
break;
case COREFREQ_TOGGLE_ON:
MiscFeatures.Turbo_IDA = 0;
ToggleFeature = 1;
break;
default:
ToggleFeature = 0;
break;
}
if (ToggleFeature == 1)
{
WRMSR(MiscFeatures, MSR_IA32_MISC_ENABLE);
RDMSR(MiscFeatures, MSR_IA32_MISC_ENABLE);
/* Refresh the Turbo capability in the leaf. */
if (PUBLIC(RO(Proc))->Features.Info.LargestStdFunc >= 0x6) {
__asm__ volatile
(
"movq $0x6, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (PUBLIC(RO(Proc))->Features.Power.EAX),
"=r" (PUBLIC(RO(Proc))->Features.Power.EBX),
"=r" (PUBLIC(RO(Proc))->Features.Power.ECX),
"=r" (PUBLIC(RO(Proc))->Features.Power.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
}
}
if (MiscFeatures.Turbo_IDA == 0)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->TurboBoost_Mask, Core->Bind);
}
}
static void SoC_Turbo_Override(CORE_RO *Core)
{
int ToggleFeature;
PKG_TURBO_CONFIG TurboCfg = {.value = 0};
RDMSR(TurboCfg, MSR_PKG_TURBO_CFG);
switch (TurboBoost_Enable[0]) {
case COREFREQ_TOGGLE_OFF:
TurboCfg.TjMax_Turbo = 0x0;
ToggleFeature = 1;
break;
case COREFREQ_TOGGLE_ON:
TurboCfg.TjMax_Turbo = 0x2;
ToggleFeature = 1;
break;
default:
ToggleFeature = 0;
break;
}
if ((ToggleFeature == 1)
&& (Core->Bind == PUBLIC(RO(Proc))->Service.Core)) /* Package */
{
WRMSR(TurboCfg, MSR_PKG_TURBO_CFG);
RDMSR(TurboCfg, MSR_PKG_TURBO_CFG);
}
if (TurboCfg.TjMax_Turbo == 0x2)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->TurboBoost_Mask, Core->Bind);
}
typedef struct {
CLOCK_ARG *pClockMod;
SET_TARGET SetTarget;
GET_TARGET GetTarget;
long rc;
} CLOCK_PPC_ARG;
static void ClockMod_PPC_PerCore(void *arg)
{
CLOCK_PPC_ARG *pClockPPC;
CORE_RO *Core;
unsigned int cpu;
pClockPPC = (CLOCK_PPC_ARG *) arg;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
pClockPPC->SetTarget(Core, (pClockPPC->pClockMod->cpu == -1) ?
pClockPPC->pClockMod->Ratio
: pClockPPC->GetTarget(Core) + pClockPPC->pClockMod->Offset);
if (WritePerformanceControl(&Core->PowerThermal.PerfControl) == true) {
pClockPPC->rc = RC_OK_COMPUTE;
} else {
pClockPPC->rc = -EINTR;
}
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
Core->Boost[BOOST(TGT)] = pClockPPC->GetTarget(Core);
}
static long For_All_PPC_Clock(CLOCK_PPC_ARG *pClockPPC)
{
long rc = RC_SUCCESS;
unsigned int cpu = PUBLIC(RO(Proc))->CPU.Count;
do {
cpu--; /* From last AP to BSP */
if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS)
&& ((pClockPPC->pClockMod->cpu == -1)||(pClockPPC->pClockMod->cpu == cpu)))
{
smp_call_function_single(cpu, ClockMod_PPC_PerCore, pClockPPC, 1);
rc = pClockPPC->rc;
}
} while ((cpu != 0) && (rc >= RC_SUCCESS)) ;
return rc;
}
static long ClockMod_Core2_PPC(CLOCK_ARG *pClockMod)
{
if (pClockMod != NULL) {
if (pClockMod->NC == CLOCK_MOD_TGT)
{
CLOCK_PPC_ARG ClockPPC = {
.pClockMod = pClockMod,
.SetTarget = Set_Core2_Target,
.GetTarget = Get_Core2_Target,
.rc = RC_SUCCESS
};
return For_All_PPC_Clock(&ClockPPC);
} else {
return -RC_UNIMPLEMENTED;
}
} else {
return -EINVAL;
}
}
static long ClockMod_Nehalem_PPC(CLOCK_ARG *pClockMod)
{
if (pClockMod != NULL) {
if (pClockMod->NC == CLOCK_MOD_TGT)
{
CLOCK_PPC_ARG ClockPPC = {
.pClockMod = pClockMod,
.SetTarget = Set_Nehalem_Target,
.GetTarget = Get_Nehalem_Target,
.rc = RC_SUCCESS
};
return For_All_PPC_Clock(&ClockPPC);
} else {
return -RC_UNIMPLEMENTED;
}
} else {
return -EINVAL;
}
}
static long ClockMod_SandyBridge_PPC(CLOCK_ARG *pClockMod)
{
if (pClockMod != NULL) {
if (pClockMod->NC == CLOCK_MOD_TGT)
{
CLOCK_PPC_ARG ClockPPC = {
.pClockMod = pClockMod,
.SetTarget = Set_SandyBridge_Target,
.GetTarget = Get_SandyBridge_Target,
.rc = RC_SUCCESS
};
return For_All_PPC_Clock(&ClockPPC);
} else {
return -RC_UNIMPLEMENTED;
}
} else {
return -EINVAL;
}
}
typedef struct {
CLOCK_ARG *pClockMod;
long rc;
} CLOCK_HWP_ARG;
static void ClockMod_HWP_PerCore(void *arg)
{
CLOCK_HWP_ARG *pClockHWP;
CORE_RO *Core;
unsigned int cpu;
unsigned short WrRdHWP;
pClockHWP = (CLOCK_HWP_ARG *) arg;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
RDMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
switch (pClockHWP->pClockMod->NC) {
case CLOCK_MOD_HWP_MIN:
if (pClockHWP->pClockMod->cpu == -1) {
Core->PowerThermal.HWP_Request.Minimum_Perf = pClockHWP->pClockMod->Ratio;
} else {
Core->PowerThermal.HWP_Request.Minimum_Perf += pClockHWP->pClockMod->Offset;
}
WrRdHWP = 1;
break;
case CLOCK_MOD_HWP_MAX:
if (pClockHWP->pClockMod->cpu == -1) {
Core->PowerThermal.HWP_Request.Maximum_Perf = pClockHWP->pClockMod->Ratio;
} else {
Core->PowerThermal.HWP_Request.Maximum_Perf += pClockHWP->pClockMod->Offset;
}
WrRdHWP = 1;
break;
case CLOCK_MOD_HWP_TGT:
if (pClockHWP->pClockMod->cpu == -1) {
Core->PowerThermal.HWP_Request.Desired_Perf = pClockHWP->pClockMod->Ratio;
} else {
Core->PowerThermal.HWP_Request.Desired_Perf += pClockHWP->pClockMod->Offset;
}
WrRdHWP = 1;
break;
default:
WrRdHWP = 0;
pClockHWP->rc = -RC_UNIMPLEMENTED;
break;
}
if (WrRdHWP == 1)
{
WRMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
RDMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
pClockHWP->rc = RC_OK_COMPUTE;
}
}
static long For_All_HWP_Clock(CLOCK_HWP_ARG *pClockHWP)
{
long rc = RC_SUCCESS;
unsigned int cpu = PUBLIC(RO(Proc))->CPU.Count;
do {
cpu--; /* From last AP to BSP */
if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS)
&& ((pClockHWP->pClockMod->cpu == -1)||(pClockHWP->pClockMod->cpu == cpu)))
{
smp_call_function_single(cpu, ClockMod_HWP_PerCore, pClockHWP, 1);
rc = pClockHWP->rc;
}
} while ((cpu != 0) && (rc >= RC_SUCCESS)) ;
return rc;
}
static long ClockMod_Intel_HWP(CLOCK_ARG *pClockMod)
{
if (PUBLIC(RO(Proc))->Features.HWP_Enable) {
if (pClockMod != NULL) {
switch (pClockMod->NC) {
case CLOCK_MOD_HWP_MIN:
case CLOCK_MOD_HWP_MAX:
case CLOCK_MOD_HWP_TGT:
{
CLOCK_HWP_ARG ClockHWP = {
.pClockMod = pClockMod,
.rc = RC_SUCCESS
};
return For_All_HWP_Clock(&ClockHWP);
}
case CLOCK_MOD_TGT:
return ClockMod_SandyBridge_PPC(pClockMod);
default:
return -RC_UNIMPLEMENTED;
}
} else {
return -EINVAL;
}
} else {
return ClockMod_SandyBridge_PPC(pClockMod);
}
}
static void AMD_Watchdog(CORE_RO *Core)
{ /* CPU Watchdog Timer. */
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1))
{
AMD_CPU_WDT_CFG CPU_WDT_CFG = {.value = 0};
RDMSR(CPU_WDT_CFG, MSR_AMD_CPU_WDT_CFG);
switch (WDT_Enable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
CPU_WDT_CFG.TmrCfgEn = WDT_Enable;
WRMSR(CPU_WDT_CFG, MSR_AMD_CPU_WDT_CFG);
RDMSR(CPU_WDT_CFG, MSR_AMD_CPU_WDT_CFG);
break;
}
if (CPU_WDT_CFG.TmrCfgEn) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->WDT, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->WDT, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->WDT_Mask, Core->Bind);
}
}
static void PerCore_AMD_CState_BAR(CORE_RO *Core)
{ /* Families: 10h, 12h, 14h, 15h, 16h, 17h: I/O C-State Base Address. */
CSTATE_BASE_ADDR CStateBaseAddr = {.value = 0};
RDMSR(CStateBaseAddr, MSR_AMD_CSTATE_BAR);
Core->Query.CStateBaseAddr = CStateBaseAddr.IOaddr;
}
static void PerCore_Query_AMD_Zen_Features(CORE_RO *Core) /* Per SMT */
{
ZEN_CSTATE_CONFIG CStateCfg = {.value = 0};
int ToggleFeature;
/* Read The Hardware Configuration Register. */
RDMSR(Core->SystemRegister.HWCR, MSR_K7_HWCR);
/* Query the SMM. */
if (Core->SystemRegister.HWCR.Family_17h.SmmLock) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->SMM, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SMM, Core->Bind);
}
/* Enable or Disable the Core Performance Boost. */
switch (TurboBoost_Enable[0]) {
case COREFREQ_TOGGLE_OFF:
Core->SystemRegister.HWCR.Family_17h.CpbDis = 1;
ToggleFeature = 1;
break;
case COREFREQ_TOGGLE_ON:
Core->SystemRegister.HWCR.Family_17h.CpbDis = 0;
ToggleFeature = 1;
break;
default:
ToggleFeature = 0;
break;
}
if (ToggleFeature == 1)
{
WRMSR(Core->SystemRegister.HWCR, MSR_K7_HWCR);
RDMSR(Core->SystemRegister.HWCR, MSR_K7_HWCR);
}
if (Core->SystemRegister.HWCR.Family_17h.CpbDis == 0)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->TurboBoost_Mask, Core->Bind);
/* Enable or Disable the Core C6 State. Bit[22,14,16] */
RDMSR(CStateCfg, MSR_AMD_F17H_CSTATE_CONFIG);
switch (CC6_Enable) {
case COREFREQ_TOGGLE_OFF:
CStateCfg.CCR2_CC6EN = 0;
CStateCfg.CCR1_CC6EN = 0;
CStateCfg.CCR0_CC6EN = 0;
ToggleFeature = 1;
break;
case COREFREQ_TOGGLE_ON:
CStateCfg.CCR2_CC6EN = 1;
CStateCfg.CCR1_CC6EN = 1;
CStateCfg.CCR0_CC6EN = 1;
ToggleFeature = 1;
break;
default:
ToggleFeature = 0;
break;
}
if (ToggleFeature == 1)
{
WRMSR(CStateCfg, MSR_AMD_F17H_CSTATE_CONFIG);
RDMSR(CStateCfg, MSR_AMD_F17H_CSTATE_CONFIG);
}
if (CStateCfg.CCR2_CC6EN && CStateCfg.CCR1_CC6EN && CStateCfg.CCR0_CC6EN
&& Core->SystemRegister.HWCR.Family_17h.INVDWBINVD)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->CC6, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->CC6, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);
/* Enable or Disable the Package C6 State . Bit[32] */
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
ZEN_PMGT_MISC PmgtMisc = {.value = 0};
RDMSR(PmgtMisc, MSR_AMD_F17H_PMGT_MISC);
switch (PC6_Enable) {
case COREFREQ_TOGGLE_OFF:
PmgtMisc.PC6En = 0;
ToggleFeature = 1;
break;
case COREFREQ_TOGGLE_ON:
PmgtMisc.PC6En = 1;
ToggleFeature = 1;
break;
default:
ToggleFeature = 0;
break;
}
if (ToggleFeature == 1) {
WRMSR(PmgtMisc, MSR_AMD_F17H_PMGT_MISC);
RDMSR(PmgtMisc, MSR_AMD_F17H_PMGT_MISC);
}
if (PmgtMisc.PC6En == 1)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->PC6, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->PC6, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PC6_Mask, Core->Bind);
PUBLIC(RO(Proc))->PowerThermal.Events[eSTS] = EVENT_THERM_NONE;
if (PUBLIC(RO(Proc))->Features.HSMP_Enable)
{
unsigned int rx;
HSMP_ARG arg[8];
RESET_ARRAY(arg, 8, 0, .value);
rx = AMD_HSMP_Exec(HSMP_RD_PROCHOT, arg);
if (rx == HSMP_RESULT_OK)
{
PUBLIC(RO(Proc))->PowerThermal.Events[eSTS] = (
((Bit64)arg[0].value & 0x1) << LSHIFT_PROCHOT_STS
);
}
else if (IS_HSMP_OOO(rx))
{
PUBLIC(RO(Proc))->Features.HSMP_Enable = 0;
}
}
}
/* SMT C-State Base Address. */
PerCore_AMD_CState_BAR(Core);
/* Package C-State: Configuration Control . */
Core->Query.CfgLock = 1;
/* Package C-State: I/O MWAIT Redirection . */
Core->Query.IORedir = 0;
AMD_Watchdog(Core);
}
static void Intel_Watchdog(CORE_RO *Core)
{
static struct pci_device_id PCI_WDT_ids[] = {
{
PCI_VDEVICE(INTEL, DID_INTEL_ICH10_LPC),
.driver_data = (kernel_ulong_t) ICH_TCO
},
{
PCI_VDEVICE(INTEL, DID_INTEL_PCH_C216_LPC),
.driver_data = (kernel_ulong_t) ICH_TCO
},
{
PCI_VDEVICE(INTEL, DID_INTEL_EHL_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_JSL_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_C620_PCH_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_C620_SUPER_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_KBL_PCH_H_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_SPT_LP_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_SPT_H_PCH_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_CNL_PCH_LP_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_CNL_PCH_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_CML_PCH_LP_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_CML_PCH_V_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_CML_H_PCH_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_ICL_LP_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_ICL_PCH_NG_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_TGL_PCH_LP_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_TGL_H_PCH_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_ADL_PCH_P_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_ADL_PCH_M_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_ADL_S_PCH_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_RPL_D_PCH_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_METEORLAKE_M_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_ARL_MTL_PCH_S_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_ARROWLAKE_S_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{
PCI_VDEVICE(INTEL, DID_INTEL_LUNARLAKE_V_SMBUS),
.driver_data = (kernel_ulong_t) TCOBASE
},
{0, }
};
if (CoreFreqK_ProbePCI(PCI_WDT_ids, NULL, NULL) < RC_SUCCESS) {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->WDT, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->WDT_Mask, Core->Bind);
}
static void Intel_Turbo_Activation_Ratio(CORE_RO *Core)
{
TURBO_ACTIVATION TurboActivation = {.value = 0};
RDMSR(TurboActivation, MSR_TURBO_ACTIVATION_RATIO);
if (!TurboActivation.Ratio_Lock && (Turbo_Activation_Ratio >= 0)
&& (Turbo_Activation_Ratio != TurboActivation.MaxRatio))
{
const short MaxRatio = \
MAXCLOCK_TO_RATIO(short, PUBLIC(RO(Core, AT(Core->Bind))->Clock.Hz));
if (Turbo_Activation_Ratio <= MaxRatio)
{
TurboActivation.MaxRatio = Turbo_Activation_Ratio;
WRMSR(TurboActivation, MSR_TURBO_ACTIVATION_RATIO);
RDMSR(TurboActivation, MSR_TURBO_ACTIVATION_RATIO);
}
}
Core->Boost[BOOST(ACT)] = TurboActivation.MaxRatio;
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PUBLIC(RO(Proc))->Features.TurboActiv_Lock = TurboActivation.Ratio_Lock;
}
}
static void Intel_Turbo_TDP_Config(CORE_RO *Core)
{
CONFIG_TDP_NOMINAL NominalTDP = {.value = 0};
CONFIG_TDP_CONTROL ControlTDP = {.value = 0};
CONFIG_TDP_LEVEL ConfigTDP[2] = {{.value = 0}, {.value = 0}};
RDMSR(NominalTDP, MSR_CONFIG_TDP_NOMINAL);
Core->Boost[BOOST(TDP)] = NominalTDP.Ratio;
RDMSR(ConfigTDP[1], MSR_CONFIG_TDP_LEVEL_2);
Core->Boost[BOOST(TDP2)] = ConfigTDP[1].Ratio;
RDMSR(ConfigTDP[0], MSR_CONFIG_TDP_LEVEL_1);
Core->Boost[BOOST(TDP1)] = ConfigTDP[0].Ratio;
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
RDMSR(ControlTDP, MSR_CONFIG_TDP_CONTROL);
if ((ControlTDP.Lock == 0) && (Config_TDP_Level >= 0))
{
ControlTDP.Level = Config_TDP_Level;
WRMSR(ControlTDP, MSR_CONFIG_TDP_CONTROL);
RDMSR(ControlTDP, MSR_CONFIG_TDP_CONTROL);
}
PUBLIC(RO(Proc))->Features.TDP_Cfg_Lock = ControlTDP.Lock;
PUBLIC(RO(Proc))->Features.TDP_Cfg_Level = ControlTDP.Level;
switch (PUBLIC(RO(Proc))->Features.TDP_Cfg_Level) {
case 2:
PUBLIC(RO(Proc))->PowerThermal.PowerInfo.ThermalSpecPower = \
ConfigTDP[1].PkgPower;
PUBLIC(RO(Proc))->PowerThermal.PowerInfo.MinimumPower = \
ConfigTDP[1].MinPower;
PUBLIC(RO(Proc))->PowerThermal.PowerInfo.MaximumPower = \
ConfigTDP[1].MaxPower;
break;
case 1:
PUBLIC(RO(Proc))->PowerThermal.PowerInfo.ThermalSpecPower = \
ConfigTDP[0].PkgPower;
PUBLIC(RO(Proc))->PowerThermal.PowerInfo.MinimumPower = \
ConfigTDP[0].MinPower;
PUBLIC(RO(Proc))->PowerThermal.PowerInfo.MaximumPower = \
ConfigTDP[0].MaxPower;
break;
case 0:
RDMSR(PUBLIC(RO(Proc))->PowerThermal.PowerInfo, MSR_PKG_POWER_INFO);
break;
}
}
}
static void Query_Intel_C1E(CORE_RO *Core) /*Per Package*/
{
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
POWER_CONTROL PowerCtrl = {.value = 0};
RDMSR(PowerCtrl, MSR_IA32_POWER_CTL);
switch (C1E_Enable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PowerCtrl.C1E = C1E_Enable;
WRMSR(PowerCtrl, MSR_IA32_POWER_CTL);
RDMSR(PowerCtrl, MSR_IA32_POWER_CTL);
break;
}
if (PowerCtrl.C1E)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->C1E, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->C1E, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1E_Mask, Core->Bind);
}
}
static void Query_AMD_Family_0Fh_C1E(CORE_RO *Core) /* Per Core */
{
INT_PENDING_MSG IntPendingMsg = {.value = 0};
RDMSR(IntPendingMsg, MSR_K8_INT_PENDING_MSG);
if (IntPendingMsg.C1eOnCmpHalt & !IntPendingMsg.SmiOnCmpHalt)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->C1E, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->C1E, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1E_Mask, Core->Bind);
}
static void ThermalMonitor2_Set(CORE_RO *Core, MISC_PROC_FEATURES MiscFeatures)
{ /* Intel Core Solo Duo. */
struct SIGNATURE allowList[] = {
_Core_Yonah , /* 06_0E */
_Core_Conroe , /* 06_0F */
_Core_Penryn , /* 06_17 */
_Atom_Bonnell , /* 06_1C */
_Atom_Silvermont, /* 06_26 */
_Atom_Lincroft , /* 06_27 */
_Atom_Clover_Trail, /* 06_35 */
_Atom_Saltwell , /* 06_36 */
_Atom_Airmont , /* 06_4C */
_Tigerlake_U , /* 06_8C */
_Alderlake_S /* 06_97 */
};
const int ids = sizeof(allowList) / sizeof(allowList[0]);
int id;
for (id = 0; id < ids; id++)
{
if((allowList[id].ExtFamily == PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily)
&& (allowList[id].Family == PUBLIC(RO(Proc))->Features.Std.EAX.Family)
&& (allowList[id].ExtModel == PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel)
&& (allowList[id].Model == PUBLIC(RO(Proc))->Features.Std.EAX.Model))
{
if (MiscFeatures.TCC)
{
THERM2_CONTROL Therm2Control = {.value = 0};
RDMSR(Therm2Control, MSR_THERM2_CTL);
if (Therm2Control.TM_SELECT)
{
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TM1, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->TM2, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TM2, Core->Bind);
}
}
break;
}
}
}
static void ThermalMonitor_IA32(CORE_RO *Core)
{
MISC_PROC_FEATURES MiscFeatures = {.value = 0};
THERM_STATUS ThermStatus = {.value = 0};
/* Query the TM1 and TM2 features state. */
RDMSR(MiscFeatures, MSR_IA32_MISC_ENABLE);
if (MiscFeatures.TCC) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->TM1, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TM1, Core->Bind);
}
if (MiscFeatures.TM2_Enable) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->TM2, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TM2, Core->Bind);
}
ThermalMonitor2_Set(Core, MiscFeatures);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->TM_Mask, Core->Bind);
if (PUBLIC(RO(Proc))->Features.Std.EDX.ACPI)
{ /* Clear Thermal Events if requested by User. */
THERM_INTERRUPT ThermInterrupt = {.value = 0};
unsigned short TjMax, ClearBit = 0;
COMPUTE_THERMAL(INTEL, TjMax, Core->PowerThermal.Param, 0);
RDMSR(ThermStatus, MSR_IA32_THERM_STATUS);
if (Clear_Events & EVENT_THERMAL_LOG) {
ThermStatus.Thermal_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_PROCHOT_LOG) {
ThermStatus.PROCHOT_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_CRITIC_LOG) {
ThermStatus.CriticalTemp_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_THOLD1_LOG) {
ThermStatus.Threshold1_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_THOLD2_LOG) {
ThermStatus.Threshold2_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_POWER_LIMIT) {
ThermStatus.PwrLimit_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_CURRENT_LIMIT) {
ThermStatus.CurLimit_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_CROSS_DOMAIN) {
ThermStatus.XDomLimit_Log = 0;
ClearBit = 1;
}
RDMSR(ThermInterrupt, MSR_IA32_THERM_INTERRUPT);
if (ThermalPoint_Count < THM_POINTS_DIM)
{
unsigned short WrRdMSR = 0;
if ((ThermalPoint_Count > THM_THRESHOLD_1)
&& (ThermalPoint[THM_THRESHOLD_1] <= TjMax))
{
if (ThermalPoint[THM_THRESHOLD_1] > 0)
{
COMPUTE_THERMAL(INVERSE_INTEL,
ThermInterrupt.Threshold1_Value,
Core->PowerThermal.Param,
ThermalPoint[THM_THRESHOLD_1]);
ThermInterrupt.Threshold1_Int = 1;
WrRdMSR = 1;
}
else if (ThermalPoint[THM_THRESHOLD_1] == 0)
{
ThermInterrupt.Threshold1_Value = 0;
ThermInterrupt.Threshold1_Int = 0;
WrRdMSR = 1;
}
}
if ((ThermalPoint_Count > THM_THRESHOLD_2)
&& (ThermalPoint[THM_THRESHOLD_2] <= TjMax))
{
if (ThermalPoint[THM_THRESHOLD_2] > 0)
{
COMPUTE_THERMAL(INVERSE_INTEL,
ThermInterrupt.Threshold2_Value,
Core->PowerThermal.Param,
ThermalPoint[THM_THRESHOLD_2]);
ThermInterrupt.Threshold2_Int = 1;
WrRdMSR = 1;
}
else if (ThermalPoint[THM_THRESHOLD_2] == 0)
{
ThermInterrupt.Threshold2_Value = 0;
ThermInterrupt.Threshold2_Int = 0;
WrRdMSR = 1;
}
}
if (WrRdMSR == 1) {
WRMSR(ThermInterrupt, MSR_IA32_THERM_INTERRUPT);
RDMSR(ThermInterrupt, MSR_IA32_THERM_INTERRUPT);
}
}
COMPUTE_THERMAL(INTEL,
Core->ThermalPoint.Value[THM_THRESHOLD_1],
Core->PowerThermal.Param,
ThermInterrupt.Threshold1_Value);
COMPUTE_THERMAL(INTEL,
Core->ThermalPoint.Value[THM_THRESHOLD_2],
Core->PowerThermal.Param,
ThermInterrupt.Threshold2_Value);
if (ThermInterrupt.Threshold1_Int) {
BITSET(LOCKLESS, Core->ThermalPoint.State, THM_THRESHOLD_1);
} else {
BITCLR(LOCKLESS, Core->ThermalPoint.State, THM_THRESHOLD_1);
}
BITSET(LOCKLESS, Core->ThermalPoint.Mask, THM_THRESHOLD_1);
BITCLR(LOCKLESS, Core->ThermalPoint.Kind, THM_THRESHOLD_1);
if (ThermInterrupt.Threshold2_Int) {
BITSET(LOCKLESS, Core->ThermalPoint.State, THM_THRESHOLD_2);
} else {
BITCLR(LOCKLESS, Core->ThermalPoint.State, THM_THRESHOLD_2);
}
BITSET(LOCKLESS, Core->ThermalPoint.Mask, THM_THRESHOLD_2);
BITCLR(LOCKLESS, Core->ThermalPoint.Kind, THM_THRESHOLD_2);
if (ClearBit)
{
if (!((ThermInterrupt.High_Temp_Int|ThermInterrupt.Low_Temp_Int)
&& PUBLIC(RO(Proc))->Features.Power.EAX.HWFB_Cap))
{
WRMSR(ThermStatus, MSR_IA32_THERM_STATUS);
RDMSR(ThermStatus, MSR_IA32_THERM_STATUS);
}
}
Core->PowerThermal.Events[eLOG] = \
((Bit64) ThermStatus.Thermal_Log << LSHIFT_THERMAL_LOG)
| ((Bit64) ThermStatus.PROCHOT_Log << LSHIFT_PROCHOT_LOG)
| ((Bit64) ThermStatus.CriticalTemp_Log << LSHIFT_CRITIC_LOG)
| ((Bit64) ThermStatus.Threshold1_Log << LSHIFT_THOLD1_LOG)
| ((Bit64) ThermStatus.Threshold2_Log << LSHIFT_THOLD2_LOG)
| ((Bit64) ThermStatus.PwrLimit_Log << LSHIFT_POWER_LIMIT)
| ((Bit64) ThermStatus.CurLimit_Log << LSHIFT_CURRENT_LIMIT)
| ((Bit64) ThermStatus.XDomLimit_Log << LSHIFT_CROSS_DOMAIN);
Core->PowerThermal.Events[eSTS] = \
((Bit64) ThermStatus.Thermal_Status << LSHIFT_THERMAL_STS)
| ((Bit64) ThermStatus.PROCHOT_Event << LSHIFT_PROCHOT_STS)
| ((Bit64) ThermStatus.CriticalTemp << LSHIFT_CRITIC_TMP)
| ((Bit64) ThermStatus.Threshold1 << LSHIFT_THOLD1_STS)
| ((Bit64) ThermStatus.Threshold2 << LSHIFT_THOLD2_STS);
if (PUBLIC(RO(Proc))->Features.Power.EAX.PTM
&& (Core->Bind == PUBLIC(RO(Proc))->Service.Core))
{
ClearBit = 0;
ThermStatus.value = 0;
RDMSR(ThermStatus, MSR_IA32_PACKAGE_THERM_STATUS);
if (Clear_Events & EVENT_THERMAL_LOG) {
ThermStatus.Thermal_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_PROCHOT_LOG) {
ThermStatus.PROCHOT_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_CRITIC_LOG) {
ThermStatus.CriticalTemp_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_THOLD1_LOG) {
ThermStatus.Threshold1_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_THOLD2_LOG) {
ThermStatus.Threshold2_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_POWER_LIMIT) {
ThermStatus.PwrLimit_Log = 0;
ClearBit = 1;
}
RDMSR(ThermInterrupt, MSR_IA32_PACKAGE_THERM_INTERRUPT);
if (PkgThermalPoint_Count < THM_POINTS_DIM)
{
unsigned short WrRdMSR = 0;
if ((PkgThermalPoint_Count > THM_THRESHOLD_1)
&& (PkgThermalPoint[THM_THRESHOLD_1] <= TjMax))
{
if (PkgThermalPoint[THM_THRESHOLD_1] > 0)
{
COMPUTE_THERMAL(INVERSE_INTEL,
ThermInterrupt.Threshold1_Value,
Core->PowerThermal.Param,
PkgThermalPoint[THM_THRESHOLD_1]);
ThermInterrupt.Threshold1_Int = 1;
WrRdMSR = 1;
}
else if (PkgThermalPoint[THM_THRESHOLD_1] == 0)
{
ThermInterrupt.Threshold1_Value = 0;
ThermInterrupt.Threshold1_Int = 0;
WrRdMSR = 1;
}
}
if ((PkgThermalPoint_Count > THM_THRESHOLD_2)
&& (PkgThermalPoint[THM_THRESHOLD_2] <= TjMax))
{
if (PkgThermalPoint[THM_THRESHOLD_2] > 0)
{
COMPUTE_THERMAL(INVERSE_INTEL,
ThermInterrupt.Threshold2_Value,
Core->PowerThermal.Param,
PkgThermalPoint[THM_THRESHOLD_2]);
ThermInterrupt.Threshold2_Int = 1;
WrRdMSR = 1;
}
else if (PkgThermalPoint[THM_THRESHOLD_2] == 0)
{
ThermInterrupt.Threshold2_Value = 0;
ThermInterrupt.Threshold2_Int = 0;
WrRdMSR = 1;
}
}
if (WrRdMSR == 1) {
WRMSR(ThermInterrupt, MSR_IA32_PACKAGE_THERM_INTERRUPT);
RDMSR(ThermInterrupt, MSR_IA32_PACKAGE_THERM_INTERRUPT);
}
}
COMPUTE_THERMAL(INTEL,
PUBLIC(RO(Proc))->ThermalPoint.Value[THM_THRESHOLD_1],
Core->PowerThermal.Param,
ThermInterrupt.Threshold1_Value);
COMPUTE_THERMAL(INTEL,
PUBLIC(RO(Proc))->ThermalPoint.Value[THM_THRESHOLD_2],
Core->PowerThermal.Param,
ThermInterrupt.Threshold2_Value);
if (ThermInterrupt.Threshold1_Int) {
BITSET(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.State,
THM_THRESHOLD_1);
} else {
BITCLR(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.State,
THM_THRESHOLD_1);
}
BITSET(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.Mask, THM_THRESHOLD_1);
BITCLR(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.Kind, THM_THRESHOLD_1);
if (ThermInterrupt.Threshold2_Int) {
BITSET(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.State,
THM_THRESHOLD_2);
} else {
BITCLR(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.State,
THM_THRESHOLD_2);
}
BITSET(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.Mask, THM_THRESHOLD_2);
BITCLR(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.Kind, THM_THRESHOLD_2);
if (ClearBit)
{
if (!((ThermInterrupt.High_Temp_Int|ThermInterrupt.Low_Temp_Int)
&& PUBLIC(RO(Proc))->Features.Power.EAX.HWFB_Cap))
{
WRMSR(ThermStatus, MSR_IA32_PACKAGE_THERM_STATUS);
RDMSR(ThermStatus, MSR_IA32_PACKAGE_THERM_STATUS);
}
}
PUBLIC(RO(Proc))->PowerThermal.Events[eLOG] = \
((Bit64) ThermStatus.Thermal_Log << LSHIFT_THERMAL_LOG)
| ((Bit64) ThermStatus.PROCHOT_Log << LSHIFT_PROCHOT_LOG)
| ((Bit64) ThermStatus.CriticalTemp_Log << LSHIFT_CRITIC_LOG)
| ((Bit64) ThermStatus.Threshold1_Log << LSHIFT_THOLD1_LOG)
| ((Bit64) ThermStatus.Threshold2_Log << LSHIFT_THOLD2_LOG)
| ((Bit64) ThermStatus.PwrLimit_Log << LSHIFT_POWER_LIMIT);
PUBLIC(RO(Proc))->PowerThermal.Events[eSTS] = \
((Bit64) ThermStatus.Thermal_Status << LSHIFT_THERMAL_STS)
| ((Bit64) ThermStatus.PROCHOT_Event << LSHIFT_PROCHOT_STS)
| ((Bit64) ThermStatus.CriticalTemp << LSHIFT_CRITIC_TMP)
| ((Bit64) ThermStatus.Threshold1 << LSHIFT_THOLD1_STS)
| ((Bit64) ThermStatus.Threshold2 << LSHIFT_THOLD2_STS);
}
}
}
static void ThermalMonitor_Atom_Bonnell(CORE_RO *Core)
{
Core->PowerThermal.Param.Offset[THERMAL_TARGET] = 100;
Core->PowerThermal.Param.Offset[THERMAL_OFFSET_P1] = 0;
ThermalMonitor_IA32(Core);
}
static void ThermalMonitor_Set(CORE_RO *Core)
{
TJMAX TjMax = {.value = 0};
PLATFORM_INFO PfInfo = {.value = 0};
/* Silvermont + Xeon[06_57] + Nehalem + Sandy Bridge & superior arch. */
RDMSR(TjMax, MSR_IA32_TEMPERATURE_TARGET);
Core->PowerThermal.Param.Offset[THERMAL_TARGET] = TjMax.Target;
if (Core->PowerThermal.Param.Offset[THERMAL_TARGET] == 0)
{
Core->PowerThermal.Param.Offset[THERMAL_TARGET] = 100;
}
RDMSR(PfInfo, MSR_PLATFORM_INFO);
if (PfInfo.ProgrammableTj)
{
switch (PUBLIC(RO(Proc))->ArchID) {
case Atom_Goldmont:
case Xeon_Phi: /* case 06_85h: */
Core->PowerThermal.Param.Offset[THERMAL_OFFSET_P1] = TjMax.Atom.Offset;
break;
case IvyBridge_EP:
case Broadwell_EP:
case Broadwell_D:
case Skylake_X:
Core->PowerThermal.Param.Offset[THERMAL_OFFSET_P1] = TjMax.EP.Offset;
break;
default:
Core->PowerThermal.Param.Offset[THERMAL_OFFSET_P1] = TjMax.Offset;
break;
case Core_Yonah ... Core_Dunnington:
break;
}
}
ThermalMonitor_IA32(Core);
}
static void CorePerfLimitReasons(CORE_RO *Core)
{
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
CORE_PERF_LIMIT_REASONS limit = {.value = 0};
unsigned short ClearBit = 0;
RDMSR(limit, MSR_SKL_CORE_PERF_LIMIT_REASONS);
if (Clear_Events & EVENT_CORE_HOT_LOG) {
limit.PROCHOT_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_CORE_THM_LOG) {
limit.Thermal_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_CORE_RES_LOG) {
limit.Residency_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_CORE_AVG_LOG) {
limit.AvgThmLimitLog = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_CORE_VRT_LOG) {
limit.VR_ThmAlertLog = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_CORE_TDC_LOG) {
limit.VR_TDC_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_CORE_PL1_LOG) {
limit.PL1_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_CORE_PL2_LOG) {
limit.PL2_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_CORE_EDP_LOG) {
limit.EDP_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_CORE_BST_LOG) {
limit.TurboLimitLog = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_CORE_ATT_LOG) {
limit.TurboAttenLog = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_CORE_TVB_LOG) {
limit.TVB_Log = 0;
ClearBit = 1;
}
if (ClearBit)
{
WRMSR(limit, MSR_SKL_CORE_PERF_LIMIT_REASONS);
RDMSR(limit, MSR_SKL_CORE_PERF_LIMIT_REASONS);
}
PUBLIC(RO(Proc))->PowerThermal.Events[eLOG] |= (
((Bit64) limit.PROCHOT_Log << LSHIFT_CORE_HOT_LOG)
| ((Bit64) limit.Thermal_Log << LSHIFT_CORE_THM_LOG)
| ((Bit64) limit.Residency_Log << LSHIFT_CORE_RES_LOG)
| ((Bit64) limit.AvgThmLimitLog << LSHIFT_CORE_AVG_LOG)
| ((Bit64) limit.VR_ThmAlertLog << LSHIFT_CORE_VRT_LOG)
| ((Bit64) limit.VR_TDC_Log << LSHIFT_CORE_TDC_LOG)
| ((Bit64) limit.PL1_Log << LSHIFT_CORE_PL1_LOG)
| ((Bit64) limit.PL2_Log << LSHIFT_CORE_PL2_LOG)
| ((Bit64) limit.EDP_Log << LSHIFT_CORE_EDP_LOG)
| ((Bit64) limit.TurboLimitLog << LSHIFT_CORE_BST_LOG)
| ((Bit64) limit.TurboAttenLog << LSHIFT_CORE_ATT_LOG)
| ((Bit64) limit.TVB_Log << LSHIFT_CORE_TVB_LOG)
);
PUBLIC(RO(Proc))->PowerThermal.Events[eSTS] |= (
((Bit64) limit.Thermal_Status << LSHIFT_CORE_THM_STS)
| ((Bit64) limit.PROCHOT_Event << LSHIFT_CORE_HOT_STS)
| ((Bit64) limit.Residency_Sts << LSHIFT_CORE_RES_STS)
| ((Bit64) limit.AvgThmLimit << LSHIFT_CORE_AVG_STS)
| ((Bit64) limit.VR_ThmAlert << LSHIFT_CORE_VRT_STS)
| ((Bit64) limit.VR_TDC_Status << LSHIFT_CORE_TDC_STS)
| ((Bit64) limit.PL1_Status << LSHIFT_CORE_PL1_STS)
| ((Bit64) limit.PL2_Status << LSHIFT_CORE_PL2_STS)
| ((Bit64) limit.EDP_Status << LSHIFT_CORE_EDP_STS)
| ((Bit64) limit.TurboLimit << LSHIFT_CORE_BST_STS)
| ((Bit64) limit.TurboAtten << LSHIFT_CORE_ATT_STS)
| ((Bit64) limit.TVB_Status << LSHIFT_CORE_TVB_STS)
);
}
}
static void GraphicsPerfLimitReasons(CORE_RO *Core)
{
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
GRAPHICS_PERF_LIMIT_REASONS limit = {.value = 0};
unsigned short ClearBit = 0;
RDMSR(limit, MSR_GRAPHICS_PERF_LIMIT_REASONS);
if (Clear_Events & EVENT_GFX_HOT_LOG) {
limit.PROCHOT_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_GFX_THM_LOG) {
limit.Thermal_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_GFX_AVG_LOG) {
limit.AvgThmLimitLog = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_GFX_VRT_LOG) {
limit.VR_ThmAlertLog = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_GFX_TDC_LOG) {
limit.VR_TDC_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_GFX_PL1_LOG) {
limit.PL1_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_GFX_PL2_LOG) {
limit.PL2_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_GFX_EDP_LOG) {
limit.EDP_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_GFX_EFF_LOG) {
limit.InefficiencyLog = 0;
ClearBit = 1;
}
if (ClearBit)
{
WRMSR(limit, MSR_GRAPHICS_PERF_LIMIT_REASONS);
RDMSR(limit, MSR_GRAPHICS_PERF_LIMIT_REASONS);
}
PUBLIC(RO(Proc))->PowerThermal.Events[eLOG] |= (
((Bit64) limit.PROCHOT_Log << LSHIFT_GFX_HOT_LOG)
| ((Bit64) limit.Thermal_Log << LSHIFT_GFX_THM_LOG)
| ((Bit64) limit.AvgThmLimitLog << LSHIFT_GFX_AVG_LOG)
| ((Bit64) limit.VR_ThmAlertLog << LSHIFT_GFX_VRT_LOG)
| ((Bit64) limit.VR_TDC_Log << LSHIFT_GFX_TDC_LOG)
| ((Bit64) limit.PL1_Log << LSHIFT_GFX_PL1_LOG)
| ((Bit64) limit.PL2_Log << LSHIFT_GFX_PL2_LOG)
| ((Bit64) limit.EDP_Log << LSHIFT_GFX_EDP_LOG)
| ((Bit64) limit.InefficiencyLog<< LSHIFT_GFX_EFF_LOG)
);
PUBLIC(RO(Proc))->PowerThermal.Events[eSTS] |= (
((Bit64) limit.Thermal_Status << LSHIFT_GFX_THM_STS)
| ((Bit64) limit.PROCHOT_Event << LSHIFT_GFX_HOT_STS)
| ((Bit64) limit.AvgThmLimit << LSHIFT_GFX_AVG_STS)
| ((Bit64) limit.VR_ThmAlert << LSHIFT_GFX_VRT_STS)
| ((Bit64) limit.VR_TDC_Status << LSHIFT_GFX_TDC_STS)
| ((Bit64) limit.PL1_Status << LSHIFT_GFX_PL1_STS)
| ((Bit64) limit.PL2_Status << LSHIFT_GFX_PL2_STS)
| ((Bit64) limit.EDP_Status << LSHIFT_GFX_EDP_STS)
| ((Bit64) limit.Inefficiency << LSHIFT_GFX_EFF_STS)
);
}
}
static void RingPerfLimitReasons(CORE_RO *Core)
{
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
RING_PERF_LIMIT_REASONS limit = {.value = 0};
unsigned short ClearBit = 0;
RDMSR(limit, MSR_RING_PERF_LIMIT_REASONS);
if (Clear_Events & EVENT_RING_HOT_LOG) {
limit.PROCHOT_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_RING_THM_LOG) {
limit.Thermal_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_RING_AVG_LOG) {
limit.AvgThmLimitLog = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_RING_VRT_LOG) {
limit.VR_ThmAlertLog = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_RING_TDC_LOG) {
limit.VR_TDC_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_RING_PL1_LOG) {
limit.PL1_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_RING_PL2_LOG) {
limit.PL2_Log = 0;
ClearBit = 1;
}
if (Clear_Events & EVENT_RING_EDP_LOG) {
limit.EDP_Log = 0;
ClearBit = 1;
}
if (ClearBit)
{
WRMSR(limit, MSR_RING_PERF_LIMIT_REASONS);
RDMSR(limit, MSR_RING_PERF_LIMIT_REASONS);
}
PUBLIC(RO(Proc))->PowerThermal.Events[eLOG] |= (
((Bit64) limit.PROCHOT_Log << LSHIFT_RING_HOT_LOG)
| ((Bit64) limit.Thermal_Log << LSHIFT_RING_THM_LOG)
| ((Bit64) limit.AvgThmLimitLog << LSHIFT_RING_AVG_LOG)
| ((Bit64) limit.VR_ThmAlertLog << LSHIFT_RING_VRT_LOG)
| ((Bit64) limit.VR_TDC_Log << LSHIFT_RING_TDC_LOG)
| ((Bit64) limit.PL1_Log << LSHIFT_RING_PL1_LOG)
| ((Bit64) limit.PL2_Log << LSHIFT_RING_PL2_LOG)
| ((Bit64) limit.EDP_Log << LSHIFT_RING_EDP_LOG)
);
PUBLIC(RO(Proc))->PowerThermal.Events[eSTS] |= (
((Bit64) limit.Thermal_Status << LSHIFT_RING_THM_STS)
| ((Bit64) limit.PROCHOT_Event << LSHIFT_RING_HOT_STS)
| ((Bit64) limit.AvgThmLimit << LSHIFT_RING_AVG_STS)
| ((Bit64) limit.VR_ThmAlert << LSHIFT_RING_VRT_STS)
| ((Bit64) limit.VR_TDC_Status << LSHIFT_RING_TDC_STS)
| ((Bit64) limit.PL1_Status << LSHIFT_RING_PL1_STS)
| ((Bit64) limit.PL2_Status << LSHIFT_RING_PL2_STS)
| ((Bit64) limit.EDP_Status << LSHIFT_RING_EDP_STS)
);
}
}
static void PowerThermal(CORE_RO *Core)
{
static struct {
struct SIGNATURE Arch;
unsigned short grantPWR_MGMT : 1-0,
grantODCM : 2-1,
experimental : 3-2,
freeToUse : 16-3;
} allowList[] = {
{_Core_Yonah, 0, 1, 1, 0},
{_Core_Conroe, 0, 1, 0, 0},
{_Core_Kentsfield, 0, 1, 1, 0},
{_Core_Conroe_616, 0, 1, 1, 0},
{_Core_Penryn, 0, 1, 0, 0},
{_Core_Dunnington, 0, 1, 1, 0},
{_Atom_Bonnell, 0, 1, 1, 0}, /* 06_1C */
{_Atom_Silvermont, 0, 1, 1, 0}, /* 06_26 */
{_Atom_Lincroft, 0, 1, 1, 0}, /* 06_27 */
{_Atom_Clover_Trail, 0, 1, 1, 0}, /* 06_35 */
{_Atom_Saltwell, 0, 1, 1, 0}, /* 06_36 */
{_Silvermont_Bay_Trail, 0, 1, 0, 0}, /* 06_37 */
{_Atom_Avoton, 0, 1, 1, 0}, /* 06_4D */
{_Atom_Airmont, 0, 1, 0, 0}, /* 06_4C */
{_Atom_Goldmont, 1, 0, 1, 0}, /* 06_5C */
{_Atom_Sofia, 0, 1, 1, 0}, /* 06_5D */
{_Atom_Merrifield, 0, 1, 1, 0}, /* 06_4A */
{_Atom_Moorefield, 0, 1, 1, 0}, /* 06_5A */
{_Nehalem_Bloomfield, 1, 1, 0, 0}, /* 06_1A */
{_Nehalem_Lynnfield, 1, 1, 0, 0}, /* 06_1E */
{_Nehalem_MB, 1, 1, 0, 0}, /* 06_1F */
{_Nehalem_EX, 1, 1, 0, 0}, /* 06_2E */
{_Westmere, 1, 1, 0, 0}, /* 06_25 */
{_Westmere_EP, 1, 1, 0, 0}, /* 06_2C */
{_Westmere_EX, 1, 1, 0, 0}, /* 06_2F */
{_SandyBridge, 1, 1, 0, 0}, /* 06_2A */
{_SandyBridge_EP, 1, 1, 0, 0}, /* 06_2D */
{_IvyBridge, 1, 1, 0, 0}, /* 06_3A */
{_IvyBridge_EP, 1, 1, 0, 0}, /* 06_3E */
{_Haswell_DT, 1, 1, 0, 0}, /* 06_3C */
{_Haswell_EP, 1, 1, 1, 0}, /* 06_3F */
{_Haswell_ULT, 1, 1, 0, 0}, /* 06_45 */
{_Haswell_ULX, 1, 1, 1, 0}, /* 06_46 */
{_Broadwell, 1, 1, 1, 0}, /* 06_3D */
{_Broadwell_D, 1, 1, 1, 0}, /* 06_56 */
{_Broadwell_H, 1, 1, 1, 0}, /* 06_47 */
{_Broadwell_EP, 1, 1, 1, 0}, /* 06_4F */
{_Skylake_UY, 1, 1, 1, 0}, /* 06_4E */
{_Skylake_S, 1, 0, 0, 0}, /* 06_5E */
{_Skylake_X, 1, 1, 1, 0}, /* 06_55 */
{_Xeon_Phi, 0, 1, 1, 0}, /* 06_57 */
{_Kabylake, 1, 1, 0, 0}, /* 06_9E */
{_Kabylake_UY, 1, 1, 1, 0}, /* 06_8E */
{_Cannonlake_U, 1, 1, 1, 0}, /* 06_66 */
{_Cannonlake_H, 1, 1, 1, 0},
{_Geminilake, 1, 0, 1, 0}, /* 06_7A */
{_Icelake_UY, 1, 1, 1, 0}, /* 06_7E */
{_Icelake_X, 1, 1, 1, 0},
{_Icelake_D, 1, 1, 1, 0},
{_Sunny_Cove, 1, 1, 1, 0},
{_Tigerlake, 1, 1, 1, 0},
{_Tigerlake_U, 1, 1, 0, 0}, /* 06_8C */
{_Cometlake, 1, 1, 1, 0},
{_Cometlake_UY, 1, 1, 1, 0},
{_Atom_Denverton, 1, 1, 1, 0},
{_Tremont_Jacobsville, 1, 1, 1, 0},
{_Tremont_Lakefield, 1, 1, 1, 0},
{_Tremont_Elkhartlake, 1, 1, 1, 0},
{_Tremont_Jasperlake, 1, 1, 1, 0},
{_Sapphire_Rapids, 1, 1, 1, 0},
{_Emerald_Rapids, 1, 1, 1, 0},
{_Granite_Rapids_X, 1, 1, 1, 0},
{_Granite_Rapids_D, 1, 1, 1, 0},
{_Sierra_Forest, 1, 1, 1, 0},
{_Grand_Ridge, 1, 1, 1, 0},
{_Rocketlake, 1, 1, 1, 0},
{_Rocketlake_U, 1, 1, 1, 0},
{_Alderlake_S, 1, 1, 0, 0}, /* 06_97 */
{_Alderlake_H, 1, 1, 0, 0},
{_Alderlake_N, 1, 1, 0, 0},
{_Meteorlake_M, 1, 1, 0, 0},
{_Meteorlake_N, 1, 1, 0, 0},
{_Meteorlake_S, 1, 1, 0, 0},
{_Raptorlake, 1, 1, 0, 0}, /* 06_B7 */
{_Raptorlake_P, 1, 1, 0, 0},
{_Raptorlake_S, 1, 1, 0, 0},
{_Lunarlake, 1, 1, 0, 0}, /* 06_BD */
{_Arrowlake, 1, 1, 0, 0}, /* 06_C6 */
{_Arrowlake_H, 1, 1, 0, 0}, /* 06_C5 */
{_Arrowlake_U, 1, 1, 0, 0}, /* 06_B5 */
{_Pantherlake, 1, 1, 1, 0}, /* 06_CC */
{_Clearwater_Forest, 1, 1, 1, 0}, /* 06_DD */
{_Bartlettlake_S, 1, 1, 1, 0} /* 06_D7 */
};
const unsigned int ids = sizeof(allowList) / sizeof(allowList[0]);
unsigned int id;
for (id = 0; id < ids; id++)
{
if((allowList[id].Arch.ExtFamily==PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily)
&& (allowList[id].Arch.Family == PUBLIC(RO(Proc))->Features.Std.EAX.Family)
&& (allowList[id].Arch.ExtModel==PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel)
&& (allowList[id].Arch.Model == PUBLIC(RO(Proc))->Features.Std.EAX.Model))
{
break;
}
}
if (PUBLIC(RO(Proc))->Features.Info.LargestStdFunc >= 0x6)
{
struct THERMAL_POWER_LEAF Power = {
.EAX = {0}, .EBX = {0}, .ECX = {{0}}, .EDX = {0}
};
__asm__ volatile
(
"movq $0x6, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (Power.EAX),
"=r" (Power.EBX),
"=r" (Power.ECX),
"=r" (Power.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
if (Power.ECX.SETBH == 1)
{
if ((id < ids) && (allowList[id].grantPWR_MGMT == 1))
{
RDMSR(Core->PowerThermal.PerfEnergyBias, MSR_IA32_ENERGY_PERF_BIAS);
RDMSR(Core->PowerThermal.PwrManagement, MSR_MISC_PWR_MGMT);
switch (PowerMGMT_Unlock) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
if (!allowList[id].experimental
|| (allowList[id].experimental
&& PUBLIC(RO(Proc))->Registration.Experimental))
{
Core->PowerThermal.PwrManagement.Perf_BIAS_Enable=PowerMGMT_Unlock;
WRMSR(Core->PowerThermal.PwrManagement, MSR_MISC_PWR_MGMT);
RDMSR(Core->PowerThermal.PwrManagement, MSR_MISC_PWR_MGMT);
}
break;
}
if (Core->PowerThermal.PwrManagement.Perf_BIAS_Enable)
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->PowerMgmt, Core->Bind);
else
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->PowerMgmt, Core->Bind);
} else
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->PowerMgmt, Core->Bind);
if ((PowerPolicy >= 0) && (PowerPolicy <= 15))
{
if (!allowList[id].experimental
|| (allowList[id].experimental
&& PUBLIC(RO(Proc))->Registration.Experimental))
{
Core->PowerThermal.PerfEnergyBias.PowerPolicy = PowerPolicy;
WRMSR(Core->PowerThermal.PerfEnergyBias, MSR_IA32_ENERGY_PERF_BIAS);
RDMSR(Core->PowerThermal.PerfEnergyBias, MSR_IA32_ENERGY_PERF_BIAS);
}
}
}
else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->PowerMgmt, Core->Bind);
}
if ((PUBLIC(RO(Proc))->Features.Std.EDX.ACPI == 1)
&& (id < ids) && (allowList[id].grantODCM == 1))
{
CLOCK_MODULATION ClockModulation = {.value = 0};
int ToggleFeature = 0;
RDMSR(Core->PowerThermal.ClockModulation, MSR_IA32_THERM_CONTROL);
ClockModulation = Core->PowerThermal.ClockModulation;
switch (ODCM_Enable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
ClockModulation.ODCM_Enable = ODCM_Enable;
ToggleFeature = 1;
break;
}
if ((ODCM_DutyCycle >= 0)
&& (ODCM_DutyCycle <= (7 << Power.EAX.ECMD)))
{
ClockModulation.DutyCycle = ODCM_DutyCycle << !Power.EAX.ECMD;
ToggleFeature = 1;
}
if (ToggleFeature == 1)
{
if (!allowList[id].experimental
|| (allowList[id].experimental
&& PUBLIC(RO(Proc))->Registration.Experimental))
{
WRMSR(ClockModulation, MSR_IA32_THERM_CONTROL);
RDMSR(Core->PowerThermal.ClockModulation, MSR_IA32_THERM_CONTROL);
}
}
Core->PowerThermal.ClockModulation.ECMD = Power.EAX.ECMD;
if (Core->PowerThermal.ClockModulation.ODCM_Enable)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->ODCM, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->ODCM, Core->Bind);
}
}
else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->ODCM, Core->Bind);
}
}
else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->PowerMgmt, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->ODCM, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->ODCM_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PowerMgmt_Mask, Core->Bind);
}
#define UNSPEC 0b11111111
struct CSTATES_ENCODING_ST {
enum CSTATES_ENCODING enc;
unsigned short int dec;
} Limit_CSTATES_NHM[CSTATES_ENCODING_COUNT] = {
{ _C0 , 0b0000 },
{ _C1 , 0b0001 },
{ _C2 , UNSPEC },
{ _C3 , 0b0010 }, /*Cannot be used to limit package C-State*/
{ _C4 , UNSPEC },
{ _C6 , 0b0011 },
{ _C6R , UNSPEC },
{ _C7 , 0b0100 },
{ _C7S , UNSPEC },
{ _C8 , UNSPEC },
{ _C9 , UNSPEC },
{ _C10 , UNSPEC }
}, IORedir_CSTATES_NHM[CSTATES_ENCODING_COUNT] = {
{ _C0 , UNSPEC },
{ _C1 , UNSPEC },
{ _C2 , UNSPEC },
{ _C3 , 0b0000 },
{ _C4 , UNSPEC },
{ _C6 , 0b0001 },
{ _C6R , UNSPEC },
{ _C7 , 0b0010 },
{ _C7S , UNSPEC },
{ _C8 , 0b0011 }, /* TODO(Undefined!) */
{ _C9 , UNSPEC },
{ _C10 , UNSPEC }
}, Limit_CSTATES_SNB[CSTATES_ENCODING_COUNT] = {
{ _C0 , 0b0000 },
{ _C1 , 0b0000 },
{ _C2 , 0b0001 },
{ _C3 , 0b0010 },
{ _C4 , UNSPEC },
{ _C6 , 0b0011 },
{ _C6R , UNSPEC },
{ _C7 , 0b0100 },
{ _C7S , 0b0101 },
{ _C8 , 0b0110 },
{ _C9 , 0b0111 },
{ _C10 , 0b1000 }
}, IORedir_CSTATES_SNB[CSTATES_ENCODING_COUNT] = {
{ _C0 , UNSPEC },
{ _C1 , UNSPEC },
{ _C2 , UNSPEC },
{ _C3 , 0b0000 },
{ _C4 , UNSPEC },
{ _C6 , 0b0001 },
{ _C6R , UNSPEC },
{ _C7 , 0b0010 },
{ _C7S , UNSPEC },
{ _C8 , 0b0011 }, /* TODO(Untested?) */
{ _C9 , UNSPEC },
{ _C10 , UNSPEC }
}, Limit_CSTATES_ULT[CSTATES_ENCODING_COUNT] = {
{ _C0 , 0b0000 },
{ _C1 , 0b0000 },
{ _C2 , 0b0001 },
{ _C3 , 0b0010 },
{ _C4 , UNSPEC },
{ _C6 , 0b0011 },
{ _C6R , UNSPEC },
{ _C7 , 0b0100 },
{ _C7S , UNSPEC },
{ _C8 , 0b0110 },
{ _C9 , 0b0111 },
{ _C10 , 0b1000 }
}, IORedir_CSTATES_ULT[CSTATES_ENCODING_COUNT] = {
{ _C0 , UNSPEC },
{ _C1 , UNSPEC },
{ _C2 , UNSPEC },
{ _C3 , 0b0000 },
{ _C4 , UNSPEC },
{ _C6 , 0b0001 },
{ _C6R , UNSPEC },
{ _C7 , 0b0010 },
{ _C7S , UNSPEC },
{ _C8 , 0b0011 }, /* TODO(Untested?) */
{ _C9 , UNSPEC },
{ _C10 , UNSPEC }
}, Limit_CSTATES_SKL[CSTATES_ENCODING_COUNT] = {
{ _C0 , 0b0000 },
{ _C1 , 0b0000 },
{ _C2 , 0b0001 },
{ _C3 , 0b0010 },
{ _C4 , UNSPEC },
{ _C6 , 0b0011 },
{ _C6R , UNSPEC },
{ _C7 , 0b0100 },
{ _C7S , UNSPEC },
{ _C8 , 0b0110 },
{ _C9 , UNSPEC },
{ _C10 , UNSPEC }
}, IORedir_CSTATES_SKL[CSTATES_ENCODING_COUNT] = {
{ _C0 , 0b0000 },
{ _C1 , 0b0001 },
{ _C2 , UNSPEC },
{ _C3 , 0b0010 },
{ _C4 , UNSPEC },
{ _C6 , 0b0011 },
{ _C6R , UNSPEC },
{ _C7 , 0b0100 },
{ _C7S , UNSPEC },
{ _C8 , 0b0101 },
{ _C9 , UNSPEC },
{ _C10 , UNSPEC }
}, Limit_CSTATES_SOC_SLM[CSTATES_ENCODING_COUNT] = {
{ _C0 , 0b0000 }, /* Silvermont, Airmont */
{ _C1 , 0b0001 }, /* Silvermont, Airmont */
{ _C2 , 0b0010 }, /* Airmont */
{ _C3 , UNSPEC },
{ _C4 , 0b0100 }, /* Silvermont */
{ _C6 , 0b0110 }, /* Silvermont, Airmont */
{ _C6R , UNSPEC },
{ _C7 , 0b0111 }, /* Silvermont, Airmont */
{ _C7S , UNSPEC },
{ _C8 , UNSPEC },
{ _C9 , UNSPEC },
{ _C10 , UNSPEC }
}, IORedir_CSTATES_SOC_SLM[CSTATES_ENCODING_COUNT] = {
{ _C0 , UNSPEC },
{ _C1 , UNSPEC },
{ _C2 , UNSPEC },
{ _C3 , 0b0000 }, /* Airmont */
{ _C4 , 0b0100 }, /* Silvermont: 0b0100 */
{ _C6 , 0b0001 }, /* Silvermont: 0b0110, Atom Deep Power Down */
{ _C6R , UNSPEC },
{ _C7 , 0b0010 }, /* Silvermont: 0b0111, Airmont: 0b0010 */
{ _C7S , UNSPEC },
{ _C8 , UNSPEC },
{ _C9 , UNSPEC },
{ _C10 , UNSPEC }
}, Limit_CSTATES_SOC_GDM[CSTATES_ENCODING_COUNT] = {
{ _C0 , 0b0000 },
{ _C1 , 0b0001 },
{ _C2 , UNSPEC },
{ _C3 , 0b0010 }, /* Goldmont, Tremont */
{ _C4 , UNSPEC },
{ _C6 , 0b0011 }, /* Goldmont, Tremont */
{ _C6R , UNSPEC },
{ _C7 , 0b0100 }, /* Goldmont, Tremont */
{ _C7S , 0b0101 }, /* Goldmont, Tremont */
{ _C8 , 0b0110 }, /* Goldmont, Tremont */
{ _C9 , 0b0111 }, /* Goldmont, Tremont */
{ _C10 , 0b1000 }
}, IORedir_CSTATES_SOC_GDM[CSTATES_ENCODING_COUNT] = {
{ _C0 , UNSPEC },
{ _C1 , UNSPEC },
{ _C2 , UNSPEC },
{ _C3 , 0b0000 },
{ _C4 , UNSPEC },
{ _C6 , 0b0001 },
{ _C6R , UNSPEC },
{ _C7 , 0b0010 },
{ _C7S , UNSPEC },
{ _C8 , 0b0011 }, /* TODO(Untested?) */
{ _C9 , UNSPEC },
{ _C10 , UNSPEC }
};
#define MAKE_TOGGLE_CSTATE_FUNC( _type, _feature, _parameter ) \
static unsigned int Toggle_CState_##_feature( _type *pConfigRegister, \
typeof(_parameter) _parameter) \
{ \
switch ( _parameter ) \
{ \
case COREFREQ_TOGGLE_OFF: \
case COREFREQ_TOGGLE_ON: \
pConfigRegister->_feature = _parameter; \
return 1; \
} \
return 0; \
}
MAKE_TOGGLE_CSTATE_FUNC(CSTATE_CONFIG, C3autoDemotion, C3A_Enable)
MAKE_TOGGLE_CSTATE_FUNC(CSTATE_CONFIG, C1autoDemotion, C1A_Enable)
MAKE_TOGGLE_CSTATE_FUNC(CSTATE_CONFIG, C3undemotion, C3U_Enable)
MAKE_TOGGLE_CSTATE_FUNC(CSTATE_CONFIG, C1undemotion, C1U_Enable)
MAKE_TOGGLE_CSTATE_FUNC(CSTATE_CONFIG, IO_MWAIT_Redir, IOMWAIT_Enable)
MAKE_TOGGLE_CSTATE_FUNC(CC6_CONFIG, CC6demotion, CC6_Enable)
MAKE_TOGGLE_CSTATE_FUNC(MC6_CONFIG, MC6demotion, PC6_Enable)
#undef MAKE_TOGGLE_CSTATE
#define Toggle_CState_Feature( _config, _feature, _parameter ) \
( \
Toggle_CState_##_feature( _config, _parameter ) \
)
#define For_All_Encodings( loopCondition, breakStatement, \
bodyStatement, closure ) \
({ \
unsigned int idx, ret = 0; \
for (idx = 0; idx < CSTATES_ENCODING_COUNT && (loopCondition); idx++)\
{ \
if (breakStatement) \
{ \
ret = 1; \
bodyStatement \
break; \
} \
} \
closure \
ret; \
})
static void Control_IO_MWAIT( struct CSTATES_ENCODING_ST IORedir[],
CORE_RO *Core )
{
CSTATE_IO_MWAIT CState_IO_MWAIT = {.value = 0};
RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
/* SMT C-State Base Address. */
Core->Query.CStateBaseAddr = CState_IO_MWAIT.LVL2_BaseAddr;
if (Core->Query.IORedir)
{
For_All_Encodings(
/* loopCondition: */
(CStateIORedir >= 0),
/* breakStatement: */
( (IORedir[idx].dec != UNSPEC)
&& (CStateIORedir == IORedir[idx].enc) ),
/* bodyStatement: */
{
CState_IO_MWAIT.CStateRange = IORedir[idx].dec;
WRMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
},
/* closure: */
{}
);
}
For_All_Encodings(
/* loopCondition: */ (1),
/* breakStatement: */
((CState_IO_MWAIT.CStateRange & 0b111) == IORedir[idx].dec),
/* bodyStatement: */
{
Core->Query.CStateInclude = IORedir[idx].enc;
},
/* closure: */
if (!ret) {
Core->Query.CStateInclude = _UNSPEC;
}
);
}
static void Control_CSTATES_NHM(struct CSTATES_ENCODING_ST Limit[],
struct CSTATES_ENCODING_ST IORedir[],
CORE_RO *Core)
{ /* Family: 06_1A, 06_1E, 06_1F, 06_25, 06_2C, 06_2E */
CSTATE_CONFIG CStateConfig = {.value = 0};
unsigned int toggleFeature = 0;
RDMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);
toggleFeature |= Toggle_CState_Feature( &CStateConfig,
C3autoDemotion,
C3A_Enable );
toggleFeature |= Toggle_CState_Feature( &CStateConfig,
C1autoDemotion,
C1A_Enable );
if (CStateConfig.CFG_Lock == 0)
{
toggleFeature |= Toggle_CState_Feature( &CStateConfig,
IO_MWAIT_Redir,
IOMWAIT_Enable);
toggleFeature |= For_All_Encodings(
/* loopCondition: */
(PkgCStateLimit >= 0),
/* breakStatement: */
( (Limit[idx].dec != UNSPEC)
&& (PkgCStateLimit == Limit[idx].enc) ),
/* bodyStatement: */
{
CStateConfig.Pkg_CStateLimit = Limit[idx].dec
| (0b1000 & CStateConfig.Pkg_CStateLimit);
},
/* closure: */
{}
);
}
if (toggleFeature == 1) {
WRMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);
RDMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);
}
if (CStateConfig.C3autoDemotion)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->C3A, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->C3A, Core->Bind);
}
if (CStateConfig.C1autoDemotion)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->C1A, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->C1A, Core->Bind);
}
Core->Query.CfgLock = CStateConfig.CFG_Lock;
Core->Query.IORedir = CStateConfig.IO_MWAIT_Redir;
For_All_Encodings(
/* loopCondition: */ (1),
/* breakStatement: */
((CStateConfig.Pkg_CStateLimit & 0b0111) == Limit[idx].dec),
/* bodyStatement: */
{
Core->Query.CStateLimit = Limit[idx].enc;
},
/* closure: */
if (!ret) {
Core->Query.CStateLimit = _UNSPEC;
}
);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3A_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1A_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3U_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1U_Mask, Core->Bind);
Control_IO_MWAIT(IORedir, Core);
}
static void Control_CSTATES_COMMON( struct CSTATES_ENCODING_ST Limit[],
struct CSTATES_ENCODING_ST IORedir[],
CORE_RO *Core,
const unsigned short bitMask,
const unsigned short unMask )
{
CSTATE_CONFIG CStateConfig = {.value = 0};
unsigned int toggleFeature = 0;
RDMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);
toggleFeature |= Toggle_CState_Feature( &CStateConfig,
C3autoDemotion,
C3A_Enable );
toggleFeature |= Toggle_CState_Feature( &CStateConfig,
C1autoDemotion,
C1A_Enable );
toggleFeature |= Toggle_CState_Feature( &CStateConfig,
C3undemotion,
C3U_Enable );
toggleFeature |= Toggle_CState_Feature( &CStateConfig,
C1undemotion,
C1U_Enable );
if (CStateConfig.CFG_Lock == 0)
{
toggleFeature |= Toggle_CState_Feature( &CStateConfig,
IO_MWAIT_Redir,
IOMWAIT_Enable);
toggleFeature |= For_All_Encodings(
/* loopCondition: */
(PkgCStateLimit >= 0),
/* breakStatement: */
( (Limit[idx].dec != UNSPEC)
&& (PkgCStateLimit == Limit[idx].enc) ),
/* bodyStatement: */
{
CStateConfig.Pkg_CStateLimit = Limit[idx].dec
| (unMask & CStateConfig.Pkg_CStateLimit);
},
/* closure: */
{}
);
}
if (toggleFeature == 1) {
WRMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);
RDMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);
}
if (CStateConfig.C3autoDemotion)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->C3A, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->C3A, Core->Bind);
}
if (CStateConfig.C1autoDemotion)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->C1A, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->C1A, Core->Bind);
}
if (CStateConfig.C3undemotion)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->C3U, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->C3U, Core->Bind);
}
if (CStateConfig.C1undemotion)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->C1U, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->C1U, Core->Bind);
}
Core->Query.CfgLock = CStateConfig.CFG_Lock;
Core->Query.IORedir = CStateConfig.IO_MWAIT_Redir;
For_All_Encodings(
/* loopCondition: */ (1),
/* breakStatement: */
((CStateConfig.Pkg_CStateLimit & bitMask) == Limit[idx].dec),
/* bodyStatement: */
{
Core->Query.CStateLimit = Limit[idx].enc;
},
/* closure: */
if (!ret) {
Core->Query.CStateLimit = _UNSPEC;
}
);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3A_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1A_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3U_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1U_Mask, Core->Bind);
Control_IO_MWAIT(IORedir, Core);
}
static void Control_CSTATES_SNB(struct CSTATES_ENCODING_ST Limit[],
struct CSTATES_ENCODING_ST IORedir[],
CORE_RO *Core)
{ /* Family: 06_2A, 06_3A, 06_3E, 06_3F, 06_4F, 06_56, 06_57, 06_85 */
Control_CSTATES_COMMON(Limit, IORedir, Core, 0b0111, 0b1000);
}
static void Control_CSTATES_ULT(struct CSTATES_ENCODING_ST Limit[],
struct CSTATES_ENCODING_ST IORedir[],
CORE_RO *Core)
{ /* Family: 06_3C, 06_3D, 06_45, 06_46, 06_47, 06_86 */
Control_CSTATES_COMMON(Limit, IORedir, Core, 0b1111, 0b0000);
}
static void Control_CSTATES_SKL(struct CSTATES_ENCODING_ST Limit[],
struct CSTATES_ENCODING_ST IORedir[],
CORE_RO *Core)
{ /* Family: 06_4E, 06_5E, 06_55, 06_66, 06_7D, 06_7E, 06_8E, 06_9E */
Control_CSTATES_COMMON(Limit, IORedir, Core, 0b0111, 0b1000);
}
static void Control_CSTATES_SOC_ATOM( struct CSTATES_ENCODING_ST Limit[],
struct CSTATES_ENCODING_ST IORedir[],
CORE_RO *Core,
const unsigned short bitMask,
const unsigned short unMask )
{
CSTATE_CONFIG CStateConfig = {.value = 0};
unsigned int toggleFeature = 0;
RDMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);
if (CStateConfig.CFG_Lock == 0)
{
toggleFeature |= Toggle_CState_Feature( &CStateConfig,
IO_MWAIT_Redir,
IOMWAIT_Enable);
toggleFeature |= For_All_Encodings(
/* loopCondition: */
(PkgCStateLimit >= 0),
/* breakStatement: */
( (Limit[idx].dec != UNSPEC)
&& (PkgCStateLimit == Limit[idx].enc) ),
/* bodyStatement: */
{
CStateConfig.Pkg_CStateLimit = Limit[idx].dec
| (unMask & CStateConfig.Pkg_CStateLimit);
},
/* closure: */
{}
);
}
if (toggleFeature == 1) {
WRMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);
RDMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);
}
Core->Query.CfgLock = CStateConfig.CFG_Lock;
Core->Query.IORedir = CStateConfig.IO_MWAIT_Redir;
For_All_Encodings(
/* loopCondition: */ (1),
/* breakStatement: */
((CStateConfig.Pkg_CStateLimit & bitMask) == Limit[idx].dec),
/* bodyStatement: */
{
Core->Query.CStateLimit = Limit[idx].enc;
},
/* closure: */
if (!ret) {
Core->Query.CStateLimit = _UNSPEC;
}
);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3A_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1A_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3U_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1U_Mask, Core->Bind);
Control_IO_MWAIT(IORedir, Core);
}
static void Control_CSTATES_SOC_SLM( struct CSTATES_ENCODING_ST Limit[],
struct CSTATES_ENCODING_ST IORedir[],
CORE_RO *Core )
{ /* Family: 06_37, 06_4A, 06_4C, 06_4D, 06_5A */
CC6_CONFIG CC6_Config = {.value = 0};
MC6_CONFIG MC6_Config = {.value = 0};
Control_CSTATES_SOC_ATOM(Limit, IORedir, Core, 0b0111, 0b1000);
RDMSR(CC6_Config, MSR_CC6_DEMOTION_POLICY_CONFIG);
if (Toggle_CState_Feature(&CC6_Config, CC6demotion, CC6_Enable))
{
WRMSR(CC6_Config, MSR_CC6_DEMOTION_POLICY_CONFIG);
RDMSR(CC6_Config, MSR_CC6_DEMOTION_POLICY_CONFIG);
}
RDMSR(MC6_Config, MSR_MC6_DEMOTION_POLICY_CONFIG);
if (Toggle_CState_Feature(&MC6_Config, MC6demotion, PC6_Enable))
{
WRMSR(MC6_Config, MSR_MC6_DEMOTION_POLICY_CONFIG);
RDMSR(MC6_Config, MSR_MC6_DEMOTION_POLICY_CONFIG);
}
if (CC6_Config.CC6demotion)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->CC6, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->CC6, Core->Bind);
}
if (MC6_Config.MC6demotion)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->PC6, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->PC6, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PC6_Mask, Core->Bind);
}
static void Control_CSTATES_SOC_GDM( struct CSTATES_ENCODING_ST Limit[],
struct CSTATES_ENCODING_ST IORedir[],
CORE_RO *Core )
{ /* Family: 06_5CH */
Control_CSTATES_SOC_ATOM(Limit, IORedir, Core, 0b1111, 0b0000);
}
#undef UNSPEC
#undef For_All_Encodings
#undef Toggle_CState_Feature
#define Intel_CStatesConfiguration( _CLASS, Core ) \
({ \
Control_##_CLASS(Limit_##_CLASS, IORedir_##_CLASS, Core); \
})
static void PerCore_AMD_Family_0Fh_PStates(CORE_RO *Core)
{
FIDVID_STATUS FidVidStatus = {.value = 0};
FIDVID_CONTROL FidVidControl = {.value = 0};
int NewFID = -1, NewVID = -1, loop = 100;
UNUSED(Core);
RDMSR(FidVidStatus, MSR_K7_FID_VID_STATUS);
NewFID = ((PState_FID >= FidVidStatus.StartFID)
&& (PState_FID <= FidVidStatus.MaxFID)) ?
PState_FID : FidVidStatus.CurrFID,
NewVID = ((PState_VID <= FidVidStatus.StartVID)
&& (PState_VID >= FidVidStatus.MaxVID)) ?
PState_VID : FidVidStatus.CurrVID;
do {
if (FidVidStatus.FidVidPending == 0) {
RDMSR(FidVidControl, MSR_K7_FID_VID_CTL);
FidVidControl.InitFidVid = 1;
FidVidControl.StpGntTOCnt = 400;
FidVidControl.NewFID = NewFID;
FidVidControl.NewVID = NewVID;
WRMSR(FidVidControl, MSR_K7_FID_VID_CTL);
loop = 0;
} else {
RDMSR(FidVidStatus, MSR_K7_FID_VID_STATUS);
}
if (loop == 0) {
break;
} else {
loop-- ;
}
} while (FidVidStatus.FidVidPending == 1) ;
}
static void SystemRegisters(CORE_RO *Core)
{
unsigned long long mem64;
__asm__ volatile
(
"xorq %%rbx, %%rbx" "\n\t"
"xorq %%rcx, %%rcx" "\n\t"
"xorq %%rdx, %%rdx" "\n\t"
"# Carry, Sign, Overflow Flags" "\n\t"
"xorq %%rax, %%rax" "\n\t"
"cmpl $-0x80000000, %%eax" "\n\t"
"# Zero Flag" "\n\t"
"movq %%rax, %1" "\n\t"
"cmpxchg8b %1" "\n\t"
"# Direction Flag" "\n\t"
"std" "\n\t"
"# Interrupt Flag" "\n\t"
"sti" "\n\t"
"# Save all RFLAGS" "\n\t"
"pushfq" "\n\t"
"popq %0" "\n\t"
"# Reset Flags" "\n\t"
"cli" "\n\t"
"cld"
: "=r" (Core->SystemRegister.RFLAGS)
: "m" (mem64)
: "%rax", "%rbx", "%rcx", "%rdx", "cc"
);
if (RDPMC_Enable) {
__asm__ volatile
(
"movq %%cr4, %%rax" "\n\t"
"btsq %0, %%rax" "\n\t"
"movq %%rax, %%cr4" "\n\t"
"wbinvd"
:
: "i" (CR4_PCE)
: "%rax"
);
}
__asm__ volatile
(
"movq %%cr0, %0" "\n\t"
"movq %%cr3, %1" "\n\t"
"movq %%cr4, %2" "\n\t"
"movq %%cr8, %3" "\n\t"
"# EFER" "\n\t"
"xorq %%rax, %%rax" "\n\t"
"xorq %%rdx, %%rdx" "\n\t"
"movq %5,%%rcx" "\n\t"
"rdmsr" "\n\t"
"shlq $32, %%rdx" "\n\t"
"orq %%rdx, %%rax" "\n\t"
"movq %%rax, %4"
: "=r" (Core->SystemRegister.CR0),
"=r" (Core->SystemRegister.CR3),
"=r" (Core->SystemRegister.CR4),
"=r" (Core->SystemRegister.CR8),
"=r" (Core->SystemRegister.EFER)
: "i" (MSR_EFER)
: "%rax", "%rcx", "%rdx"
);
if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL) {
__asm__ volatile
(
"# EFCR" "\n\t"
"xorq %%rax, %%rax" "\n\t"
"xorq %%rdx, %%rdx" "\n\t"
"movq %1,%%rcx" "\n\t"
"rdmsr" "\n\t"
"shlq $32, %%rdx" "\n\t"
"orq %%rdx, %%rax" "\n\t"
"movq %%rax, %0"
: "=r" (Core->SystemRegister.EFCR)
: "i" (MSR_IA32_FEAT_CTL)
: "%rax", "%rcx", "%rdx"
);
/* Virtualization Technology. */
if (BITVAL(Core->SystemRegister.EFCR, EXFCR_VMX_IN_SMX)
| BITVAL(Core->SystemRegister.EFCR, EXFCR_VMXOUT_SMX))
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->VM, Core->Bind);
}
else if ( (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
||(PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON) )
{
RDMSR(Core->SystemRegister.VMCR, MSR_VM_CR);
/* Secure Virtual Machine . */
if (!Core->SystemRegister.VMCR.SVME_Disable
&& Core->SystemRegister.VMCR.SVM_Lock)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->VM, Core->Bind);
}
/* System Configuration Register. */
__asm__ volatile
(
"# SYSCFG" "\n\t"
"xorq %%rax, %%rax" "\n\t"
"xorq %%rdx, %%rdx" "\n\t"
"movq %1,%%rcx" "\n\t"
"rdmsr" "\n\t"
"shlq $32, %%rdx" "\n\t"
"orq %%rdx, %%rax" "\n\t"
"movq %%rax, %0"
: "=r" (Core->SystemRegister.SYSCFG)
: "i" (MSR_AMD64_SYSCFG)
: "%rax", "%rcx", "%rdx"
);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->CR_Mask, Core->Bind);
if (PUBLIC(RO(Proc))->Features.Std.ECX.XSAVE
&& BITVAL(Core->SystemRegister.CR4, CR4_OSXSAVE)) {
__asm__ volatile
(
"# XCR0" "\n\t"
"xorq %%rcx, %%rcx" "\n\t"
"xgetbv" "\n\t"
"shlq $32, %%rdx" "\n\t"
"orq %%rdx, %%rax" "\n\t"
"movq %%rax, %0"
: "=r" (Core->SystemRegister.XCR0)
:
: "%rax", "%rcx", "%rdx"
);
}
if (PUBLIC(RO(Proc))->Features.ExtState_Leaf1.EAX.IA32_XSS) {
__asm__ volatile
(
"# XSS" "\n\t"
"xorq %%rax, %%rax" "\n\t"
"xorq %%rdx, %%rdx" "\n\t"
"movq %1,%%rcx" "\n\t"
"rdmsr" "\n\t"
"shlq $32, %%rdx" "\n\t"
"orq %%rdx, %%rax" "\n\t"
"movq %%rax, %0"
: "=r" (Core->SystemRegister.XSS)
: "i" (MSR_IA32_XSS)
: "%rax", "%rcx", "%rdx"
);
}
}
static void Intel_Mitigation_Mechanisms(CORE_RO *Core)
{
SPEC_CTRL Spec_Ctrl = {.value = 0};
PRED_CMD Pred_Cmd = {.value = 0};
FLUSH_CMD Flush_Cmd = {.value = 0};
unsigned short WrRdMSR = 0;
if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.IBRS_IBPB_Cap
|| PUBLIC(RO(Proc))->Features.ExtFeature.EDX.STIBP_Cap
|| PUBLIC(RO(Proc))->Features.ExtFeature.EDX.SSBD_Cap)
{
RDMSR(Spec_Ctrl, MSR_IA32_SPEC_CTRL);
}
if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.IBRS_IBPB_Cap
&& ((Mech_IBRS == COREFREQ_TOGGLE_OFF)
|| (Mech_IBRS == COREFREQ_TOGGLE_ON)))
{
Spec_Ctrl.IBRS = Mech_IBRS;
WrRdMSR = 1;
}
if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.STIBP_Cap
&& ((Mech_STIBP == COREFREQ_TOGGLE_OFF)
|| (Mech_STIBP == COREFREQ_TOGGLE_ON)))
{
Spec_Ctrl.STIBP = Mech_STIBP;
WrRdMSR = 1;
}
if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.SSBD_Cap
&& ((Mech_SSBD == COREFREQ_TOGGLE_OFF)
|| (Mech_SSBD == COREFREQ_TOGGLE_ON)))
{
Spec_Ctrl.SSBD = Mech_SSBD;
WrRdMSR = 1;
}
if (WrRdMSR == 1)
{
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 16, 11)
x86_spec_ctrl_base = Spec_Ctrl.value;
#endif
WRMSR(Spec_Ctrl, MSR_IA32_SPEC_CTRL);
RDMSR(Spec_Ctrl, MSR_IA32_SPEC_CTRL);
}
if (Spec_Ctrl.IBRS) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->IBRS, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->IBRS, Core->Bind);
}
if (Spec_Ctrl.STIBP) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->STIBP, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->STIBP, Core->Bind);
}
if (Spec_Ctrl.SSBD) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->SSBD, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SSBD, Core->Bind);
}
if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.IBRS_IBPB_Cap
&& ((Mech_IBPB == COREFREQ_TOGGLE_OFF)
|| (Mech_IBPB == COREFREQ_TOGGLE_ON)))
{
Pred_Cmd.IBPB = Mech_IBPB;
WRMSR(Pred_Cmd, MSR_IA32_PRED_CMD);
}
if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.L1D_FLUSH_Cap
&& ((Mech_L1D_FLUSH == COREFREQ_TOGGLE_OFF)
|| (Mech_L1D_FLUSH == COREFREQ_TOGGLE_ON)))
{
Flush_Cmd.L1D_FLUSH_CMD = Mech_L1D_FLUSH;
WRMSR(Flush_Cmd, MSR_IA32_FLUSH_CMD);
}
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->PSFD, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->IPRED_DIS_U, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->IPRED_DIS_S, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RRSBA_DIS_U, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RRSBA_DIS_S, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->DDPD_U_DIS, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->BHI_DIS_S, Core->Bind);
if (PUBLIC(RO(Proc))->Features.ExtFeature.EAX.MaxSubLeaf >= 2)
{
if (PUBLIC(RO(Proc))->Features.ExtFeature_Leaf2_EDX.PSFD_SPEC_CTRL) {
RDMSR(Spec_Ctrl, MSR_IA32_SPEC_CTRL);
if (Spec_Ctrl.PSFD) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->PSFD, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->PSFD, Core->Bind);
}
}
if (PUBLIC(RO(Proc))->Features.ExtFeature_Leaf2_EDX.IPRED_SPEC_CTRL) {
RDMSR(Spec_Ctrl, MSR_IA32_SPEC_CTRL);
if (Spec_Ctrl.IPRED_DIS_U) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->IPRED_DIS_U, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->IPRED_DIS_U, Core->Bind);
}
if (Spec_Ctrl.IPRED_DIS_S) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->IPRED_DIS_S, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->IPRED_DIS_S, Core->Bind);
}
}
if (PUBLIC(RO(Proc))->Features.ExtFeature_Leaf2_EDX.RRSBA_SPEC_CTRL) {
RDMSR(Spec_Ctrl, MSR_IA32_SPEC_CTRL);
if (Spec_Ctrl.RRSBA_DIS_U) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->RRSBA_DIS_U, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RRSBA_DIS_U, Core->Bind);
}
if (Spec_Ctrl.RRSBA_DIS_S) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->RRSBA_DIS_S, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RRSBA_DIS_S, Core->Bind);
}
}
if (PUBLIC(RO(Proc))->Features.ExtFeature_Leaf2_EDX.DDPD_U_SPEC_CTRL) {
RDMSR(Spec_Ctrl, MSR_IA32_SPEC_CTRL);
if (Spec_Ctrl.DDPD_U_DIS) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->DDPD_U_DIS, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->DDPD_U_DIS, Core->Bind);
}
}
if (PUBLIC(RO(Proc))->Features.ExtFeature_Leaf2_EDX.BHI_SPEC_CTRL) {
RDMSR(Spec_Ctrl, MSR_IA32_SPEC_CTRL);
if (Spec_Ctrl.BHI_DIS_S) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->BHI_DIS_S, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->BHI_DIS_S, Core->Bind);
}
}
}
if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.IA32_ARCH_CAP)
{
ARCH_CAPABILITIES Arch_Cap = {.value = 0};
RDMSR(Arch_Cap, MSR_IA32_ARCH_CAPABILITIES);
if (Arch_Cap.RDCL_NO) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->RDCL_NO, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RDCL_NO, Core->Bind);
}
if (Arch_Cap.IBRS_ALL) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->IBRS_ALL, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->IBRS_ALL, Core->Bind);
}
if (Arch_Cap.RSBA) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->RSBA, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RSBA, Core->Bind);
}
if (Arch_Cap.L1DFL_VMENTRY_NO) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1DFL_VMENTRY_NO,Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1DFL_VMENTRY_NO,Core->Bind);
}
if (Arch_Cap.SSB_NO) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->SSB_NO, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SSB_NO, Core->Bind);
}
if (Arch_Cap.MDS_NO) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->MDS_NO, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->MDS_NO, Core->Bind);
}
if (Arch_Cap.PSCHANGE_MC_NO) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->PSCHANGE_MC_NO, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->PSCHANGE_MC_NO, Core->Bind);
}
if (Arch_Cap.TAA_NO) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->TAA_NO, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TAA_NO, Core->Bind);
}
if (Arch_Cap.DOITM_UARCH_MISC_CTRL) {
UARCH_MISC_CTRL uARCH_Ctrl = {.value = 0};
RDMSR(uARCH_Ctrl, MSR_IA32_UARCH_MISC_CTRL);
if (uARCH_Ctrl.DOITM) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->DOITM_EN, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->DOITM_EN, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->DOITM_MSR, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->DOITM_MSR, Core->Bind);
}
if (Arch_Cap.SBDR_SSDP_NO) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->SBDR_SSDP_NO, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SBDR_SSDP_NO, Core->Bind);
}
if (Arch_Cap.FBSDP_NO) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->FBSDP_NO, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->FBSDP_NO, Core->Bind);
}
if (Arch_Cap.PSDP_NO) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->PSDP_NO, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->PSDP_NO, Core->Bind);
}
if (Arch_Cap.FB_CLEAR) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->FB_CLEAR, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->FB_CLEAR, Core->Bind);
}
if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.SRBDS_CTRL) {
if (Arch_Cap.FB_CLEAR_CTRL) {
MCU_OPT_CTRL MCU_Ctrl = {.value = 0};
RDMSR(MCU_Ctrl, MSR_IA32_MCU_OPT_CTRL);
if (MCU_Ctrl._RNGDS_MITG_DIS) {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RNGDS, Core->Bind);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->RNGDS, Core->Bind);
}
if (MCU_Ctrl._RTM_ALLOW && !MCU_Ctrl._RTM_LOCKED) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->RTM, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RTM, Core->Bind);
}
if (MCU_Ctrl._FB_CLEAR_DIS) {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->VERW, Core->Bind);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->VERW, Core->Bind);
}
}
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->SRBDS_MSR, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SRBDS_MSR, Core->Bind);
}
if (Arch_Cap.RRSBA) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->RRSBA, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RRSBA, Core->Bind);
}
if (Arch_Cap.BHI_NO) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->BHI_NO, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->BHI_NO, Core->Bind);
}
if (Arch_Cap.XAPIC_DISABLE_STATUS_MSR) {
XAPIC_DISABLE_STATUS xAPIC_Status = {.value = 0};
RDMSR(xAPIC_Status, MSR_IA32_XAPIC_DISABLE_STATUS);
if (xAPIC_Status.LEGACY_XAPIC_DIS) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->XAPIC_DIS, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->XAPIC_DIS, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->XAPIC_MSR, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->XAPIC_MSR, Core->Bind);
}
if (Arch_Cap.PBRSB_NO) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->PBRSB_NO, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->PBRSB_NO, Core->Bind);
}
if (Arch_Cap.OVERCLOCKING_STATUS_MSR) {
OVERCLOCKING_STATUS OC_Status = {.value = 0};
RDMSR(OC_Status, MSR_IA32_OVERCLOCKING_STATUS);
if (OC_Status.OC_Utilized) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->OC_UTILIZED, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->OC_UTILIZED, Core->Bind);
}
if (OC_Status.OC_Undervolt) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->OC_UNDERVOLT, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->OC_UNDERVOLT, Core->Bind);
}
if (OC_Status.OC_Unlocked) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->OC_UNLOCKED, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->OC_UNLOCKED, Core->Bind);
}
}
if (Arch_Cap.GDS_NO) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->GDS_NO, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->GDS_NO, Core->Bind);
}
if (Arch_Cap.RFDS_NO) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->RFDS_NO, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RFDS_NO, Core->Bind);
}
}
if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.IA32_CORE_CAP)
{
CORE_CAPABILITIES Core_Cap = {.value = 0};
RDMSR(Core_Cap, MSR_IA32_CORE_CAPABILITIES);
if (Core_Cap.STLB_SUPPORTED) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->STLB, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->STLB, Core->Bind);
}
if (Core_Cap.FUSA_SUPPORTED) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->FUSA, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->FUSA, Core->Bind);
}
if (Core_Cap.RSM_IN_CPL0_ONLY) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->RSM_CPL0, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RSM_CPL0, Core->Bind);
}
if (Core_Cap.SPLA_EXCEPTION) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->SPLA, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SPLA, Core->Bind);
}
if (Core_Cap.SNOOP_FILTER_SUP) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->SNOOP_FILTER, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SNOOP_FILTER, Core->Bind);
}
} else {
/* Source: arch/x86/kernel/cpu/intel.c */
struct SIGNATURE allowList[] = {
_Icelake_UY, /* 06_7E */
_Icelake_X, /* 06_6A */
_Tremont_Jacobsville, /* 06_86 */
_Tremont_Elkhartlake, /* 06_96 */
_Tremont_Jasperlake /* 06_9C */
};
const int ids = sizeof(allowList) / sizeof(allowList[0]);
int id;
for (id = 0; id < ids; id++) {
if((allowList[id].ExtFamily==PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily)
&& (allowList[id].Family == PUBLIC(RO(Proc))->Features.Std.EAX.Family)
&& (allowList[id].ExtModel == PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel)
&& (allowList[id].Model == PUBLIC(RO(Proc))->Features.Std.EAX.Model))
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->SPLA, Core->Bind);
break;
}
}
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->ARCH_CAP_Mask, Core->Bind);
}
static void AMD_Mitigation_Mechanisms(CORE_RO *Core)
{
AMD_SPEC_CTRL Spec_Ctrl = {.value = 0};
AMD_PRED_CMD Pred_Cmd = {.value = 0};
unsigned short WrRdMSR = 0;
if (PUBLIC(RO(Proc))->Features.leaf80000008.EBX.IBRS
|| PUBLIC(RO(Proc))->Features.leaf80000008.EBX.STIBP
|| PUBLIC(RO(Proc))->Features.leaf80000008.EBX.SSBD
|| PUBLIC(RO(Proc))->Features.leaf80000008.EBX.PSFD)
{
RDMSR(Spec_Ctrl, MSR_AMD_SPEC_CTRL);
if ((Mech_IBRS == COREFREQ_TOGGLE_OFF)
|| (Mech_IBRS == COREFREQ_TOGGLE_ON))
{
Spec_Ctrl.IBRS = Mech_IBRS;
WrRdMSR = 1;
}
if ((Mech_STIBP == COREFREQ_TOGGLE_OFF)
|| (Mech_STIBP == COREFREQ_TOGGLE_ON))
{
Spec_Ctrl.STIBP = Mech_STIBP;
WrRdMSR = 1;
}
if ((Mech_SSBD == COREFREQ_TOGGLE_OFF)
|| (Mech_SSBD == COREFREQ_TOGGLE_ON))
{
Spec_Ctrl.SSBD = Mech_SSBD;
WrRdMSR = 1;
}
if ((Mech_PSFD == COREFREQ_TOGGLE_OFF)
|| (Mech_PSFD == COREFREQ_TOGGLE_ON))
{
if (PUBLIC(RO(Proc))->Features.leaf80000008.EBX.PSFD) {
Spec_Ctrl.PSFD = Mech_PSFD;
WrRdMSR = 1;
}
}
if (WrRdMSR == 1)
{
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 16, 11)
x86_spec_ctrl_base = Spec_Ctrl.value;
#endif
WRMSR(Spec_Ctrl, MSR_AMD_SPEC_CTRL);
RDMSR(Spec_Ctrl, MSR_AMD_SPEC_CTRL);
}
if (Spec_Ctrl.IBRS) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->IBRS, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->IBRS, Core->Bind);
}
if (Spec_Ctrl.STIBP) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->STIBP, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->STIBP, Core->Bind);
}
if (Spec_Ctrl.SSBD) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->SSBD, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SSBD, Core->Bind);
}
if (Spec_Ctrl.PSFD) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->PSFD, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->PSFD, Core->Bind);
}
}
if (PUBLIC(RO(Proc))->Features.leaf80000008.EBX.IBPB
&& ((Mech_IBPB == COREFREQ_TOGGLE_OFF)
|| (Mech_IBPB == COREFREQ_TOGGLE_ON)))
{
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1))
{
Pred_Cmd.IBPB = Mech_IBPB;
WRMSR(Pred_Cmd, MSR_AMD_PRED_CMD);
}
}
if (PUBLIC(RO(Proc))->Features.ExtFeature2_EAX.SBPB
&& ((Mech_SBPB == COREFREQ_TOGGLE_OFF)
|| (Mech_SBPB == COREFREQ_TOGGLE_ON)))
{
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1))
{
Pred_Cmd.SBPB = Mech_SBPB;
WRMSR(Pred_Cmd, MSR_AMD_PRED_CMD);
}
}
if ((PUBLIC(RO(Proc))->Features.leaf80000008.EBX.SSBD_VirtSpecCtrl == 0)
&& (BITVAL_CC(PUBLIC(RW(Proc))->SSBD, Core->Bind) == 0))
{
AMD_LS_CFG LS_CFG = {.value = 0};
CPUID_0x8000001e leaf8000001e = {
.EAX = {0}, .EBX = {{0}}, .ECX = {0}, .EDX = {0}
};
__asm__ volatile
(
"movq $0x8000001e, %%rax \n\t"
"xorq %%rbx, %%rbx \n\t"
"xorq %%rcx, %%rcx \n\t"
"xorq %%rdx, %%rdx \n\t"
"cpuid \n\t"
"mov %%eax, %0 \n\t"
"mov %%ebx, %1 \n\t"
"mov %%ecx, %2 \n\t"
"mov %%edx, %3"
: "=r" (leaf8000001e.EAX),
"=r" (leaf8000001e.EBX),
"=r" (leaf8000001e.ECX),
"=r" (leaf8000001e.EDX)
:
: "%rax", "%rbx", "%rcx", "%rdx"
);
if (leaf8000001e.EBX.ThreadsPerCore == 1)
{
RDMSR(LS_CFG, MSR_AMD64_LS_CFG);
if ((Mech_SSBD == COREFREQ_TOGGLE_OFF)
|| (Mech_SSBD == COREFREQ_TOGGLE_ON))
{
LS_CFG.F17h_SSBD_EN = Mech_SSBD;
WRMSR(LS_CFG, MSR_AMD64_LS_CFG);
RDMSR(LS_CFG, MSR_AMD64_LS_CFG);
}
if (LS_CFG.F17h_SSBD_EN == 1) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->AMD_LS_CFG_SSBD, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->AMD_LS_CFG_SSBD, Core->Bind);
}
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->AMD_LS_CFG_SSBD, Core->Bind);
}
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->AMD_LS_CFG_SSBD, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->ARCH_CAP_Mask, Core->Bind);
switch (PUBLIC(RO(Proc))->ArchID) {
case AMD_EPYC_Rome_CPK:
case AMD_Zen2_Renoir:
case AMD_Zen2_LCN:
case AMD_Zen2_MTS:
case AMD_Zen2_Ariel:
case AMD_Zen2_Jupiter:
case AMD_Zen2_Galileo:
case AMD_Zen2_MDN:
{
AMD_LS_CFG LS_CFG = {.value = 0};
AMD_DE_CFG2 DE_CFG = {.value = 0};
RDMSR(DE_CFG, MSR_AMD_DE_CFG2);
if (((Mech_BTC_NOBR == COREFREQ_TOGGLE_OFF)
|| (Mech_BTC_NOBR == COREFREQ_TOGGLE_ON))
&& (PUBLIC(RO(Proc))->Features.leaf80000008.EBX.STIBP == 1))
{
DE_CFG.SuppressBPOnNonBr = Mech_BTC_NOBR;
WRMSR(DE_CFG, MSR_AMD_DE_CFG2);
RDMSR(DE_CFG, MSR_AMD_DE_CFG2);
}
if (DE_CFG.SuppressBPOnNonBr) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->BTC_NOBR, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->BTC_NOBR, Core->Bind);
}
RDMSR(LS_CFG, MSR_AMD64_LS_CFG);
if ( (Mech_AGENPICK == COREFREQ_TOGGLE_OFF)
|| (Mech_AGENPICK == COREFREQ_TOGGLE_ON) )
{
LS_CFG.F17h_AgenPick = Mech_AGENPICK;
WRMSR(LS_CFG, MSR_AMD64_LS_CFG);
RDMSR(LS_CFG, MSR_AMD64_LS_CFG);
}
if (LS_CFG.F17h_AgenPick == 1) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->AGENPICK, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->AGENPICK, Core->Bind);
}
}
break;
default:
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->BTC_NOBR, Core->Bind);
break;
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->BTC_NOBR_Mask, Core->Bind);
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
switch (PUBLIC(RO(Proc))->ArchID) {
case AMD_EPYC_Rome_CPK:
case AMD_Zen2_Renoir:
case AMD_Zen2_LCN:
case AMD_Zen2_MTS:
case AMD_Zen2_Ariel:
case AMD_Zen2_Jupiter:
case AMD_Zen2_Galileo:
case AMD_Zen2_MDN:
{
AMD_DE_CFG DE_CFG = {.value = 0};
RDMSR(DE_CFG, MSR_AMD64_DE_CFG);
if ((Mech_XPROC_LEAK == COREFREQ_TOGGLE_OFF)
|| (Mech_XPROC_LEAK == COREFREQ_TOGGLE_ON))
{
DE_CFG.Cross_Proc_Leak = Mech_XPROC_LEAK;
WRMSR(DE_CFG, MSR_AMD64_DE_CFG);
RDMSR(DE_CFG, MSR_AMD64_DE_CFG);
}
if (DE_CFG.Cross_Proc_Leak) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->XPROC_LEAK, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->XPROC_LEAK, Core->Bind);
}
}
break;
default:
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->XPROC_LEAK, Core->Bind);
break;
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->XPROC_LEAK_Mask, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->XPROC_LEAK_Mask, Core->Bind);
}
}
static void Intel_VirtualMachine(CORE_RO *Core)
{
if (PUBLIC(RO(Proc))->Features.Std.ECX.VMX) {
VMX_BASIC VMX_Basic = {.value = 0};
/* Basic VMX Information. */
RDMSR(VMX_Basic, MSR_IA32_VMX_BASIC);
if (VMX_Basic.SMM_DualMon)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->SMM, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SMM, Core->Bind);
}
}
}
static void Intel_Microcode(CORE_RO *Core)
{
MICROCODE_ID Microcode = {.value = 0};
RDMSR(Microcode, MSR_IA32_UCODE_REV);
Core->Query.Microcode = Microcode.Signature;
}
static void AMD_Microcode(CORE_RO *Core)
{
unsigned long long value = 0;
RDMSR64(value, MSR_AMD64_PATCH_LEVEL);
Core->Query.Microcode = (unsigned int) value;
}
#define Pkg_Reset_ThermalPoint(Pkg) \
({ \
BITWISECLR(BUS_LOCK, Pkg->ThermalPoint.Mask); \
BITWISECLR(BUS_LOCK, Pkg->ThermalPoint.Kind); \
BITWISECLR(BUS_LOCK, Pkg->ThermalPoint.State); \
})
static void PerCore_Reset(CORE_RO *Core)
{
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->TM_Mask , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->ODCM_Mask , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->DCU_Mask , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->L1_Scrub_Mask, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->L2_AMP_Mask, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->ECORE_Mask, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->PowerMgmt_Mask, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->SpeedStep_Mask, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->TurboBoost_Mask,Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->HWP_Mask , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1E_Mask , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3A_Mask , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1A_Mask , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3U_Mask , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1U_Mask , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->CC6_Mask , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->PC6_Mask , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->ARCH_CAP_Mask , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->BTC_NOBR_Mask , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->XPROC_LEAK_Mask,Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->CR_Mask , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RO(Proc))->WDT_Mask , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TM1 , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TM2 , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->ODCM , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_HW_Prefetch , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_HW_IP_Prefetch , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_NPP_Prefetch , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_Scrubbing , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_HW_Prefetch , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_HW_CL_Prefetch , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_AMP_Prefetch , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_Stride_Pf , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_Region_Pf , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1_Burst_Pf , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_Stream_HW_Pf , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L2_UpDown_Pf , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->LLC_Streamer , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->PowerMgmt , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SpeedStep , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->HWP , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->C1E , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->C3A , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->C1A , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->C3U , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->C1U , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->CC6 , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->PC6 , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->IBRS , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->STIBP , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SSBD , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->PSFD , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RDCL_NO , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->IBRS_ALL , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RSBA , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->L1DFL_VMENTRY_NO,Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SSB_NO , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->MDS_NO , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->PSCHANGE_MC_NO, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TAA_NO , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->STLB , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->FUSA , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RSM_CPL0 , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SPLA , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SNOOP_FILTER, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SMM , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->VM , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->WDT , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->AMD_LS_CFG_SSBD, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->DOITM_EN , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->DOITM_MSR , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SBDR_SSDP_NO, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->FBSDP_NO , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->PSDP_NO , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->FB_CLEAR , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->SRBDS_MSR , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RNGDS , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RTM , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->VERW , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RRSBA , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->BHI_NO , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->XAPIC_MSR , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->XAPIC_DIS , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->PBRSB_NO , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->IPRED_DIS_U, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->IPRED_DIS_S, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RRSBA_DIS_U, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RRSBA_DIS_S, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->DDPD_U_DIS, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->BHI_DIS_S , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->BTC_NOBR , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->XPROC_LEAK, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->OC_UTILIZED, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->OC_UNDERVOLT, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->OC_UNLOCKED, Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->GDS_NO , Core->Bind);
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->RFDS_NO , Core->Bind);
BITWISECLR(LOCKLESS, Core->ThermalPoint.Mask);
BITWISECLR(LOCKLESS, Core->ThermalPoint.Kind);
BITWISECLR(LOCKLESS, Core->ThermalPoint.State);
}
static void PerCore_VirtualMachine(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL)
{
PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(MIN)] = 10;
Intel_VirtualMachine(Core);
Intel_Microcode(Core);
SpeedStep_Technology(Core);
}
else if ( (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
||(PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON) )
{
PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(MIN)] = 8;
if (PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily >= 1) {
AMD_Microcode(Core);
}
BITSET_CC(BUS_LOCK,PUBLIC(RO(Proc))->SpeedStep_Mask,Core->Bind);
BITSET_CC(BUS_LOCK,PUBLIC(RO(Proc))->ODCM_Mask, Core->Bind);
BITSET_CC(BUS_LOCK,PUBLIC(RO(Proc))->PowerMgmt_Mask,Core->Bind);
}
SystemRegisters(Core);
Dump_CPUID(Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->TM_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->TurboBoost_Mask,Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1E_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3A_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1A_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3U_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1U_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->CC6_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PC6_Mask,
PUBLIC(RO(Proc))->Service.Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->ARCH_CAP_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->BTC_NOBR_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->WDT_Mask,
PUBLIC(RO(Proc))->Service.Core);
}
static void PerCore_Intel_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
Intel_Core_Platform_Info(Core->Bind);
SystemRegisters(Core);
Intel_VirtualMachine(Core);
Intel_Microcode(Core);
Dump_CPUID(Core);
SpeedStep_Technology(Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->TurboBoost_Mask,Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1E_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3A_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1A_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3U_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1U_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->CC6_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PC6_Mask,
PUBLIC(RO(Proc))->Service.Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->ARCH_CAP_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->BTC_NOBR_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->WDT_Mask,
PUBLIC(RO(Proc))->Service.Core);
PowerThermal(Core);
ThermalMonitor_Set(Core);
}
static void PerCore_AuthenticAMD_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
Compute_AuthenticAMD_Boost(Core->Bind);
SystemRegisters(Core);
if (PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily >= 1) {
AMD_Microcode(Core);
}
Dump_CPUID(Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->TM_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->ODCM_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PowerMgmt_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->SpeedStep_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->TurboBoost_Mask,Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1E_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3A_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1A_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3U_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1U_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->CC6_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PC6_Mask,
PUBLIC(RO(Proc))->Service.Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->ARCH_CAP_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->BTC_NOBR_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->WDT_Mask, Core->Bind);
}
static void PerCore_Core2_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
Intel_Core_Platform_Info(Core->Bind);
SystemRegisters(Core);
Intel_VirtualMachine(Core);
Intel_Microcode(Core);
Dump_CPUID(Core);
SpeedStep_Technology(Core);
DynamicAcceleration(Core); /* Unique */
Compute_Intel_Core_Burst(Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1E_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3A_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1A_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3U_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1U_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PC6_Mask,
PUBLIC(RO(Proc))->Service.Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->ARCH_CAP_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->BTC_NOBR_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->WDT_Mask,
PUBLIC(RO(Proc))->Service.Core);
PowerThermal(Core); /* Shared | Unique */
ThermalMonitor_Set(Core);
}
static void PerCore_Atom_Bonnell_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
Intel_Core_Platform_Info(Core->Bind);
SystemRegisters(Core);
Intel_VirtualMachine(Core);
Intel_Microcode(Core);
Dump_CPUID(Core);
SpeedStep_Technology(Core);
DynamicAcceleration(Core); /* Unique */
Compute_Intel_Core_Burst(Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1E_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3A_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1A_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3U_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1U_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PC6_Mask,
PUBLIC(RO(Proc))->Service.Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->ARCH_CAP_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->BTC_NOBR_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->WDT_Mask,
PUBLIC(RO(Proc))->Service.Core);
PowerThermal(Core); /* Shared | Unique */
ThermalMonitor_Atom_Bonnell(Core);
}
static void Compute_Intel_Silvermont_Burst(CORE_RO *Core)
{
enum RATIO_BOOST boost;
unsigned int burstRatio;
PLATFORM_ID PfID = {.value = 0};
PERF_STATUS PerfStatus = {.value = 0};
bool initialize = false;
burstRatio = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(MAX)];
if (Intel_MaxBusRatio(&PfID) == 0) {
burstRatio = KMAX(burstRatio, PfID.MaxBusRatio);
}
PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(ACT)] = burstRatio;
RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
burstRatio = KMAX(burstRatio, PerfStatus.CORE.MaxBusRatio);
if (PRIVATE(OF(Specific)) != NULL)
{
unsigned int XtraBoost = PRIVATE(OF(Specific))->Boost[0]
+ PRIVATE(OF(Specific))->Boost[1];
if (XtraBoost > 0) {
burstRatio = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(MAX)];
burstRatio = burstRatio + XtraBoost;
}
}
if (PUBLIC(RO(Proc))->Features.Turbo_Unlock == 1)
{ /* Read the Turbo Boost register from any programmed ratio */
Intel_Turbo_Config(Core, Intel_Turbo_Cfg8C_PerCore, Assign_8C_Boost);
boost = BOOST(SIZE) - PUBLIC(RO(Proc))->Features.SpecTurboRatio;
do
{
if (PUBLIC(RO(Core, AT(Core->Bind)))->Boost[boost] == 0)
{
PUBLIC(RO(Core, AT(Core->Bind)))->Boost[boost] = burstRatio;
initialize = true;
}
} while ( ++boost < BOOST(SIZE) );
if ((initialize == true) && (PUBLIC(RO(Proc))->ArchID != Atom_Airmont))
{ /* Re-program the register if at least one value was zero */
TURBO_RATIO_CONFIG0 Cfg0 = {
.MaxRatio_1C = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(1C)],
.MaxRatio_2C = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(2C)],
.MaxRatio_3C = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(3C)],
.MaxRatio_4C = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(4C)],
.MaxRatio_5C = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(5C)],
.MaxRatio_6C = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(6C)],
.MaxRatio_7C = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(7C)],
.MaxRatio_8C = PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(8C)]
};
WRMSR(Cfg0, MSR_TURBO_RATIO_LIMIT);
}
}
}
static void PerCore_Silvermont_Query(void *arg)
{ /* Query the Min and Max frequency ratios per CPU */
CORE_RO *Core = (CORE_RO *) arg;
PLATFORM_INFO PfInfo = {.value = 0};
RDMSR(PfInfo, MSR_PLATFORM_INFO);
PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(MIN)]=PfInfo.MinimumRatio;
PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(MAX)]=PfInfo.MaxNonTurboRatio;
SystemRegisters(Core);
Intel_Mitigation_Mechanisms(Core);
Intel_VirtualMachine(Core);
Intel_Microcode(Core);
Dump_CPUID(Core);
SpeedStep_Technology(Core);
DynamicAcceleration(Core); /* Unique */
SoC_Turbo_Override(Core);
Compute_Intel_Silvermont_Burst(Core);
Query_Intel_C1E(Core);
Intel_CStatesConfiguration(CSTATES_SOC_SLM, Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->BTC_NOBR_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->WDT_Mask,
PUBLIC(RO(Proc))->Service.Core);
PowerThermal(Core); /* Shared | Unique */
ThermalMonitor_Set(Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
Intel_DomainPowerLimit( MSR_PKG_POWER_LIMIT, /* Table 2-8 */
PKG_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(PKG) );
Intel_Watchdog(Core);
}
}
static void PerCore_Airmont_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
PerCore_Silvermont_Query(arg);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
Intel_DomainPowerLimit( MSR_PP0_POWER_LIMIT, /* Table 2-11 */
PPn_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(CORES) );
}
}
static void PerCore_Atom_Goldmont_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
Intel_Platform_Info(Core->Bind);
Intel_Turbo_Config(Core, Intel_Turbo_Cfg8C_PerCore, Assign_8C_Boost);
SystemRegisters(Core);
Intel_Mitigation_Mechanisms(Core);
Intel_VirtualMachine(Core);
Intel_Microcode(Core);
Dump_CPUID(Core);
Intel_DCU_Technology(Core);
SpeedStep_Technology(Core);
TurboBoost_Technology( Core,
Set_SandyBridge_Target,
Get_SandyBridge_Target,
Cmp_SandyBridge_Target,
Core->Boost[BOOST(1C)],
Core->Boost[BOOST(MAX)] );
Query_Intel_C1E(Core);
if (Core->T.ThreadID == 0) { /* Per Core */
Intel_CStatesConfiguration(CSTATES_SOC_GDM, Core);
} else {
CSTATE_IO_MWAIT CState_IO_MWAIT = {.value = 0};
RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
/* Store the C-State Base Address used by I/O-MWAIT */
Core->Query.CStateBaseAddr = CState_IO_MWAIT.LVL2_BaseAddr;
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PC6_Mask,
PUBLIC(RO(Proc))->Service.Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->BTC_NOBR_Mask , Core->Bind);
PowerThermal(Core);
ThermalMonitor_Set(Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
Intel_DomainPowerLimit( MSR_PKG_POWER_LIMIT, /* Table 2-12 */
PKG_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(PKG) );
Intel_DomainPowerLimit( MSR_DRAM_POWER_LIMIT, /* Table 2-12 */
PPn_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(RAM) );
if (PUBLIC(RO(Proc))->Registration.Experimental) {
Intel_Pkg_CST_IRTL(MSR_PKGC3_IRTL,
&PUBLIC(RO(Proc))->PowerThermal.IRTL.PC03);
Intel_Pkg_CST_IRTL(MSR_PKGC6_IRTL,
&PUBLIC(RO(Proc))->PowerThermal.IRTL.PC06);
Intel_Pkg_CST_IRTL(MSR_PKGC7_IRTL,
&PUBLIC(RO(Proc))->PowerThermal.IRTL.PC07);
}
Intel_Watchdog(Core);
}
}
static void PerCore_Goldmont_Query(void *arg)
{
PerCore_Atom_Goldmont_Query(arg);
}
static void PerCore_Geminilake_Query(void *arg)
{
PerCore_Atom_Goldmont_Query(arg);
}
static void PerCore_Tremont_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
PerCore_Atom_Goldmont_Query(arg);
Intel_Turbo_Activation_Ratio(Core);
}
static void PerCore_Nehalem_Same_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
SystemRegisters(Core);
Intel_Mitigation_Mechanisms(Core);
Intel_VirtualMachine(Core);
Intel_Microcode(Core);
Dump_CPUID(Core);
Intel_DCU_Technology(Core);
SpeedStep_Technology(Core);
TurboBoost_Technology( Core,
Set_Nehalem_Target,
Get_Nehalem_Target,
Cmp_Nehalem_Target,
1 + Core->Boost[BOOST(MAX)],
1 + Core->Boost[BOOST(MAX)] );
Query_Intel_C1E(Core);
if (Core->T.ThreadID == 0) { /* Per Core */
Intel_CStatesConfiguration(CSTATES_NHM, Core);
} else {
CSTATE_IO_MWAIT CState_IO_MWAIT = {.value = 0};
RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
/* Store the C-State Base Address used by I/O-MWAIT */
Core->Query.CStateBaseAddr = CState_IO_MWAIT.LVL2_BaseAddr;
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PC6_Mask,
PUBLIC(RO(Proc))->Service.Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->BTC_NOBR_Mask , Core->Bind);
PowerThermal(Core);
ThermalMonitor_Set(Core);
}
static void PerCore_Nehalem_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
Intel_Platform_Info(Core->Bind);
Intel_Turbo_Config(Core, Intel_Turbo_Cfg8C_PerCore, Assign_8C_Boost);
PerCore_Nehalem_Same_Query(Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
Nehalem_PowerLimit(); /* Table 2-15 */
Intel_Watchdog(Core);
}
}
static void PerCore_Nehalem_EX_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
Intel_Platform_Info(Core->Bind);
Intel_Core_Platform_Info(Core->Bind);
PerCore_Nehalem_Same_Query(Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
Nehalem_PowerLimit(); /* Table 2-15 */
Intel_Watchdog(Core);
}
}
static void PerCore_Avoton_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
PerCore_Silvermont_Query(arg);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
Intel_DomainPowerLimit( MSR_PKG_POWER_LIMIT, /* Table 2-10 */
PKG_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(PKG) );
}
}
static void PerCore_SandyBridge_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
Intel_Platform_Info(Core->Bind);
Intel_Turbo_Config(Core, Intel_Turbo_Cfg8C_PerCore, Assign_8C_Boost);
SystemRegisters(Core);
Intel_Mitigation_Mechanisms(Core);
Intel_VirtualMachine(Core);
Intel_Microcode(Core);
Dump_CPUID(Core);
Intel_DCU_Technology(Core);
SpeedStep_Technology(Core);
TurboBoost_Technology( Core,
Set_SandyBridge_Target,
Get_SandyBridge_Target,
Cmp_SandyBridge_Target,
Core->Boost[BOOST(1C)],
Core->Boost[BOOST(MAX)] );
Query_Intel_C1E(Core);
if (Core->T.ThreadID == 0) { /* Per Core */
Intel_CStatesConfiguration(CSTATES_SNB, Core);
} else {
CSTATE_IO_MWAIT CState_IO_MWAIT = {.value = 0};
RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
/* Store the C-State Base Address used by I/O-MWAIT */
Core->Query.CStateBaseAddr = CState_IO_MWAIT.LVL2_BaseAddr;
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PC6_Mask,
PUBLIC(RO(Proc))->Service.Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->BTC_NOBR_Mask , Core->Bind);
PowerThermal(Core);
ThermalMonitor_Set(Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
Intel_DomainPowerLimit( MSR_PKG_POWER_LIMIT,
PKG_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(PKG) );
Intel_DomainPowerLimit( MSR_PP0_POWER_LIMIT,
PPn_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(CORES) );
if (PUBLIC(RO(Proc))->Registration.Experimental) {
Intel_Pkg_CST_IRTL(MSR_PKGC3_IRTL,
&PUBLIC(RO(Proc))->PowerThermal.IRTL.PC03);
Intel_Pkg_CST_IRTL(MSR_PKGC6_IRTL,
&PUBLIC(RO(Proc))->PowerThermal.IRTL.PC06);
Intel_Pkg_CST_IRTL(MSR_PKGC7_IRTL,
&PUBLIC(RO(Proc))->PowerThermal.IRTL.PC07);
Intel_Watchdog(Core);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->WDT_Mask, Core->Bind);
}
}
}
static void PerCore_SandyBridge_EP_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
PerCore_SandyBridge_Query(arg);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
Intel_DomainPowerLimit( MSR_DRAM_POWER_LIMIT,
PPn_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(RAM) );
}
}
static void PerCore_IvyBridge_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
PerCore_SandyBridge_Query(arg);
Intel_Turbo_Activation_Ratio( (CORE_RO*) arg );
Intel_Turbo_TDP_Config( (CORE_RO*) arg );
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
Intel_DomainPowerLimit( MSR_PP1_POWER_LIMIT,
PPn_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(UNCORE) );
}
}
static void PerCore_IvyBridge_EP_Query(void *arg)
{
Intel_Turbo_Config( (CORE_RO*) arg,
Intel_Turbo_Cfg15C_PerCore,
Assign_15C_Boost );
PerCore_SandyBridge_EP_Query(arg);
}
static void PerCore_Haswell_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
PerCore_SandyBridge_Query(arg);
Intel_Turbo_Activation_Ratio(Core);
Intel_Turbo_TDP_Config(Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
Intel_DomainPowerLimit( MSR_PP1_POWER_LIMIT,
PPn_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(UNCORE) );
}
}
static void PerCore_Haswell_EP_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
Intel_Platform_Info(Core->Bind);
Intel_Turbo_Config(Core, Intel_Turbo_Cfg8C_PerCore, Assign_8C_Boost);
Intel_Turbo_Config( (CORE_RO*) arg,
Intel_Turbo_Cfg16C_PerCore,
Assign_16C_Boost );
Intel_Turbo_Config( (CORE_RO*) arg,
Intel_Turbo_Cfg18C_PerCore,
Assign_18C_Boost );
SystemRegisters(Core);
Intel_Mitigation_Mechanisms(Core);
Intel_VirtualMachine(Core);
if (PUBLIC(RO(Proc))->Registration.Experimental) {
Intel_Microcode(Core);
}
Dump_CPUID(Core);
Intel_DCU_Technology(Core);
SpeedStep_Technology(Core);
TurboBoost_Technology( Core,
Set_SandyBridge_Target,
Get_SandyBridge_Target,
Cmp_SandyBridge_Target,
Core->Boost[BOOST(1C)],
Core->Boost[BOOST(MAX)] );
Query_Intel_C1E(Core);
if (Core->T.ThreadID == 0) { /* Per Core */
Intel_CStatesConfiguration(CSTATES_SNB, Core);
} else {
CSTATE_IO_MWAIT CState_IO_MWAIT = {.value = 0};
RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
/* Store the C-State Base Address used by I/O-MWAIT */
Core->Query.CStateBaseAddr = CState_IO_MWAIT.LVL2_BaseAddr;
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PC6_Mask,
PUBLIC(RO(Proc))->Service.Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->BTC_NOBR_Mask , Core->Bind);
PowerThermal(Core);
ThermalMonitor_Set(Core);
Intel_Turbo_Activation_Ratio(Core);
Intel_Turbo_TDP_Config(Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
Intel_DomainPowerLimit( MSR_PKG_POWER_LIMIT,
PKG_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(PKG) );
Intel_DomainPowerLimit( MSR_PP0_POWER_LIMIT,
PPn_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(CORES) );
Intel_DomainPowerLimit( MSR_DRAM_POWER_LIMIT,
PPn_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(RAM) );
if (PUBLIC(RO(Proc))->Registration.Experimental) {
Intel_Pkg_CST_IRTL(MSR_PKGC6_IRTL, /* Table 2-29 */
&PUBLIC(RO(Proc))->PowerThermal.IRTL.PC06);
Intel_Pkg_CST_IRTL(MSR_PKGC7_IRTL, /* Table 2-29 */
&PUBLIC(RO(Proc))->PowerThermal.IRTL.PC07);
}
Intel_Watchdog(Core);
}
}
static void PerCore_Haswell_ULT_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
Intel_Platform_Info(Core->Bind);
Intel_Turbo_Config(Core, Intel_Turbo_Cfg8C_PerCore, Assign_8C_Boost);
SystemRegisters(Core);
Intel_Mitigation_Mechanisms(Core);
Intel_VirtualMachine(Core);
Intel_Microcode(Core);
Dump_CPUID(Core);
Intel_DCU_Technology(Core);
SpeedStep_Technology(Core);
TurboBoost_Technology( Core,
Set_SandyBridge_Target,
Get_SandyBridge_Target,
Cmp_SandyBridge_Target,
Core->Boost[BOOST(1C)],
Core->Boost[BOOST(MAX)] );
Query_Intel_C1E(Core);
if (Core->T.ThreadID == 0) { /* Per Core */
Intel_CStatesConfiguration(CSTATES_ULT, Core);
} else {
CSTATE_IO_MWAIT CState_IO_MWAIT = {.value = 0};
RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
/* Store the C-State Base Address used by I/O-MWAIT */
Core->Query.CStateBaseAddr = CState_IO_MWAIT.LVL2_BaseAddr;
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PC6_Mask,
PUBLIC(RO(Proc))->Service.Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->BTC_NOBR_Mask , Core->Bind);
PowerThermal(Core);
ThermalMonitor_Set(Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
Intel_DomainPowerLimit( MSR_PKG_POWER_LIMIT,
PKG_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(PKG) );
Intel_DomainPowerLimit( MSR_PP0_POWER_LIMIT,
PPn_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(CORES) );
Intel_DomainPowerLimit( MSR_PP1_POWER_LIMIT,
PPn_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(UNCORE) );
Intel_Watchdog(Core);
}
}
static void PerCore_Haswell_ULX(void *arg)
{
PerCore_IvyBridge_Query(arg);
}
static void PerCore_Broadwell_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
PerCore_SandyBridge_Query(arg);
Intel_Turbo_Activation_Ratio(Core);
Intel_Turbo_TDP_Config(Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
Intel_DomainPowerLimit( MSR_PP1_POWER_LIMIT,
PPn_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(UNCORE) );
}
}
static void PerCore_Skylake_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
Intel_Platform_Info(Core->Bind);
Intel_Turbo_Config(Core, Intel_Turbo_Cfg8C_PerCore, Assign_8C_Boost);
SystemRegisters(Core);
Intel_Mitigation_Mechanisms(Core);
Intel_VirtualMachine(Core);
Intel_Microcode(Core);
Dump_CPUID(Core);
Intel_DCU_Technology(Core);
SpeedStep_Technology(Core);
TurboBoost_Technology( Core,
Set_SandyBridge_Target,
Get_SandyBridge_Target,
Cmp_Skylake_Target,
Core->Boost[BOOST(TDP)] > 0 ?
Core->Boost[BOOST(TDP)]:Core->Boost[BOOST(1C)],
Core->Boost[BOOST(MAX)] );
Query_Intel_C1E(Core);
if (Core->T.ThreadID == 0) { /* Per Core */
Intel_CStatesConfiguration(CSTATES_SKL, Core);
} else {
CSTATE_IO_MWAIT CState_IO_MWAIT = {.value = 0};
RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
/* Store the C-State Base Address used by I/O-MWAIT */
Core->Query.CStateBaseAddr = CState_IO_MWAIT.LVL2_BaseAddr;
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PC6_Mask,
PUBLIC(RO(Proc))->Service.Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->BTC_NOBR_Mask , Core->Bind);
PowerThermal(Core);
ThermalMonitor_Set(Core);
CorePerfLimitReasons(Core);
GraphicsPerfLimitReasons(Core);
RingPerfLimitReasons(Core);
Intel_Turbo_Activation_Ratio(Core);
Intel_Turbo_TDP_Config(Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
Intel_DomainPowerLimit( MSR_PKG_POWER_LIMIT,
PKG_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(PKG) );
Intel_DomainPowerLimit( MSR_PP0_POWER_LIMIT,
PPn_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(CORES) );
Intel_DomainPowerLimit( MSR_PP1_POWER_LIMIT,
PPn_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(UNCORE) );
Intel_DomainPowerLimit( MSR_PLATFORM_POWER_LIMIT,
PKG_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(PLATFORM) );
Intel_Watchdog(Core);
}
}
static void PerCore_Skylake_X_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
Intel_Turbo_Config( Core,
Intel_Turbo_Cfg_SKL_X_PerCore,
Assign_SKL_X_Boost );
Intel_Platform_Info(Core->Bind);
Intel_Turbo_Config(Core, Intel_Turbo_Cfg8C_PerCore, Assign_8C_Boost);
SystemRegisters(Core);
Intel_Mitigation_Mechanisms(Core);
Intel_VirtualMachine(Core);
Intel_Microcode(Core);
Dump_CPUID(Core);
Intel_DCU_Technology(Core);
SpeedStep_Technology(Core);
TurboBoost_Technology( Core,
Set_SandyBridge_Target,
Get_SandyBridge_Target,
Cmp_Skylake_Target,
Core->Boost[BOOST(TDP)] > 0 ?
Core->Boost[BOOST(TDP)]:Core->Boost[BOOST(1C)],
Core->Boost[BOOST(MAX)] );
Query_Intel_C1E(Core);
if (Core->T.ThreadID == 0) { /* Per Core */
Intel_CStatesConfiguration(CSTATES_SKL, Core);
} else {
CSTATE_IO_MWAIT CState_IO_MWAIT = {.value = 0};
RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
/* Store the C-State Base Address used by I/O-MWAIT */
Core->Query.CStateBaseAddr = CState_IO_MWAIT.LVL2_BaseAddr;
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->CC6_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PC6_Mask,
PUBLIC(RO(Proc))->Service.Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->BTC_NOBR_Mask , Core->Bind);
PowerThermal(Core);
ThermalMonitor_Set(Core);
CorePerfLimitReasons(Core);
/*TODO(Unsolved)
GraphicsPerfLimitReasons(Core);
*/
RingPerfLimitReasons(Core);
Intel_Turbo_Activation_Ratio(Core);
Intel_Turbo_TDP_Config(Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
Intel_DomainPowerLimit( MSR_PKG_POWER_LIMIT,
PKG_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(PKG) );
Intel_DomainPowerLimit( MSR_PP0_POWER_LIMIT,
PPn_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(CORES) );
/*TODO(Unsupported MSR and CSR registers)
Intel_DomainPowerLimit( MSR_PLATFORM_POWER_LIMIT,
PKG_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(PLATFORM) );
Intel_DomainPowerLimit( MSR_DRAM_POWER_LIMIT,
PPn_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(RAM) );
*/
if (PUBLIC(RO(Proc))->Registration.Experimental) {
Intel_Watchdog(Core);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->WDT_Mask, Core->Bind);
}
}
}
static void PerCore_Kaby_Lake_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
PerCore_Skylake_Query(arg);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
Intel_DomainPowerLimit( MSR_DRAM_POWER_LIMIT,
PPn_POWER_LIMIT_LOCK_MASK,
PWR_DOMAIN(RAM) );
}
}
static void PerCore_Icelake_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
PerCore_Skylake_Query(arg);
Intel_Core_MicroArchControl(Core);
}
static void PerCore_Tigerlake_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
PerCore_Kaby_Lake_Query(arg);
Intel_Core_MicroArchControl(Core);
}
static void PerCore_Alderlake_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
PerCore_Kaby_Lake_Query(arg);
Intel_Core_MicroArchitecture(Core);
}
static void PerCore_Raptorlake_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
PerCore_Skylake_Query(arg);
Intel_Core_MicroArchitecture(Core);
}
static void PerCore_Meteorlake_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
PerCore_Kaby_Lake_Query(arg);
Intel_Core_MicroArchitecture(Core);
Intel_Ultra7_MicroArchitecture(Core);
}
static void PerCore_AMD_Family_0Fh_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
PerCore_AMD_Family_0Fh_PStates(Core); /* Alter P-States */
Compute_AMD_Family_0Fh_Boost(Core->Bind); /* Query P-States */
SystemRegisters(Core);
Dump_CPUID(Core);
Query_AMD_Family_0Fh_C1E(Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->TM_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->ODCM_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PowerMgmt_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->SpeedStep_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->TurboBoost_Mask,Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3A_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1A_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3U_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1U_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->CC6_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PC6_Mask,
PUBLIC(RO(Proc))->Service.Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->ARCH_CAP_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->BTC_NOBR_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->WDT_Mask, Core->Bind);
}
static void PerCore_AMD_Family_Same_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
SystemRegisters(Core);
AMD_Microcode(Core);
Dump_CPUID(Core);
if (PUBLIC(RO(Proc))->Registration.Experimental) {
Query_AMD_Family_0Fh_C1E(Core);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1E_Mask , Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->TM_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->ODCM_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PowerMgmt_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->SpeedStep_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->TurboBoost_Mask,Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3A_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1A_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3U_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1U_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->CC6_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PC6_Mask,
PUBLIC(RO(Proc))->Service.Core);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->ARCH_CAP_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->BTC_NOBR_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->WDT_Mask, Core->Bind);
}
static void PerCore_AMD_Family_10h_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
Compute_AMD_Family_10h_Boost(Core->Bind);
PerCore_AMD_Family_Same_Query(Core);
/*TODO(Errata 438) PerCore_AMD_CState_BAR(Core); */
AMD_Watchdog(Core);
}
static void PerCore_AMD_Family_11h_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
Compute_AMD_Family_11h_Boost(Core->Bind);
PerCore_AMD_Family_Same_Query(Core);
/*TODO(Unspecified) PerCore_AMD_CState_BAR(Core); */
/*TODO(Watchdog): D18F3x{40,44} MCA NB Registers - Hardware needed */
}
static void PerCore_AMD_Family_12h_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
Compute_AMD_Family_12h_Boost(Core->Bind);
PerCore_AMD_Family_Same_Query(Core);
PerCore_AMD_CState_BAR(Core);
AMD_Watchdog(Core);
}
static void PerCore_AMD_Family_14h_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
Compute_AMD_Family_14h_Boost(Core->Bind);
PerCore_AMD_Family_Same_Query(Core);
PerCore_AMD_CState_BAR(Core);
AMD_Watchdog(Core);
}
static void PerCore_AMD_Family_15h_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
Compute_AMD_Family_15h_Boost(Core->Bind);
PerCore_AMD_Family_Same_Query(Core);
PerCore_AMD_CState_BAR(Core);
switch (PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel) {
case 0x0:
case 0x1:
/*TODO(Watchdog): D18F3x{40,44} MCA NB Registers - Hardware needed */
break;
case 0x3:
case 0x6:
case 0x7:
if ((PUBLIC(RO(Proc))->Features.Std.EAX.Model >= 0x0)
&& (PUBLIC(RO(Proc))->Features.Std.EAX.Model <= 0xf))
{
AMD_Watchdog(Core);
}
break;
}
}
static void PerCore_AMD_Family_16h_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
Compute_AMD_Family_15h_Boost(Core->Bind);
PerCore_AMD_Family_Same_Query(Core);
PerCore_AMD_CState_BAR(Core);
AMD_Watchdog(Core);
}
static void PerCore_AMD_Family_17h_Query(void *arg)
{
CORE_RO *Core = (CORE_RO *) arg;
PM16 PM = {.value = 0};
int ToggleFeature = 0;
bool CPB_State;
/* Query the Min, Max, Target & Turbo P-States */
PerCore_Query_AMD_Zen_Features(Core);
CPB_State = Compute_AMD_Zen_Boost(Core->Bind);
/* Query and set features per Package */
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
TCTL_THERM_TRIP ThermTrip = {.value = 0};
TCTL_HTC HTC = {.value = 0};
unsigned int rx;
HSMP_ARG arg[8];
/* Query the HTC and THERM_TRIP features from SMUTHM */
Core_AMD_SMN_Read( HTC,
SMU_AMD_THM_TCTL_REGISTER_F17H + 0x4,
PRIVATE(OF(Zen)).Device.DF );
PUBLIC(RO(Proc))->ThermalPoint.Value[THM_HTC_LIMIT] = HTC.HTC_TMP_LIMIT;
PUBLIC(RO(Proc))->ThermalPoint.Value[THM_HTC_HYST] = HTC.HTC_HYST_LIMIT;
if (HTC.HTC_EN) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->TM2, Core->Bind);
BITSET(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.State,
THM_HTC_LIMIT);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TM2, Core->Bind);
BITCLR(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.State,
THM_HTC_LIMIT);
}
BITSET(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.Mask, THM_HTC_LIMIT);
BITSET(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.Kind, THM_HTC_LIMIT);
BITSET(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.State, THM_HTC_HYST);
BITSET(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.Mask, THM_HTC_HYST);
BITCLR(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.Kind, THM_HTC_HYST);
Core_AMD_SMN_Read( ThermTrip,
SMU_AMD_THM_TCTL_REGISTER_F17H + 0x8,
PRIVATE(OF(Zen)).Device.DF );
PUBLIC(RO(Proc))->ThermalPoint.Value[THM_TRIP_LIMIT] = \
ThermTrip.THERM_TP_LIMIT - 49;
if (ThermTrip.THERM_TP_EN) {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->TM1, Core->Bind);
BITSET(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.State,
THM_TRIP_LIMIT);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TM1, Core->Bind);
BITCLR(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.State,
THM_TRIP_LIMIT);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->TM_Mask, Core->Bind);
BITSET(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.Mask, THM_TRIP_LIMIT);
BITSET(BUS_LOCK, PUBLIC(RO(Proc))->ThermalPoint.Kind, THM_TRIP_LIMIT);
/* Count CPB and XFR ratios */
if (CPB_State == true)
{
PUBLIC(RO(Proc))->Features.XtraCOF = 2;
}
else {
PUBLIC(RO(Proc))->Features.XtraCOF = 0;
}
#define _lt (PWR_LIMIT_SIZE * PWR_DOMAIN(PKG))
if (Custom_TDP_Count > _lt) {
if (Custom_TDP_Offset[_lt] != 0)
{
signed int TDP_Limit;
TDP_Limit = PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Domain_Limit1;
TDP_Limit = TDP_Limit + Custom_TDP_Offset[_lt];
if (PUBLIC(RO(Proc))->Features.HSMP_Enable && (TDP_Limit >= 0))
{
RESET_ARRAY(arg, 8, 0, .value);
arg[0].value = 1000 * TDP_Limit;
rx = AMD_HSMP_Exec(HSMP_WR_PKG_PL1, arg);
if (rx == HSMP_RESULT_OK)
{
RESET_ARRAY(arg, 8, 0, .value);
rx = AMD_HSMP_Exec(HSMP_RD_PKG_PL1, arg);
}
if (rx == HSMP_RESULT_OK)
{
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Domain_Limit1 = arg[0].value / 1000;
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Enable_Limit1 = \
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Clamping1 = \
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].Unlock = \
PUBLIC(RO(Proc))->PowerThermal.Domain[
PWR_DOMAIN(PKG)
].PowerLimit.Domain_Limit1 > 0 ? 1 : 0;
}
else if (IS_HSMP_OOO(rx))
{
PUBLIC(RO(Proc))->Features.HSMP_Enable = 0;
}
}
}
}
#undef _lt
}
SystemRegisters(Core);
AMD_Microcode(Core);
Dump_CPUID(Core);
AMD_F17h_DCU_Technology(Core);
AMD_Mitigation_Mechanisms(Core);
/* Query the FCH for various registers */
AMD_FCH_PM_Read16(AMD_FCH_PM_CSTATE_EN, PM);
switch (C1E_Enable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PM.CStateEn.C1eToC3En = \
PM.CStateEn.C1eToC2En = !C1E_Enable;
ToggleFeature = 1;
break;
}
switch (C3U_Enable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PM.CStateEn.C1eToC3En = C3U_Enable;
ToggleFeature = 1;
break;
}
switch (C1U_Enable) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
PM.CStateEn.C1eToC2En = C1U_Enable;
ToggleFeature = 1;
break;
}
if (ToggleFeature == 1) {
AMD_FCH_PM_Write16(AMD_FCH_PM_CSTATE_EN, PM);
AMD_FCH_PM_Read16(AMD_FCH_PM_CSTATE_EN, PM);
}
if (PM.CStateEn.C1eToC2En)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->C1U, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->C1U, Core->Bind);
}
if (PM.CStateEn.C1eToC3En)
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->C3U, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->C3U, Core->Bind);
}
if (PM.CStateEn.C1eToC2En || PM.CStateEn.C1eToC3En)
{
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->C1E, Core->Bind);
} else {
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->C1E, Core->Bind);
}
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->ODCM_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->PowerMgmt_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->SpeedStep_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1E_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3A_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1A_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C3U_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->C1U_Mask , Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->SPEC_CTRL_Mask, Core->Bind);
BITSET_CC(BUS_LOCK, PUBLIC(RO(Proc))->ARCH_CAP_Mask , Core->Bind);
/* Per SMT, initialize with the saved thermal parameters */
if (Core->PowerThermal.Sensor == 0) {
Core->PowerThermal.Param = PUBLIC(RO(Proc))->PowerThermal.Param;
}
/* Collaborative Processor Performance Control */
if (PUBLIC(RO(Proc))->Features.HWP_Enable)
{
AMD_CPPC_CAP1 CPPC_Cap = {.value = 0};
AMD_CPPC_REQUEST CPPC_Req = {.value = 0};
RDMSR(Core->SystemRegister.HWCR, MSR_K7_HWCR);
RDMSR(CPPC_Cap, MSR_AMD_CPPC_CAP1);
Core->PowerThermal.HWP_Capabilities.Highest = \
CPPC_AMD_Zen_ScaleRatio(
Core, 255U, CPPC_Cap.Highest,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
Core->PowerThermal.HWP_Capabilities.Guaranteed = \
CPPC_AMD_Zen_ScaleRatio(
Core, 255U, CPPC_Cap.Nominal,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
Core->PowerThermal.HWP_Capabilities.Most_Efficient = \
CPPC_AMD_Zen_ScaleRatio(
Core, 255U, CPPC_Cap.LowNonlinear,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
Core->PowerThermal.HWP_Capabilities.Lowest = \
CPPC_AMD_Zen_ScaleRatio(
Core, 255U, CPPC_Cap.Lowest,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
RDMSR(CPPC_Req, MSR_AMD_CPPC_REQ);
if ((HWP_EPP >= 0) && (HWP_EPP <= 0xff))
{
CPPC_Req.Energy_Pref = HWP_EPP;
WRMSR(CPPC_Req, MSR_AMD_CPPC_REQ);
RDMSR(CPPC_Req, MSR_AMD_CPPC_REQ);
}
Core->PowerThermal.HWP_Request.Minimum_Perf = \
CPPC_AMD_Zen_ScaleRatio(
Core, 255U, CPPC_Req.Minimum_Perf,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
Core->PowerThermal.HWP_Request.Maximum_Perf = \
CPPC_AMD_Zen_ScaleRatio(
Core, 255U, CPPC_Req.Maximum_Perf,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
Core->PowerThermal.HWP_Request.Desired_Perf = \
CPPC_AMD_Zen_ScaleRatio(
Core, 255U, CPPC_Req.Desired_Perf,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
Core->PowerThermal.HWP_Request.Energy_Pref = CPPC_Req.Energy_Pref;
Core->Boost[BOOST(HWP_MIN)]=Core->PowerThermal.HWP_Request.Minimum_Perf;
Core->Boost[BOOST(HWP_MAX)]=Core->PowerThermal.HWP_Request.Maximum_Perf;
Core->Boost[BOOST(HWP_TGT)]=Core->PowerThermal.HWP_Request.Desired_Perf;
}
else if (PUBLIC(RO(Proc))->Features.ACPI_CPPC)
{
RDMSR(Core->SystemRegister.HWCR, MSR_K7_HWCR);
Core->PowerThermal.HWP_Capabilities.Highest = \
CPPC_AMD_Zen_ScaleRatio(Core,
PUBLIC(RO(Proc))->PowerThermal.ACPI_CPPC.Maximum,
Core->PowerThermal.ACPI_CPPC.Highest,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
Core->PowerThermal.HWP_Capabilities.Guaranteed = \
CPPC_AMD_Zen_ScaleRatio(Core,
PUBLIC(RO(Proc))->PowerThermal.ACPI_CPPC.Maximum,
Core->PowerThermal.ACPI_CPPC.Guaranteed,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0)
Core->PowerThermal.HWP_Capabilities.Most_Efficient = \
Core->PowerThermal.ACPI_CPPC.Efficient / PRECISION;
Core->PowerThermal.HWP_Capabilities.Lowest = \
Core->PowerThermal.ACPI_CPPC.Lowest / PRECISION;
Core->PowerThermal.HWP_Request.Minimum_Perf = \
Core->PowerThermal.ACPI_CPPC.Minimum / PRECISION;
#else
Core->PowerThermal.HWP_Capabilities.Most_Efficient = \
CPPC_AMD_Zen_ScaleRatio(Core,
PUBLIC(RO(Proc))->PowerThermal.ACPI_CPPC.Maximum,
Core->PowerThermal.ACPI_CPPC.Efficient,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
Core->PowerThermal.HWP_Capabilities.Lowest = \
CPPC_AMD_Zen_ScaleRatio(Core,
PUBLIC(RO(Proc))->PowerThermal.ACPI_CPPC.Maximum,
Core->PowerThermal.ACPI_CPPC.Lowest,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
Core->PowerThermal.HWP_Request.Minimum_Perf = \
CPPC_AMD_Zen_ScaleRatio(Core,
Core->PowerThermal.ACPI_CPPC.Highest,
Core->PowerThermal.ACPI_CPPC.Minimum,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
#endif
Core->PowerThermal.HWP_Request.Maximum_Perf = \
CPPC_AMD_Zen_ScaleRatio(Core,
Core->PowerThermal.ACPI_CPPC.Highest,
Core->PowerThermal.ACPI_CPPC.Maximum,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
Core->PowerThermal.HWP_Request.Desired_Perf = \
CPPC_AMD_Zen_ScaleRatio(Core,
Core->PowerThermal.ACPI_CPPC.Highest,
Core->PowerThermal.ACPI_CPPC.Desired,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
Core->PowerThermal.HWP_Request.Energy_Pref = \
Core->PowerThermal.ACPI_CPPC.Energy;
Core->Boost[BOOST(HWP_MIN)]=Core->PowerThermal.HWP_Request.Minimum_Perf;
Core->Boost[BOOST(HWP_MAX)]=Core->PowerThermal.HWP_Request.Maximum_Perf;
Core->Boost[BOOST(HWP_TGT)]=Core->PowerThermal.HWP_Request.Desired_Perf;
}
}
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 56)
static void Sys_DumpTask(SYSGATE_RO *SysGate)
{
SysGate->taskCount = 0;
}
#else /* KERNEL_VERSION(3, 10, 56) */
static void Sys_DumpTask(SYSGATE_RO *SysGate)
{ /* Source: /include/linux/sched.h */
struct task_struct *process, *thread;
int cnt = 0;
rcu_read_lock();
for_each_process_thread(process, thread) {
if (cnt < TASK_LIMIT) {
#if defined(CONFIG_SCHED_MUQSS) \
|| defined(CONFIG_SCHED_BMQ) \
|| defined(CONFIG_SCHED_PDS)
SysGate->taskList[cnt].runtime = tsk_seruntime(thread);
#else
SysGate->taskList[cnt].runtime = thread->se.sum_exec_runtime;
#endif /* CONFIG_SCHED_* */
SysGate->taskList[cnt].usertime = thread->utime;
SysGate->taskList[cnt].systime = thread->stime;
SysGate->taskList[cnt].pid = thread->pid;
SysGate->taskList[cnt].tgid = thread->tgid;
SysGate->taskList[cnt].ppid = thread->parent->pid;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0) \
|| (defined(RHEL_MAJOR) && (RHEL_MAJOR == 8) && (RHEL_MINOR >= 9))
SysGate->taskList[cnt].state = (short int) thread->__state;
#else
SysGate->taskList[cnt].state = (short int) thread->state;
#endif
#if defined(CONFIG_SCHED_ALT)
SysGate->taskList[cnt].wake_cpu = (short int) task_cpu(thread);
#elif defined(CONFIG_SCHED_BMQ) || defined(CONFIG_SCHED_PDS)
SysGate->taskList[cnt].wake_cpu = (short int) thread->cpu;
#else
SysGate->taskList[cnt].wake_cpu = (short int) thread->wake_cpu;
#endif /* CONFIG_SCHED_BMQ */
memcpy(SysGate->taskList[cnt].comm, thread->comm,TASK_COMM_LEN);
cnt++;
}
}
rcu_read_unlock();
SysGate->taskCount = cnt;
}
#endif /* KERNEL_VERSION(3, 10, 56) */
#define Sys_Tick(Pkg, ...) \
({ \
if (PUBLIC(OF(Gate)) != NULL) \
{ \
Pkg->tickStep--; \
if (!Pkg->tickStep) { \
Pkg->tickStep = Pkg->tickReset ; \
Sys_DumpTask( PUBLIC(OF(Gate)) ); \
__VA_ARGS__ \
} \
} \
})
static void InitTimer(void *Cycle_Function)
{
unsigned int cpu = smp_processor_id();
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, CREATED) == 0)
{
#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 15, 0)
hrtimer_init( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
CLOCK_MONOTONIC,
HRTIMER_MODE_REL_PINNED);
PRIVATE(OF(Core, AT(cpu)))->Join.Timer.function = Cycle_Function;
#else
hrtimer_setup( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
Cycle_Function,
CLOCK_MONOTONIC,
HRTIMER_MODE_REL_PINNED);
#endif
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, CREATED);
}
}
static void Controller_Init(void)
{
CLOCK sClock = {.Q = 0, .R = 0, .Hz = 0};
unsigned int cpu = PUBLIC(RO(Proc))->CPU.Count, ratio = 0;
Package_Init_Reset();
if (Arch[PUBLIC(RO(Proc))->ArchID].Query != NULL)
{
Arch[PUBLIC(RO(Proc))->ArchID].Query(PUBLIC(RO(Proc))->Service.Core);
}
ratio=PUBLIC(RO(Core,AT(PUBLIC(RO(Proc))->Service.Core)))->Boost[BOOST(MAX)];
if (Arch[PUBLIC(RO(Proc))->ArchID].BaseClock != NULL)
{
sClock = Arch[PUBLIC(RO(Proc))->ArchID].BaseClock(ratio);
}
if (sClock.Hz == 0) { /* Fallback to 100 MHz */
sClock.Q = 100;
sClock.R = 0;
sClock.Hz = 100000000LLU;
}
ratio = FixMissingRatioAndFrequency(ratio, &sClock);
if ((AutoClock & 0b01) || PUBLIC(RO(Proc))->Features.Std.ECX.Hyperv)
{
CLOCK vClock = {.Q = 0, .R =0, .Hz = 0};
COMPUTE_ARG Compute;
struct kmem_cache *hwCache = NULL;
/* Allocate Cache aligned resources. */
hwCache = kmem_cache_create( "CoreFreqCache",
STRUCT_SIZE, 0,
SLAB_HWCACHE_ALIGN, NULL);
if (hwCache != NULL)
{
do { /* from last AP to BSP */
cpu--;
if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS))
{
Compute.TSC[0] = kmem_cache_alloc(hwCache, GFP_ATOMIC);
if (Compute.TSC[0] != NULL)
{
Compute.TSC[1] = kmem_cache_alloc(hwCache, GFP_ATOMIC);
if (Compute.TSC[1] != NULL)
{
if (ratio != 0)
{
Compute.Clock.Q = ratio;
Compute.Clock.R = 0;
Compute.Clock.Hz = 0;
PUBLIC(RO(Core, AT(cpu)))->Clock = Compute_Clock(cpu,&Compute);
if (PUBLIC(RO(Proc))->Features.Std.ECX.Hyperv)
{
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = ratio;
}
}
else
{
vClock.Q = 0; vClock.R = 0; vClock.Hz = 0;
Compute.Clock.Q = sClock.Q;
Compute.Clock.R = 0;
Compute.Clock.Hz = 0;
vClock = Compute_Clock(cpu, &Compute);
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)] = vClock.Q;
}
/* Release memory resources. */
kmem_cache_free(hwCache, Compute.TSC[1]);
}
kmem_cache_free(hwCache, Compute.TSC[0]);
}
}
} while (cpu != 0) ;
kmem_cache_destroy(hwCache);
}
if ((ratio == 0) && (vClock.Hz != 0)) {
ratio = FixMissingRatioAndFrequency(vClock.Q, &sClock);
}
}
/* Launch a high resolution timer per online CPU. */
for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++)
{
if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS)) {
if (!PUBLIC(RO(Core, AT(cpu)))->Clock.Hz)
{
PUBLIC(RO(Core, AT(cpu)))->Clock = sClock;
}
if (Arch[PUBLIC(RO(Proc))->ArchID].Timer != NULL)
{
Arch[PUBLIC(RO(Proc))->ArchID].Timer(cpu);
}
}
}
}
static void Controller_Start(int wait)
{
if (Arch[PUBLIC(RO(Proc))->ArchID].Start != NULL)
{
unsigned int cpu;
for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++)
{
if ((BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, CREATED) == 1)
&& (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED) == 0))
{
smp_call_function_single(cpu,
Arch[PUBLIC(RO(Proc))->ArchID].Start,
NULL, wait);
}
}
}
}
static void Controller_Stop(int wait)
{
if (Arch[PUBLIC(RO(Proc))->ArchID].Stop != NULL)
{
unsigned int cpu;
for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++)
if ((BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, CREATED) == 1)
&& (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED) == 1))
{
smp_call_function_single(cpu,
Arch[PUBLIC(RO(Proc))->ArchID].Stop,
NULL, wait);
}
}
}
static void Controller_Exit(void)
{
unsigned int cpu;
if (Arch[PUBLIC(RO(Proc))->ArchID].Exit != NULL)
{
Arch[PUBLIC(RO(Proc))->ArchID].Exit();
}
for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++)
{
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, CREATED);
}
}
static void Intel_Core_Counters_Set(union SAVE_AREA_CORE *Save, CORE_RO *Core)
{
UNUSED(Core);
if (PUBLIC(RO(Proc))->Features.PerfMon.EAX.Version >= 2) {
CORE_GLOBAL_PERF_CONTROL Core_GlobalPerfControl = {.value = 0};
CORE_FIXED_PERF_CONTROL Core_FixedPerfControl = {.value = 0};
CORE_GLOBAL_PERF_STATUS Core_PerfOverflow = {.value = 0};
CORE_GLOBAL_PERF_OVF_CTRL Core_PerfOvfControl = {.value = 0};
RDMSR(Core_GlobalPerfControl, MSR_CORE_PERF_GLOBAL_CTRL);
Save->Core_GlobalPerfControl = Core_GlobalPerfControl;
Core_GlobalPerfControl.EN_FIXED_CTR0 = 1;
#if defined(MSR_CORE_PERF_UCC) && MSR_CORE_PERF_UCC == MSR_CORE_PERF_FIXED_CTR1
Core_GlobalPerfControl.EN_FIXED_CTR1 = 1;
#endif
#if defined(MSR_CORE_PERF_URC) && MSR_CORE_PERF_URC == MSR_CORE_PERF_FIXED_CTR2
Core_GlobalPerfControl.EN_FIXED_CTR2 = 1;
#endif
WRMSR(Core_GlobalPerfControl, MSR_CORE_PERF_GLOBAL_CTRL);
RDMSR(Core_FixedPerfControl, MSR_CORE_PERF_FIXED_CTR_CTRL);
Save->Core_FixedPerfControl = Core_FixedPerfControl;
Core_FixedPerfControl.EN0_OS = 1;
#if defined(MSR_CORE_PERF_UCC) && MSR_CORE_PERF_UCC == MSR_CORE_PERF_FIXED_CTR1
Core_FixedPerfControl.EN1_OS = 1;
#endif
#if defined(MSR_CORE_PERF_URC) && MSR_CORE_PERF_URC == MSR_CORE_PERF_FIXED_CTR2
Core_FixedPerfControl.EN2_OS = 1;
#endif
Core_FixedPerfControl.EN0_Usr = 1;
#if defined(MSR_CORE_PERF_UCC) && MSR_CORE_PERF_UCC == MSR_CORE_PERF_FIXED_CTR1
Core_FixedPerfControl.EN1_Usr = 1;
#endif
#if defined(MSR_CORE_PERF_URC) && MSR_CORE_PERF_URC == MSR_CORE_PERF_FIXED_CTR2
Core_FixedPerfControl.EN2_Usr = 1;
#endif
if (PUBLIC(RO(Proc))->Features.PerfMon.EAX.Version >= 3) {
if (!PUBLIC(RO(Proc))->Features.HTT_Enable) {
Core_FixedPerfControl.AnyThread_EN0 = 1;
#if defined(MSR_CORE_PERF_UCC) && MSR_CORE_PERF_UCC == MSR_CORE_PERF_FIXED_CTR1
Core_FixedPerfControl.AnyThread_EN1 = 1;
#endif
#if defined(MSR_CORE_PERF_URC) && MSR_CORE_PERF_URC == MSR_CORE_PERF_FIXED_CTR2
Core_FixedPerfControl.AnyThread_EN2 = 1;
#endif
} else {
/* Per Thread */
Core_FixedPerfControl.AnyThread_EN0 = 0;
#if defined(MSR_CORE_PERF_UCC) && MSR_CORE_PERF_UCC == MSR_CORE_PERF_FIXED_CTR1
Core_FixedPerfControl.AnyThread_EN1 = 0;
#endif
#if defined(MSR_CORE_PERF_URC) && MSR_CORE_PERF_URC == MSR_CORE_PERF_FIXED_CTR2
Core_FixedPerfControl.AnyThread_EN2 = 0;
#endif
}
}
WRMSR(Core_FixedPerfControl, MSR_CORE_PERF_FIXED_CTR_CTRL);
RDMSR(Core_PerfOverflow, MSR_CORE_PERF_GLOBAL_STATUS);
RDMSR(Core_PerfOvfControl, MSR_CORE_PERF_GLOBAL_OVF_CTRL);
if (Core_PerfOverflow.Overflow_CTR0)
Core_PerfOvfControl.Clear_Ovf_CTR0 = 1;
#if defined(MSR_CORE_PERF_UCC) && MSR_CORE_PERF_UCC == MSR_CORE_PERF_FIXED_CTR1
if (Core_PerfOverflow.Overflow_CTR1)
Core_PerfOvfControl.Clear_Ovf_CTR1 = 1;
#endif
#if defined(MSR_CORE_PERF_URC) && MSR_CORE_PERF_URC == MSR_CORE_PERF_FIXED_CTR2
if (Core_PerfOverflow.Overflow_CTR2)
Core_PerfOvfControl.Clear_Ovf_CTR2 = 1;
#endif
if (Core_PerfOverflow.Overflow_CTR0
| Core_PerfOverflow.Overflow_CTR1
| Core_PerfOverflow.Overflow_CTR2)
WRMSR(Core_PerfOvfControl, MSR_CORE_PERF_GLOBAL_OVF_CTRL);
}
}
#define AMD_Core_Counters_Set(Save, Core, PMC) \
({ \
if (PUBLIC(RO(Proc))->Features.PerfMon.EBX.InstrRetired == 0) \
{ \
RDMSR(Core->SystemRegister.HWCR, MSR_K7_HWCR); \
Save->Core_HardwareConfiguration = Core->SystemRegister.HWCR; \
Core->SystemRegister.HWCR.PMC.IRPerfEn = 1 ; \
WRMSR(Core->SystemRegister.HWCR, MSR_K7_HWCR); \
} \
})
#define AMD_Zen_PMC_L3_Set(Save, Core) \
({ \
const unsigned int bitwiseID = Core->T.ApicID \
& PUBLIC(RO(Proc))->Features.leaf80000008.ECX.ApicIdCoreIdSize; \
\
if ((PUBLIC(RO(Proc))->Features.ExtInfo.ECX.PerfLLC == 1) \
&& (Core->T.ThreadID == 0) && (bitwiseID == 0)) \
{ \
ZEN_L3_PERF_CTL Zen_L3_Cache_PerfControl = {.value = 0}; \
\
RDMSR(Zen_L3_Cache_PerfControl, MSR_AMD_F17H_L3_PERF_CTL); \
Save->Zen_L3_Cache_PerfControl=Zen_L3_Cache_PerfControl; \
Zen_L3_Cache_PerfControl.EventSelect = 0x90; \
Zen_L3_Cache_PerfControl.UnitMask = 0x00; \
Zen_L3_Cache_PerfControl.CounterEn = 1; \
Zen_L3_Cache_PerfControl.CoreID = 0; \
Zen_L3_Cache_PerfControl.EnAllSlices = 0; \
Zen_L3_Cache_PerfControl.EnAllCores = 0; \
Zen_L3_Cache_PerfControl.SliceMask = 0x0f; \
Zen_L3_Cache_PerfControl.ThreadMask = 0xff; \
WRMSR(Zen_L3_Cache_PerfControl, MSR_AMD_F17H_L3_PERF_CTL); \
} \
})
#define AMD_Zen_PMC_PERF_Set(Save, Core) \
({ \
const unsigned int bitwiseID = Core->T.ApicID \
& PUBLIC(RO(Proc))->Features.leaf80000008.ECX.ApicIdCoreIdSize; \
\
if ((Core->T.ThreadID == 0) && (bitwiseID == 0)) \
{ \
ZEN_PERF_CTL Zen_PerformanceControl = {.value = 0}; \
\
RDMSR(Zen_PerformanceControl, MSR_AMD_F17H_PERF_CTL); \
Save->Zen_PerformanceControl = Zen_PerformanceControl; \
Zen_PerformanceControl.EventSelect00 = 0xc1; \
Zen_PerformanceControl.UnitMask = 0x00; \
Zen_PerformanceControl.OsUserMode = 0x03; \
Zen_PerformanceControl.EdgeDetect = 0; \
Zen_PerformanceControl.APIC_Interrupt = 0; \
Zen_PerformanceControl.CounterEn = 1; \
Zen_PerformanceControl.InvCntMask = 0; \
Zen_PerformanceControl.CntMask = 0x00; \
Zen_PerformanceControl.EventSelect08 = 0x00; \
Zen_PerformanceControl.HostGuestOnly = 0x00; \
WRMSR(Zen_PerformanceControl, MSR_AMD_F17H_PERF_CTL); \
} \
})
#define AMD_Zen_PMC_UMC_Set(Save, Core) ({ /* NOP */ })
#define AMD_Zen_PMC_PCU_Set(Save, Core) ({ /* NOP */ })
#define _AMD_Zen_PMC_Set_(Save, Core, _PMC_) \
({ \
AMD_Zen_PMC_##_PMC_##_Set(Save, Core); \
})
#define AMD_Zen_PMC_Set(Save, Core, _PMC_) \
_AMD_Zen_PMC_Set_(Save, Core, _PMC_)
#define AMD_Zen_PMC_ARCH_PMC_Set(Save, Core) ({ /* NOP */ })
#define Uncore_Counters_Set(PMU) \
({ \
if (PUBLIC(RO(Proc))->Features.PerfMon.EAX.Version >= 3) \
{ \
UNCORE_GLOBAL_PERF_CONTROL Uncore_GlobalPerfControl; \
UNCORE_FIXED_PERF_CONTROL Uncore_FixedPerfControl; \
\
RDMSR( Uncore_GlobalPerfControl, \
MSR_##PMU##_UNCORE_PERF_GLOBAL_CTRL ); \
\
PRIVATE(OF(SaveArea)).Intel.Uncore_GlobalPerfControl = \
Uncore_GlobalPerfControl;\
\
Uncore_GlobalPerfControl.PMU.EN_FIXED_CTR0 = 1; \
WRMSR( Uncore_GlobalPerfControl, \
MSR_##PMU##_UNCORE_PERF_GLOBAL_CTRL ); \
\
RDMSR( Uncore_FixedPerfControl, \
MSR_##PMU##_UNCORE_PERF_FIXED_CTR_CTRL ); \
\
PRIVATE(OF(SaveArea)).Intel.Uncore_FixedPerfControl = \
Uncore_FixedPerfControl;\
\
Uncore_FixedPerfControl.PMU.EN_CTR0 = 1; \
WRMSR( Uncore_FixedPerfControl, \
MSR_##PMU##_UNCORE_PERF_FIXED_CTR_CTRL ); \
} \
})
#define Uncore_Counters_Ovf(PMU) \
({ \
if (PUBLIC(RO(Proc))->Features.PerfMon.EAX.Version >= 3) \
{ \
UNCORE_GLOBAL_PERF_STATUS Uncore_PerfOverflow = {.value = 0}; \
UNCORE_GLOBAL_PERF_OVF_CTRL Uncore_PerfOvfControl = {.value = 0};\
\
RDMSR(Uncore_PerfOverflow, MSR_##PMU##_UNCORE_PERF_GLOBAL_STATUS);\
\
if (Uncore_PerfOverflow.PMU.Overflow_CTR0) { \
/* MSR_UNCORE_PERF_GLOBAL_OVF_CTRL is a write-only register */ \
Uncore_PerfOvfControl.Clear_Ovf_CTR0 = 1; \
WRMSR(Uncore_PerfOvfControl, MSR_UNCORE_PERF_GLOBAL_OVF_CTRL); \
} \
} \
})
static void Intel_Core_Counters_Clear(union SAVE_AREA_CORE *Save, CORE_RO *Core)
{
UNUSED(Core);
if (PUBLIC(RO(Proc))->Features.PerfMon.EAX.Version >= 2)
{
WRMSR(Save->Core_FixedPerfControl, MSR_CORE_PERF_FIXED_CTR_CTRL);
WRMSR(Save->Core_GlobalPerfControl, MSR_CORE_PERF_GLOBAL_CTRL);
}
}
static void AMD_Core_Counters_Clear(union SAVE_AREA_CORE *Save, CORE_RO *Core)
{
UNUSED(Core);
if (PUBLIC(RO(Proc))->Features.PerfMon.EBX.InstrRetired == 0)
{
WRMSR(Save->Core_HardwareConfiguration, MSR_K7_HWCR);
}
}
#define AMD_Zen_PMC_L3_Clear(Save, Core) \
({ \
const unsigned int bitwiseID = Core->T.ApicID \
& PUBLIC(RO(Proc))->Features.leaf80000008.ECX.ApicIdCoreIdSize; \
\
if ((PUBLIC(RO(Proc))->Features.ExtInfo.ECX.PerfLLC == 1) \
&& (Core->T.ThreadID == 0) && (bitwiseID == 0)) \
{ \
WRMSR(Save->Zen_L3_Cache_PerfControl, MSR_AMD_F17H_L3_PERF_CTL);\
} \
})
#define AMD_Zen_PMC_PERF_Clear(Save, Core) \
({ \
const unsigned int bitwiseID = Core->T.ApicID \
& PUBLIC(RO(Proc))->Features.leaf80000008.ECX.ApicIdCoreIdSize; \
\
if ((Core->T.ThreadID == 0) && (bitwiseID == 0)) \
{ \
WRMSR(Save->Zen_PerformanceControl, MSR_AMD_F17H_PERF_CTL); \
} \
})
#define AMD_Zen_PMC_UMC_Clear(Save, Core) ({ /* NOP */ })
#define AMD_Zen_PMC_PCU_Clear(Save, Core) ({ /* NOP */ })
#define _AMD_Zen_PMC_Clear_(Save, Core, _PMC_) \
({ \
AMD_Zen_PMC_##_PMC_##_Clear(Save, Core); \
})
#define AMD_Zen_PMC_Clear(Save, Core, _PMC_) \
_AMD_Zen_PMC_Clear_(Save, Core, _PMC_)
#define AMD_Zen_PMC_ARCH_PMC_Clear(Save, Core) ({ /* NOP */ })
#define Uncore_Counters_Clear(PMU) \
({ \
if (PUBLIC(RO(Proc))->Features.PerfMon.EAX.Version >= 3) \
{ \
WRMSR( PRIVATE(OF(SaveArea)).Intel.Uncore_FixedPerfControl, \
MSR_##PMU##_UNCORE_PERF_FIXED_CTR_CTRL ); \
\
WRMSR( PRIVATE(OF(SaveArea)).Intel.Uncore_GlobalPerfControl, \
MSR_##PMU##_UNCORE_PERF_GLOBAL_CTRL ); \
} \
})
#define Counters_VirtualMachine(Core, T) \
({ \
if (!PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC) { \
RDTSC64(Core->Counter[T].TSC); \
} else { \
RDTSCP64(Core->Counter[T].TSC); \
} \
switch (PUBLIC(RO(Proc))->Features.Info.Hypervisor.CRC) { \
HCF_MSR: \
case CRC_VBOX: \
case CRC_KBOX: \
default: \
if (PUBLIC(RO(Proc))->Features.Power.ECX.HCF_Cap \
|| PUBLIC(RO(Proc))->Features.AdvPower.EDX.EffFrqRO) \
{ \
RDCOUNTER(Core->Counter[T].C0.UCC, MSR_CORE_PERF_UCC); \
RDCOUNTER(Core->Counter[T].C0.URC, MSR_CORE_PERF_URC); \
} \
break; \
case CRC_KVM: \
case CRC_VMWARE: \
case CRC_HYPERV: \
/* HV_PARTITION_PRIVILEGE_MASK: AccessVpRunTimeReg */ \
if (BITVAL(Core->CpuID[ \
CPUID_40000003_00000000_HYPERVISOR_FEATURES \
].reg[REG_CPUID_EAX], 0)) \
{/* HV_X64_MSR_VP_RUNTIME: vcpu runtime in 100ns units */ \
RDCOUNTER(Core->Counter[T].C0.URC, 0x40000010); \
\
Core->Counter[T].C0.URC = Core->Counter[T].C0.URC \
* PUBLIC(RO(Proc))->Features.Factory.Ratio * 10; \
\
Core->Counter[T].C0.UCC = Core->Counter[T].C0.URC; \
} else \
goto HCF_MSR; \
break; \
} \
/* Derive C1: */ \
Core->Counter[T].C1 = \
(Core->Counter[T].TSC > Core->Counter[T].C0.URC) ? \
Core->Counter[T].TSC - Core->Counter[T].C0.URC : 0; \
})
#define Counters_Generic(Core, T) \
({ \
RDTSC_COUNTERx2(Core->Counter[T].TSC, \
MSR_CORE_PERF_UCC, Core->Counter[T].C0.UCC, \
MSR_CORE_PERF_URC, Core->Counter[T].C0.URC); \
/* Derive C1: */ \
Core->Counter[T].C1 = \
(Core->Counter[T].TSC > Core->Counter[T].C0.URC) ? \
Core->Counter[T].TSC - Core->Counter[T].C0.URC \
: 0; \
})
#define Counters_Core2(Core, T) \
({ \
if (!PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC) \
{ \
RDTSC_COUNTERx3(Core->Counter[T].TSC, \
MSR_CORE_PERF_UCC, Core->Counter[T].C0.UCC, \
MSR_CORE_PERF_URC, Core->Counter[T].C0.URC, \
MSR_CORE_PERF_FIXED_CTR0,Core->Counter[T].INST);\
} else { \
RDTSCP_COUNTERx3(Core->Counter[T].TSC, \
MSR_CORE_PERF_UCC, Core->Counter[T].C0.UCC, \
MSR_CORE_PERF_URC, Core->Counter[T].C0.URC, \
MSR_CORE_PERF_FIXED_CTR0,Core->Counter[T].INST);\
} \
/* Derive C1: */ \
Core->Counter[T].C1 = \
(Core->Counter[T].TSC > Core->Counter[T].C0.URC) ? \
Core->Counter[T].TSC - Core->Counter[T].C0.URC \
: 0; \
})
#define Counters_SLM(Core, T) \
({ \
RDTSC_COUNTERx6(Core->Counter[T].TSC, \
MSR_CORE_PERF_UCC, Core->Counter[T].C0.UCC, \
MSR_CORE_PERF_URC, Core->Counter[T].C0.URC, \
MSR_CORE_PERF_FIXED_CTR0,Core->Counter[T].INST, \
MSR_CORE_C1_RESIDENCY, Core->Counter[T].C1, \
MSR_CORE_C3_RESIDENCY, Core->Counter[T].C3, \
MSR_CORE_C6_RESIDENCY, Core->Counter[T].C6); \
})
#define Counters_Goldmont(Core, T) Counters_SLM(Core, T)
#define SMT_Counters_Nehalem(Core, T) \
({ \
register unsigned long long Cx; \
\
RDTSCP_COUNTERx5(Core->Counter[T].TSC, \
MSR_CORE_PERF_UCC, Core->Counter[T].C0.UCC, \
MSR_CORE_PERF_URC, Core->Counter[T].C0.URC, \
MSR_CORE_C3_RESIDENCY,Core->Counter[T].C3, \
MSR_CORE_C6_RESIDENCY,Core->Counter[T].C6, \
MSR_CORE_PERF_FIXED_CTR0,Core->Counter[T].INST);\
/* Derive C1: */ \
Cx = Core->Counter[T].C6 \
+ Core->Counter[T].C3 \
+ Core->Counter[T].C0.URC; \
\
Core->Counter[T].C1 = \
(Core->Counter[T].TSC > Cx) ? \
Core->Counter[T].TSC - Cx \
: 0; \
})
#define SMT_Counters_SandyBridge(Core, T) \
({ \
register unsigned long long Cx; \
\
RDTSCP_COUNTERx6(Core->Counter[T].TSC, \
MSR_CORE_PERF_UCC, Core->Counter[T].C0.UCC, \
MSR_CORE_PERF_URC, Core->Counter[T].C0.URC, \
MSR_CORE_C3_RESIDENCY,Core->Counter[T].C3, \
MSR_CORE_C6_RESIDENCY,Core->Counter[T].C6, \
MSR_CORE_C7_RESIDENCY,Core->Counter[T].C7, \
MSR_CORE_PERF_FIXED_CTR0,Core->Counter[T].INST);\
/* Derive C1: */ \
Cx = Core->Counter[T].C7 \
+ Core->Counter[T].C6 \
+ Core->Counter[T].C3 \
+ Core->Counter[T].C0.URC; \
\
Core->Counter[T].C1 = \
(Core->Counter[T].TSC > Cx) ? \
Core->Counter[T].TSC - Cx \
: 0; \
})
#define SMT_Counters_Alderlake_Ecore(Core, T) \
({ \
RDTSC_COUNTERx7(Core->Counter[T].TSC, \
MSR_CORE_PERF_UCC, Core->Counter[T].C0.UCC, \
MSR_CORE_PERF_URC, Core->Counter[T].C0.URC, \
MSR_CORE_C1_RESIDENCY, Core->Counter[T].C1, \
MSR_CORE_C3_RESIDENCY, Core->Counter[T].C3, \
MSR_CORE_C6_RESIDENCY, Core->Counter[T].C6, \
MSR_CORE_C7_RESIDENCY, Core->Counter[T].C7, \
MSR_CORE_PERF_FIXED_CTR0,Core->Counter[T].INST);\
})
#define SMT_Counters_Alderlake_Pcore(Core, T) \
({ \
RDTSC_COUNTERx7(Core->Counter[T].TSC, \
MSR_CORE_PERF_UCC, Core->Counter[T].C0.UCC, \
MSR_CORE_PERF_URC, Core->Counter[T].C0.URC, \
MSR_CORE_C1_RESIDENCY, Core->Counter[T].C1, \
MSR_CORE_C3_RESIDENCY, Core->Counter[T].C3, \
MSR_CORE_C6_RESIDENCY, Core->Counter[T].C6, \
MSR_CORE_C7_RESIDENCY, Core->Counter[T].C7, \
MSR_CORE_PERF_FIXED_CTR0,Core->Counter[T].INST);\
})
#define SMT_Counters_Arrowlake(Core, T) \
({ \
RDTSC_COUNTERx7(Core->Counter[T].TSC, \
MSR_CORE_PERF_UCC, Core->Counter[T].C0.UCC, \
MSR_CORE_PERF_URC, Core->Counter[T].C0.URC, \
MSR_CORE_C1_RESIDENCY, Core->Counter[T].C1, \
MSR_CORE_C3_RESIDENCY, Core->Counter[T].C3, \
MSR_CORE_C6_RESIDENCY, Core->Counter[T].C6, \
MSR_CORE_C7_RESIDENCY, Core->Counter[T].C7, \
MSR_CORE_PERF_FIXED_CTR0,Core->Counter[T].INST);\
})
#define SMT_Counters_AMD_Family_17h(Core, T) \
({ \
RDTSCP_COUNTERx3(Core->Counter[T].TSC, \
MSR_CORE_PERF_UCC, Core->Counter[T].C0.UCC, \
MSR_CORE_PERF_URC, Core->Counter[T].C0.URC, \
MSR_AMD_F17H_IRPERF, Core->Counter[T].INST); \
\
if(PUBLIC(RO(Proc))->Registration.Driver.CPUidle == REGISTRATION_ENABLE)\
{ /* Read Virtual PMC and cumulative store: */ \
Atomic_Read_VPMC(LOCKLESS, Core->Counter[T].C1, Core->VPMC.C1); \
Atomic_Read_VPMC(LOCKLESS, Core->Counter[T].C3, Core->VPMC.C2); \
Atomic_Add_VPMC (LOCKLESS, Core->Counter[T].C3, Core->VPMC.C3); \
Atomic_Read_VPMC(LOCKLESS, Core->Counter[T].C6, Core->VPMC.C4); \
Atomic_Add_VPMC (LOCKLESS, Core->Counter[T].C6, Core->VPMC.C5); \
Atomic_Add_VPMC (LOCKLESS, Core->Counter[T].C6, Core->VPMC.C6); \
} \
else \
{ \
Core->Counter[T].C1 = \
(Core->Counter[T].TSC > Core->Counter[T].C0.URC) ? \
Core->Counter[T].TSC - Core->Counter[T].C0.URC \
: 0; \
} \
})
#define Mark_OVH(Core) \
({ \
RDTSCP64(Core->Overhead.TSC); \
})
#define Core_OVH(Core) \
({ \
Core->Delta.TSC -= (Core->Counter[1].TSC - Core->Overhead.TSC); \
})
#define Delta_TSC(Core) \
({ \
Core->Delta.TSC = Core->Counter[1].TSC \
- Core->Counter[0].TSC; \
})
#define Delta_TSC_OVH(Core) \
({ \
Delta_TSC(Core); \
\
if (AutoClock & 0b10) { \
Core_OVH(Core); \
\
REL_BCLK(Core->Clock, \
Core->Boost[BOOST(MAX)], \
Core->Delta.TSC, \
PUBLIC(RO(Proc))->SleepInterval); \
} \
})
#define Delta_C0(Core) \
({ /* Absolute Delta of Unhalted (Core & Ref) C0 Counter. */ \
Core->Delta.C0.UCC = \
(Core->Counter[0].C0.UCC > \
Core->Counter[1].C0.UCC) ? \
Core->Counter[0].C0.UCC \
- Core->Counter[1].C0.UCC \
: Core->Counter[1].C0.UCC \
- Core->Counter[0].C0.UCC; \
\
Core->Delta.C0.URC = Core->Counter[1].C0.URC \
- Core->Counter[0].C0.URC; \
})
#define Delta_C1(Core) \
({ \
Core->Delta.C1 = \
(Core->Counter[0].C1 > \
Core->Counter[1].C1) ? \
Core->Counter[0].C1 \
- Core->Counter[1].C1 \
: Core->Counter[1].C1 \
- Core->Counter[0].C1; \
})
#define Delta_C3(Core) \
({ \
Core->Delta.C3 = Core->Counter[1].C3 \
- Core->Counter[0].C3; \
})
#define Delta_C6(Core) \
({ \
Core->Delta.C6 = Core->Counter[1].C6 \
- Core->Counter[0].C6; \
})
#define Delta_C7(Core) \
({ \
Core->Delta.C7 = Core->Counter[1].C7 \
- Core->Counter[0].C7; \
})
#define Delta_INST(Core) \
({ /* Delta of Retired Instructions */ \
Core->Delta.INST = Core->Counter[1].INST \
- Core->Counter[0].INST; \
})
#define PKG_Counters_VirtualMachine(Core, T) \
({ \
PUBLIC(RO(Proc))->Counter[T].PCLK = Core->Counter[T].TSC; \
})
#define PKG_Counters_Generic(Core, T) \
({ \
PUBLIC(RO(Proc))->Counter[T].PCLK = Core->Counter[T].TSC; \
})
#define PKG_Counters_SLM(Core, T) \
({ \
RDTSCP_COUNTERx5(PUBLIC(RO(Proc))->Counter[T].PCLK , \
MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02, \
MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03, \
MSR_ATOM_PKG_C4_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC04, \
MSR_ATOM_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06, \
MSR_ATOM_MC6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].MC6); \
})
#define PKG_Counters_Nehalem(Core, T) \
({ \
RDTSCP_COUNTERx4(PUBLIC(RO(Proc))->Counter[T].PCLK , \
MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,\
MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,\
MSR_PKG_C7_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC07,\
MSR_NHM_UNCORE_PERF_FIXED_CTR0 , \
PUBLIC(RO(Proc))->Counter[T].Uncore.FC0); \
})
#define PKG_Counters_SandyBridge(Core, T) \
({ \
RDTSCP_COUNTERx5(PUBLIC(RO(Proc))->Counter[T].PCLK , \
MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02,\
MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,\
MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,\
MSR_PKG_C7_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC07,\
MSR_SNB_UNCORE_PERF_FIXED_CTR0 , \
PUBLIC(RO(Proc))->Counter[T].Uncore.FC0); \
})
#define PKG_COUNTERS_SANDYBRIDGE_EP(Core, T) \
({ \
RDTSCP_COUNTERx5(PUBLIC(RO(Proc))->Counter[T].PCLK , \
MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02,\
MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,\
MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,\
MSR_PKG_C7_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC07,\
MSR_SNB_EP_UNCORE_PERF_FIXED_CTR0, \
PUBLIC(RO(Proc))->Counter[T].Uncore.FC0); \
})
static void PKG_Counters_SandyBridge_EP(CORE_RO *Core, unsigned int T)
{
UNUSED(Core);
PKG_COUNTERS_SANDYBRIDGE_EP(Core, T);
}
#define PKG_COUNTERS_IVYBRIDGE_EP(Core, T) \
({ \
RDTSCP_COUNTERx5(PUBLIC(RO(Proc))->Counter[T].PCLK , \
MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02,\
MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,\
MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,\
MSR_PKG_C7_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC07,\
MSR_SNB_EP_UNCORE_PERF_FIXED_CTR0, \
PUBLIC(RO(Proc))->Counter[T].Uncore.FC0); \
})
static void PKG_Counters_IvyBridge_EP(CORE_RO *Core, unsigned int T)
{
UNUSED(Core);
PKG_COUNTERS_IVYBRIDGE_EP(Core, T);
}
#define PKG_Counters_Haswell_EP(Core, T) \
({ \
RDTSCP_COUNTERx5(PUBLIC(RO(Proc))->Counter[T].PCLK , \
MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02,\
MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,\
MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,\
MSR_PKG_C7_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC07,\
MSR_HSW_EP_UNCORE_PERF_FIXED_CTR0, \
PUBLIC(RO(Proc))->Counter[T].Uncore.FC0); \
})
#define PKG_Counters_Haswell_ULT(Core, T) \
({ \
RDTSCP_COUNTERx7(PUBLIC(RO(Proc))->Counter[T].PCLK , \
MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02,\
MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,\
MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,\
MSR_PKG_C7_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC07,\
MSR_PKG_C8_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC08,\
MSR_PKG_C9_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC09,\
MSR_PKG_C10_RESIDENCY,PUBLIC(RO(Proc))->Counter[T].PC10);\
\
RDCOUNTER( PUBLIC(RO(Proc))->Counter[T].Uncore.FC0, \
MSR_SNB_UNCORE_PERF_FIXED_CTR0 ); \
})
#define PKG_Counters_Goldmont(Core, T) \
({ \
RDTSCP_COUNTERx4(PUBLIC(RO(Proc))->Counter[T].PCLK , \
MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02,\
MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,\
MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,\
MSR_PKG_C10_RESIDENCY,PUBLIC(RO(Proc))->Counter[T].PC10);\
})
#define PKG_Counters_Skylake(Core, T) \
({ \
RDTSCP_COUNTERx5(PUBLIC(RO(Proc))->Counter[T].PCLK , \
MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02,\
MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,\
MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,\
MSR_PKG_C7_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC07,\
MSR_SKL_UNCORE_PERF_FIXED_CTR0 , \
PUBLIC(RO(Proc))->Counter[T].Uncore.FC0); \
})
#define PKG_Counters_Skylake_X(Core, T) \
({ \
RDTSCP_COUNTERx4(PUBLIC(RO(Proc))->Counter[T].PCLK , \
MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02,\
MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03,\
MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06,\
MSR_PKG_C7_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC07);\
})
#define PKG_Counters_Alderlake_Ecore(Core, T) \
({ \
RDCOUNTER(PUBLIC(RO(Proc))->Counter[T].PC04, MSR_ATOM_PKG_C4_RESIDENCY);\
RDCOUNTER(PUBLIC(RO(Proc))->Counter[T].PC10, MSR_PKG_C10_RESIDENCY);\
RDCOUNTER(PUBLIC(RO(Proc))->Counter[T].MC6, MSR_ATOM_MC6_RESIDENCY);\
})
#define PKG_Counters_Alderlake_Pcore(Core, T) \
({ \
RDTSCP_COUNTERx7(PUBLIC(RO(Proc))->Counter[T].PCLK , \
MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02, \
MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03, \
MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06, \
MSR_PKG_C7_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC07, \
MSR_PKG_C8_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC08, \
MSR_PKG_C9_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC09, \
MSR_ADL_UNCORE_PERF_FIXED_CTR0 , \
PUBLIC(RO(Proc))->Counter[T].Uncore.FC0);\
})
#define PKG_Counters_Arrowlake(Core, T) \
({ /* P-core */ \
RDTSCP_COUNTERx6(PUBLIC(RO(Proc))->Counter[T].PCLK , \
MSR_PKG_C2_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC02, \
MSR_PKG_C3_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC03, \
MSR_PKG_C6_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC06, \
MSR_PKG_C7_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC07, \
MSR_PKG_C8_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC08, \
MSR_PKG_C9_RESIDENCY, PUBLIC(RO(Proc))->Counter[T].PC09); \
/*TODO(Uncore) MSR_ARL_UNCORE_PERF_FIXED_CTR0 , \
PUBLIC(RO(Proc))->Counter[T].Uncore.FC0 */ \
/* E-core */ \
RDCOUNTER(PUBLIC(RO(Proc))->Counter[T].PC04, MSR_ATOM_PKG_C4_RESIDENCY);\
RDCOUNTER(PUBLIC(RO(Proc))->Counter[T].PC10, MSR_PKG_C10_RESIDENCY);\
RDCOUNTER(PUBLIC(RO(Proc))->Counter[T].MC6, MSR_ATOM_MC6_RESIDENCY);\
})
/*
* Counter | Description
* --------|------------
* 0x5828 | Cycle Sum of All Active Cores
* 0x5830 | Cycle Sum of Any Active Core
* 0x5838 | Cycle Sum of Active Graphics
* 0x5840 | Cycle Sum of Overlapping Active GT and Core
* 0x5848 | Cycle Sum of Any Active GT Slice
* 0x5850 | Cycle Sum of All Active GT Slice
* 0x5858 | Cycle Sum of Any GT Media Engine
* 0x5860 | Ratio Sum of Any Active Core
* 0x5868 | Ratio Sum of Active GT
* 0x5870 | Ratio Sum of Active GT Slice
*/
#define Pkg_Intel_PMC_PCU_Set(...) \
({ \
if (GetMemoryBAR(0, 0, 0, 0, 0x48, 64, 0x10000, 0, \
&PRIVATE(OF(PCU)).HB, &PRIVATE(OF(PCU)).BAR) == 0) \
{ \
PRIVATE(OF(PCU)).ADDR = __VA_ARGS__; \
} else { \
PRIVATE(OF(PCU)).ADDR = 0x0; \
} \
})
#define Pkg_Intel_PMC_L3_Set(...) ({ /* NOP */ })
#define Pkg_Intel_PMC_PERF_Set(...) ({ /* NOP */ })
#define Pkg_Intel_PMC_UMC_Set(...) ({ /* NOP */ })
#define _Pkg_Intel_PMC_Set_(_PMC_ , ...) \
({ \
Pkg_Intel_PMC_##_PMC_##_Set(__VA_ARGS__); \
})
#define Pkg_Intel_PMC_Set(_PMC_, ...) \
_Pkg_Intel_PMC_Set_(_PMC_ , __VA_ARGS__)
#define Pkg_Intel_PMC_ARCH_PMC_Set(_PMC_, ...) ({ /* NOP */ })
#define Pkg_Intel_PMC_L3_Clear() ({ /* NOP */ })
#define Pkg_Intel_PMC_PERF_Clear() ({ /* NOP */ })
#define Pkg_Intel_PMC_UMC_Clear() ({ /* NOP */ })
#define Pkg_Intel_PMC_PCU_Clear() \
({ \
PutMemoryBAR(&PRIVATE(OF(PCU)).HB, &PRIVATE(OF(PCU)).BAR); \
PRIVATE(OF(PCU)).ADDR = 0x0; \
})
#define _Pkg_Intel_PMC_Clear_(_PMC_) \
({ \
Pkg_Intel_PMC_##_PMC_##_Clear(); \
})
#define Pkg_Intel_PMC_Clear(_PMC_) \
_Pkg_Intel_PMC_Clear_(_PMC_)
#define Pkg_Intel_PMC_ARCH_PMC_Clear() ({ /* NOP */ })
#define Pkg_Intel_PMC_PCU_Counters(Core, _T) \
({ \
if (PRIVATE(OF(PCU)).BAR != NULL) { \
if (PRIVATE(OF(PCU)).ADDR != 0x0) \
{ \
PUBLIC(RO(Proc))->Counter[_T].MC6 = \
readq(PRIVATE(OF(PCU)).BAR + PRIVATE(OF(PCU)).ADDR); \
} \
switch (PUBLIC(RO(Proc))->ArchID) { \
case Tigerlake: \
case Tigerlake_U: \
TGL_SA(PRIVATE(OF(PCU)).BAR); \
break; \
case Rocketlake: \
case Rocketlake_U: \
RKL_SA(PRIVATE(OF(PCU)).BAR); \
break; \
case Alderlake_S: \
case Alderlake_H: \
case Alderlake_N: \
case Raptorlake: \
case Raptorlake_P: \
case Raptorlake_S: \
ADL_SA(PRIVATE(OF(PCU)).BAR); \
break; \
case Meteorlake_M: \
case Meteorlake_N: \
case Meteorlake_S: \
case LunarLake: \
case ArrowLake: \
case ArrowLake_H: \
case ArrowLake_U: \
case PantherLake: \
MTL_SA(PRIVATE(OF(PCU)).BAR); \
break; \
} \
} \
})
#define Pkg_Intel_PMC_L3_Counters(Core, _T) ({ /* NOP */ })
#define Pkg_Intel_PMC_PERF_Counters(Core, _T) ({ /* NOP */ })
#define Pkg_Intel_PMC_UMC_Counters(Core, _T) ({ /* NOP */ })
#define _Pkg_Intel_PMC_Counters_(Core, _T, _PMC_) \
({ \
Pkg_Intel_PMC_##_PMC_##_Counters(Core, _T); \
})
#define Pkg_Intel_PMC_Counters(Core, _T, _PMC_) \
_Pkg_Intel_PMC_Counters_(Core, _T, _PMC_)
#define Pkg_Intel_PMC_ARCH_PMC_Counters(Core, _T) ({ /* NOP */ })
#define AMD_Zen_PMC_L3_Closure(Core, _T) \
({ \
PUBLIC(RO(Proc))->Delta.CTR[CCX] = \
PUBLIC(RO(Proc))->Counter[1].CTR[CCX] \
- PUBLIC(RO(Proc))->Counter[0].CTR[CCX];\
\
PUBLIC(RO(Proc))->Counter[0].CTR[CCX] = \
PUBLIC(RO(Proc))->Counter[1].CTR[CCX]; \
})
#define AMD_Zen_PMC_L3_Counters(Core, _T, ...) \
({ /* Read the Cache L3 performance counter per Complex */ \
const unsigned int bitwiseID = Core->T.ApicID \
& PUBLIC(RO(Proc))->Features.leaf80000008.ECX.ApicIdCoreIdSize; \
\
if ((PUBLIC(RO(Proc))->Features.ExtInfo.ECX.PerfLLC == 1) \
&& (Core->T.PackageID == 0) && (bitwiseID == 0)) \
{ \
const unsigned short CCX = Core->T.Cluster.CCX & 0b111; \
\
RDCOUNTER( PUBLIC(RO(Proc))->Counter[_T].CTR[CCX], \
MSR_AMD_F17H_L3_PERF_CTR ); \
\
PUBLIC(RO(Proc))->Counter[_T].CTR[CCX] &= 0xffffffffffff; \
/* Closure statement */ \
__VA_ARGS__; \
} \
})
#define AMD_Zen_PMC_PERF_Closure(Core, _T) \
({ \
PUBLIC(RO(Proc))->Delta.CTR[CCX] = \
PUBLIC(RO(Proc))->Counter[1].CTR[CCX] \
- PUBLIC(RO(Proc))->Counter[0].CTR[CCX];\
\
PUBLIC(RO(Proc))->Counter[0].CTR[CCX] = \
PUBLIC(RO(Proc))->Counter[1].CTR[CCX]; \
})
#define AMD_Zen_PMC_PERF_Counters(Core, _T, ...) \
({ \
const unsigned int bitwiseID = Core->T.ApicID \
& PUBLIC(RO(Proc))->Features.leaf80000008.ECX.ApicIdCoreIdSize; \
\
if ((Core->T.PackageID == 0) && (bitwiseID == 0)) \
{ \
const unsigned short CCX = Core->T.Cluster.CCX & 0b111; \
\
RDCOUNTER( PUBLIC(RO(Proc))->Counter[_T].CTR[CCX], \
MSR_AMD_F17H_PERF_CTR ); \
\
PUBLIC(RO(Proc))->Counter[_T].CTR[CCX] &= 0xffffffffffff; \
/* Closure statement */ \
__VA_ARGS__; \
} \
})
#define AMD_Zen_PMC_UMC_Closure(Core, _T) ({ /* NOP */ })
#define AMD_Zen_PMC_UMC_Counters(Core, _T , ...) \
({ \
/* Closure statement */ \
__VA_ARGS__; \
})
#define AMD_Zen_PMC_PCU_Closure(Core, _T) ({ /* NOP */ })
#define AMD_Zen_PMC_PCU_Counters(Core, _T , ...) \
({ \
/* Closure statement */ \
__VA_ARGS__; \
})
#define _AMD_Zen_PMC_Closure_(Core, _T, _PMC_) \
AMD_Zen_PMC_##_PMC_##_Closure(Core, _T)
#define AMD_Zen_PMC_Closure(Core, _T, _PMC_) \
_AMD_Zen_PMC_Closure_(Core, _T, _PMC_)
#define _AMD_Zen_PMC_Counters_(Core, _T, _PMC_, ...) \
({ \
AMD_Zen_PMC_##_PMC_##_Counters(Core, _T, __VA_ARGS__); \
})
#define AMD_Zen_PMC_Counters(Core, _T, _PMC_, ...) \
_AMD_Zen_PMC_Counters_(Core, _T, _PMC_, __VA_ARGS__)
#define AMD_Zen_PMC_ARCH_PMC_Counters(Core, _T, ...) ({ /* NOP */ })
#define Pkg_AMD_Zen_PMC_L3_Set(Core) ({ /* NOP */ })
#define Pkg_AMD_Zen_PMC_PERF_Set(Core) ({ /* NOP */ })
#define Pkg_AMD_Zen_PMC_UMC_Set(Core) \
({ \
ZEN_UMC_PERF_CTL_CLK Zen_UMC_PerfCtlClk = { \
.value=0, .GlblReset=1, .GlblMonEn=1, .CtrClkEn=1 \
}; \
\
unsigned short mc; \
for (mc = 0; mc < PUBLIC(RO(Proc))->Uncore.CtrlCount; mc++) \
{ \
unsigned short cha; \
for (cha=0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount;cha++)\
{ \
unsigned int slot; \
for (slot=0;slot < PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount;slot++)\
{ \
ZEN_UMC_PERF_CTL Zen_UMC_PerfControl = { \
.value = 0, .EventSelect = 0x13 + slot, .CounterEn = 1 \
}; \
\
Core_AMD_SMN_Write( Zen_UMC_PerfControl, \
SMU_AMD_ZEN_UMC_PERF_CTL(cha, slot), \
PRIVATE(OF(Zen)).Device.DF ); \
} \
Core_AMD_SMN_Write( Zen_UMC_PerfCtlClk, \
SMU_AMD_UMC_PERF_CTL_CLK(cha), \
PRIVATE(OF(Zen)).Device.DF ); \
} \
} \
})
#define Pkg_AMD_Zen_PMC_PCU_Set(Core) ({ /* NOP */ })
#define _Pkg_AMD_Zen_PMC_Set_(Core, _PMC_) \
({ \
Pkg_AMD_Zen_PMC_##_PMC_##_Set(Core); \
})
#define Pkg_AMD_Zen_PMC_Set(Core, _PMC_) \
_Pkg_AMD_Zen_PMC_Set_(Core, _PMC_)
#define Pkg_AMD_Zen_PMC_ARCH_PMC_Set(Core) ({ /* NOP */ })
#define Pkg_AMD_Zen_PMC_L3_Clear(Core) ({ /* NOP */ })
#define Pkg_AMD_Zen_PMC_PERF_Clear(Core) ({ /* NOP */ })
#define Pkg_AMD_Zen_PMC_UMC_Clear(Core) \
({ \
ZEN_UMC_PERF_CTL_CLK Zen_UMC_PerfCtlClk = { .value = 0 }; \
\
unsigned short mc; \
for (mc = 0; mc < PUBLIC(RO(Proc))->Uncore.CtrlCount; mc++) \
{ \
unsigned short cha; \
for (cha=0; cha < PUBLIC(RO(Proc))->Uncore.MC[mc].ChannelCount;cha++)\
{ \
unsigned int slot; \
for (slot=0;slot < PUBLIC(RO(Proc))->Uncore.MC[mc].SlotCount;slot++)\
{ \
ZEN_UMC_PERF_CTL Zen_UMC_PerfControl = { .value = 0 }; \
\
Core_AMD_SMN_Write( Zen_UMC_PerfControl, \
SMU_AMD_ZEN_UMC_PERF_CTL(cha, slot), \
PRIVATE(OF(Zen)).Device.DF ); \
} \
Core_AMD_SMN_Write( Zen_UMC_PerfCtlClk, \
SMU_AMD_UMC_PERF_CTL_CLK(cha), \
PRIVATE(OF(Zen)).Device.DF ); \
} \
} \
})
#define Pkg_AMD_Zen_PMC_PCU_Clear(Core) ({ /* NOP */ })
#define _Pkg_AMD_Zen_PMC_Clear_(Core, _PMC_) \
({ \
Pkg_AMD_Zen_PMC_##_PMC_##_Clear(Core); \
})
#define Pkg_AMD_Zen_PMC_Clear(Core, _PMC_) \
_Pkg_AMD_Zen_PMC_Clear_(Core, _PMC_)
#define Pkg_AMD_Zen_PMC_ARCH_PMC_Clear(Core) ({ /* NOP */ })
#define Pkg_OVH(Pkg, Core) \
({ \
Pkg->Delta.PCLK -= (Pkg->Counter[1].PCLK - Core->Overhead.TSC); \
})
#define Delta_PTSC(Pkg) \
({ \
Pkg->Delta.PCLK = Pkg->Counter[1].PCLK \
- Pkg->Counter[0].PCLK; \
})
#define Delta_PTSC_OVH(Pkg, Core) \
({ \
Delta_PTSC(Pkg); \
\
if (AutoClock & 0b10) \
Pkg_OVH(Pkg, Core); \
})
#define Delta_PC02(Pkg) \
({ \
Pkg->Delta.PC02 = Pkg->Counter[1].PC02 \
- Pkg->Counter[0].PC02; \
})
#define Delta_PC03(Pkg) \
({ \
Pkg->Delta.PC03 = Pkg->Counter[1].PC03 \
- Pkg->Counter[0].PC03; \
})
#define Delta_PC04(Pkg) \
({ \
Pkg->Delta.PC04 = Pkg->Counter[1].PC04 \
- Pkg->Counter[0].PC04; \
})
#define Delta_PC06(Pkg) \
({ \
Pkg->Delta.PC06 = Pkg->Counter[1].PC06 \
- Pkg->Counter[0].PC06; \
})
#define Delta_PC07(Pkg) \
({ \
Pkg->Delta.PC07 = Pkg->Counter[1].PC07 \
- Pkg->Counter[0].PC07; \
})
#define Delta_PC08(Pkg) \
({ \
Pkg->Delta.PC08 = Pkg->Counter[1].PC08 \
- Pkg->Counter[0].PC08; \
})
#define Delta_PC09(Pkg) \
({ \
Pkg->Delta.PC09 = Pkg->Counter[1].PC09 \
- Pkg->Counter[0].PC09; \
})
#define Delta_PC10(Pkg) \
({ \
Pkg->Delta.PC10 = Pkg->Counter[1].PC10 \
- Pkg->Counter[0].PC10; \
})
#define Delta_MC6(Pkg) \
({ \
Pkg->Delta.MC6 = Pkg->Counter[1].MC6 \
- Pkg->Counter[0].MC6; \
})
#define Delta_UNCORE_FC0(Pkg) \
({ \
Pkg->Delta.Uncore.FC0 = \
(Pkg->Counter[0].Uncore.FC0 > \
Pkg->Counter[1].Uncore.FC0) ? \
Pkg->Counter[0].Uncore.FC0 \
- Pkg->Counter[1].Uncore.FC0 \
: Pkg->Counter[1].Uncore.FC0 \
- Pkg->Counter[0].Uncore.FC0; \
})
#define Save_TSC(Core) \
({ /* Save Time Stamp Counter. */ \
Core->Counter[0].TSC = Core->Counter[1].TSC; \
})
#define Save_C0(Core) \
({ /* Save the Unhalted Core & Reference Counter */ \
Core->Counter[0].C0.UCC = Core->Counter[1].C0.UCC; \
Core->Counter[0].C0.URC = Core->Counter[1].C0.URC; \
})
#define Save_C1(Core) \
({ \
Core->Counter[0].C1 = Core->Counter[1].C1; \
})
#define Save_C3(Core) \
({ \
Core->Counter[0].C3 = Core->Counter[1].C3; \
})
#define Save_C6(Core) \
({ \
Core->Counter[0].C6 = Core->Counter[1].C6; \
})
#define Save_C7(Core) \
({ \
Core->Counter[0].C7 = Core->Counter[1].C7; \
})
#define Save_INST(Core) \
({ /* Save the Instructions counter. */ \
Core->Counter[0].INST = Core->Counter[1].INST; \
})
#define Save_PTSC(Pkg) \
({ \
Pkg->Counter[0].PCLK = Pkg->Counter[1].PCLK; \
})
#define Save_PC02(Pkg) \
({ \
Pkg->Counter[0].PC02 = Pkg->Counter[1].PC02; \
})
#define Save_PC03(Pkg) \
({ \
Pkg->Counter[0].PC03 = Pkg->Counter[1].PC03; \
})
#define Save_PC04(Pkg) \
({ \
Pkg->Counter[0].PC04 = Pkg->Counter[1].PC04; \
})
#define Save_PC06(Pkg) \
({ \
Pkg->Counter[0].PC06 = Pkg->Counter[1].PC06; \
})
#define Save_PC07(Pkg) \
({ \
Pkg->Counter[0].PC07 = Pkg->Counter[1].PC07; \
})
#define Save_PC08(Pkg) \
({ \
Pkg->Counter[0].PC08 = Pkg->Counter[1].PC08; \
})
#define Save_PC09(Pkg) \
({ \
Pkg->Counter[0].PC09 = Pkg->Counter[1].PC09; \
})
#define Save_PC10(Pkg) \
({ \
Pkg->Counter[0].PC10 = Pkg->Counter[1].PC10; \
})
#define Save_MC6(Pkg) \
({ \
Pkg->Counter[0].MC6 = Pkg->Counter[1].MC6; \
})
#define Save_UNCORE_FC0(Pkg) \
({ \
Pkg->Counter[0].Uncore.FC0 = Pkg->Counter[1].Uncore.FC0; \
})
#define PWR_ACCU_Goldmont(Pkg, T) \
({ \
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(PKG)], \
MSR_PKG_ENERGY_STATUS); \
\
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(CORES)], \
MSR_PP0_ENERGY_STATUS); \
\
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(UNCORE)], \
MSR_PP1_ENERGY_STATUS); \
\
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(RAM)], \
MSR_DRAM_ENERGY_STATUS);\
})
#define PWR_ACCU_SandyBridge(Pkg, T) \
({ \
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(PKG)], \
MSR_PKG_ENERGY_STATUS); \
\
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(CORES)], \
MSR_PP0_ENERGY_STATUS); \
\
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(UNCORE)], \
MSR_PP1_ENERGY_STATUS); \
})
#define PWR_ACCU_SandyBridge_EP(Pkg, T) \
({ \
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(PKG)], \
MSR_PKG_ENERGY_STATUS); \
\
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(CORES)], \
MSR_PP0_ENERGY_STATUS); \
\
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(RAM)], \
MSR_DRAM_ENERGY_STATUS);\
})
#define PWR_ACCU_Skylake(Pkg, T) \
({ \
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(PKG)], \
MSR_PKG_ENERGY_STATUS); \
\
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(CORES)], \
MSR_PP0_ENERGY_STATUS); \
\
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(UNCORE)], \
MSR_PP1_ENERGY_STATUS); \
\
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(RAM)], \
MSR_DRAM_ENERGY_STATUS);\
})
static void Power_ACCU_SKL_DEFAULT(PROC_RO *Pkg, unsigned int T)
{
PWR_ACCU_Skylake(Pkg, T);
}
#define PWR_ACCU_SKL_PLATFORM(Pkg, T) \
({ \
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(PKG)], \
MSR_PKG_ENERGY_STATUS); \
\
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(CORES)], \
MSR_PP0_ENERGY_STATUS); \
\
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(PLATFORM)], \
MSR_PLATFORM_ENERGY_STATUS); \
\
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(RAM)], \
MSR_DRAM_ENERGY_STATUS);\
})
static void Power_ACCU_SKL_PLATFORM(PROC_RO *Pkg, unsigned int T)
{
PWR_ACCU_SKL_PLATFORM(Pkg, T);
}
#define PWR_ACCU_Alderlake(Pkg, T) \
({ \
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(PKG)], \
MSR_PKG_ENERGY_STATUS); \
\
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(CORES)], \
MSR_PP0_ENERGY_STATUS); \
\
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(UNCORE)], \
MSR_PP1_ENERGY_STATUS); \
\
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(RAM)], \
MSR_DRAM_ENERGY_STATUS);\
\
RDCOUNTER(Pkg->Counter[T].Power.ACCU[PWR_DOMAIN(PLATFORM)], \
MSR_PLATFORM_ENERGY_STATUS); \
})
#define PWR_ACCU_Arrowlake PWR_ACCU_Alderlake
#define Delta_PWR_ACCU(Pkg, PwrDomain) \
({ \
PUBLIC(RW(Pkg))->Delta.Power.ACCU[PWR_DOMAIN(PwrDomain)] = \
(PUBLIC(RO(Pkg))->Counter[1].Power.ACCU[PWR_DOMAIN(PwrDomain)] \
- PUBLIC(RO(Pkg))->Counter[0].Power.ACCU[PWR_DOMAIN(PwrDomain)])\
& 0x7fffffffU; \
})
#define Save_PWR_ACCU(Pkg, PwrDomain) \
({ \
Pkg->Counter[0].Power.ACCU[PWR_DOMAIN(PwrDomain)] = \
Pkg->Counter[1].Power.ACCU[PWR_DOMAIN(PwrDomain)]; \
})
#define Pkg_AMD_Zen_PMC_L3_Closure(Pkg, Core, _T) \
({ \
Delta_PTSC_OVH(Pkg, Core); \
Save_PTSC(Pkg); \
})
#define Pkg_AMD_Zen_PMC_ARCH_PMC_Closure(Pkg, Core, _T) \
({ \
Delta_PTSC_OVH(Pkg, Core); \
Save_PTSC(Pkg); \
})
#define Pkg_AMD_Zen_PMC_L3_Counters(Pkg, Core, T, ...) \
({ \
Pkg->Counter[T].PCLK = Core->Counter[T].TSC; \
/* Closure statement */ \
__VA_ARGS__; \
\
RDCOUNTER(Pkg->Counter[T].Uncore.FC0, MSR_AMD_F17H_DF_PERF_CTR);\
})
#define Pkg_AMD_Zen_PMC_PERF_Closure(Pkg, Core, _T) \
({ \
Delta_PTSC_OVH(Pkg, Core); \
Save_PTSC(Pkg); \
})
#define Pkg_AMD_Zen_PMC_PERF_Counters(Pkg, Core, _T, ...) \
({ \
Pkg->Counter[_T].PCLK = Core->Counter[_T].TSC; \
/* Closure statement */ \
__VA_ARGS__; \
\
RDCOUNTER(Pkg->Counter[_T].Uncore.FC0, MSR_AMD_F17H_DF_PERF_CTR);\
})
#define Pkg_AMD_Zen_PMC_UMC_Closure(Pkg, Core, _T) \
({ \
/* Delta with previous counter */ \
Pkg->Delta.CTR[idx] = Pkg->Counter[1].CTR[idx] \
- Pkg->Counter[0].CTR[idx]; \
/* Save current counter */ \
Pkg->Counter[0].CTR[idx] = Pkg->Counter[1].CTR[idx]; \
})
#define Pkg_AMD_Zen_PMC_UMC_Counters(Pkg, Core, _T, ...) \
({ \
unsigned short umc; \
bool Got_Mem_Clock = false; \
\
for (umc = 0; umc < Pkg->Uncore.CtrlCount; umc++) \
{ \
unsigned short cha; \
for (cha = 0; cha < Pkg->Uncore.MC[umc].ChannelCount; cha++) \
{ \
const unsigned short idx = MC_VECTOR_TO_SCALAR(umc, cha); \
\
union { \
unsigned long long ctr48; \
struct { \
struct { \
unsigned int value; \
} low32; \
struct { \
unsigned int value; \
} high16; \
}; \
} data; \
\
/* 48-bits UMC counter */ \
Core_AMD_SMN_Read(data.low32, SMU_AMD_ZEN_UMC_PERF_CLK_LOW(cha),\
PRIVATE(OF(Zen)).Device.DF); \
\
Core_AMD_SMN_Read(data.high16, SMU_AMD_ZEN_UMC_PERF_CLK_HIGH(cha),\
PRIVATE(OF(Zen)).Device.DF); \
data.high16.value &= 0xffff; \
\
Pkg->Counter[_T].CTR[idx] = data.ctr48 ; \
/* Closure statement */ \
__VA_ARGS__; \
/* Update memory clock frequency (Hz) */ \
if (Got_Mem_Clock == false) \
{ \
switch(Pkg->Uncore.MC[umc].Channel[cha].AMD17h.CONFIG.BurstLength) {\
case 0x0: /* BL2 */ \
case 0x1: /* BL4 */ \
case 0x2: /* BL8 */ \
Pkg->Counter[_T].PCLK = Core->Clock.Hz \
* Pkg->Uncore.MC[umc].Channel[cha].AMD17h.MISC.DDR4.MEMCLK;\
\
Pkg->Counter[_T].PCLK = DIV_ROUND_CLOSEST(Pkg->Counter[_T].PCLK, 3);\
/* Apply the memory clock divisor */ \
Pkg->Counter[_T].PCLK >>= \
!Pkg->Uncore.MC[umc].Channel[cha].AMD17h.DbgMisc.UCLK_Divisor;\
\
break; \
case 0x3: /* BL16 */ \
Pkg->Counter[_T].PCLK = Core->Clock.Hz * 10LLU \
* Pkg->Uncore.MC[umc].Channel[cha].AMD17h.MISC.DDR5.MEMCLK;\
break; \
} \
Pkg->Delta.PCLK = Pkg->Counter[_T].PCLK; \
\
Got_Mem_Clock = true; \
} \
} \
} \
/* Data Fabric Counter */ \
RDCOUNTER(Pkg->Counter[_T].Uncore.FC0, MSR_AMD_F17H_DF_PERF_CTR);\
})
#define Pkg_AMD_Zen_PMC_PCU_Closure(Pkg, Core, _T) \
({ \
Delta_PTSC_OVH(Pkg, Core); \
Save_PTSC(Pkg); \
})
#define Pkg_AMD_Zen_PMC_PCU_Counters(Pkg, Core, _T, ...) \
({ \
Pkg->Counter[_T].PCLK = Core->Counter[_T].TSC; \
/* Closure statement */ \
__VA_ARGS__; \
})
#define _Pkg_AMD_Zen_PMC_Closure_(Pkg, Core, _T, _PMC_) \
Pkg_AMD_Zen_PMC_##_PMC_##_Closure(Pkg, Core, _T)
#define Pkg_AMD_Zen_PMC_Closure(Pkg, Core, _T, _PMC_) \
_Pkg_AMD_Zen_PMC_Closure_(Pkg, Core, _T, _PMC_)
#define _Pkg_AMD_Zen_PMC_Counters_(Pkg, Core , _T, _PMC_, ...) \
({ \
Pkg_AMD_Zen_PMC_##_PMC_##_Counters(Pkg, Core, _T, __VA_ARGS__); \
})
#define Pkg_AMD_Zen_PMC_Counters(Pkg, Core, _T, _PMC_, ...) \
_Pkg_AMD_Zen_PMC_Counters_(Pkg, Core, _T, _PMC_, __VA_ARGS__)
#define Pkg_AMD_Zen_PMC_ARCH_PMC_Counters(Pkg, Core, _T, ...) \
({ \
Pkg->Counter[_T].PCLK = Core->Counter[_T].TSC; \
/* Closure statement */ \
__VA_ARGS__; \
})
static void Core_Intel_Temp(CORE_RO *Core)
{
THERM_STATUS ThermStatus = {.value = 0};
RDMSR(ThermStatus, MSR_IA32_THERM_STATUS); /*All Intel families.*/
Core->PowerThermal.Sensor = ThermStatus.DTS;
Core->PowerThermal.Events[eLOG] = \
((Bit64) ThermStatus.Thermal_Log << LSHIFT_THERMAL_LOG)
| ((Bit64) ThermStatus.PROCHOT_Log << LSHIFT_PROCHOT_LOG)
| ((Bit64) ThermStatus.CriticalTemp_Log << LSHIFT_CRITIC_LOG)
| ((Bit64) ThermStatus.Threshold1_Log << LSHIFT_THOLD1_LOG)
| ((Bit64) ThermStatus.Threshold2_Log << LSHIFT_THOLD2_LOG)
| ((Bit64) ThermStatus.PwrLimit_Log << LSHIFT_POWER_LIMIT)
| ((Bit64) ThermStatus.CurLimit_Log << LSHIFT_CURRENT_LIMIT)
| ((Bit64) ThermStatus.XDomLimit_Log << LSHIFT_CROSS_DOMAIN);
Core->PowerThermal.Events[eSTS] = \
((Bit64) ThermStatus.Thermal_Status << LSHIFT_THERMAL_STS)
| ((Bit64) ThermStatus.PROCHOT_Event << LSHIFT_PROCHOT_STS)
| ((Bit64) ThermStatus.CriticalTemp << LSHIFT_CRITIC_TMP)
| ((Bit64) ThermStatus.Threshold1 << LSHIFT_THOLD1_STS)
| ((Bit64) ThermStatus.Threshold2 << LSHIFT_THOLD2_STS);
}
#define Pkg_Intel_Temp(Pkg) \
({ \
if (Pkg->Features.Power.EAX.PTM) \
{ \
THERM_STATUS ThermStatus = {.value = 0}; \
RDMSR(ThermStatus, MSR_IA32_PACKAGE_THERM_STATUS); \
Pkg->PowerThermal.Sensor = ThermStatus.DTS; \
\
Pkg->PowerThermal.Events[eLOG] = \
((Bit64) ThermStatus.Thermal_Log << LSHIFT_THERMAL_LOG) \
| ((Bit64) ThermStatus.PROCHOT_Log << LSHIFT_PROCHOT_LOG) \
| ((Bit64) ThermStatus.CriticalTemp_Log << LSHIFT_CRITIC_LOG) \
| ((Bit64) ThermStatus.Threshold1_Log << LSHIFT_THOLD1_LOG) \
| ((Bit64) ThermStatus.Threshold2_Log << LSHIFT_THOLD2_LOG) \
| ((Bit64) ThermStatus.PwrLimit_Log << LSHIFT_POWER_LIMIT); \
\
Pkg->PowerThermal.Events[eSTS] = \
((Bit64) ThermStatus.Thermal_Status << LSHIFT_THERMAL_STS) \
| ((Bit64) ThermStatus.PROCHOT_Event << LSHIFT_PROCHOT_STS) \
| ((Bit64) ThermStatus.CriticalTemp << LSHIFT_CRITIC_TMP) \
| ((Bit64) ThermStatus.Threshold1 << LSHIFT_THOLD1_STS) \
| ((Bit64) ThermStatus.Threshold2 << LSHIFT_THOLD2_STS); \
} \
})
static void Monitor_CorePerfLimitReasons(PROC_RO *Pkg)
{
CORE_PERF_LIMIT_REASONS limit = {.value = 0};
RDMSR(limit, MSR_SKL_CORE_PERF_LIMIT_REASONS);
Pkg->PowerThermal.Events[eLOG] |= (
((Bit64) limit.PROCHOT_Log << LSHIFT_CORE_HOT_LOG)
| ((Bit64) limit.Thermal_Log << LSHIFT_CORE_THM_LOG)
| ((Bit64) limit.Residency_Log << LSHIFT_CORE_RES_LOG)
| ((Bit64) limit.AvgThmLimitLog << LSHIFT_CORE_AVG_LOG)
| ((Bit64) limit.VR_ThmAlertLog << LSHIFT_CORE_VRT_LOG)
| ((Bit64) limit.VR_TDC_Log << LSHIFT_CORE_TDC_LOG)
| ((Bit64) limit.PL1_Log << LSHIFT_CORE_PL1_LOG)
| ((Bit64) limit.PL2_Log << LSHIFT_CORE_PL2_LOG)
| ((Bit64) limit.EDP_Log << LSHIFT_CORE_EDP_LOG)
| ((Bit64) limit.TurboLimitLog << LSHIFT_CORE_BST_LOG)
| ((Bit64) limit.TurboAttenLog << LSHIFT_CORE_ATT_LOG)
| ((Bit64) limit.TVB_Log << LSHIFT_CORE_TVB_LOG)
);
Pkg->PowerThermal.Events[eSTS] |= (
((Bit64) limit.Thermal_Status << LSHIFT_CORE_THM_STS)
| ((Bit64) limit.PROCHOT_Event << LSHIFT_CORE_HOT_STS)
| ((Bit64) limit.Residency_Sts << LSHIFT_CORE_RES_STS)
| ((Bit64) limit.AvgThmLimit << LSHIFT_CORE_AVG_STS)
| ((Bit64) limit.VR_ThmAlert << LSHIFT_CORE_VRT_STS)
| ((Bit64) limit.VR_TDC_Status << LSHIFT_CORE_TDC_STS)
| ((Bit64) limit.PL1_Status << LSHIFT_CORE_PL1_STS)
| ((Bit64) limit.PL2_Status << LSHIFT_CORE_PL2_STS)
| ((Bit64) limit.EDP_Status << LSHIFT_CORE_EDP_STS)
| ((Bit64) limit.TurboLimit << LSHIFT_CORE_BST_STS)
| ((Bit64) limit.TurboAtten << LSHIFT_CORE_ATT_STS)
| ((Bit64) limit.TVB_Status << LSHIFT_CORE_TVB_STS)
);
}
static void Monitor_GraphicsPerfLimitReasons(PROC_RO *Pkg)
{
GRAPHICS_PERF_LIMIT_REASONS limit = {.value = 0};
RDMSR(limit, MSR_GRAPHICS_PERF_LIMIT_REASONS);
Pkg->PowerThermal.Events[eLOG] |= (
((Bit64) limit.PROCHOT_Log << LSHIFT_GFX_HOT_LOG)
| ((Bit64) limit.Thermal_Log << LSHIFT_GFX_THM_LOG)
| ((Bit64) limit.AvgThmLimitLog << LSHIFT_GFX_AVG_LOG)
| ((Bit64) limit.VR_ThmAlertLog << LSHIFT_GFX_VRT_LOG)
| ((Bit64) limit.VR_TDC_Log << LSHIFT_GFX_TDC_LOG)
| ((Bit64) limit.PL1_Log << LSHIFT_GFX_PL1_LOG)
| ((Bit64) limit.PL2_Log << LSHIFT_GFX_PL2_LOG)
| ((Bit64) limit.EDP_Log << LSHIFT_GFX_EDP_LOG)
| ((Bit64) limit.InefficiencyLog<< LSHIFT_GFX_EFF_LOG)
);
Pkg->PowerThermal.Events[eSTS] |= (
((Bit64) limit.Thermal_Status << LSHIFT_GFX_THM_STS)
| ((Bit64) limit.PROCHOT_Event << LSHIFT_GFX_HOT_STS)
| ((Bit64) limit.AvgThmLimit << LSHIFT_GFX_AVG_STS)
| ((Bit64) limit.VR_ThmAlert << LSHIFT_GFX_VRT_STS)
| ((Bit64) limit.VR_TDC_Status << LSHIFT_GFX_TDC_STS)
| ((Bit64) limit.PL1_Status << LSHIFT_GFX_PL1_STS)
| ((Bit64) limit.PL2_Status << LSHIFT_GFX_PL2_STS)
| ((Bit64) limit.EDP_Status << LSHIFT_GFX_EDP_STS)
| ((Bit64) limit.Inefficiency << LSHIFT_GFX_EFF_STS)
);
}
static void Monitor_RingPerfLimitReasons(PROC_RO *Pkg)
{
RING_PERF_LIMIT_REASONS limit = {.value = 0};
RDMSR(limit, MSR_RING_PERF_LIMIT_REASONS);
Pkg->PowerThermal.Events[eLOG] |= (
((Bit64) limit.PROCHOT_Log << LSHIFT_RING_HOT_LOG)
| ((Bit64) limit.Thermal_Log << LSHIFT_RING_THM_LOG)
| ((Bit64) limit.AvgThmLimitLog << LSHIFT_RING_AVG_LOG)
| ((Bit64) limit.VR_ThmAlertLog << LSHIFT_RING_VRT_LOG)
| ((Bit64) limit.VR_TDC_Log << LSHIFT_RING_TDC_LOG)
| ((Bit64) limit.PL1_Log << LSHIFT_RING_PL1_LOG)
| ((Bit64) limit.PL2_Log << LSHIFT_RING_PL2_LOG)
| ((Bit64) limit.EDP_Log << LSHIFT_RING_EDP_LOG)
);
Pkg->PowerThermal.Events[eSTS] |= (
((Bit64) limit.Thermal_Status << LSHIFT_RING_THM_STS)
| ((Bit64) limit.PROCHOT_Event << LSHIFT_RING_HOT_STS)
| ((Bit64) limit.AvgThmLimit << LSHIFT_RING_AVG_STS)
| ((Bit64) limit.VR_ThmAlert << LSHIFT_RING_VRT_STS)
| ((Bit64) limit.VR_TDC_Status << LSHIFT_RING_TDC_STS)
| ((Bit64) limit.PL1_Status << LSHIFT_RING_PL1_STS)
| ((Bit64) limit.PL2_Status << LSHIFT_RING_PL2_STS)
| ((Bit64) limit.EDP_Status << LSHIFT_RING_EDP_STS)
);
}
static void Core_AMD_Family_0Fh_Temp(CORE_RO *Core)
{
if (PUBLIC(RO(Proc))->Features.AdvPower.EDX.TTP == 1) {
THERMTRIP_STATUS ThermTrip = {0};
RDPCI(ThermTrip, PCI_AMD_THERMTRIP_STATUS);
/* Select Core to read sensor from: */
ThermTrip.SensorCoreSelect = Core->Bind;
WRPCI(ThermTrip, PCI_AMD_THERMTRIP_STATUS);
RDPCI(ThermTrip, PCI_AMD_THERMTRIP_STATUS);
/* Formula is " CurTmp - (TjOffset * 2) - 49 " */
Core->PowerThermal.Param.Target = ThermTrip.TjOffset;
Core->PowerThermal.Sensor = ThermTrip.CurrentTemp;
Core->PowerThermal.Events[eSTS] = \
(Bit64)ThermTrip.SensorTrip << LSHIFT_THERMAL_STS;
}
}
static void Core_AMD_Family_15h_00h_Temp(CORE_RO *Core)
{
TCTL_REGISTER TctlSensor = {.value = 0};
RDPCI(TctlSensor, PCI_AMD_TEMPERATURE_TCTL);
Core->PowerThermal.Sensor = TctlSensor.CurTmp;
if (PUBLIC(RO(Proc))->Features.AdvPower.EDX.TTP == 1) {
THERMTRIP_STATUS ThermTrip = {0};
RDPCI(ThermTrip, PCI_AMD_THERMTRIP_STATUS);
Core->PowerThermal.Events[eSTS] = \
(Bit64)ThermTrip.SensorTrip << LSHIFT_THERMAL_STS;
}
}
/* Bulldozer/Excavator through SMU interface */
static void Core_AMD_Family_15_60h_Temp(CORE_RO *Core)
{
TCTL_REGISTER TctlSensor = {.value = 0};
PCI_AMD_SMN_Read( TctlSensor,
SMU_AMD_THM_TCTL_REGISTER_F15H,
SMU_AMD_INDEX_REGISTER_F15H,
SMU_AMD_DATA_REGISTER_F15H );
Core->PowerThermal.Sensor = TctlSensor.CurTmp;
if (PUBLIC(RO(Proc))->Features.AdvPower.EDX.TTP == 1) {
THERMTRIP_STATUS ThermTrip = {0};
WRPCI( SMU_AMD_THM_TRIP_REGISTER_F15H,
SMU_AMD_INDEX_REGISTER_F15H);
RDPCI(ThermTrip, SMU_AMD_DATA_REGISTER_F15H);
Core->PowerThermal.Events[eSTS] = \
(Bit64)ThermTrip.SensorTrip << LSHIFT_THERMAL_STS;
}
}
static void Core_AMD_Family_15h_Temp(CORE_RO *Core)
{
switch (PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel) {
case 0x6:
case 0x7:
if ((PUBLIC(RO(Proc))->Features.Std.EAX.Model >= 0x0)
&& (PUBLIC(RO(Proc))->Features.Std.EAX.Model <= 0xf)) {
Core_AMD_Family_15_60h_Temp(Core);
} else {
Core_AMD_Family_15h_00h_Temp(Core);
}
break;
default:
Core_AMD_Family_15h_00h_Temp(Core);
break;
}
}
static void Core_AMD_Family_17h_ThermTrip(CORE_RO *Core)
{
TCTL_THERM_TRIP ThermTrip = {.value = 0};
Core_AMD_SMN_Read( ThermTrip,
SMU_AMD_THM_TCTL_REGISTER_F17H + 0x8,
PRIVATE(OF(Zen)).Device.DF );
if (ThermTrip.THERM_TP_EN) {
Core->PowerThermal.Events[eSTS] = \
(Bit64)ThermTrip.THERM_TP << LSHIFT_THERMAL_STS;
}
Core->PowerThermal.Events[eSTS] |= \
(Bit64)ThermTrip.CTF_THRESHOLD << LSHIFT_CRITIC_TMP;
}
static void Core_AMD_Zen_Filter_Temp( CORE_RO *Core, unsigned int CurTmp,
bool scaleRangeSel )
{
/* Register: SMU::THM::THM_TCON_CUR_TMP - Bit 19: CUR_TEMP_RANGE_SEL
0 = Report on 0C to 225C scale range.
1 = Report on -49C to 206C scale range.
*/
Core->PowerThermal.Sensor = CurTmp;
if (scaleRangeSel == true) {
if (Core->PowerThermal.Sensor >= (49 << 3))
{
Core->PowerThermal.Param.Offset[THERMAL_OFFSET_P1] = 49;
if (Core->PowerThermal.Sensor > (255 << 3)) {
Core->PowerThermal.Events[eLOG] |= EVENT_THOLD2_LOG;
}
} else {
Core->PowerThermal.Param.Offset[THERMAL_OFFSET_P1] = 0;
Core->PowerThermal.Events[eLOG] |= EVENT_THOLD1_LOG;
}
} else {
Core->PowerThermal.Param.Offset[THERMAL_OFFSET_P1] = 0;
if (Core->PowerThermal.Sensor > (225 << 3)) {
Core->PowerThermal.Events[eLOG] |= EVENT_THOLD2_LOG;
}
}
}
static void CTL_AMD_Family_17h_Temp(CORE_RO *Core)
{
TCTL_REGISTER TctlSensor = {.value = 0};
Core_AMD_SMN_Read( TctlSensor,
SMU_AMD_THM_TCTL_REGISTER_F17H,
PRIVATE(OF(Zen)).Device.DF );
Core_AMD_Zen_Filter_Temp( Core, TctlSensor.CurTmp,
TctlSensor.CurTempRangeSel == 1 );
Core_AMD_Family_17h_ThermTrip(Core);
}
static void CCD_AMD_Family_17h_Zen2_Temp(CORE_RO *Core)
{
TCCD_REGISTER TccdSensor = {.value = 0};
Core_AMD_SMN_Read( TccdSensor,
(SMU_AMD_THM_TCTL_CCD_REGISTER_F17H
+ (Core->T.Cluster.CCD << 2)),
PRIVATE(OF(Zen)).Device.DF );
Core_AMD_Zen_Filter_Temp( Core, TccdSensor.CurTmp,
TccdSensor.CurTempRangeSel == 1 );
Core_AMD_Family_17h_ThermTrip(Core);
}
static void Pkg_AMD_Family_19h_Genoa_Temp(PROC_RO *Pkg, CORE_RO* Core)
{
CTL_AMD_Family_17h_Temp(Core);
Pkg->PowerThermal.Sensor = Core->PowerThermal.Sensor;
}
static void CCD_AMD_Family_19h_Genoa_Temp(CORE_RO *Core)
{
TCCD_REGISTER TccdSensor = {.value = 0};
Core_AMD_SMN_Read( TccdSensor,
(SMU_AMD_THM_TCTL_CCD_REGISTER_F19H_11H
+ (Core->T.Cluster.CCD << 2)),
PRIVATE(OF(Zen)).Device.DF );
Core_AMD_Zen_Filter_Temp( Core, TccdSensor.CurTmp,
TccdSensor.CurTempRangeSel == 1 );
Core_AMD_Family_17h_ThermTrip(Core);
}
static void CCD_AMD_Family_19h_Zen4_Temp(CORE_RO *Core)
{
TCCD_REGISTER TccdSensor = {.value = 0};
Core_AMD_SMN_Read( TccdSensor,
(SMU_AMD_THM_TCTL_CCD_REGISTER_F19H_61H
+ (Core->T.Cluster.CCD << 2)),
PRIVATE(OF(Zen)).Device.DF );
Core_AMD_Zen_Filter_Temp( Core, TccdSensor.CurTmp,
TccdSensor.CurTempRangeSel == 1 );
Core_AMD_Family_17h_ThermTrip(Core);
}
static enum hrtimer_restart Cycle_VirtualMachine(struct hrtimer *pTimer)
{
CORE_RO *Core;
unsigned int cpu;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
if (!PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC) {
RDTSC64(Core->Overhead.TSC);
} else {
RDTSCP64(Core->Overhead.TSC);
}
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
Counters_VirtualMachine(Core, 1);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PKG_Counters_VirtualMachine(Core, 1);
Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);
Save_PTSC(PUBLIC(RO(Proc)));
Sys_Tick(PUBLIC(RO(Proc)));
}
Delta_C0(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C1(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static void InitTimer_VirtualMachine(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_VirtualMachine, 1);
}
static void Start_VirtualMachine(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Counters_VirtualMachine(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_VirtualMachine(Core, 0);
}
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static enum hrtimer_restart Cycle_Safe_VirtualMachine(struct hrtimer *pTimer)
{
CORE_RO *Core;
unsigned int cpu;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
if (!PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC) {
RDTSC64(Core->Overhead.TSC);
} else {
RDTSCP64(Core->Overhead.TSC);
}
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
Counters_Generic(Core, 1);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PKG_Counters_Generic(Core, 1);
Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);
Save_PTSC(PUBLIC(RO(Proc)));
Sys_Tick(PUBLIC(RO(Proc)));
}
Delta_C0(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C1(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static void InitTimer_Safe_VirtualMachine(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_Safe_VirtualMachine, 1);
}
static void Start_Safe_VirtualMachine(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Counters_Generic(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Generic(Core, 0);
}
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Stop_VirtualMachine(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
UNUSED(arg);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_cancel(&PRIVATE(OF(Core, AT(cpu)))->Join.Timer);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
}
Pkg_Reset_ThermalPoint(PUBLIC(RO(Proc)));
}
PerCore_Reset(Core);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static enum hrtimer_restart Cycle_GenuineIntel(struct hrtimer *pTimer)
{
CORE_RO *Core;
unsigned int cpu;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
if (!PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC) {
RDTSC64(Core->Overhead.TSC);
} else {
RDTSCP64(Core->Overhead.TSC);
}
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
Counters_Generic(Core, 1);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PKG_Counters_Generic(Core, 1);
Pkg_Intel_Temp(PUBLIC(RO(Proc)));
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
{
case FORMULA_SCOPE_PKG:
Core_Intel_Temp(Core);
break;
}
Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);
Save_PTSC(PUBLIC(RO(Proc)));
Sys_Tick(PUBLIC(RO(Proc)));
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core_Intel_Temp(Core);
}
break;
case FORMULA_SCOPE_SMT:
Core_Intel_Temp(Core);
break;
}
Delta_C0(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C1(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static void InitTimer_GenuineIntel(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_GenuineIntel, 1);
}
static void Start_GenuineIntel(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Counters_Generic(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Generic(Core, 0);
}
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Stop_GenuineIntel(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
UNUSED(arg);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_cancel(&PRIVATE(OF(Core, AT(cpu)))->Join.Timer);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
}
Pkg_Reset_ThermalPoint(PUBLIC(RO(Proc)));
}
PerCore_Reset(Core);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static enum hrtimer_restart Cycle_AuthenticAMD(struct hrtimer *pTimer)
{
CORE_RO *Core;
unsigned int cpu;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
if (!PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC) {
RDTSC64(Core->Overhead.TSC);
} else {
RDTSCP64(Core->Overhead.TSC);
}
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
Counters_Generic(Core, 1);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PKG_Counters_Generic(Core, 1);
Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);
Save_PTSC(PUBLIC(RO(Proc)));
Sys_Tick(PUBLIC(RO(Proc)));
}
Delta_C0(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C1(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static void InitTimer_AuthenticAMD(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_AuthenticAMD, 1);
}
static void Start_AuthenticAMD(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Counters_Generic(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Generic(Core, 0);
}
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Stop_AuthenticAMD(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
UNUSED(arg);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_cancel(&PRIVATE(OF(Core, AT(cpu)))->Join.Timer);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
}
Pkg_Reset_ThermalPoint(PUBLIC(RO(Proc)));
}
PerCore_Reset(Core);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static enum hrtimer_restart Cycle_Core2(struct hrtimer *pTimer)
{
PERF_STATUS PerfStatus = {.value = 0};
CORE_RO *Core;
unsigned int cpu;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
if (!PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC) {
RDTSC64(Core->Overhead.TSC);
} else {
RDTSCP64(Core->Overhead.TSC);
}
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
Counters_Core2(Core, 1);
RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
Core->Ratio.Perf = PerfStatus.CORE.CurrFID;
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
Core->Boost[BOOST(TGT)] = GET_CORE2_TARGET(Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PKG_Counters_Generic(Core, 1);
Pkg_Intel_Temp(PUBLIC(RO(Proc)));
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
{
case FORMULA_SCOPE_PKG:
Core_Intel_Temp(Core);
break;
}
PUBLIC(RO(Proc))->PowerThermal.VID.CPU=PerfStatus.CORE.CurrVID;
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
{
case FORMULA_SCOPE_PKG:
Core->PowerThermal.VID = PerfStatus.CORE.CurrVID;
break;
}
Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);
Save_PTSC(PUBLIC(RO(Proc)));
Sys_Tick(PUBLIC(RO(Proc)));
} else {
Core->PowerThermal.VID = 0;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core_Intel_Temp(Core);
}
break;
case FORMULA_SCOPE_SMT:
Core_Intel_Temp(Core);
break;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core->PowerThermal.VID = PerfStatus.CORE.CurrVID;
}
break;
case FORMULA_SCOPE_SMT:
Core->PowerThermal.VID = PerfStatus.CORE.CurrVID;
break;
}
Delta_INST(Core);
Delta_C0(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Save_INST(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C1(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static void InitTimer_Core2(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_Core2, 1);
}
static void Start_Core2(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Intel_Core_Counters_Set(Save, Core);
Counters_Core2(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Generic(Core, 0);
}
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Stop_Core2(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_cancel(&PRIVATE(OF(Core, AT(cpu)))->Join.Timer);
Intel_Core_Counters_Clear(Save, Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
}
Pkg_Reset_ThermalPoint(PUBLIC(RO(Proc)));
}
PerCore_Reset(Core);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static enum hrtimer_restart Cycle_Silvermont(struct hrtimer *pTimer)
{
PERF_STATUS PerfStatus = {.value = 0};
CORE_RO *Core;
unsigned int cpu;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
Mark_OVH(Core);
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
Counters_SLM(Core, 1);
RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
Core->Ratio.Perf = PerfStatus.CORE.CurrFID;
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
Core->Boost[BOOST(TGT)] = GET_CORE2_TARGET(Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PKG_Counters_SLM(Core, 1);
Pkg_Intel_Temp(PUBLIC(RO(Proc)));
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
{
case FORMULA_SCOPE_PKG:
Core_Intel_Temp(Core);
break;
}
PUBLIC(RO(Proc))->PowerThermal.VID.CPU=PerfStatus.CORE.CurrVID;
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
{
case FORMULA_SCOPE_PKG:
Core->PowerThermal.VID = PerfStatus.CORE.CurrVID;
break;
}
RDCOUNTER( PUBLIC(RO(Proc))->Counter[1].Power.ACCU[PWR_DOMAIN(PKG)],
MSR_PKG_ENERGY_STATUS );
RDCOUNTER( PUBLIC(RO(Proc))->Counter[1].Power.ACCU[PWR_DOMAIN(CORES)],
MSR_PP0_ENERGY_STATUS );
Delta_PC02(PUBLIC(RO(Proc)));
Delta_PC03(PUBLIC(RO(Proc)));
Delta_PC04(PUBLIC(RO(Proc)));
Delta_PC06(PUBLIC(RO(Proc)));
Delta_MC6(PUBLIC(RO(Proc)));
Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);
Delta_PWR_ACCU(Proc, PKG);
Delta_PWR_ACCU(Proc, CORES);
Save_PC02(PUBLIC(RO(Proc)));
Save_PC03(PUBLIC(RO(Proc)));
Save_PC04(PUBLIC(RO(Proc)));
Save_PC06(PUBLIC(RO(Proc)));
Save_MC6(PUBLIC(RO(Proc)));
Save_PTSC(PUBLIC(RO(Proc)));
Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);
Save_PWR_ACCU(PUBLIC(RO(Proc)), CORES);
Sys_Tick(PUBLIC(RO(Proc)));
} else {
Core->PowerThermal.VID = 0;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core_Intel_Temp(Core);
}
break;
case FORMULA_SCOPE_SMT:
Core_Intel_Temp(Core);
break;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core->PowerThermal.VID = PerfStatus.CORE.CurrVID;
}
break;
case FORMULA_SCOPE_SMT:
Core->PowerThermal.VID = PerfStatus.CORE.CurrVID;
break;
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
Delta_INST(Core);
Delta_C0(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Delta_C3(Core);
Delta_C6(Core);
Save_INST(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C1(Core);
Save_C3(Core);
Save_C6(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static void InitTimer_Silvermont(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_Silvermont, 1);
}
static void Start_Silvermont(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Intel_Core_Counters_Set(Save, Core);
Counters_SLM(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_SLM(Core, 0);
RDCOUNTER( PUBLIC(RO(Proc))->Counter[0].Power.ACCU[PWR_DOMAIN(PKG)],
MSR_PKG_ENERGY_STATUS );
RDCOUNTER( PUBLIC(RO(Proc))->Counter[0].Power.ACCU[PWR_DOMAIN(CORES)],
MSR_PP0_ENERGY_STATUS );
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Stop_Silvermont(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_cancel(&PRIVATE(OF(Core, AT(cpu)))->Join.Timer);
Intel_Core_Counters_Clear(Save, Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
}
Pkg_Reset_ThermalPoint(PUBLIC(RO(Proc)));
}
PerCore_Reset(Core);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static enum hrtimer_restart Cycle_Nehalem(struct hrtimer *pTimer)
{
PERF_STATUS PerfStatus = {.value = 0};
CORE_RO *Core;
unsigned int cpu;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
Mark_OVH(Core);
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
SMT_Counters_Nehalem(Core, 1);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PKG_Counters_Nehalem(Core, 1);
Pkg_Intel_Temp(PUBLIC(RO(Proc)));
#if defined(HWM_CHIPSET) \
&& ((HWM_CHIPSET == W83627) || (HWM_CHIPSET == IT8720))
RDSIO( PUBLIC(RO(Proc))->PowerThermal.VID.CPU,
HWM_SIO_CPUVCORE,
HWM_SIO_INDEX_PORT, HWM_SIO_DATA_PORT );
#endif
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
{
case FORMULA_SCOPE_PKG:
Core_Intel_Temp(Core);
break;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
{
case FORMULA_SCOPE_PKG:
#if defined(HWM_CHIPSET) \
&& ((HWM_CHIPSET == W83627) || (HWM_CHIPSET == IT8720))
Core->PowerThermal.VID = PUBLIC(RO(Proc))->PowerThermal.VID.CPU;
#endif
break;
}
Delta_PC03(PUBLIC(RO(Proc)));
Delta_PC06(PUBLIC(RO(Proc)));
Delta_PC07(PUBLIC(RO(Proc)));
Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);
Delta_UNCORE_FC0(PUBLIC(RO(Proc)));
Save_PC03(PUBLIC(RO(Proc)));
Save_PC06(PUBLIC(RO(Proc)));
Save_PC07(PUBLIC(RO(Proc)));
Save_PTSC(PUBLIC(RO(Proc)));
Save_UNCORE_FC0(PUBLIC(RO(Proc)));
Sys_Tick(PUBLIC(RO(Proc)));
} else {
#if defined(HWM_CHIPSET) \
&& ((HWM_CHIPSET == W83627) || (HWM_CHIPSET == IT8720))
Core->PowerThermal.VID = 0;
#endif
}
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
Core->Boost[BOOST(TGT)] = GET_NEHALEM_TARGET(Core);
RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
Core->Ratio.Perf = PerfStatus.NHM.CurrentRatio;
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core_Intel_Temp(Core);
}
break;
case FORMULA_SCOPE_SMT:
Core_Intel_Temp(Core);
break;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
#if defined(HWM_CHIPSET) \
&& ((HWM_CHIPSET == W83627) || (HWM_CHIPSET == IT8720))
RDSIO( Core->PowerThermal.VID, HWM_SIO_CPUVCORE,
HWM_SIO_INDEX_PORT, HWM_SIO_DATA_PORT );
#endif
}
break;
case FORMULA_SCOPE_SMT:
#if defined(HWM_CHIPSET) \
&& ((HWM_CHIPSET == W83627) || (HWM_CHIPSET == IT8720))
RDSIO( Core->PowerThermal.VID, HWM_SIO_CPUVCORE,
HWM_SIO_INDEX_PORT, HWM_SIO_DATA_PORT );
#endif
break;
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
Delta_INST(Core);
Delta_C0(Core);
Delta_C3(Core);
Delta_C6(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Save_INST(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C3(Core);
Save_C6(Core);
Save_C1(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static void InitTimer_Nehalem(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_Nehalem, 1);
}
static void Start_Nehalem(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Intel_Core_Counters_Set(Save, Core);
SMT_Counters_Nehalem(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Nehalem(Core, 0);
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Stop_Nehalem(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_cancel(&PRIVATE(OF(Core, AT(cpu)))->Join.Timer);
Intel_Core_Counters_Clear(Save, Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
}
Pkg_Reset_ThermalPoint(PUBLIC(RO(Proc)));
}
PerCore_Reset(Core);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Start_Uncore_Nehalem(void *arg)
{
UNUSED(arg);
Uncore_Counters_Set(NHM);
Uncore_Counters_Ovf(NHM);
}
static void Stop_Uncore_Nehalem(void *arg)
{
UNUSED(arg);
Uncore_Counters_Clear(NHM);
}
static enum hrtimer_restart Cycle_SandyBridge(struct hrtimer *pTimer)
{
PERF_STATUS PerfStatus = {.value = 0};
CORE_RO *Core;
unsigned int cpu;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
Mark_OVH(Core);
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
SMT_Counters_SandyBridge(Core, 1);
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
Core->Boost[BOOST(TGT)] = GET_SANDYBRIDGE_TARGET(Core);
RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
Core->Ratio.Perf = PerfStatus.SNB.CurrentRatio;
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PKG_Counters_SandyBridge(Core, 1);
Pkg_Intel_Temp(PUBLIC(RO(Proc)));
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
{
case FORMULA_SCOPE_PKG:
Core_Intel_Temp(Core);
break;
}
PUBLIC(RO(Proc))->PowerThermal.VID.CPU = PerfStatus.SNB.CurrVID;
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
{
case FORMULA_SCOPE_PKG:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
PWR_ACCU_SandyBridge(PUBLIC(RO(Proc)), 1);
Delta_PC02(PUBLIC(RO(Proc)));
Delta_PC03(PUBLIC(RO(Proc)));
Delta_PC06(PUBLIC(RO(Proc)));
Delta_PC07(PUBLIC(RO(Proc)));
Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);
Delta_UNCORE_FC0(PUBLIC(RO(Proc)));
Delta_PWR_ACCU(Proc, PKG);
Delta_PWR_ACCU(Proc, CORES);
Delta_PWR_ACCU(Proc, UNCORE);
Save_PC02(PUBLIC(RO(Proc)));
Save_PC03(PUBLIC(RO(Proc)));
Save_PC06(PUBLIC(RO(Proc)));
Save_PC07(PUBLIC(RO(Proc)));
Save_PTSC(PUBLIC(RO(Proc)));
Save_UNCORE_FC0(PUBLIC(RO(Proc)));
Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);
Save_PWR_ACCU(PUBLIC(RO(Proc)), CORES);
Save_PWR_ACCU(PUBLIC(RO(Proc)), UNCORE);
Sys_Tick(PUBLIC(RO(Proc)));
} else {
Core->PowerThermal.VID = 0;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core_Intel_Temp(Core);
}
break;
case FORMULA_SCOPE_SMT:
Core_Intel_Temp(Core);
break;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
}
break;
case FORMULA_SCOPE_SMT:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
Delta_INST(Core);
Delta_C0(Core);
Delta_C3(Core);
Delta_C6(Core);
Delta_C7(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Save_INST(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C3(Core);
Save_C6(Core);
Save_C7(Core);
Save_C1(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static void InitTimer_SandyBridge(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_SandyBridge, 1);
}
static void Start_SandyBridge(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Intel_Core_Counters_Set(Save, Core);
SMT_Counters_SandyBridge(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_SandyBridge(Core, 0);
PWR_ACCU_SandyBridge(PUBLIC(RO(Proc)), 0);
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Stop_SandyBridge(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_cancel(&PRIVATE(OF(Core, AT(cpu)))->Join.Timer);
Intel_Core_Counters_Clear(Save, Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
}
Pkg_Reset_ThermalPoint(PUBLIC(RO(Proc)));
}
PerCore_Reset(Core);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Start_Uncore_SandyBridge(void *arg)
{
UNUSED(arg);
Uncore_Counters_Set(SNB);
Uncore_Counters_Ovf(SNB);
}
static void Stop_Uncore_SandyBridge(void *arg)
{
UNUSED(arg);
Uncore_Counters_Clear(SNB);
}
static enum hrtimer_restart Cycle_Intel_Xeon_EP(struct hrtimer *pTimer,
void (*PKG_Counters_Intel_EP)(CORE_RO*, unsigned int))
{
PERF_STATUS PerfStatus = {.value = 0};
CORE_RO *Core;
unsigned int cpu;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
Mark_OVH(Core);
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
SMT_Counters_SandyBridge(Core, 1);
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
Core->Boost[BOOST(TGT)] = GET_SANDYBRIDGE_TARGET(Core);
RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
Core->Ratio.Perf = PerfStatus.SNB.CurrentRatio;
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PKG_Counters_Intel_EP(Core, 1);
Pkg_Intel_Temp(PUBLIC(RO(Proc)));
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
{
case FORMULA_SCOPE_PKG:
Core_Intel_Temp(Core);
break;
}
PUBLIC(RO(Proc))->PowerThermal.VID.CPU = PerfStatus.SNB.CurrVID;
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
{
case FORMULA_SCOPE_PKG:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
PWR_ACCU_SandyBridge_EP(PUBLIC(RO(Proc)), 1);
Delta_PC02(PUBLIC(RO(Proc)));
Delta_PC03(PUBLIC(RO(Proc)));
Delta_PC06(PUBLIC(RO(Proc)));
Delta_PC07(PUBLIC(RO(Proc)));
Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);
Delta_UNCORE_FC0(PUBLIC(RO(Proc)));
Delta_PWR_ACCU(Proc, PKG);
Delta_PWR_ACCU(Proc, CORES);
Delta_PWR_ACCU(Proc, RAM);
Save_PC02(PUBLIC(RO(Proc)));
Save_PC03(PUBLIC(RO(Proc)));
Save_PC06(PUBLIC(RO(Proc)));
Save_PC07(PUBLIC(RO(Proc)));
Save_PTSC(PUBLIC(RO(Proc)));
Save_UNCORE_FC0(PUBLIC(RO(Proc)));
Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);
Save_PWR_ACCU(PUBLIC(RO(Proc)), CORES);
Save_PWR_ACCU(PUBLIC(RO(Proc)), RAM);
Sys_Tick(PUBLIC(RO(Proc)));
} else {
Core->PowerThermal.VID = 0;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core_Intel_Temp(Core);
}
break;
case FORMULA_SCOPE_SMT:
Core_Intel_Temp(Core);
break;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
}
break;
case FORMULA_SCOPE_SMT:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
Delta_INST(Core);
Delta_C0(Core);
Delta_C3(Core);
Delta_C6(Core);
Delta_C7(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Save_INST(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C3(Core);
Save_C6(Core);
Save_C7(Core);
Save_C1(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static enum hrtimer_restart Cycle_SandyBridge_EP(struct hrtimer *pTimer)
{
return Cycle_Intel_Xeon_EP(pTimer, PKG_Counters_SandyBridge_EP);
}
static void InitTimer_SandyBridge_EP(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_SandyBridge_EP, 1);
}
static void Entry_Intel_Xeon_EP(void *arg,
void (*PKG_Counters_Intel_EP)(CORE_RO*, unsigned int))
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Intel_Core_Counters_Set(Save, Core);
SMT_Counters_SandyBridge(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Intel_EP(Core, 0);
PWR_ACCU_SandyBridge_EP(PUBLIC(RO(Proc)), 0);
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Stop_SandyBridge_EP(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_cancel(&PRIVATE(OF(Core, AT(cpu)))->Join.Timer);
Intel_Core_Counters_Clear(Save, Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
}
Pkg_Reset_ThermalPoint(PUBLIC(RO(Proc)));
}
PerCore_Reset(Core);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Start_SandyBridge_EP(void *arg)
{
Entry_Intel_Xeon_EP(arg, PKG_Counters_SandyBridge_EP);
}
/* SandyBridge/EP Uncore PMU MSR for Xeon family 06_2Dh */
static void Start_Uncore_SandyBridge_EP(void *arg)
{
UNUSED(arg);
if (PUBLIC(RO(Proc))->Features.PerfMon.EAX.Version > 3)
{
/* Tables 2-20 , 2-23 , 2-24 */
UNCORE_FIXED_PERF_CONTROL Uncore_FixedPerfControl;
UNCORE_PMON_GLOBAL_CONTROL Uncore_PMonGlobalControl;
RDMSR(Uncore_FixedPerfControl, MSR_SNB_EP_UNCORE_PERF_FIXED_CTR_CTRL);
PRIVATE(OF(SaveArea)).Intel.Uncore_FixedPerfControl = \
Uncore_FixedPerfControl;
Uncore_FixedPerfControl.SNB.EN_CTR0 = 1;
WRMSR(Uncore_FixedPerfControl, MSR_SNB_EP_UNCORE_PERF_FIXED_CTR_CTRL);
RDMSR(Uncore_PMonGlobalControl, MSR_SNB_UNCORE_PERF_GLOBAL_CTRL);
PRIVATE(OF(SaveArea)).Intel.Uncore_PMonGlobalControl = \
Uncore_PMonGlobalControl;
Uncore_PMonGlobalControl.Unfreeze_All = 1;
WRMSR(Uncore_PMonGlobalControl, MSR_SNB_UNCORE_PERF_GLOBAL_CTRL);
}
}
static void Stop_Uncore_SandyBridge_EP(void *arg)
{
UNUSED(arg);
if (PUBLIC(RO(Proc))->Features.PerfMon.EAX.Version > 3)
{
if(PRIVATE(OF(SaveArea)).Intel.Uncore_FixedPerfControl.SNB.EN_CTR0 == 0)
{
PRIVATE(OF(SaveArea)).Intel.Uncore_PMonGlobalControl.Freeze_All = 1;
}
WRMSR( PRIVATE(OF(SaveArea)).Intel.Uncore_PMonGlobalControl,
MSR_SNB_UNCORE_PERF_GLOBAL_CTRL);
WRMSR( PRIVATE(OF(SaveArea)).Intel.Uncore_FixedPerfControl,
MSR_SNB_EP_UNCORE_PERF_FIXED_CTR_CTRL);
}
}
static enum hrtimer_restart Cycle_IvyBridge_EP(struct hrtimer *pTimer)
{
return Cycle_Intel_Xeon_EP(pTimer, PKG_Counters_IvyBridge_EP);
}
static void InitTimer_IvyBridge_EP(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_IvyBridge_EP, 1);
}
static void Start_IvyBridge_EP(void *arg)
{
Entry_Intel_Xeon_EP(arg, PKG_Counters_IvyBridge_EP);
}
/* IvyBridge/EP Uncore PMU MSR for Xeon family 06_3E */
static void Start_Uncore_IvyBridge_EP(void *arg)
{ /* Tables 2-20 , 2-24 , 2-26 , 2-27 , 2-28 */
UNCORE_FIXED_PERF_CONTROL Uncore_FixedPerfControl;
UNCORE_PMON_GLOBAL_CONTROL Uncore_PMonGlobalControl;
UNUSED(arg);
RDMSR(Uncore_FixedPerfControl, MSR_SNB_EP_UNCORE_PERF_FIXED_CTR_CTRL);
PRIVATE(OF(SaveArea)).Intel.Uncore_FixedPerfControl = \
Uncore_FixedPerfControl;
Uncore_FixedPerfControl.SNB.EN_CTR0 = 1;
WRMSR(Uncore_FixedPerfControl, MSR_SNB_EP_UNCORE_PERF_FIXED_CTR_CTRL);
RDMSR(Uncore_PMonGlobalControl, MSR_IVB_EP_PMON_GLOBAL_CTRL);
PRIVATE(OF(SaveArea)).Intel.Uncore_PMonGlobalControl = \
Uncore_PMonGlobalControl;
Uncore_PMonGlobalControl.Unfreeze_All = 1;
WRMSR(Uncore_PMonGlobalControl, MSR_IVB_EP_PMON_GLOBAL_CTRL);
}
static void Stop_Uncore_IvyBridge_EP(void *arg)
{
UNUSED(arg);
/* If fixed counter was disable at entry, force freezing */
if (PRIVATE(OF(SaveArea)).Intel.Uncore_FixedPerfControl.SNB.EN_CTR0 == 0)
{
PRIVATE(OF(SaveArea)).Intel.Uncore_PMonGlobalControl.Freeze_All = 1;
}
WRMSR( PRIVATE(OF(SaveArea)).Intel.Uncore_PMonGlobalControl,
MSR_IVB_EP_PMON_GLOBAL_CTRL);
WRMSR( PRIVATE(OF(SaveArea)).Intel.Uncore_FixedPerfControl,
MSR_SNB_EP_UNCORE_PERF_FIXED_CTR_CTRL);
}
static enum hrtimer_restart Cycle_Haswell_ULT(struct hrtimer *pTimer)
{
PERF_STATUS PerfStatus = {.value = 0};
CORE_RO *Core;
unsigned int cpu;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
Mark_OVH(Core);
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
SMT_Counters_SandyBridge(Core, 1);
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
Core->Boost[BOOST(TGT)] = GET_SANDYBRIDGE_TARGET(Core);
RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
Core->Ratio.Perf = PerfStatus.SNB.CurrentRatio;
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PKG_Counters_Haswell_ULT(Core, 1);
Pkg_Intel_Temp(PUBLIC(RO(Proc)));
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
{
case FORMULA_SCOPE_PKG:
Core_Intel_Temp(Core);
break;
}
PUBLIC(RO(Proc))->PowerThermal.VID.CPU = PerfStatus.SNB.CurrVID;
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
{
case FORMULA_SCOPE_PKG:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
PWR_ACCU_SandyBridge(PUBLIC(RO(Proc)), 1);
Delta_PC02(PUBLIC(RO(Proc)));
Delta_PC03(PUBLIC(RO(Proc)));
Delta_PC06(PUBLIC(RO(Proc)));
Delta_PC07(PUBLIC(RO(Proc)));
Delta_PC08(PUBLIC(RO(Proc)));
Delta_PC09(PUBLIC(RO(Proc)));
Delta_PC10(PUBLIC(RO(Proc)));
Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);
Delta_UNCORE_FC0(PUBLIC(RO(Proc)));
Delta_PWR_ACCU(Proc, PKG);
Delta_PWR_ACCU(Proc, CORES);
Delta_PWR_ACCU(Proc, UNCORE);
Save_PC02(PUBLIC(RO(Proc)));
Save_PC03(PUBLIC(RO(Proc)));
Save_PC06(PUBLIC(RO(Proc)));
Save_PC07(PUBLIC(RO(Proc)));
Save_PC08(PUBLIC(RO(Proc)));
Save_PC09(PUBLIC(RO(Proc)));
Save_PC10(PUBLIC(RO(Proc)));
Save_PTSC(PUBLIC(RO(Proc)));
Save_UNCORE_FC0(PUBLIC(RO(Proc)));
Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);
Save_PWR_ACCU(PUBLIC(RO(Proc)), CORES);
Save_PWR_ACCU(PUBLIC(RO(Proc)), UNCORE);
Sys_Tick(PUBLIC(RO(Proc)));
} else {
Core->PowerThermal.VID = 0;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core_Intel_Temp(Core);
}
break;
case FORMULA_SCOPE_SMT:
Core_Intel_Temp(Core);
break;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
}
break;
case FORMULA_SCOPE_SMT:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
Delta_INST(Core);
Delta_C0(Core);
Delta_C3(Core);
Delta_C6(Core);
Delta_C7(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Save_INST(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C3(Core);
Save_C6(Core);
Save_C7(Core);
Save_C1(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static void InitTimer_Haswell_ULT(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_Haswell_ULT, 1);
}
static void Start_Haswell_ULT(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Intel_Core_Counters_Set(Save, Core);
SMT_Counters_SandyBridge(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Haswell_ULT(Core, 0);
PWR_ACCU_SandyBridge(PUBLIC(RO(Proc)), 0);
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Stop_Haswell_ULT(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_cancel(&PRIVATE(OF(Core, AT(cpu)))->Join.Timer);
Intel_Core_Counters_Clear(Save, Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
}
Pkg_Reset_ThermalPoint(PUBLIC(RO(Proc)));
}
PerCore_Reset(Core);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Start_Uncore_Haswell_ULT(void *arg)
{
UNUSED(arg);
if (PUBLIC(RO(Proc))->Registration.Experimental) {
Uncore_Counters_Set(SNB);
Uncore_Counters_Ovf(SNB);
}
}
static void Stop_Uncore_Haswell_ULT(void *arg)
{
UNUSED(arg);
if (PUBLIC(RO(Proc))->Registration.Experimental) {
Uncore_Counters_Clear(SNB);
}
}
static enum hrtimer_restart Cycle_Goldmont(struct hrtimer *pTimer)
{
PERF_STATUS PerfStatus = {.value = 0};
CORE_RO *Core;
unsigned int cpu;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
Mark_OVH(Core);
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
Counters_Goldmont(Core, 1);
RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
Core->Ratio.Perf = PerfStatus.SNB.CurrentRatio;
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
Core->Boost[BOOST(TGT)] = GET_SANDYBRIDGE_TARGET(Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PKG_Counters_Goldmont(Core, 1);
Pkg_Intel_Temp(PUBLIC(RO(Proc)));
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
{
case FORMULA_SCOPE_PKG:
Core_Intel_Temp(Core);
break;
}
PUBLIC(RO(Proc))->PowerThermal.VID.CPU = PerfStatus.SNB.CurrVID;
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
{
case FORMULA_SCOPE_PKG:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
PWR_ACCU_Goldmont(PUBLIC(RO(Proc)), 1);
Delta_PC02(PUBLIC(RO(Proc)));
Delta_PC03(PUBLIC(RO(Proc)));
Delta_PC06(PUBLIC(RO(Proc)));
Delta_PC10(PUBLIC(RO(Proc)));
Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);
Delta_PWR_ACCU(Proc, PKG);
Delta_PWR_ACCU(Proc, CORES);
Delta_PWR_ACCU(Proc, UNCORE);
Delta_PWR_ACCU(Proc, RAM);
Save_PC02(PUBLIC(RO(Proc)));
Save_PC03(PUBLIC(RO(Proc)));
Save_PC06(PUBLIC(RO(Proc)));
Save_PC10(PUBLIC(RO(Proc)));
Save_PTSC(PUBLIC(RO(Proc)));
Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);
Save_PWR_ACCU(PUBLIC(RO(Proc)), CORES);
Save_PWR_ACCU(PUBLIC(RO(Proc)), UNCORE);
Save_PWR_ACCU(PUBLIC(RO(Proc)), RAM);
Sys_Tick(PUBLIC(RO(Proc)));
} else {
Core->PowerThermal.VID = 0;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core_Intel_Temp(Core);
}
break;
case FORMULA_SCOPE_SMT:
Core_Intel_Temp(Core);
break;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
}
break;
case FORMULA_SCOPE_SMT:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
Delta_INST(Core);
Delta_C0(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Delta_C3(Core);
Delta_C6(Core);
Save_INST(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C1(Core);
Save_C3(Core);
Save_C6(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static void InitTimer_Goldmont(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_Goldmont, 1);
}
static void Start_Goldmont(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Intel_Core_Counters_Set(Save, Core);
Counters_Goldmont(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Goldmont(Core, 0);
PWR_ACCU_Goldmont(PUBLIC(RO(Proc)), 0);
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Stop_Goldmont(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_cancel(&PRIVATE(OF(Core, AT(cpu)))->Join.Timer);
Intel_Core_Counters_Clear(Save, Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
}
Pkg_Reset_ThermalPoint(PUBLIC(RO(Proc)));
}
PerCore_Reset(Core);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static enum hrtimer_restart Cycle_Haswell_EP(struct hrtimer *pTimer)
{
PERF_STATUS PerfStatus = {.value = 0};
CORE_RO *Core;
unsigned int cpu;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
Mark_OVH(Core);
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
SMT_Counters_SandyBridge(Core, 1);
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
Core->Boost[BOOST(TGT)] = GET_SANDYBRIDGE_TARGET(Core);
RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
Core->Ratio.Perf = PerfStatus.SNB.CurrentRatio;
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PKG_Counters_Haswell_EP(Core, 1);
Pkg_Intel_Temp(PUBLIC(RO(Proc)));
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
{
case FORMULA_SCOPE_PKG:
Core_Intel_Temp(Core);
break;
}
PUBLIC(RO(Proc))->PowerThermal.VID.CPU = PerfStatus.SNB.CurrVID;
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
{
case FORMULA_SCOPE_PKG:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
PWR_ACCU_SandyBridge_EP(PUBLIC(RO(Proc)), 1);
Delta_PC02(PUBLIC(RO(Proc)));
Delta_PC03(PUBLIC(RO(Proc)));
Delta_PC06(PUBLIC(RO(Proc)));
Delta_PC07(PUBLIC(RO(Proc)));
Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);
Delta_UNCORE_FC0(PUBLIC(RO(Proc)));
Delta_PWR_ACCU(Proc, PKG);
Delta_PWR_ACCU(Proc, CORES);
Delta_PWR_ACCU(Proc, RAM);
Save_PC02(PUBLIC(RO(Proc)));
Save_PC03(PUBLIC(RO(Proc)));
Save_PC06(PUBLIC(RO(Proc)));
Save_PC07(PUBLIC(RO(Proc)));
Save_PTSC(PUBLIC(RO(Proc)));
Save_UNCORE_FC0(PUBLIC(RO(Proc)));
Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);
Save_PWR_ACCU(PUBLIC(RO(Proc)), CORES);
Save_PWR_ACCU(PUBLIC(RO(Proc)), RAM);
Sys_Tick(PUBLIC(RO(Proc)));
} else {
Core->PowerThermal.VID = 0;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core_Intel_Temp(Core);
}
break;
case FORMULA_SCOPE_SMT:
Core_Intel_Temp(Core);
break;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
}
break;
case FORMULA_SCOPE_SMT:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
Delta_INST(Core);
Delta_C0(Core);
Delta_C3(Core);
Delta_C6(Core);
Delta_C7(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Save_INST(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C3(Core);
Save_C6(Core);
Save_C7(Core);
Save_C1(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static void InitTimer_Haswell_EP(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_Haswell_EP, 1);
}
static void Start_Haswell_EP(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Intel_Core_Counters_Set(Save, Core);
SMT_Counters_SandyBridge(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Haswell_EP(Core, 0);
PWR_ACCU_SandyBridge_EP(PUBLIC(RO(Proc)), 0);
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Stop_Haswell_EP(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_cancel(&PRIVATE(OF(Core, AT(cpu)))->Join.Timer);
Intel_Core_Counters_Clear(Save, Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
}
Pkg_Reset_ThermalPoint(PUBLIC(RO(Proc)));
}
PerCore_Reset(Core);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Start_Uncore_Haswell_EP(void *arg)
{
UNCORE_FIXED_PERF_CONTROL Uncore_FixedPerfControl;
UNCORE_PMON_GLOBAL_CONTROL Uncore_PMonGlobalControl;
UNUSED(arg);
RDMSR(Uncore_FixedPerfControl, MSR_HSW_EP_UNCORE_PERF_FIXED_CTR_CTRL);
PRIVATE(OF(SaveArea)).Intel.Uncore_FixedPerfControl = \
Uncore_FixedPerfControl;
Uncore_FixedPerfControl.HSW_EP.EN_CTR0 = 1;
WRMSR(Uncore_FixedPerfControl, MSR_HSW_EP_UNCORE_PERF_FIXED_CTR_CTRL);
RDMSR(Uncore_PMonGlobalControl, MSR_HSW_EP_PMON_GLOBAL_CTRL);
PRIVATE(OF(SaveArea)).Intel.Uncore_PMonGlobalControl = \
Uncore_PMonGlobalControl;
Uncore_PMonGlobalControl.Unfreeze_All = 1;
WRMSR(Uncore_PMonGlobalControl, MSR_HSW_EP_PMON_GLOBAL_CTRL);
}
static void Stop_Uncore_Haswell_EP(void *arg)
{
UNUSED(arg);
if(PRIVATE(OF(SaveArea)).Intel.Uncore_FixedPerfControl.HSW_EP.EN_CTR0 == 0)
{
PRIVATE(OF(SaveArea)).Intel.Uncore_PMonGlobalControl.Freeze_All = 1;
}
WRMSR( PRIVATE(OF(SaveArea)).Intel.Uncore_PMonGlobalControl,
MSR_HSW_EP_PMON_GLOBAL_CTRL );
WRMSR( PRIVATE(OF(SaveArea)).Intel.Uncore_FixedPerfControl,
MSR_HSW_EP_UNCORE_PERF_FIXED_CTR_CTRL );
}
static enum hrtimer_restart Cycle_Skylake(struct hrtimer *pTimer)
{
PERF_STATUS PerfStatus = {.value = 0};
CORE_RO *Core;
unsigned int cpu;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
Mark_OVH(Core);
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
SMT_Counters_SandyBridge(Core, 1);
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
Core->Boost[BOOST(TGT)] = GET_SANDYBRIDGE_TARGET(Core);
RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
Core->Ratio.Perf = PerfStatus.SNB.CurrentRatio;
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PKG_Counters_Skylake(Core, 1);
Pkg_Intel_PMC_Counters(Core, 1, ARCH_PMC);
Pkg_Intel_Temp(PUBLIC(RO(Proc)));
Monitor_CorePerfLimitReasons(PUBLIC(RO(Proc)));
Monitor_GraphicsPerfLimitReasons(PUBLIC(RO(Proc)));
Monitor_RingPerfLimitReasons(PUBLIC(RO(Proc)));
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
{
case FORMULA_SCOPE_PKG:
Core_Intel_Temp(Core);
break;
}
PUBLIC(RO(Proc))->PowerThermal.VID.CPU = PerfStatus.SNB.CurrVID;
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
{
case FORMULA_SCOPE_PKG:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
Power_ACCU_Skylake(PUBLIC(RO(Proc)), 1);
Delta_PC02(PUBLIC(RO(Proc)));
Delta_PC03(PUBLIC(RO(Proc)));
Delta_PC06(PUBLIC(RO(Proc)));
Delta_PC07(PUBLIC(RO(Proc)));
Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);
Delta_UNCORE_FC0(PUBLIC(RO(Proc)));
Delta_MC6(PUBLIC(RO(Proc)));
Delta_PWR_ACCU(Proc, PKG);
Delta_PWR_ACCU(Proc, CORES);
Delta_PWR_ACCU(Proc, UNCORE);
Delta_PWR_ACCU(Proc, RAM);
Save_PC02(PUBLIC(RO(Proc)));
Save_PC03(PUBLIC(RO(Proc)));
Save_PC06(PUBLIC(RO(Proc)));
Save_PC07(PUBLIC(RO(Proc)));
Save_PTSC(PUBLIC(RO(Proc)));
Save_UNCORE_FC0(PUBLIC(RO(Proc)));
Save_MC6(PUBLIC(RO(Proc)));
Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);
Save_PWR_ACCU(PUBLIC(RO(Proc)), CORES);
Save_PWR_ACCU(PUBLIC(RO(Proc)), UNCORE);
Save_PWR_ACCU(PUBLIC(RO(Proc)), RAM);
Sys_Tick(PUBLIC(RO(Proc)));
} else {
Core->PowerThermal.VID = 0;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core_Intel_Temp(Core);
}
break;
case FORMULA_SCOPE_SMT:
Core_Intel_Temp(Core);
break;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
}
break;
case FORMULA_SCOPE_SMT:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
Delta_INST(Core);
Delta_C0(Core);
Delta_C3(Core);
Delta_C6(Core);
Delta_C7(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Save_INST(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C3(Core);
Save_C6(Core);
Save_C7(Core);
Save_C1(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static void InitTimer_Skylake(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_Skylake, 1);
}
static void Start_Skylake(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Intel_Core_Counters_Set(Save, Core);
SMT_Counters_SandyBridge(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Skylake(Core, 0);
Pkg_Intel_PMC_Counters(Core, 0, ARCH_PMC);
Power_ACCU_Skylake(PUBLIC(RO(Proc)), 0);
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Stop_Skylake(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_cancel(&PRIVATE(OF(Core, AT(cpu)))->Join.Timer);
Intel_Core_Counters_Clear(Save, Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
}
Pkg_Reset_ThermalPoint(PUBLIC(RO(Proc)));
}
PerCore_Reset(Core);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Start_Uncore_Skylake(void *arg)
{
UNUSED(arg);
Uncore_Counters_Set(SKL);
Uncore_Counters_Ovf(SKL);
Pkg_Intel_PMC_Set(ARCH_PMC, 0x5838);
}
static void Stop_Uncore_Skylake(void *arg)
{
UNUSED(arg);
Uncore_Counters_Clear(SKL);
Pkg_Intel_PMC_Clear(ARCH_PMC);
}
static enum hrtimer_restart Cycle_Skylake_X(struct hrtimer *pTimer)
{
PERF_STATUS PerfStatus = {.value = 0};
CORE_RO *Core;
unsigned int cpu;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
Mark_OVH(Core);
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
SMT_Counters_SandyBridge(Core, 1);
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
Core->Boost[BOOST(TGT)] = GET_SANDYBRIDGE_TARGET(Core);
RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
Core->Ratio.Perf = PerfStatus.SNB.CurrentRatio;
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PKG_Counters_Skylake_X(Core, 1);
Pkg_Intel_PMC_Counters(Core, 1, ARCH_PMC);
Pkg_Intel_Temp(PUBLIC(RO(Proc)));
Monitor_CorePerfLimitReasons(PUBLIC(RO(Proc)));
/*TODO(Unsolved)
Monitor_GraphicsPerfLimitReasons(PUBLIC(RO(Proc)));
*/
Monitor_RingPerfLimitReasons(PUBLIC(RO(Proc)));
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
{
case FORMULA_SCOPE_PKG:
Core_Intel_Temp(Core);
break;
}
PUBLIC(RO(Proc))->PowerThermal.VID.CPU = PerfStatus.SNB.CurrVID;
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
{
case FORMULA_SCOPE_PKG:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
PWR_ACCU_SandyBridge_EP(PUBLIC(RO(Proc)), 1);
Delta_PC02(PUBLIC(RO(Proc)));
Delta_PC03(PUBLIC(RO(Proc)));
Delta_PC06(PUBLIC(RO(Proc)));
Delta_PC07(PUBLIC(RO(Proc)));
Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);
Delta_UNCORE_FC0(PUBLIC(RO(Proc)));
Delta_MC6(PUBLIC(RO(Proc)));
Delta_PWR_ACCU(Proc, PKG);
Delta_PWR_ACCU(Proc, CORES);
Delta_PWR_ACCU(Proc, RAM);
Save_PC02(PUBLIC(RO(Proc)));
Save_PC03(PUBLIC(RO(Proc)));
Save_PC06(PUBLIC(RO(Proc)));
Save_PC07(PUBLIC(RO(Proc)));
Save_PTSC(PUBLIC(RO(Proc)));
Save_UNCORE_FC0(PUBLIC(RO(Proc)));
Save_MC6(PUBLIC(RO(Proc)));
Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);
Save_PWR_ACCU(PUBLIC(RO(Proc)), CORES);
Save_PWR_ACCU(PUBLIC(RO(Proc)), RAM);
Sys_Tick(PUBLIC(RO(Proc)));
} else {
Core->PowerThermal.VID = 0;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core_Intel_Temp(Core);
}
break;
case FORMULA_SCOPE_SMT:
Core_Intel_Temp(Core);
break;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
}
break;
case FORMULA_SCOPE_SMT:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
Delta_INST(Core);
Delta_C0(Core);
Delta_C3(Core);
Delta_C6(Core);
Delta_C7(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Save_INST(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C3(Core);
Save_C6(Core);
Save_C7(Core);
Save_C1(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static void InitTimer_Skylake_X(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_Skylake_X, 1);
}
static void Start_Skylake_X(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Intel_Core_Counters_Set(Save, Core);
SMT_Counters_SandyBridge(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Skylake_X(Core, 0);
Pkg_Intel_PMC_Counters(Core, 0, ARCH_PMC);
PWR_ACCU_SandyBridge_EP(PUBLIC(RO(Proc)), 0);
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Stop_Skylake_X(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_cancel(&PRIVATE(OF(Core, AT(cpu)))->Join.Timer);
Intel_Core_Counters_Clear(Save, Core);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
}
Pkg_Reset_ThermalPoint(PUBLIC(RO(Proc)));
}
PerCore_Reset(Core);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Start_Uncore_Skylake_X(void *arg)
{
UNUSED(arg);
/*TODO(Unsolved)
Uncore_Counters_Set(SKL_X);
Uncore_Counters_Ovf(SKL_X);
*/
Pkg_Intel_PMC_Set(ARCH_PMC, 0x5838);
}
static void Stop_Uncore_Skylake_X(void *arg)
{
UNUSED(arg);
/*TODO(Unsolved)
Uncore_Counters_Clear(SKL_X);
*/
Pkg_Intel_PMC_Clear(ARCH_PMC);
}
static enum hrtimer_restart Cycle_Alderlake_Ecore(struct hrtimer *pTimer)
{
PERF_STATUS PerfStatus = {.value = 0};
CORE_RO *Core;
unsigned int cpu;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
Mark_OVH(Core);
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
SMT_Counters_Alderlake_Ecore(Core, 1);
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
Core->Boost[BOOST(TGT)] = GET_SANDYBRIDGE_TARGET(Core);
RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
Core->Ratio.Perf = PerfStatus.SNB.CurrentRatio;
if (Core->Bind == PUBLIC(RO(Proc))->Service.Hybrid)
{
PKG_Counters_Alderlake_Ecore(Core, 1);
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
{
case FORMULA_SCOPE_PKG:
Core_Intel_Temp(Core);
break;
}
PUBLIC(RO(Proc))->PowerThermal.VID.CPU = PerfStatus.SNB.CurrVID;
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
{
case FORMULA_SCOPE_PKG:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
Delta_PC04(PUBLIC(RO(Proc)));
Delta_PC10(PUBLIC(RO(Proc)));
Delta_MC6(PUBLIC(RO(Proc)));
Save_PC04(PUBLIC(RO(Proc)));
Save_PC10(PUBLIC(RO(Proc)));
Save_MC6(PUBLIC(RO(Proc)));
} else {
Core->PowerThermal.VID = 0;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core_Intel_Temp(Core);
}
break;
case FORMULA_SCOPE_SMT:
Core_Intel_Temp(Core);
break;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
}
break;
case FORMULA_SCOPE_SMT:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
Delta_INST(Core);
Delta_C0(Core);
Delta_C3(Core);
Delta_C6(Core);
Delta_C7(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Save_INST(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C3(Core);
Save_C6(Core);
Save_C7(Core);
Save_C1(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static enum hrtimer_restart Cycle_Alderlake_Pcore(struct hrtimer *pTimer)
{
PERF_STATUS PerfStatus = {.value = 0};
CORE_RO *Core;
unsigned int cpu;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
Mark_OVH(Core);
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
SMT_Counters_Alderlake_Pcore(Core, 1);
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
Core->Boost[BOOST(TGT)] = GET_SANDYBRIDGE_TARGET(Core);
RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
Core->Ratio.Perf = PerfStatus.SNB.CurrentRatio;
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PKG_Counters_Alderlake_Pcore(Core, 1);
Pkg_Intel_Temp(PUBLIC(RO(Proc)));
Monitor_CorePerfLimitReasons(PUBLIC(RO(Proc)));
Monitor_GraphicsPerfLimitReasons(PUBLIC(RO(Proc)));
Monitor_RingPerfLimitReasons(PUBLIC(RO(Proc)));
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
{
case FORMULA_SCOPE_PKG:
Core_Intel_Temp(Core);
break;
}
PUBLIC(RO(Proc))->PowerThermal.VID.CPU = PerfStatus.SNB.CurrVID;
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
{
case FORMULA_SCOPE_PKG:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
PWR_ACCU_Alderlake(PUBLIC(RO(Proc)), 1);
Delta_PC02(PUBLIC(RO(Proc)));
Delta_PC03(PUBLIC(RO(Proc)));
Delta_PC06(PUBLIC(RO(Proc)));
Delta_PC07(PUBLIC(RO(Proc)));
Delta_PC08(PUBLIC(RO(Proc)));
Delta_PC09(PUBLIC(RO(Proc)));
Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);
Delta_UNCORE_FC0(PUBLIC(RO(Proc)));
Delta_PWR_ACCU(Proc, PKG);
Delta_PWR_ACCU(Proc, CORES);
Delta_PWR_ACCU(Proc, UNCORE);
Delta_PWR_ACCU(Proc, RAM);
Delta_PWR_ACCU(Proc, PLATFORM);
Save_PC02(PUBLIC(RO(Proc)));
Save_PC03(PUBLIC(RO(Proc)));
Save_PC06(PUBLIC(RO(Proc)));
Save_PC07(PUBLIC(RO(Proc)));
Save_PC08(PUBLIC(RO(Proc)));
Save_PC09(PUBLIC(RO(Proc)));
Save_PTSC(PUBLIC(RO(Proc)));
Save_UNCORE_FC0(PUBLIC(RO(Proc)));
Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);
Save_PWR_ACCU(PUBLIC(RO(Proc)), CORES);
Save_PWR_ACCU(PUBLIC(RO(Proc)), UNCORE);
Save_PWR_ACCU(PUBLIC(RO(Proc)), RAM);
Save_PWR_ACCU(PUBLIC(RO(Proc)), PLATFORM);
Sys_Tick(PUBLIC(RO(Proc)));
} else {
Core->PowerThermal.VID = 0;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core_Intel_Temp(Core);
}
break;
case FORMULA_SCOPE_SMT:
Core_Intel_Temp(Core);
break;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
}
break;
case FORMULA_SCOPE_SMT:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
Delta_INST(Core);
Delta_C0(Core);
Delta_C3(Core);
Delta_C6(Core);
Delta_C7(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Save_INST(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C3(Core);
Save_C6(Core);
Save_C7(Core);
Save_C1(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static void InitTimer_Alderlake(unsigned int cpu)
{
switch (PUBLIC(RO(Core, AT(cpu)))->T.Cluster.Hybrid.CoreType) {
case Hybrid_Atom:
smp_call_function_single(cpu, InitTimer, Cycle_Alderlake_Ecore, 1);
break;
case Hybrid_Core:
case Hybrid_RSVD1:
case Hybrid_RSVD2:
default:
smp_call_function_single(cpu, InitTimer, Cycle_Alderlake_Pcore, 1);
break;
}
}
static void Start_Alderlake(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Intel_Core_Counters_Set(Save, Core);
switch (PUBLIC(RO(Core, AT(cpu)))->T.Cluster.Hybrid.CoreType) {
case Hybrid_Atom:
SMT_Counters_Alderlake_Ecore(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Hybrid)
{
PKG_Counters_Alderlake_Ecore(Core, 0);
}
break;
case Hybrid_Core:
case Hybrid_RSVD1:
case Hybrid_RSVD2:
default:
SMT_Counters_Alderlake_Pcore(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Alderlake_Pcore(Core, 0);
PWR_ACCU_Alderlake(PUBLIC(RO(Proc)), 0);
}
break;
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Start_Uncore_Alderlake(void *arg)
{
UNUSED(arg);
Uncore_Counters_Set(ADL);
Uncore_Counters_Ovf(ADL);
}
static void Stop_Uncore_Alderlake(void *arg)
{
UNUSED(arg);
Uncore_Counters_Clear(ADL);
}
static enum hrtimer_restart Cycle_Arrowlake(struct hrtimer *pTimer)
{
PERF_STATUS PerfStatus = {.value = 0};
CORE_RO *Core;
unsigned int cpu;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
Mark_OVH(Core);
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
SMT_Counters_Arrowlake(Core, 1);
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
Core->Boost[BOOST(TGT)] = GET_SANDYBRIDGE_TARGET(Core);
RDMSR(PerfStatus, MSR_IA32_PERF_STATUS);
Core->Ratio.Perf = PerfStatus.SNB.CurrentRatio;
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PKG_Counters_Arrowlake(Core, 1);
Pkg_Intel_Temp(PUBLIC(RO(Proc)));
Monitor_CorePerfLimitReasons(PUBLIC(RO(Proc)));
Monitor_GraphicsPerfLimitReasons(PUBLIC(RO(Proc)));
Monitor_RingPerfLimitReasons(PUBLIC(RO(Proc)));
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
{
case FORMULA_SCOPE_PKG:
Core_Intel_Temp(Core);
break;
}
PUBLIC(RO(Proc))->PowerThermal.VID.CPU = PerfStatus.SNB.CurrVID;
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
{
case FORMULA_SCOPE_PKG:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
PWR_ACCU_Arrowlake(PUBLIC(RO(Proc)), 1);
/* P-core */
Delta_PC02(PUBLIC(RO(Proc)));
Delta_PC03(PUBLIC(RO(Proc)));
Delta_PC06(PUBLIC(RO(Proc)));
Delta_PC07(PUBLIC(RO(Proc)));
Delta_PC08(PUBLIC(RO(Proc)));
Delta_PC09(PUBLIC(RO(Proc)));
/* TSC & Uncore */
Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);
Delta_UNCORE_FC0(PUBLIC(RO(Proc)));
/* E-core */
Delta_PC04(PUBLIC(RO(Proc)));
Delta_PC10(PUBLIC(RO(Proc)));
Delta_MC6(PUBLIC(RO(Proc)));
/* Power */
Delta_PWR_ACCU(Proc, PKG);
Delta_PWR_ACCU(Proc, CORES);
Delta_PWR_ACCU(Proc, UNCORE);
Delta_PWR_ACCU(Proc, RAM);
Delta_PWR_ACCU(Proc, PLATFORM);
/* P-Core */
Save_PC02(PUBLIC(RO(Proc)));
Save_PC03(PUBLIC(RO(Proc)));
Save_PC06(PUBLIC(RO(Proc)));
Save_PC07(PUBLIC(RO(Proc)));
Save_PC08(PUBLIC(RO(Proc)));
Save_PC09(PUBLIC(RO(Proc)));
/* TSC & Uncore */
Save_PTSC(PUBLIC(RO(Proc)));
Save_UNCORE_FC0(PUBLIC(RO(Proc)));
/* E-core */
Save_PC04(PUBLIC(RO(Proc)));
Save_PC10(PUBLIC(RO(Proc)));
Save_MC6(PUBLIC(RO(Proc)));
/* Power */
Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);
Save_PWR_ACCU(PUBLIC(RO(Proc)), CORES);
Save_PWR_ACCU(PUBLIC(RO(Proc)), UNCORE);
Save_PWR_ACCU(PUBLIC(RO(Proc)), RAM);
Save_PWR_ACCU(PUBLIC(RO(Proc)), PLATFORM);
Sys_Tick(PUBLIC(RO(Proc)));
} else {
Core->PowerThermal.VID = 0;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core_Intel_Temp(Core);
}
break;
case FORMULA_SCOPE_SMT:
Core_Intel_Temp(Core);
break;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
}
break;
case FORMULA_SCOPE_SMT:
Core->PowerThermal.VID = PerfStatus.SNB.CurrVID;
break;
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
Delta_INST(Core);
Delta_C0(Core);
Delta_C3(Core);
Delta_C6(Core);
Delta_C7(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Save_INST(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C3(Core);
Save_C6(Core);
Save_C7(Core);
Save_C1(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static void InitTimer_Arrowlake(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_Arrowlake, 1);
}
static void Start_Arrowlake(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Intel_Core_Counters_Set(Save, Core);
SMT_Counters_Arrowlake(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Arrowlake(Core, 0);
PWR_ACCU_Arrowlake(PUBLIC(RO(Proc)), 0);
}
RDCOUNTER(Core->Interrupt.SMI, MSR_SMI_COUNT);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Start_Uncore_Arrowlake(void *arg)
{
UNUSED(arg);
/*TODO Uncore_Counters_Set(ARL); */
}
static void Stop_Uncore_Arrowlake(void *arg)
{
UNUSED(arg);
/*TODO Uncore_Counters_Clear(ARL); */
}
static enum hrtimer_restart Cycle_AMD_Family_0Fh(struct hrtimer *pTimer)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
FIDVID_CONTROL FidVidControl = {.value = 0};
FIDVID_STATUS FidVidStatus = {.value = 0};
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
RDMSR(FidVidControl, MSR_K7_FID_VID_CTL);
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(TGT)] = FidVidControl.NewFID;
RDMSR(FidVidStatus, MSR_K7_FID_VID_STATUS);
Core->PowerThermal.VID = FidVidStatus.CurrVID;
Core->Ratio.Perf = 8 + FidVidStatus.CurrFID;
/* P-States */
Core->Counter[1].C0.UCC = Core->Counter[0].C0.UCC
+ Core->Ratio.Perf
* Core->Clock.Hz;
Core->Counter[1].C0.URC = Core->Counter[1].C0.UCC;
Core->Counter[1].TSC = Core->Counter[0].TSC
+ (Core->Boost[BOOST(MAX)]
* Core->Clock.Hz);
/* Derive C1 */
Core->Counter[1].C1 =
(Core->Counter[1].TSC > Core->Counter[1].C0.URC) ?
Core->Counter[1].TSC - Core->Counter[1].C0.URC
: 0;
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PKG_Counters_Generic(Core, 1);
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
{
case FORMULA_SCOPE_PKG:
Core_AMD_Family_0Fh_Temp(Core);
break;
}
PUBLIC(RO(Proc))->PowerThermal.VID.CPU = FidVidStatus.CurrVID;
Delta_PTSC(PUBLIC(RO(Proc)));
Save_PTSC(PUBLIC(RO(Proc)));
Sys_Tick(PUBLIC(RO(Proc)));
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core_AMD_Family_0Fh_Temp(Core);
}
break;
case FORMULA_SCOPE_SMT:
Core_AMD_Family_0Fh_Temp(Core);
break;
}
Delta_C0(Core);
Delta_TSC(Core);
Delta_C1(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C1(Core);
if (AutoClock & 0b10)
{
REL_BCLK(Core->Clock,
Core->Boost[BOOST(MAX)],
Core->Delta.TSC,
PUBLIC(RO(Proc))->SleepInterval);
}
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static void InitTimer_AMD_Family_0Fh(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_AMD_Family_0Fh, 1);
}
static void Start_AMD_Family_0Fh(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Generic(Core, 0);
}
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Stop_AMD_Family_0Fh(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
UNUSED(arg);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_cancel(&PRIVATE(OF(Core, AT(cpu)))->Join.Timer);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
}
Pkg_Reset_ThermalPoint(PUBLIC(RO(Proc)));
}
PerCore_Reset(Core);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Start_AMD_Family_10h(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Counters_Generic(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Generic(Core, 0);
}
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Stop_AMD_Family_10h(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
UNUSED(arg);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_cancel(&PRIVATE(OF(Core, AT(cpu)))->Join.Timer);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
}
Pkg_Reset_ThermalPoint(PUBLIC(RO(Proc)));
}
PerCore_Reset(Core);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Start_AMD_Family_11h(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Counters_Generic(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Generic(Core, 0);
}
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Start_AMD_Family_12h(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Counters_Generic(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Generic(Core, 0);
}
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Start_AMD_Family_14h(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Counters_Generic(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Generic(Core, 0);
}
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static enum hrtimer_restart Cycle_AMD_Family_15h(struct hrtimer *pTimer)
{
PSTATECTRL PstateCtrl = {.value = 0};
PSTATESTAT PstateStat = {.value = 0};
PSTATEDEF PstateDef = {.value = 0};
CORE_RO *Core;
unsigned int cpu;
cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
Mark_OVH(Core);
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
Counters_Generic(Core, 1);
if (PUBLIC(RO(Proc))->Features.AdvPower.EDX.HwPstate)
{
unsigned int pstate, COF;
/* Read the Target & Status P-State. */
RDMSR(PstateCtrl, MSR_AMD_PERF_CTL);
RDMSR(PstateDef,MSR_AMD_PSTATE_DEF_BASE + PstateCtrl.PstateCmd);
COF = AMD_F15h_CoreCOF( PstateDef.Family_15h.CpuFid,
PstateDef.Family_15h.CpuDid );
PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(TGT)] = COF;
/* Read the current P-State number. */
RDMSR(PstateStat, MSR_AMD_PERF_STATUS);
/* Offset the P-State base register. */
pstate = MSR_AMD_PSTATE_DEF_BASE + PstateStat.Current;
/* Read the voltage ID at the offset */
RDMSR(PstateDef, pstate);
COF = AMD_F15h_CoreCOF( PstateDef.Family_15h.CpuFid,
PstateDef.Family_15h.CpuDid );
Core->Ratio.Perf = COF;
}
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
PKG_Counters_Generic(Core, 1);
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula))
{
case FORMULA_SCOPE_PKG:
Core_AMD_Family_15h_Temp(Core);
break;
}
PUBLIC(RO(Proc))->PowerThermal.VID.CPU=PstateDef.Family_15h.CpuVid;
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula))
{
case FORMULA_SCOPE_PKG:
Core->PowerThermal.VID = PstateDef.Family_15h.CpuVid;
break;
}
Delta_PTSC_OVH(PUBLIC(RO(Proc)), Core);
Save_PTSC(PUBLIC(RO(Proc)));
Sys_Tick(PUBLIC(RO(Proc)));
} else {
Core->PowerThermal.VID = 0;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
case FORMULA_SCOPE_CORE:
if (Core->T.CoreID == 0) {
Core_AMD_Family_15h_Temp(Core);
}
break;
case FORMULA_SCOPE_SMT:
Core_AMD_Family_15h_Temp(Core);
break;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)) {
Core->PowerThermal.VID = PstateDef.Family_15h.CpuVid;
}
break;
case FORMULA_SCOPE_SMT:
Core->PowerThermal.VID = PstateDef.Family_15h.CpuVid;
break;
}
Delta_C0(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C1(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static void InitTimer_AMD_Family_15h(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_AMD_Family_15h, 1);
}
static void Start_AMD_Family_15h(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
UNUSED(arg);
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
Counters_Generic(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core) {
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
PKG_Counters_Generic(Core, 0);
}
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Cycle_AMD_Family_17h(CORE_RO *Core,
void (*Call_PWR)(CORE_RO *Core),
void (*Call_SMU)(const unsigned int, const unsigned int,
const unsigned long long),
const unsigned int plane0, const unsigned int plane1,
const unsigned long long factor)
{
PSTATEDEF PstateDef;
PSTATECTRL PstateCtrl;
COF_ST COF;
SMT_Counters_AMD_Family_17h(Core, 1);
/* Read the Target P-State. */
RDMSR(PstateCtrl, MSR_AMD_PERF_CTL);
RDMSR(PstateDef, MSR_AMD_PSTATE_DEF_BASE + PstateCtrl.PstateCmd);
COF = AMD_Zen_CoreCOF(PstateDef);
PUBLIC(RO(Core, AT(Core->Bind)))->Boost[BOOST(TGT)] = COF.Q;
/* Read the Boosted Frequency and voltage VID. */
RDMSR(PstateDef, MSR_AMD_F17H_HW_PSTATE_STATUS);
COF = AMD_Zen_CoreCOF(PstateDef);
Core->Ratio.COF = COF;
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
Pkg_AMD_Zen_PMC_Counters(PUBLIC(RO(Proc)), Core, 1, ARCH_PMC,
Pkg_AMD_Zen_PMC_Closure(PUBLIC(RO(Proc)), Core, 1, ARCH_PMC)
);
Pkg_AMD_Family_17h_Temp(PUBLIC(RO(Proc)), Core);
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
case FORMULA_SCOPE_PKG:
Core_AMD_Family_17h_Temp(Core);
break;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
case FORMULA_SCOPE_PKG:
Core->PowerThermal.VID = PstateDef.Family_17h.CpuVid;
break;
}
Call_SMU(plane0, plane1, factor);
RDCOUNTER( PUBLIC(RO(Proc))->Counter[1].Power.ACCU[PWR_DOMAIN(PKG)],
MSR_AMD_PKG_ENERGY_STATUS );
Delta_UNCORE_FC0(PUBLIC(RO(Proc)));
Delta_PWR_ACCU(Proc, PKG);
Save_UNCORE_FC0(PUBLIC(RO(Proc)));
Save_PWR_ACCU(PUBLIC(RO(Proc)), PKG);
Sys_Tick(PUBLIC(RO(Proc)));
} else {
Core->PowerThermal.VID = 0;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->thermalFormula)) {
case FORMULA_SCOPE_CORE:
if ((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1))
{
Core_AMD_Family_17h_Temp(Core);
break;
}
fallthrough;
case FORMULA_SCOPE_SMT:
Core_AMD_Family_17h_Temp(Core);
break;
}
switch (SCOPE_OF_FORMULA(PUBLIC(RO(Proc))->voltageFormula)) {
case FORMULA_SCOPE_CORE:
if (!((Core->T.ThreadID == 0) || (Core->T.ThreadID == -1)))
{
break;
}
fallthrough;
case FORMULA_SCOPE_SMT:
Core->PowerThermal.VID = PstateDef.Family_17h.CpuVid;
break;
}
Call_PWR(Core);
AMD_Zen_PMC_Counters(Core, 1, ARCH_PMC,
AMD_Zen_PMC_Closure(Core, 1, ARCH_PMC)
);
Delta_INST(Core);
Delta_C0(Core);
Delta_C3(Core);
Delta_C6(Core);
Delta_TSC_OVH(Core);
Delta_C1(Core);
Save_INST(Core);
Save_TSC(Core);
Save_C0(Core);
Save_C3(Core);
Save_C6(Core);
Save_C1(Core);
BITSET(LOCKLESS, PUBLIC(RW(Core, AT(Core->Bind)))->Sync.V, NTFY);
}
static void Call_MSR_ACCU(CORE_RO *Core)
{ /* Read the Physical Core RAPL counter. */
if (Core->T.ThreadID == 0)
{
RDCOUNTER(Core->Counter[1].Power.ACCU, MSR_AMD_PP0_ENERGY_STATUS);
Core->Counter[1].Power.ACCU &= 0xffffffff;
Core->Delta.Power.ACCU = Core->Counter[1].Power.ACCU
- Core->Counter[0].Power.ACCU;
Core->Delta.Power.ACCU &= 0xffffffff;
Core->Counter[0].Power.ACCU = Core->Counter[1].Power.ACCU;
}
}
static void Call_HSMP_ACCU(CORE_RO *Core)
{ /* Convert DIMM Power from HSMP to RAPL. */
if (PUBLIC(RO(Proc))->Features.HSMP_Enable)
{
ZEN_HSMP_DIMM_PWR DIMM_PWR = {
.mWatt = 0, .ms = 0, .addr = Core->T.ApicID
};
HSMP_ARG arg[8] = {
[7] = {0x0}, [6] = {0x0}, [5] = {0x0}, [4] = {0x0},
[3] = {0x0}, [2] = {0x0}, [1] = {0x0}, [0] = {DIMM_PWR.value}
};
unsigned int rx;
if ((rx = AMD_HSMP_Exec(HSMP_RD_DIMM_PWR, arg)) == HSMP_RESULT_OK)
{
DIMM_PWR.value = arg[0].value;
if ((DIMM_PWR.mWatt > 0) && (DIMM_PWR.mWatt != 0b111111111111111))
{
Core->Delta.RAM.ACCU = (unsigned long long) DIMM_PWR.mWatt;
Core->Delta.RAM.ACCU <<= PUBLIC(RO(Proc))->PowerThermal.Unit.ESU;
Core->Delta.RAM.ACCU *= PUBLIC(RO(Proc))->SleepInterval;
Core->Delta.RAM.ACCU = Core->Delta.RAM.ACCU / (1000LLU * 1000LLU);
}
}
else if (IS_HSMP_OOO(rx))
{
PUBLIC(RO(Proc))->Features.HSMP_Enable = 0;
}
}
}
static void Call_Genoa_ACCU(CORE_RO *Core)
{
Call_MSR_ACCU(Core);
Call_HSMP_ACCU(Core);
}
static void SoC_RAPL(AMD_F17H_SVI SVI, const unsigned long long factor)
{
unsigned long long VCC, ICC, ACCU;
/* PLATFORM RAPL workaround to provide the SoC power */
VCC = 155000LLU - (625LLU * SVI.VID);
ICC = SVI.IDD * factor;
ACCU = VCC * ICC;
ACCU = ACCU << PUBLIC(RO(Proc))->PowerThermal.Unit.ESU;
ACCU = ACCU / (100000LLU * 1000000LLU);
ACCU = (PUBLIC(RO(Proc))->SleepInterval * ACCU) / 1000LLU;
PUBLIC(RW(Proc))->Delta.Power.ACCU[PWR_DOMAIN(UNCORE)] = ACCU;
}
static void Call_SVI( const unsigned int plane0, const unsigned int plane1,
const unsigned long long factor )
{
AMD_F17H_SVI SVI = {.value = 0};
Core_AMD_SMN_Read( SVI,
SMU_AMD_F17H_SVI(plane0),
PRIVATE(OF(Zen)).Device.DF );
PUBLIC(RO(Proc))->PowerThermal.VID.CPU = SVI.VID;
Core_AMD_SMN_Read( SVI,
SMU_AMD_F17H_SVI(plane1),
PRIVATE(OF(Zen)).Device.DF );
PUBLIC(RO(Proc))->PowerThermal.VID.SOC = SVI.VID;
SoC_RAPL(SVI, factor);
}
static void Call_SVI_APU(const unsigned int plane0, const unsigned int plane1,
const unsigned long long factor)
{
AMD_F17H_SVI SVI = {.value = 0};
Core_AMD_SMN_Read( SVI,
SMU_AMD_F17_60H_SVI(plane0),
PRIVATE(OF(Zen)).Device.DF );
PUBLIC(RO(Proc))->PowerThermal.VID.CPU = SVI.VID;
Core_AMD_SMN_Read( SVI,
SMU_AMD_F17_60H_SVI(plane1),
PRIVATE(OF(Zen)).Device.DF );
PUBLIC(RO(Proc))->PowerThermal.VID.SOC = SVI.VID;
SoC_RAPL(SVI, factor);
}
static void Call_SVI_RMB(const unsigned int plane0, const unsigned int plane1,
const unsigned long long factor)
{
AMD_RMB_SVI SVI = {.value = 0};
UNUSED(factor);
Core_AMD_SMN_Read( SVI,
SMU_AMD_RMB_SVI(plane0),
PRIVATE(OF(Zen)).Device.DF );
PUBLIC(RO(Proc))->PowerThermal.VID.CPU = SVI.SVI1;
Core_AMD_SMN_Read( SVI,
SMU_AMD_RMB_SVI(plane1),
PRIVATE(OF(Zen)).Device.DF );
PUBLIC(RO(Proc))->PowerThermal.VID.SOC = SVI.SVI0;
}
static void Call_DFLT( const unsigned int plane0, const unsigned int plane1,
const unsigned long long factor )
{
UNUSED(plane0);
UNUSED(plane1);
UNUSED(factor);
PUBLIC(RO(Proc))->PowerThermal.VID.CPU = \
PUBLIC(RO(Core,AT( PUBLIC(RO(Proc))->Service.Core )))->PowerThermal.VID;
}
static void Call_Raphael(const unsigned int plane0, const unsigned int plane1,
const unsigned long long factor)
{
AMD_F19H_SVI SVI = {.value = 0};
Call_DFLT(plane0, plane1, factor);
Core_AMD_SMN_Read(SVI,
PUBLIC(RO(Core, AT(PUBLIC(RO(Proc))->Service.Core)))->T.PackageID == 0 ?
SMU_AMD_F19H_SVI(plane0) : SMU_AMD_F19H_SVI(plane1),
PRIVATE(OF(Zen)).Device.DF);
PUBLIC(RO(Proc))->PowerThermal.VID.SOC = SVI.SVI1;
}
static void Call_Genoa( const unsigned int plane0, const unsigned int plane1,
const unsigned long long factor )
{
AMD_F19H_SVI SVI = {.value = 0};
Call_DFLT(plane0, plane1, factor);
Core_AMD_SMN_Read(SVI,
PUBLIC(RO(Core, AT(PUBLIC(RO(Proc))->Service.Core)))->T.PackageID == 0 ?
SMU_AMD_F17H_SVI(plane0) : SMU_AMD_F17H_SVI(plane1),
PRIVATE(OF(Zen)).Device.DF);
PUBLIC(RO(Proc))->PowerThermal.VID.SOC = SVI.SVI1;
}
static enum hrtimer_restart Entry_AMD_F17h(struct hrtimer *pTimer,
void (*Call_PWR)(CORE_RO *Core),
void (*Call_SMU)(const unsigned int, const unsigned int,
const unsigned long long),
const unsigned int plane0, const unsigned int plane1,
const unsigned long long factor)
{
CORE_RO *Core;
unsigned int cpu = smp_processor_id();
Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
Mark_OVH(Core);
if (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD) == 1)
{
hrtimer_forward(pTimer,
hrtimer_cb_get_time(pTimer),
RearmTheTimer);
Cycle_AMD_Family_17h(Core, Call_PWR, Call_SMU, plane0, plane1, factor);
return HRTIMER_RESTART;
} else
return HRTIMER_NORESTART;
}
static enum hrtimer_restart Cycle_AMD_F17h_Zen(struct hrtimer *pTimer)
{
return Entry_AMD_F17h(pTimer, Call_MSR_ACCU, Call_SVI, 0, 1, 360772LLU);
}
static enum hrtimer_restart Cycle_AMD_F17h_Zen2_SP(struct hrtimer *pTimer)
{
return Entry_AMD_F17h(pTimer, Call_MSR_ACCU, Call_SVI, 1, 0, 294300LLU);
}
static enum hrtimer_restart Cycle_AMD_F17h_Zen2_MP(struct hrtimer *pTimer)
{
return Entry_AMD_F17h(pTimer, Call_MSR_ACCU, Call_SVI, 2, 1, 294300LLU);
}
static enum hrtimer_restart Cycle_AMD_F17h_Zen2_APU(struct hrtimer *pTimer)
{
return Entry_AMD_F17h(pTimer, Call_MSR_ACCU,Call_SVI_APU,0,1,294300LLU);
}
static enum hrtimer_restart Cycle_AMD_Zen3Plus_RMB(struct hrtimer *pTimer)
{
return Entry_AMD_F17h(pTimer, Call_MSR_ACCU, Call_SVI_RMB, 0, 2, 0LLU);
}
static enum hrtimer_restart Cycle_AMD_Zen4_RPL(struct hrtimer *pTimer)
{
return Entry_AMD_F17h(pTimer, Call_MSR_ACCU, Call_Raphael, 1, 2, 0LLU);
}
static enum hrtimer_restart Cycle_AMD_Zen4_Genoa(struct hrtimer *pTimer)
{
return Entry_AMD_F17h(pTimer, Call_Genoa_ACCU, Call_Genoa, 1, 2, 0LLU);
}
static enum hrtimer_restart Cycle_AMD_F17h(struct hrtimer *pTimer)
{
return Entry_AMD_F17h(pTimer, Call_MSR_ACCU, Call_DFLT, 0, 0, 0LLU);
}
static void InitTimer_AMD_Family_17h(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_AMD_F17h, 1);
}
static void InitTimer_AMD_F17h_Zen(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_AMD_F17h_Zen, 1);
}
static void InitTimer_AMD_F17h_Zen2_SP(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_AMD_F17h_Zen2_SP, 1);
}
static void InitTimer_AMD_F17h_Zen2_MP(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_AMD_F17h_Zen2_MP, 1);
}
static void InitTimer_AMD_F17h_Zen2_APU(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_AMD_F17h_Zen2_APU, 1);
}
static void InitTimer_AMD_Zen3Plus_RMB(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_AMD_Zen3Plus_RMB, 1);
}
static void InitTimer_AMD_Zen4_RPL(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_AMD_Zen4_RPL, 1);
}
static void InitTimer_AMD_Zen4_Genoa(unsigned int cpu)
{
smp_call_function_single(cpu, InitTimer, Cycle_AMD_Zen4_Genoa, 1);
}
static void Start_AMD_Family_17h(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
#ifdef CONFIG_PM_SLEEP
if (CoreFreqK.ResumeFromSuspend == true) {
WRCOUNTER(0x0LLU, MSR_IA32_MPERF);
WRCOUNTER(0x0LLU, MSR_IA32_APERF);
}
#endif /* CONFIG_PM_SLEEP */
if (Arch[PUBLIC(RO(Proc))->ArchID].Update != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Update(Core);
}
AMD_Core_Counters_Set(Save, Core, Family_17h);
AMD_Zen_PMC_Set(Save, Core, ARCH_PMC);
SMT_Counters_AMD_Family_17h(Core, 0);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
Pkg_AMD_Zen_PMC_Set(Core, ARCH_PMC);
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Start(NULL);
}
Pkg_AMD_Zen_PMC_Counters(PUBLIC(RO(Proc)), Core, 0, ARCH_PMC);
RDCOUNTER(PUBLIC(RO(Proc))->Counter[0].Power.ACCU[PWR_DOMAIN(PKG)],
MSR_AMD_PKG_ENERGY_STATUS );
}
if (Core->T.ThreadID == 0)
{
RDCOUNTER(Core->Counter[0].Power.ACCU, MSR_AMD_PP0_ENERGY_STATUS);
Core->Counter[0].Power.ACCU &= 0xffffffff;
}
AMD_Zen_PMC_Counters(Core, 0, ARCH_PMC);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_start( &PRIVATE(OF(Core, AT(cpu)))->Join.Timer,
RearmTheTimer,
HRTIMER_MODE_REL_PINNED);
BITSET(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Stop_AMD_Family_17h(void *arg)
{
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
union SAVE_AREA_CORE *Save = &PRIVATE(OF(Core, AT(cpu)))->SaveArea;
UNUSED(arg);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, MUSTFWD);
hrtimer_cancel(&PRIVATE(OF(Core, AT(cpu)))->Join.Timer);
AMD_Core_Counters_Clear(Save, Core);
AMD_Zen_PMC_Clear(Save, Core, ARCH_PMC);
if (Core->Bind == PUBLIC(RO(Proc))->Service.Core)
{
Pkg_AMD_Zen_PMC_Clear(Core, ARCH_PMC);
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Uncore.Stop(NULL);
}
Pkg_Reset_ThermalPoint(PUBLIC(RO(Proc)));
}
PerCore_Reset(Core);
BITCLR(LOCKLESS, PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED);
}
static void Start_Uncore_AMD_Family_17h(void *arg)
{
ZEN_DF_PERF_CTL Zen_DataFabricPerfControl;
UNUSED(arg);
RDMSR(Zen_DataFabricPerfControl, MSR_AMD_F17H_DF_PERF_CTL);
PRIVATE(OF(SaveArea)).AMD.Zen_DataFabricPerfControl = \
Zen_DataFabricPerfControl;
Zen_DataFabricPerfControl.EventSelect00 = 0x0f;
Zen_DataFabricPerfControl.UnitMask = 0xff;
Zen_DataFabricPerfControl.CounterEn = 1;
Zen_DataFabricPerfControl.EventSelect08 = 0x00;
Zen_DataFabricPerfControl.EventSelect12 = 0x00;
WRMSR(Zen_DataFabricPerfControl, MSR_AMD_F17H_DF_PERF_CTL);
}
static void Stop_Uncore_AMD_Family_17h(void *arg)
{
UNUSED(arg);
WRMSR( PRIVATE(OF(SaveArea)).AMD.Zen_DataFabricPerfControl,
MSR_AMD_F17H_DF_PERF_CTL );
}
#ifdef CONFIG_CPU_FREQ
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 16, 0)
static int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
{
struct cpufreq_policy *cpu_policy __free(put_cpufreq_policy);
if (!policy)
return -EINVAL;
cpu_policy = cpufreq_cpu_get(cpu);
if (!cpu_policy)
return -EINVAL;
memcpy(policy, cpu_policy, sizeof(*policy));
return 0;
}
#endif
#endif /* CONFIG_CPU_FREQ */
static long Sys_OS_Driver_Query(void)
{
int rc = RC_SUCCESS;
#ifdef CONFIG_CPU_FREQ
const char *pFreqDriver;
struct cpufreq_policy *pFreqPolicy;
#endif /* CONFIG_CPU_FREQ */
#ifdef CONFIG_CPU_IDLE
struct cpuidle_driver *pIdleDriver;
#endif /* CONFIG_CPU_IDLE */
memset(&PUBLIC(RO(Proc))->OS, 0, sizeof(OS_DRIVER));
#ifdef CONFIG_CPU_IDLE
if ((pIdleDriver = cpuidle_get_driver()) != NULL)
{
int idx;
StrCopy(PUBLIC(RO(Proc))->OS.IdleDriver.Name,
pIdleDriver->name,
CPUIDLE_NAME_LEN);
if (pIdleDriver->state_count < CPUIDLE_STATE_MAX) {
PUBLIC(RO(Proc))->OS.IdleDriver.stateCount = pIdleDriver->state_count;
} else {
PUBLIC(RO(Proc))->OS.IdleDriver.stateCount = CPUIDLE_STATE_MAX;
}
PUBLIC(RO(Proc))->OS.IdleDriver.stateLimit = pIdleDriver->state_count;
for (idx = 0; idx < PUBLIC(RO(Proc))->OS.IdleDriver.stateCount; idx++)
{
StrCopy(PUBLIC(RO(Proc))->OS.IdleDriver.State[idx].Name,
pIdleDriver->states[idx].name, CPUIDLE_NAME_LEN);
StrCopy(PUBLIC(RO(Proc))->OS.IdleDriver.State[idx].Desc,
pIdleDriver->states[idx].desc, CPUIDLE_NAME_LEN);
PUBLIC(RO(Proc))->OS.IdleDriver.State[idx].exitLatency = \
pIdleDriver->states[idx].exit_latency;
PUBLIC(RO(Proc))->OS.IdleDriver.State[idx].powerUsage = \
pIdleDriver->states[idx].power_usage;
PUBLIC(RO(Proc))->OS.IdleDriver.State[idx].targetResidency = \
pIdleDriver->states[idx].target_residency;
}
if(PUBLIC(RO(Proc))->Registration.Driver.CPUidle == REGISTRATION_ENABLE)
{
for (idx = SUB_CSTATE_COUNT - 1; idx >= 0 ; idx--)
{
if (CoreFreqK.SubCstate[idx] > 0)
{
PUBLIC(RO(Proc))->OS.IdleDriver.stateLimit = 1 + idx;
break;
}
}
}
}
#endif /* CONFIG_CPU_IDLE */
#ifdef CONFIG_CPU_FREQ
if ((pFreqDriver = cpufreq_get_current_driver()) != NULL) {
StrCopy(PUBLIC(RO(Proc))->OS.FreqDriver.Name,
pFreqDriver, CPUFREQ_NAME_LEN);
}
if ((pFreqPolicy=kzalloc(sizeof(struct cpufreq_policy),GFP_KERNEL)) != NULL) {
if((rc=cpufreq_get_policy(pFreqPolicy,PUBLIC(RO(Proc))->Service.Core)) == 0)
{
struct cpufreq_governor *pGovernor = pFreqPolicy->governor;
if (pGovernor != NULL) {
StrCopy(PUBLIC(RO(Proc))->OS.FreqDriver.Governor,
pGovernor->name, CPUFREQ_NAME_LEN);
} else {
PUBLIC(RO(Proc))->OS.FreqDriver.Governor[0] = '\0';
}
} else {
PUBLIC(RO(Proc))->OS.FreqDriver.Governor[0] = '\0';
}
kfree(pFreqPolicy);
} else {
rc = -ENOMEM;
}
#endif /* CONFIG_CPU_FREQ */
return rc;
}
static long Sys_Kernel(SYSGATE_RO *SysGate)
{ /* Sources: /include/generated/uapi/linux/version.h
/include/uapi/linux/utsname.h */
if (SysGate != NULL) {
SysGate->kernelVersionNumber = LINUX_VERSION_CODE;
memcpy(SysGate->sysname, utsname()->sysname, MAX_UTS_LEN);
memcpy(SysGate->release, utsname()->release, MAX_UTS_LEN);
memcpy(SysGate->version, utsname()->version, MAX_UTS_LEN);
memcpy(SysGate->machine, utsname()->machine, MAX_UTS_LEN);
return RC_OK_SYSGATE;
} else {
return -ENXIO;
}
}
static long SysGate_OnDemand(void)
{
long rc = -1;
if (PUBLIC(OF(Gate)) == NULL)
{ /* On-demand allocation. */
PUBLIC(OF(Gate)) = alloc_pages_exact(PUBLIC(RO(Proc))->Gate.ReqMem.Size,
GFP_KERNEL);
if (PUBLIC(OF(Gate)) != NULL)
{
memset(PUBLIC(OF(Gate)), 0, PUBLIC(RO(Proc))->Gate.ReqMem.Size);
rc = 0;
}
} else { /* Already allocated */
rc = 1;
}
return rc;
}
#define Atomic_Write_VPMC( _Core, cycles, _lvl) \
{ \
switch (_lvl) { \
case 0: \
Atomic_Add_VPMC(LOCKLESS, _Core->VPMC.C1, cycles); \
break; \
case 1: \
Atomic_Add_VPMC(LOCKLESS, _Core->VPMC.C2, cycles); \
break; \
case 2: \
Atomic_Add_VPMC(LOCKLESS, _Core->VPMC.C3, cycles); \
break; \
case 3: \
Atomic_Add_VPMC(LOCKLESS, _Core->VPMC.C4, cycles); \
break; \
case 4: \
Atomic_Add_VPMC(LOCKLESS, _Core->VPMC.C5, cycles); \
break; \
case 5: \
Atomic_Add_VPMC(LOCKLESS, _Core->VPMC.C6, cycles); \
break; \
case 6: \
Atomic_Add_VPMC(LOCKLESS, _Core->VPMC.C7, cycles); \
break; \
}; \
}
#if defined(CONFIG_CPU_IDLE) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
/* MWAIT Idle methods */
static int CoreFreqK_MWAIT_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{/* Source: /drivers/cpuidle/cpuidle.c */
unsigned long MWAIT=(CoreFreqK.IdleDriver.states[index].flags>>24)&0xff;
UNUSED(pIdleDevice);
UNUSED(pIdleDriver);
__asm__ volatile
(
"xorq %%rcx , %%rcx" "\n\t"
"xorq %%rdx , %%rdx" "\n\t"
"leaq %[addr] , %%rax" "\n\t"
"monitor" "\n\t"
"# Bit 0 of ECX must be set" "\n\t"
"movq $0x1 , %%rcx" "\n\t"
"movq %[hint] , %%rax" "\n\t"
"mwait"
:
: [addr] "m" (current_thread_info()->flags),
[hint] "ir" (MWAIT)
: "%rax", "%rcx", "%rdx", "memory"
);
return index;
}
static int CoreFreqK_MWAIT_AMD_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
return CoreFreqK_MWAIT_Handler(pIdleDevice, pIdleDriver, index);
}
#if ((LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)) && (RHEL_MAJOR == 0)) \
|| ((RHEL_MAJOR == 8) && (RHEL_MINOR < 4))
static void CoreFreqK_S2_MWAIT_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
#else
static int CoreFreqK_S2_MWAIT_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
#endif /* 5.9.0 */
{
unsigned long MWAIT=(CoreFreqK.IdleDriver.states[index].flags>>24)&0xff;
UNUSED(pIdleDevice);
UNUSED(pIdleDriver);
__asm__ volatile
(
"xorq %%rcx , %%rcx" "\n\t"
"xorq %%rdx , %%rdx" "\n\t"
"leaq %[addr] , %%rax" "\n\t"
"monitor" "\n\t"
"# Bit 0 of ECX must be set" "\n\t"
"movq $0x1 , %%rcx" "\n\t"
"movq %[hint] , %%rax" "\n\t"
"mwait"
:
: [addr] "m" (current_thread_info()->flags),
[hint] "ir" (MWAIT)
: "%rax", "%rcx", "%rdx", "memory"
);
#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0)) || (RHEL_MINOR >= 4))
return index;
#endif /* 5.9.0 */
}
#if ((LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)) && (RHEL_MAJOR == 0)) \
|| ((RHEL_MAJOR == 8) && (RHEL_MINOR < 4))
static void CoreFreqK_S2_MWAIT_AMD_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
CoreFreqK_S2_MWAIT_Handler(pIdleDevice, pIdleDriver, index);
}
#else
static int CoreFreqK_S2_MWAIT_AMD_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
return CoreFreqK_S2_MWAIT_Handler(pIdleDevice, pIdleDriver, index);
}
#endif /* 5.9.0 */
/* HALT Idle methods */
static int CoreFreqK_HALT_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
UNUSED(pIdleDevice);
UNUSED(pIdleDriver);
/* Source: /arch/x86/include/asm/irqflags.h: native_safe_halt(); */
__asm__ volatile
(
"sti" "\n\t"
"hlt" "\n\t"
"cli"
:
:
: "cc"
);
return index;
}
static int CoreFreqK_HALT_AMD_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
return CoreFreqK_HALT_Handler(pIdleDevice, pIdleDriver, index);
}
#if ((LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)) && (RHEL_MAJOR == 0)) \
|| ((RHEL_MAJOR == 8) && (RHEL_MINOR < 4))
static void CoreFreqK_S2_HALT_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
#else
static int CoreFreqK_S2_HALT_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
#endif /* 5.9.0 */
{
UNUSED(pIdleDevice);
UNUSED(pIdleDriver);
__asm__ volatile
(
"sti" "\n\t"
"hlt" "\n\t"
"cli"
:
:
: "cc"
);
#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0)) || (RHEL_MINOR >= 4))
return index;
#endif /* 5.9.0 */
}
#if ((LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)) && (RHEL_MAJOR == 0)) \
|| ((RHEL_MAJOR == 8) && (RHEL_MINOR < 4))
static void CoreFreqK_S2_HALT_AMD_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
CoreFreqK_S2_HALT_Handler(pIdleDevice, pIdleDriver, index);
}
#else
static int CoreFreqK_S2_HALT_AMD_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
return CoreFreqK_S2_HALT_Handler(pIdleDevice, pIdleDriver, index);
}
#endif /* 5.9.0 */
/* I/O Idle methods */
static int CoreFreqK_IO_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
const unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
UNUSED(pIdleDevice);
UNUSED(pIdleDriver);
__asm__ volatile
(
"xorw %%ax, %%ax" "\n\t"
"movw %0, %%dx" "\n\t"
"inb %%dx, %%al" "\n\t"
:
: "ir" (Core->Query.CStateBaseAddr)
: "%ax", "%dx"
);
return index;
}
static int CoreFreqK_IO_AMD_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
return CoreFreqK_IO_Handler(pIdleDevice, pIdleDriver, index);
}
#if ((LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)) && (RHEL_MAJOR == 0)) \
|| ((RHEL_MAJOR == 8) && (RHEL_MINOR < 4))
static void CoreFreqK_S2_IO_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
#else
static int CoreFreqK_S2_IO_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
#endif /* 5.9.0 */
{
const unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
UNUSED(pIdleDevice);
UNUSED(pIdleDriver);
__asm__ volatile
(
"xorw %%ax, %%ax" "\n\t"
"movw %0, %%dx" "\n\t"
"inb %%dx, %%al" "\n\t"
:
: "ir" (Core->Query.CStateBaseAddr)
: "%ax", "%dx"
);
#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0)) || (RHEL_MINOR >= 4))
return index;
#endif /* 5.9.0 */
}
#if ((LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)) && (RHEL_MAJOR == 0)) \
|| ((RHEL_MAJOR == 8) && (RHEL_MINOR < 4))
static void CoreFreqK_S2_IO_AMD_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
CoreFreqK_S2_IO_Handler(pIdleDevice, pIdleDriver, index);
}
#else
static int CoreFreqK_S2_IO_AMD_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
return CoreFreqK_S2_IO_Handler(pIdleDevice, pIdleDriver, index);
}
#endif /* 5.9.0 */
/* Idle Cycles callback functions */
static int Alternative_Computation_Of_Cycles(
int (*Handler)(struct cpuidle_device*, struct cpuidle_driver*, int),
struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index
)
{
unsigned long long TSC[3] __attribute__ ((aligned (8)));
const unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
const unsigned short lvl = \
(CoreFreqK.IdleDriver.states[index].flags >> 28) & 0xf;
if ((PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC == 1)
|| (PUBLIC(RO(Proc))->Features.ExtInfo.EDX.RDTSCP == 1))
{
RDTSCP64(TSC[0]);
RDTSCP64(TSC[1]);
Handler(pIdleDevice, pIdleDriver, index);
RDTSCP64(TSC[2]);
}
else
{
RDTSC64(TSC[0]);
RDTSCP64(TSC[1]);
Handler(pIdleDevice, pIdleDriver, index);
RDTSC64(TSC[2]);
}
TSC[2] = TSC[2] - TSC[1]
- (TSC[1] > TSC[0] ? TSC[1] - TSC[0] : TSC[0] - TSC[1]);
Atomic_Write_VPMC(Core, TSC[2], lvl);
return index;
}
#if ((LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)) && (RHEL_MAJOR == 0)) \
|| ((RHEL_MAJOR == 8) && (RHEL_MINOR < 4))
static void Alternative_Computation_Of_Cycles_S2(
void (*S2_Handler)(struct cpuidle_device*, struct cpuidle_driver*, int),
struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index
)
#else
static int Alternative_Computation_Of_Cycles_S2(
int (*S2_Handler)(struct cpuidle_device*, struct cpuidle_driver*, int),
struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index
)
#endif /* 5.9.0 */
{
unsigned long long TSC[3] __attribute__ ((aligned (8)));
const unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
const unsigned short lvl = \
(CoreFreqK.IdleDriver.states[index].flags >> 28) & 0xf;
if ((PUBLIC(RO(Proc))->Features.AdvPower.EDX.Inv_TSC == 1)
|| (PUBLIC(RO(Proc))->Features.ExtInfo.EDX.RDTSCP == 1))
{
RDTSCP64(TSC[0]);
RDTSCP64(TSC[1]);
S2_Handler(pIdleDevice, pIdleDriver, index);
RDTSCP64(TSC[2]);
}
else
{
RDTSC64(TSC[0]);
RDTSC64(TSC[1]);
S2_Handler(pIdleDevice, pIdleDriver, index);
RDTSCP64(TSC[2]);
}
TSC[2] = TSC[2] - TSC[1]
- (TSC[1] > TSC[0] ? TSC[1] - TSC[0] : TSC[0] - TSC[1]);
Atomic_Write_VPMC(Core, TSC[2], lvl);
#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0)) || (RHEL_MINOR >= 4))
return index;
#endif /* 5.9.0 */
}
#undef Atomic_Write_VPMC
/* Alternative Idle methods */
static int CoreFreqK_Alt_MWAIT_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
return Alternative_Computation_Of_Cycles( CoreFreqK_MWAIT_Handler,
pIdleDevice,
pIdleDriver,
index );
}
static int CoreFreqK_Alt_HALT_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
return Alternative_Computation_Of_Cycles( CoreFreqK_HALT_Handler,
pIdleDevice,
pIdleDriver,
index );
}
static int CoreFreqK_Alt_IO_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
return Alternative_Computation_Of_Cycles( CoreFreqK_IO_Handler,
pIdleDevice,
pIdleDriver,
index );
}
static int CoreFreqK_Alt_MWAIT_AMD_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
return Alternative_Computation_Of_Cycles( CoreFreqK_MWAIT_AMD_Handler,
pIdleDevice,
pIdleDriver,
index );
}
static int CoreFreqK_Alt_HALT_AMD_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
return Alternative_Computation_Of_Cycles( CoreFreqK_HALT_AMD_Handler,
pIdleDevice,
pIdleDriver,
index );
}
static int CoreFreqK_Alt_IO_AMD_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
return Alternative_Computation_Of_Cycles( CoreFreqK_IO_AMD_Handler,
pIdleDevice,
pIdleDriver,
index );
}
#if ((LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)) && (RHEL_MAJOR == 0)) \
|| ((RHEL_MAJOR == 8) && (RHEL_MINOR < 4))
static void CoreFreqK_Alt_S2_MWAIT_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_MWAIT_Handler,
pIdleDevice,
pIdleDriver,
index );
}
static void CoreFreqK_Alt_S2_HALT_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_HALT_Handler,
pIdleDevice,
pIdleDriver,
index );
}
static void CoreFreqK_Alt_S2_IO_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_IO_Handler,
pIdleDevice,
pIdleDriver,
index );
}
static void CoreFreqK_Alt_S2_MWAIT_AMD_Handler(
struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver,
int index)
{
Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_MWAIT_AMD_Handler,
pIdleDevice,
pIdleDriver,
index );
}
static void CoreFreqK_Alt_S2_HALT_AMD_Handler(
struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver,
int index)
{
Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_HALT_AMD_Handler,
pIdleDevice,
pIdleDriver,
index );
}
static void CoreFreqK_Alt_S2_IO_AMD_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_IO_AMD_Handler,
pIdleDevice,
pIdleDriver,
index );
}
#else
static int CoreFreqK_Alt_S2_MWAIT_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
#if (RHEL_MAJOR == 8)
int rx = index;
#else
int rx =
#endif
Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_MWAIT_Handler,
pIdleDevice,
pIdleDriver,
index );
return rx;
}
static int CoreFreqK_Alt_S2_HALT_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
#if (RHEL_MAJOR == 8)
int rx = index;
#else
int rx =
#endif
Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_HALT_Handler,
pIdleDevice,
pIdleDriver,
index );
return rx;
}
static int CoreFreqK_Alt_S2_IO_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
#if (RHEL_MAJOR == 8)
int rx = index;
#else
int rx =
#endif
Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_IO_Handler,
pIdleDevice,
pIdleDriver,
index );
return rx;
}
static int CoreFreqK_Alt_S2_MWAIT_AMD_Handler(
struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver,
int index)
{
#if (RHEL_MAJOR == 8)
int rx = index;
#else
int rx =
#endif
Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_MWAIT_AMD_Handler,
pIdleDevice,
pIdleDriver,
index );
return rx;
}
static int CoreFreqK_Alt_S2_HALT_AMD_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
#if (RHEL_MAJOR == 8)
int rx = index;
#else
int rx =
#endif
Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_HALT_AMD_Handler,
pIdleDevice,
pIdleDriver,
index );
return rx;
}
static int CoreFreqK_Alt_S2_IO_AMD_Handler(struct cpuidle_device *pIdleDevice,
struct cpuidle_driver *pIdleDriver, int index)
{
#if (RHEL_MAJOR == 8)
int rx = index;
#else
int rx =
#endif
Alternative_Computation_Of_Cycles_S2( CoreFreqK_S2_IO_AMD_Handler,
pIdleDevice,
pIdleDriver,
index );
return rx;
}
#endif /* 5.9.0 */
#endif /* CONFIG_CPU_IDLE and 4.14.0 */
static void CoreFreqK_IdleDriver_UnInit(void)
{
#if defined(CONFIG_CPU_IDLE) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
struct cpuidle_device *device;
unsigned int cpu;
for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++) {
if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, HW)) {
if ((device=per_cpu_ptr(CoreFreqK.IdleDevice, cpu)) != NULL)
{
cpuidle_unregister_device(device);
}
}
}
cpuidle_unregister_driver(&CoreFreqK.IdleDriver);
free_percpu(CoreFreqK.IdleDevice);
#endif /* CONFIG_CPU_IDLE and 4.14.0 */
PUBLIC(RO(Proc))->Registration.Driver.Route = ROUTE_DEFAULT;
}
static int CoreFreqK_IdleDriver_Init(void)
{
int rc = -RC_UNIMPLEMENTED;
#if defined(CONFIG_CPU_IDLE) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
if (Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.IdleState != NULL)
{
IDLE_STATE *pIdleState;
pIdleState = Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.IdleState;
if ((pIdleState != NULL) && PUBLIC(RO(Proc))->Features.Std.ECX.MONITOR)
{
if ((CoreFreqK.IdleDevice=alloc_percpu(struct cpuidle_device)) == NULL)
{
rc = -ENOMEM;
}
else
{
struct cpuidle_device *device;
unsigned int cpu, enroll = 0;
/* Kernel polling loop */
cpuidle_poll_state_init(&CoreFreqK.IdleDriver);
CoreFreqK.IdleDriver.state_count = 1;
/* Idle States */
while (pIdleState->Name != NULL)
{
if (CoreFreqK.IdleDriver.state_count < SUB_CSTATE_COUNT)
{
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].flags = pIdleState->flags;
if (CoreFreqK.SubCstate[CoreFreqK.IdleDriver.state_count] == 0)
{
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].flags |= CPUIDLE_FLAG_UNUSABLE;
}
StrCopy(CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].name, pIdleState->Name, CPUIDLE_NAME_LEN);
StrCopy(CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].desc, pIdleState->Desc, CPUIDLE_NAME_LEN);
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].exit_latency = pIdleState->Latency;
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].target_residency = pIdleState->Residency;
switch (Idle_Route) {
case ROUTE_MWAIT:
if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL)
{
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].enter = CoreFreqK_Alt_MWAIT_Handler;
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].enter_s2idle = CoreFreqK_Alt_S2_MWAIT_Handler;
}
else if ((PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
|| (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON))
{
HWCR HwCfgRegister;
RDMSR(HwCfgRegister, MSR_K7_HWCR);
if (BITVAL(HwCfgRegister.value, 9) == 0)
{
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].enter = CoreFreqK_Alt_MWAIT_AMD_Handler;
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].enter_s2idle = CoreFreqK_Alt_S2_MWAIT_AMD_Handler;
} else {
goto IDLE_WARNING;
}
}
else {
goto IDLE_DEFAULT;
}
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].desc[0] = 'M';
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].desc[1] = 'W';
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].desc[2] = 'T';
PUBLIC(RO(Proc))->Registration.Driver.Route = ROUTE_MWAIT;
break;
case ROUTE_HALT:
if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL)
{
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].enter = CoreFreqK_Alt_HALT_Handler;
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].enter_s2idle = CoreFreqK_Alt_S2_HALT_Handler;
}
else if ((PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
|| (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON))
{
HWCR HwCfgRegister;
RDMSR(HwCfgRegister, MSR_K7_HWCR);
if (BITVAL(HwCfgRegister.value, 9) == 0)
{
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].enter = CoreFreqK_Alt_HALT_AMD_Handler;
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].enter_s2idle = CoreFreqK_Alt_S2_HALT_AMD_Handler;
} else {
goto IDLE_WARNING;
}
}
else {
goto IDLE_DEFAULT;
}
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].desc[0] = 'H';
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].desc[1] = 'L';
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].desc[2] = 'T';
PUBLIC(RO(Proc))->Registration.Driver.Route = ROUTE_HALT;
break;
case ROUTE_IO:
{
if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL)
{
CSTATE_IO_MWAIT CState_IO_MWAIT = {.value = 0};
CSTATE_CONFIG CStateConfig = {.value = 0};
RDMSR(CState_IO_MWAIT, MSR_PMG_IO_CAPTURE_BASE);
RDMSR(CStateConfig, MSR_PKG_CST_CONFIG_CONTROL);
if ((CState_IO_MWAIT.LVL2_BaseAddr != 0x0)
&& (CStateConfig.IO_MWAIT_Redir == 1))
{
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].enter = CoreFreqK_Alt_IO_Handler;
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].enter_s2idle = CoreFreqK_Alt_S2_IO_Handler;
} else {
goto IDLE_DEFAULT;
}
}
else if ((PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
|| (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON))
{
HWCR HwCfgRegister;
RDMSR(HwCfgRegister, MSR_K7_HWCR);
if (BITVAL(HwCfgRegister.value, 9) == 0)
{
CSTATE_BASE_ADDR CStateBaseAddr = {.value = 0};
RDMSR(CStateBaseAddr, MSR_AMD_CSTATE_BAR);
if (CStateBaseAddr.IOaddr != 0x0)
{
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].enter = CoreFreqK_Alt_IO_AMD_Handler;
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].enter_s2idle=CoreFreqK_Alt_S2_IO_AMD_Handler;
} else {
goto IDLE_DEFAULT;
}
} else {
goto IDLE_WARNING;
}
}
else {
goto IDLE_DEFAULT;
}
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].desc[0] = 'I';
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].desc[1] = '/';
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].desc[2] = 'O';
}
PUBLIC(RO(Proc))->Registration.Driver.Route = ROUTE_IO;
break;
case ROUTE_DEFAULT:
IDLE_DEFAULT:
default:
PUBLIC(RO(Proc))->Registration.Driver.Route = ROUTE_DEFAULT;
if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL)
{
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].enter = CoreFreqK_MWAIT_Handler;
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].enter_s2idle = CoreFreqK_S2_MWAIT_Handler;
PUBLIC(RO(Proc))->Registration.Driver.Route = ROUTE_MWAIT;
}
else if ((PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
|| (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON))
{ /* Avoid kernel crash if the MWAIT opcode has been disabled */
HWCR HwCfgRegister;
RDMSR(HwCfgRegister, MSR_K7_HWCR);
if (BITVAL(HwCfgRegister.value, 9) == 0)
{
CSTATE_BASE_ADDR CStateBaseAddr = {.value = 0};
RDMSR(CStateBaseAddr, MSR_AMD_CSTATE_BAR);
if (CStateBaseAddr.IOaddr != 0x0)
{
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].enter = CoreFreqK_Alt_IO_AMD_Handler;
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].enter_s2idle = CoreFreqK_Alt_S2_IO_AMD_Handler;
PUBLIC(RO(Proc))->Registration.Driver.Route = ROUTE_IO;
}
else
{
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].enter = CoreFreqK_Alt_HALT_AMD_Handler;
CoreFreqK.IdleDriver.states[
CoreFreqK.IdleDriver.state_count
].enter_s2idle = CoreFreqK_Alt_S2_HALT_AMD_Handler;
PUBLIC(RO(Proc))->Registration.Driver.Route=ROUTE_HALT;
}
} else {
goto IDLE_WARNING;
}
}
else {
IDLE_WARNING: pr_warn("CoreFreq: " \
"No Idle implementation for Vendor CRC 0x%x\n",
PUBLIC(RO(Proc))->Features.Info.Vendor.CRC);
}
break;
}
CoreFreqK.IdleDriver.state_count++;
}
pIdleState++;
}
if ((rc = cpuidle_register_driver(&CoreFreqK.IdleDriver)) == 0)
{
for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++)
{
if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, HW))
{
device = per_cpu_ptr(CoreFreqK.IdleDevice, cpu);
if (device != NULL)
{
device->cpu = cpu;
if ((rc = cpuidle_register_device(device)) == 0) {
continue;
}
}
break;
}
}
enroll = cpu;
}
if (rc != 0)
{ /* Cancel the registration if the driver and/or a device failed */
for (cpu = 0; cpu < enroll; cpu++)
{
device = per_cpu_ptr(CoreFreqK.IdleDevice, cpu);
if (device != NULL)
{
cpuidle_unregister_device(device);
}
}
cpuidle_unregister_driver(&CoreFreqK.IdleDriver);
free_percpu(CoreFreqK.IdleDevice);
}
}
}
}
#endif /* CONFIG_CPU_IDLE and 4.14.0 */
return rc;
}
#ifdef CONFIG_CPU_IDLE
static void CoreFreqK_Idle_State_Withdraw(int idx, bool disable)
{
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0) || (RHEL_MAJOR == 8)
struct cpuidle_device *device;
unsigned int cpu;
for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++)
{
if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, HW)
&& ((device = per_cpu_ptr(CoreFreqK.IdleDevice, cpu)) != NULL))
{
if (disable) {
device->states_usage[idx].disable |= CPUIDLE_STATE_DISABLED_BY_DRIVER;
} else {
device->states_usage[idx].disable &= ~CPUIDLE_STATE_DISABLED_BY_DRIVER;
}
}
}
#else
CoreFreqK.IdleDriver.states[idx].disabled = disable;
#endif /* 5.5.0 */
}
#endif /* CONFIG_CPU_IDLE */
static long CoreFreqK_Limit_Idle(int target)
{
long rc = -EINVAL;
#ifdef CONFIG_CPU_IDLE
int idx, floor = -1;
if ((target > 0) && (target <= CoreFreqK.IdleDriver.state_count))
{
for (idx = 0; idx < CoreFreqK.IdleDriver.state_count; idx++)
{
if (idx < target)
{
CoreFreqK_Idle_State_Withdraw(idx, false);
floor = idx;
} else {
CoreFreqK_Idle_State_Withdraw(idx, true);
}
}
rc = RC_SUCCESS;
}
else if (target == 0)
{
for (idx = 0; idx < CoreFreqK.IdleDriver.state_count; idx++)
{
CoreFreqK_Idle_State_Withdraw(idx, false);
floor = idx;
}
rc = RC_SUCCESS;
}
if (floor != -1) {
PUBLIC(RO(Proc))->OS.IdleDriver.stateLimit = 1 + floor;
}
#endif /* CONFIG_CPU_IDLE */
return rc;
}
#ifdef CONFIG_CPU_FREQ
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 11, 0)
static void CoreFreqK_Policy_Exit(struct cpufreq_policy *policy)
{
UNUSED(policy);
}
#else
static int CoreFreqK_Policy_Exit(struct cpufreq_policy *policy)
{
UNUSED(policy);
return 0;
}
#endif
static int CoreFreqK_Policy_Init(struct cpufreq_policy *policy)
{
if (policy != NULL) {
if (policy->cpu < PUBLIC(RO(Proc))->CPU.Count)
{
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(policy->cpu)));
policy->cpuinfo.min_freq =(Core->Boost[BOOST(MIN)]
* Core->Clock.Hz) / 1000LLU;
policy->cpuinfo.max_freq =(Core->Boost[BOOST(MAX)]
* Core->Clock.Hz) / 1000LLU;
/* MANDATORY Per-CPU Initialization */
policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
policy->cur = policy->cpuinfo.max_freq;
policy->min = policy->cpuinfo.min_freq;
policy->max = policy->cpuinfo.max_freq;
policy->policy = CPUFREQ_POLICY_PERFORMANCE;
if (Register_Governor == 1) {
policy->governor = &CoreFreqK.FreqGovernor;
} else {
policy->governor = NULL;
}
}
}
return 0;
}
#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 19)) \
&& (LINUX_VERSION_CODE <= KERNEL_VERSION(5, 5, 0))) \
|| (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 3)) \
|| (RHEL_MAJOR == 8)
static int CoreFreqK_Policy_Verify(struct cpufreq_policy_data *policy)
#else
static int CoreFreqK_Policy_Verify(struct cpufreq_policy *policy)
#endif
{
if (policy != NULL) {
cpufreq_verify_within_cpu_limits(policy);
}
return 0;
}
static int CoreFreqK_SetPolicy(struct cpufreq_policy *policy)
{
UNUSED(policy);
return 0;
}
static int CoreFreqK_Bios_Limit(int cpu, unsigned int *limit)
{
if ((cpu >= 0) && (cpu < PUBLIC(RO(Proc))->CPU.Count) && (limit != NULL))
{
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
(*limit) = (Core->Boost[BOOST(MAX)] * Core->Clock.Hz) / 1000LLU;
}
return 0;
}
static void Policy_Aggregate_Turbo(void)
{
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0)
if (PUBLIC(RO(Proc))->Registration.Driver.CPUfreq & REGISTRATION_ENABLE) {
CoreFreqK.FreqDriver.boost_enabled = (
BITWISEAND_CC( BUS_LOCK,
PUBLIC(RW(Proc))->TurboBoost,
PUBLIC(RO(Proc))->TurboBoost_Mask ) != 0
);
}
#endif
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0)
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0) \
|| ((RHEL_MAJOR == 8) && (RHEL_MINOR > 3))
static int CoreFreqK_SetBoost(struct cpufreq_policy *policy, int state)
{
UNUSED(policy);
#else
static int CoreFreqK_SetBoost(int state)
{
#endif /* 5.8.0 */
Controller_Stop(1);
TurboBoost_Enable[0] = (state != 0);
Controller_Start(1);
TurboBoost_Enable[0] = -1;
Policy_Aggregate_Turbo();
BITSET(BUS_LOCK, PUBLIC(RW(Proc))->OS.Signal, NTFY); /* Notify Daemon*/
return 0;
}
#endif /* 3.14.0 */
static ssize_t CoreFreqK_Show_SetSpeed(struct cpufreq_policy *policy,char *buf)
{
if (policy != NULL)
{
CORE_RO *Core;
enum RATIO_BOOST boost;
if (policy->cpu < PUBLIC(RO(Proc))->CPU.Count)
{
Core = (CORE_RO *) PUBLIC(RO(Core, AT(policy->cpu)));
} else {
Core = (CORE_RO *) PUBLIC(RO(Core, AT(PUBLIC(RO(Proc))->Service.Core)));
}
if (PUBLIC(RO(Proc))->Features.HWP_Enable
&& (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL)) {
boost = BOOST(HWP_TGT);
} else {
boost = BOOST(TGT);
}
return sprintf( buf, "%7llu\n",
(Core->Boost[boost] * Core->Clock.Hz) / 1000LLU );
}
return 0;
}
static int CoreFreqK_Store_SetSpeed(struct cpufreq_policy *policy,
unsigned int freq)
{
if (policy != NULL)
{
void (*SetTarget)(void *arg) = NULL;
if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL) {
SetTarget = Policy_HWP_SetTarget;
} else if ( (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
|| (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON) ) {
SetTarget = Policy_Zen_CPPC_SetTarget;
} else {
SetTarget = Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.SetTarget;
}
if ((policy->cpu < PUBLIC(RO(Proc))->CPU.Count) && (SetTarget != NULL))
{
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(policy->cpu)));
unsigned int ratio = (freq * 1000LLU) / Core->Clock.Hz;
if (ratio > 0) {
if (smp_call_function_single( policy->cpu,
SetTarget,
&ratio, 1) == 0 )
{
BITSET(BUS_LOCK, PUBLIC(RW(Proc))->OS.Signal, NTFY);
}
return 0;
}
}
}
return -EINVAL;
}
#endif /* CONFIG_CPU_FREQ */
static unsigned int Policy_GetFreq(unsigned int cpu)
{
unsigned int CPU_Freq = PUBLIC(RO(Proc))->Features.Factory.Freq * 1000U;
if (cpu < PUBLIC(RO(Proc))->CPU.Count)
{
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
unsigned int Freq_MHz = Core->Delta.C0.UCC
/ PUBLIC(RO(Proc))->SleepInterval;
if (Freq_MHz > 0) { /* at least 1 interval must have been elapsed */
CPU_Freq = Freq_MHz;
}
}
return CPU_Freq;
}
static void Policy_Core2_SetTarget(void *arg)
{
#ifdef CONFIG_CPU_FREQ
unsigned int *ratio = (unsigned int*) arg;
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
if ((*ratio) <= Core->Boost[BOOST(1C)])
{
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
Set_Core2_Target(Core, (*ratio));
WritePerformanceControl(&Core->PowerThermal.PerfControl);
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA) {
if (Cmp_Core2_Target(Core, 1 + Core->Boost[BOOST(MAX)]))
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
}
}
Core->Boost[BOOST(TGT)] = Get_Core2_Target(Core);
}
#endif /* CONFIG_CPU_FREQ */
}
static void Policy_Nehalem_SetTarget(void *arg)
{
#ifdef CONFIG_CPU_FREQ
unsigned int *ratio = (unsigned int*) arg;
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
if ((*ratio) <= Core->Boost[BOOST(1C)])
{
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
Set_Nehalem_Target(Core, (*ratio));
WritePerformanceControl(&Core->PowerThermal.PerfControl);
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA) {
if (Cmp_Nehalem_Target(Core, 1 + Core->Boost[BOOST(MAX)]))
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
}
}
Core->Boost[BOOST(TGT)] = Get_Nehalem_Target(Core);
}
#endif /* CONFIG_CPU_FREQ */
}
static void Policy_SandyBridge_SetTarget(void *arg)
{
#ifdef CONFIG_CPU_FREQ
unsigned int *ratio = (unsigned int*) arg;
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
if ((*ratio) <= Core->Boost[BOOST(1C)])
{
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
Set_SandyBridge_Target(Core, (*ratio));
WritePerformanceControl(&Core->PowerThermal.PerfControl);
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA) {
if (Cmp_SandyBridge_Target(Core, Core->Boost[BOOST(MAX)]))
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
}
}
Core->Boost[BOOST(TGT)] = Get_SandyBridge_Target(Core);
}
#endif /* CONFIG_CPU_FREQ */
}
static void Policy_Skylake_SetTarget(void *arg)
{
#ifdef CONFIG_CPU_FREQ
unsigned int *ratio = (unsigned int*) arg;
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
if ((*ratio) <= Core->Boost[BOOST(1C)])
{
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
Set_SandyBridge_Target(Core, (*ratio));
WritePerformanceControl(&Core->PowerThermal.PerfControl);
RDMSR(Core->PowerThermal.PerfControl, MSR_IA32_PERF_CTL);
if (PUBLIC(RO(Proc))->Features.Power.EAX.TurboIDA) {
if (Cmp_Skylake_Target(Core, Core->Boost[BOOST(TDP)] > 0 ?
Core->Boost[BOOST(TDP)] : Core->Boost[BOOST(MAX)]))
{
BITSET_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
} else {
BITCLR_CC(BUS_LOCK, PUBLIC(RW(Proc))->TurboBoost, Core->Bind);
}
}
Core->Boost[BOOST(TGT)] = Get_SandyBridge_Target(Core);
}
#endif /* CONFIG_CPU_FREQ */
}
static void Policy_HWP_SetTarget(void *arg)
{
#ifdef CONFIG_CPU_FREQ
if (PUBLIC(RO(Proc))->Features.HWP_Enable)
{
unsigned int *ratio = (unsigned int*) arg;
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
RDMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
if (((*ratio) >= Core->PowerThermal.HWP_Capabilities.Lowest)
&& ((*ratio) <= Core->PowerThermal.HWP_Capabilities.Highest))
{
Core->PowerThermal.HWP_Request.Maximum_Perf = \
Core->PowerThermal.HWP_Request.Desired_Perf = (*ratio);
WRMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
RDMSR(Core->PowerThermal.HWP_Request, MSR_IA32_HWP_REQUEST);
Core->Boost[BOOST(HWP_MAX)]=Core->PowerThermal.HWP_Request.Maximum_Perf;
Core->Boost[BOOST(HWP_TGT)]=Core->PowerThermal.HWP_Request.Desired_Perf;
}
} else {
if (Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.SetTarget != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.SetTarget(arg);
}
}
#endif /* CONFIG_CPU_FREQ */
}
static void Policy_Zen_SetTarget(void *arg)
{
#ifdef CONFIG_CPU_FREQ
unsigned int *ratio = (unsigned int*) arg;
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
PSTATEDEF PstateDef;
COF_ST COF = {.Q = 0, .R = 0};
unsigned int pstate;
unsigned short WrModRd = 0;
/* Look-up for the first enabled P-State with the same target ratio */
for (pstate = 0; pstate <= 7; pstate++)
{
PstateDef.value = 0;
RDMSR(PstateDef, MSR_AMD_PSTATE_DEF_BASE + pstate);
if (PstateDef.Family_17h.PstateEn)
{
COF = AMD_Zen_CoreCOF(PstateDef);
if (COF.Q == (*ratio)) {
WrModRd = 1;
break;
}
}
}
if (WrModRd == 1)
{
PSTATECTRL PstateCtrl;
/* Write-Modify-Read the new target P-state */
PstateCtrl.value = 0;
RDMSR(PstateCtrl, MSR_AMD_PERF_CTL);
PstateCtrl.PstateCmd = pstate;
WRMSR(PstateCtrl, MSR_AMD_PERF_CTL);
PstateCtrl.value = 0;
RDMSR(PstateCtrl, MSR_AMD_PERF_CTL);
PstateDef.value = 0;
RDMSR(PstateDef, MSR_AMD_PSTATE_DEF_BASE + pstate);
COF = AMD_Zen_CoreCOF(PstateDef);
Core->Boost[BOOST(TGT)] = COF.Q;
}
#endif /* CONFIG_CPU_FREQ */
}
static void Policy_Zen_CPPC_SetTarget(void *arg)
{
#ifdef CONFIG_CPU_FREQ
if (PUBLIC(RO(Proc))->Features.HWP_Enable)
{
unsigned int *ratio = (unsigned int*) arg;
unsigned int cpu = smp_processor_id();
CORE_RO *Core = (CORE_RO *) PUBLIC(RO(Core, AT(cpu)));
if (((*ratio) >= Core->PowerThermal.HWP_Capabilities.Lowest)
&& ((*ratio) <= Core->PowerThermal.HWP_Capabilities.Highest))
{
AMD_CPPC_REQUEST CPPC_Req = {.value = 0};
unsigned int hint;
RDMSR(Core->SystemRegister.HWCR, MSR_K7_HWCR);
RDMSR(CPPC_Req, MSR_AMD_CPPC_REQ);
hint = CPPC_AMD_Zen_ScaleHint(
Core, 255U, (*ratio),
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
if ((hint & (1 << 31)) != (1 << 31))
{
CPPC_Req.Maximum_Perf = CPPC_Req.Desired_Perf = hint & 0xff;
WRMSR(CPPC_Req, MSR_AMD_CPPC_REQ);
RDMSR(CPPC_Req, MSR_AMD_CPPC_REQ);
Core->PowerThermal.HWP_Request.Maximum_Perf = \
CPPC_AMD_Zen_ScaleRatio(
Core, 255U, CPPC_Req.Maximum_Perf,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
Core->PowerThermal.HWP_Request.Desired_Perf = \
CPPC_AMD_Zen_ScaleRatio(
Core, 255U, CPPC_Req.Desired_Perf,
!Core->SystemRegister.HWCR.Family_17h.CpbDis
);
Core->Boost[BOOST(HWP_MAX)]=Core->PowerThermal.HWP_Request.Maximum_Perf;
Core->Boost[BOOST(HWP_TGT)]=Core->PowerThermal.HWP_Request.Desired_Perf;
}
}
} else {
if (Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.SetTarget != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.SetTarget(arg);
}
}
#endif /* CONFIG_CPU_FREQ */
}
static int CoreFreqK_FreqDriver_UnInit(void)
{
int rc = -EINVAL;
#ifdef CONFIG_CPU_FREQ
#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 3, 0)) && (!defined(CONFIG_CACHY)) \
&& (!defined(RHEL_MAJOR))
rc =
#else
rc = 0;
#endif
cpufreq_unregister_driver(&CoreFreqK.FreqDriver);
#endif /* CONFIG_CPU_FREQ */
return rc;
}
static int CoreFreqK_FreqDriver_Init(void)
{
int rc = -RC_UNIMPLEMENTED;
#ifdef CONFIG_CPU_FREQ
if (Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.IdleState != NULL)
{
if (Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.GetFreq != NULL)
{
CoreFreqK.FreqDriver.get=Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.GetFreq;
rc = cpufreq_register_driver(&CoreFreqK.FreqDriver);
}
}
#endif /* CONFIG_CPU_FREQ */
return rc;
}
static void CoreFreqK_Governor_UnInit(void)
{
#ifdef CONFIG_CPU_FREQ
cpufreq_unregister_governor(&CoreFreqK.FreqGovernor);
#endif /* CONFIG_CPU_FREQ */
}
static int CoreFreqK_Governor_Init(void)
{
int rc = -RC_UNIMPLEMENTED;
#ifdef CONFIG_CPU_FREQ
if (Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.IdleState != NULL)
{
if (Arch[PUBLIC(RO(Proc))->ArchID].SystemDriver.SetTarget != NULL)
{
rc = cpufreq_register_governor(&CoreFreqK.FreqGovernor);
}
}
#endif /* CONFIG_CPU_FREQ */
return rc;
}
static signed int Seek_Topology_Core_Peer(unsigned int cpu, signed int exclude)
{
unsigned int seek;
for (seek = 0; seek < PUBLIC(RO(Proc))->CPU.Count; seek++) {
if ( ((exclude ^ cpu) > 0)
&& (PUBLIC(RO(Core, AT(seek)))->T.ApicID \
!= PUBLIC(RO(Core, AT(cpu)))->T.ApicID)
&& (PUBLIC(RO(Core, AT(seek)))->T.CoreID \
== PUBLIC(RO(Core, AT(cpu)))->T.CoreID)
&& (PUBLIC(RO(Core, AT(seek)))->T.ThreadID \
!= PUBLIC(RO(Core, AT(cpu)))->T.ThreadID)
&& (PUBLIC(RO(Core, AT(seek)))->T.PackageID \
== PUBLIC(RO(Core, AT(cpu)))->T.PackageID)
&& (PUBLIC(RO(Core, AT(seek)))->T.ThreadID == 0)
&& !BITVAL(PUBLIC(RO(Core, AT(seek)))->OffLine, OS) )
{
return (signed int) seek;
}
}
return -1;
}
static signed int Seek_Topology_Thread_Peer(unsigned int cpu,signed int exclude)
{
unsigned int seek;
for (seek = 0; seek < PUBLIC(RO(Proc))->CPU.Count; seek++) {
if ( ((exclude ^ cpu) > 0)
&& (PUBLIC(RO(Core, AT(seek)))->T.ApicID \
!= PUBLIC(RO(Core, AT(cpu)))->T.ApicID)
&& (PUBLIC(RO(Core, AT(seek)))->T.CoreID \
== PUBLIC(RO(Core, AT(cpu)))->T.CoreID)
&& (PUBLIC(RO(Core, AT(seek)))->T.ThreadID \
!= PUBLIC(RO(Core, AT(cpu)))->T.ThreadID)
&& (PUBLIC(RO(Core, AT(seek)))->T.PackageID \
== PUBLIC(RO(Core, AT(cpu)))->T.PackageID)
&& (PUBLIC(RO(Core, AT(seek)))->T.ThreadID > 0)
&& !BITVAL(PUBLIC(RO(Core, AT(seek)))->OffLine, OS) )
{
return (signed int) seek;
}
}
return -1;
}
static signed int Seek_Topology_Hybrid_Core(unsigned int cpu)
{
signed int any = (signed int) PUBLIC(RO(Proc))->CPU.Count, seek;
do {
any--;
} while (BITVAL(PUBLIC(RO(Core, AT(any)))->OffLine, OS) && (any > 0)) ;
for (seek = any; seek != 0; seek--)
{
if ((PUBLIC(RO(Core, AT(seek)))->T.Cluster.Hybrid.CoreType == Hybrid_Atom)
&& (PUBLIC(RO(Core, AT(seek)))->T.PackageID \
== PUBLIC(RO(Core, AT(cpu)))->T.PackageID)
&& !BITVAL(PUBLIC(RO(Core, AT(seek)))->OffLine, OS))
{
any = seek;
break;
}
}
return any;
}
static void MatchCoreForService(SERVICE_PROC *pService,
unsigned int cpi, signed int cpx)
{
unsigned int cpu;
for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++) {
if ( ((cpx ^ cpu) > 0)
&& (PUBLIC(RO(Core, AT(cpu)))->T.PackageID \
== PUBLIC(RO(Core, AT(cpi)))->T.PackageID)
&& !BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS) )
{
pService->Core = cpu;
pService->Thread = -1;
break;
}
}
}
static int MatchPeerForService(SERVICE_PROC *pService,
unsigned int cpi, signed int cpx)
{
unsigned int cpu = cpi, cpn = 0;
signed int seek;
MATCH:
if (PUBLIC(RO(Core, AT(cpu)))->T.ThreadID == 0)
{
if ((seek = Seek_Topology_Thread_Peer(cpu, cpx)) != -1) {
pService->Core = cpu;
pService->Thread = seek;
return 0;
}
}
else if (PUBLIC(RO(Core, AT(cpu)))->T.ThreadID > 0)
{
if ((seek = Seek_Topology_Core_Peer(cpu, cpx)) != -1) {
pService->Core = seek;
pService->Thread = cpu;
return 0;
}
}
while (cpn < PUBLIC(RO(Proc))->CPU.Count) {
cpu = cpn++;
if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS)) {
goto MATCH;
}
}
return -1;
}
static void MatchPeerForDefaultService(SERVICE_PROC *pService, unsigned int cpu)
{
if (PUBLIC(RO(Proc))->Features.HTT_Enable) {
if (MatchPeerForService(pService, cpu, -1) == -1)
{
MatchCoreForService(pService, cpu, -1);
}
} else {
pService->Core = cpu;
pService->Thread = -1;
}
if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.Hybrid) {
pService->Hybrid = Seek_Topology_Hybrid_Core(cpu);
} else {
pService->Hybrid = -1;
}
if (ServiceProcessor != -1) {
DefaultSMT.Core = pService->Core;
DefaultSMT.Thread = pService->Thread;
}
}
static void MatchPeerForUpService(SERVICE_PROC *pService, unsigned int cpu)
{ /* Try to restore the initial Service affinity or move to SMT peer. */
SERVICE_PROC hService = {
.Core = cpu,
.Thread = -1,
.Hybrid = -1
};
if (PUBLIC(RO(Proc))->Features.HTT_Enable)
{
signed int seek;
if ((PUBLIC(RO(Core, AT(cpu)))->T.ThreadID == 0)
&& ((seek = Seek_Topology_Thread_Peer(cpu, -1)) != -1))
{
hService.Core = cpu;
hService.Thread = seek;
} else {
if ((PUBLIC(RO(Core, AT(cpu)))->T.ThreadID > 0)
&& ((seek = Seek_Topology_Core_Peer(cpu, -1)) != -1))
{
hService.Core = seek;
hService.Thread = cpu;
}
}
}
if ((pService->Core != DefaultSMT.Core)
|| (pService->Thread != DefaultSMT.Thread))
{
if ((hService.Core == DefaultSMT.Core)
&& (hService.Thread == DefaultSMT.Thread))
{
pService->Core = hService.Core;
pService->Thread = hService.Thread;
} else {
if ((pService->Thread == -1) && (hService.Thread > 0))
{
pService->Core = hService.Core;
pService->Thread = hService.Thread;
}
}
}
if (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.Hybrid) {
pService->Hybrid = Seek_Topology_Hybrid_Core(cpu);
}
}
static void MatchPeerForDownService(SERVICE_PROC *pService, unsigned int cpu)
{
int rc = -1;
if (PUBLIC(RO(Proc))->Features.HTT_Enable) {
rc = MatchPeerForService(pService, cpu, cpu);
}
if (rc == -1) {
MatchCoreForService(pService, cpu, cpu);
}
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
#ifdef CONFIG_HAVE_NMI
static int CoreFreqK_NMI_Handler(unsigned int type, struct pt_regs *pRegs)
{
unsigned int cpu = smp_processor_id();
UNUSED(pRegs);
switch (type) {
case NMI_LOCAL:
PUBLIC(RO(Core, AT(cpu)))->Interrupt.NMI.LOCAL++;
break;
case NMI_UNKNOWN:
PUBLIC(RO(Core, AT(cpu)))->Interrupt.NMI.UNKNOWN++;
break;
case NMI_SERR:
PUBLIC(RO(Core, AT(cpu)))->Interrupt.NMI.PCISERR++;
break;
case NMI_IO_CHECK:
PUBLIC(RO(Core, AT(cpu)))->Interrupt.NMI.IOCHECK++;
break;
}
return NMI_DONE;
}
#endif /* CONFIG_HAVE_NMI */
static long CoreFreqK_UnRegister_CPU_Idle(void)
{
long rc = -EINVAL;
if (PUBLIC(RO(Proc))->Registration.Driver.CPUidle & REGISTRATION_ENABLE)
{
CoreFreqK_IdleDriver_UnInit();
PUBLIC(RO(Proc))->Registration.Driver.CPUidle = REGISTRATION_DISABLE;
rc = RC_SUCCESS;
}
return rc;
}
static long CoreFreqK_Register_CPU_Idle(void)
{
long rc = -EINVAL;
if (Register_CPU_Idle == 1)
{
int rx = CoreFreqK_IdleDriver_Init();
switch ( rx ) {
default:
fallthrough;
case -ENODEV:
case -ENOMEM:
PUBLIC(RO(Proc))->Registration.Driver.CPUidle = REGISTRATION_DISABLE;
rc = (long) rx;
break;
case 0: /* Registration succeeded. */
PUBLIC(RO(Proc))->Registration.Driver.CPUidle = REGISTRATION_ENABLE;
rc = RC_SUCCESS;
break;
}
} else { /* Nothing requested by User. */
PUBLIC(RO(Proc))->Registration.Driver.CPUidle = REGISTRATION_DISABLE;
}
return rc;
}
static long CoreFreqK_UnRegister_CPU_Freq(void)
{
long rc = -EINVAL;
if (PUBLIC(RO(Proc))->Registration.Driver.CPUfreq & REGISTRATION_ENABLE)
{
int rx = CoreFreqK_FreqDriver_UnInit();
switch ( rx ) {
case 0:
PUBLIC(RO(Proc))->Registration.Driver.CPUfreq=REGISTRATION_DISABLE;
rc = RC_SUCCESS;
break;
default:
rc = (long) rx;
break;
}
}
return rc;
}
static long CoreFreqK_Register_CPU_Freq(void)
{ /* Source: cpufreq_register_driver @ /drivers/cpufreq/cpufreq.c */
long rc = -EINVAL;
if (Register_CPU_Freq == 1)
{
int rx = CoreFreqK_FreqDriver_Init();
switch ( rx ) {
default:
fallthrough;
case -EEXIST: /* Another driver is in control. */
PUBLIC(RO(Proc))->Registration.Driver.CPUfreq = REGISTRATION_DISABLE;
rc = (long) rx;
break;
case -ENODEV: /* Missing CPU-Freq or Interfaces. */
case -EPROBE_DEFER: /* CPU probing failed */
case -EINVAL: /* Missing CPU-Freq prerequisites. */
PUBLIC(RO(Proc))->Registration.Driver.CPUfreq = REGISTRATION_FULLCTRL;
rc = (long) rx;
break;
case 0: /* Registration succeeded . */
PUBLIC(RO(Proc))->Registration.Driver.CPUfreq = REGISTRATION_ENABLE;
rc = RC_SUCCESS;
break;
}
} else { /* Invalid or no User request. */
#ifdef CONFIG_CPU_FREQ
PUBLIC(RO(Proc))->Registration.Driver.CPUfreq = REGISTRATION_DISABLE;
#else /* No CPU-FREQ built in Kernel, presume we have the full control. */
PUBLIC(RO(Proc))->Registration.Driver.CPUfreq = REGISTRATION_FULLCTRL;
#endif /* CONFIG_CPU_FREQ */
}
return rc;
}
static long CoreFreqK_UnRegister_Governor(void)
{
long rc = EINVAL;
if (PUBLIC(RO(Proc))->Registration.Driver.Governor & REGISTRATION_ENABLE)
{
CoreFreqK_Governor_UnInit();
PUBLIC(RO(Proc))->Registration.Driver.Governor = REGISTRATION_DISABLE;
rc = RC_SUCCESS;
}
return rc;
}
static long CoreFreqK_Register_Governor(void)
{
long rc = -EINVAL;
if (Register_Governor == 1)
{
int rx = CoreFreqK_Governor_Init();
switch ( rx ) {
default:
case -ENODEV:
PUBLIC(RO(Proc))->Registration.Driver.Governor = REGISTRATION_DISABLE;
rc = (long) rx;
break;
case 0: /* Registration succeeded . */
PUBLIC(RO(Proc))->Registration.Driver.Governor = REGISTRATION_ENABLE;
rc = RC_SUCCESS;
break;
}
} else { /* Nothing requested by User. */
PUBLIC(RO(Proc))->Registration.Driver.Governor = REGISTRATION_DISABLE;
}
return rc;
}
static void CoreFreqK_Register_NMI(void)
{
#ifdef CONFIG_HAVE_NMI
if (BITVAL(PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_LOCAL) == 0)
{
if(register_nmi_handler(NMI_LOCAL,
CoreFreqK_NMI_Handler,
0,
"corefreqk") == 0)
{
BITSET(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_LOCAL);
} else {
BITCLR(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_LOCAL);
}
}
if (BITVAL(PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_UNKNOWN) == 0)
{
if(register_nmi_handler(NMI_UNKNOWN,
CoreFreqK_NMI_Handler,
0,
"corefreqk") == 0)
{
BITSET(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_UNKNOWN);
} else {
BITCLR(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_UNKNOWN);
}
}
if (BITVAL(PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_SERR) == 0)
{
if(register_nmi_handler(NMI_SERR,
CoreFreqK_NMI_Handler,
0,
"corefreqk") == 0)
{
BITSET(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_SERR);
} else {
BITCLR(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_SERR);
}
}
if (BITVAL(PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_IO_CHECK) == 0)
{
if(register_nmi_handler(NMI_IO_CHECK,
CoreFreqK_NMI_Handler,
0,
"corefreqk") == 0)
{
BITSET(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_IO_CHECK);
} else {
BITCLR(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_IO_CHECK);
}
}
#endif /* CONFIG_HAVE_NMI */
}
static void CoreFreqK_UnRegister_NMI(void)
{
#ifdef CONFIG_HAVE_NMI
if (BITVAL(PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_LOCAL) == 1)
{
unregister_nmi_handler(NMI_LOCAL, "corefreqk");
BITCLR(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_LOCAL);
}
if (BITVAL(PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_UNKNOWN) == 1)
{
unregister_nmi_handler(NMI_UNKNOWN, "corefreqk");
BITCLR(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_UNKNOWN);
}
if (BITVAL(PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_SERR) == 1)
{
unregister_nmi_handler(NMI_SERR, "corefreqk");
BITCLR(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_SERR);
}
if (BITVAL(PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_IO_CHECK) == 1)
{
unregister_nmi_handler(NMI_IO_CHECK, "corefreqk");
BITCLR(LOCKLESS, PUBLIC(RO(Proc))->Registration.NMI, BIT_NMI_IO_CHECK);
}
#endif /* CONFIG_HAVE_NMI */
}
#else
static void CoreFreqK_Register_NMI(void) {}
static void CoreFreqK_UnRegister_NMI(void) {}
#endif
static void For_All_CPU_Compute_Clock(void)
{
unsigned int cpu = PUBLIC(RO(Proc))->CPU.Count;
do {
CLOCK Clock = {
.Q = PUBLIC(RO(Proc))->Features.Factory.Clock.Q,
.R = PUBLIC(RO(Proc))->Features.Factory.Clock.R,
.Hz = PUBLIC(RO(Proc))->Features.Factory.Clock.Hz
};
/* from last AP to BSP */
cpu--;
if (!BITVAL(PUBLIC(RO(Core, AT(cpu)))->OffLine, OS))
{
COMPUTE_ARG Compute = {
.TSC = {NULL, NULL},
.Clock = {
.Q = PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MAX)],
.R = 0, .Hz = 0
}
};
if ((Compute.TSC[0] = kmalloc(STRUCT_SIZE, GFP_KERNEL)) != NULL)
{
if ((Compute.TSC[1] = kmalloc(STRUCT_SIZE, GFP_KERNEL)) != NULL)
{
Clock = Compute_Clock(cpu, &Compute);
kfree(Compute.TSC[1]);
}
kfree(Compute.TSC[0]);
}
}
PUBLIC(RO(Core, AT(cpu)))->Clock.Q = Clock.Q;
PUBLIC(RO(Core, AT(cpu)))->Clock.R = Clock.R;
PUBLIC(RO(Core, AT(cpu)))->Clock.Hz = Clock.Hz;
} while (cpu != 0) ;
}
#define SYSGATE_UPDATE(_rc) \
({ \
_rc = Sys_OS_Driver_Query(); \
_rc = (_rc != -ENXIO) ? RC_OK_SYSGATE : _rc; \
})
static long CoreFreqK_ioctl( struct file *filp,
unsigned int cmd,
unsigned long arg )
{
long rc = -EPERM;
UNUSED(filp);
switch ((enum COREFREQ_MAGIC_COMMAND) cmd)
{
case COREFREQ_IOCTL_SYSUPDT:
Controller_Stop(1);
SYSGATE_UPDATE(rc);
Controller_Start(1);
BITSET(BUS_LOCK, PUBLIC(RW(Proc))->OS.Signal, NTFY);
break;
case COREFREQ_IOCTL_SYSONCE:
rc = Sys_OS_Driver_Query();
rc = (rc != -ENXIO) ? Sys_Kernel(PUBLIC(OF(Gate))) : rc;
break;
case COREFREQ_IOCTL_MACHINE:
{
RING_ARG_QWORD prm = {.arg = arg};
switch (prm.dl.hi)
{
case MACHINE_CONTROLLER:
switch (prm.dl.lo)
{
case COREFREQ_TOGGLE_OFF:
Controller_Stop(1);
rc = RC_SUCCESS;
break;
case COREFREQ_TOGGLE_ON:
Controller_Start(1);
#ifdef CONFIG_CPU_FREQ
Policy_Aggregate_Turbo();
#endif /* CONFIG_CPU_FREQ */
rc = RC_OK_COMPUTE;
break;
}
break;
case MACHINE_INTERVAL:
Controller_Stop(1);
SleepInterval = prm.dl.lo;
Compute_Interval();
Controller_Start(1);
rc = RC_SUCCESS;
break;
case MACHINE_AUTOCLOCK:
switch (prm.dl.lo)
{
case COREFREQ_TOGGLE_OFF:
Controller_Stop(1);
For_All_CPU_Compute_Clock();
BITCLR(LOCKLESS, AutoClock, 1);
PUBLIC(RO(Proc))->Registration.AutoClock = AutoClock;
Controller_Start(1);
rc = RC_SUCCESS;
break;
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
BITSET(LOCKLESS, AutoClock, 1);
PUBLIC(RO(Proc))->Registration.AutoClock = AutoClock;
Controller_Start(1);
rc = RC_SUCCESS;
break;
}
break;
case MACHINE_EXPERIMENTAL:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
PUBLIC(RO(Proc))->Registration.Experimental = prm.dl.lo;
Controller_Start(1);
if (PUBLIC(RO(Proc))->Registration.Experimental)
{
if ( !PUBLIC(RO(Proc))->Registration.PCI ) {
PUBLIC(RO(Proc))->Registration.PCI = \
CoreFreqK_ProbePCI(
Arch[PUBLIC(RO(Proc))->ArchID].PCI_ids,
CoreFreqK_ResetChip, CoreFreqK_AppendChip
) == 0;
rc = RC_OK_COMPUTE;
} else {
rc = RC_SUCCESS;
}
} else {
rc = RC_SUCCESS;
}
break;
}
break;
case MACHINE_INTERRUPTS:
switch (prm.dl.lo)
{
case COREFREQ_TOGGLE_OFF:
Controller_Stop(1);
CoreFreqK_UnRegister_NMI();
Controller_Start(1);
rc = RC_SUCCESS;
break;
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
CoreFreqK_Register_NMI();
Controller_Start(1);
rc = RC_SUCCESS;
break;
}
break;
case MACHINE_LIMIT_IDLE:
if (PUBLIC(RO(Proc))->Registration.Driver.CPUidle & REGISTRATION_ENABLE)
{
rc = CoreFreqK_Limit_Idle(prm.dl.lo);
}
break;
case MACHINE_CPU_IDLE:
switch (prm.dl.lo)
{
case COREFREQ_TOGGLE_OFF:
Controller_Stop(1);
rc = CoreFreqK_UnRegister_CPU_Idle();
Register_CPU_Idle = -1;
if (rc == RC_SUCCESS) {
SYSGATE_UPDATE(rc);
}
Controller_Start(1);
break;
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
Register_CPU_Idle = 1;
rc = CoreFreqK_Register_CPU_Idle();
if (rc == RC_SUCCESS) {
SYSGATE_UPDATE(rc);
}
Controller_Start(1);
break;
}
break;
case MACHINE_IDLE_ROUTE:
if (PUBLIC(RO(Proc))->Registration.Driver.CPUidle & REGISTRATION_ENABLE)
{
Controller_Stop(1);
rc = CoreFreqK_UnRegister_CPU_Idle();
Register_CPU_Idle = -1;
if (rc == RC_SUCCESS)
{
Register_CPU_Idle = 1;
Idle_Route = prm.dl.lo;
rc = CoreFreqK_Register_CPU_Idle();
if (rc == RC_SUCCESS) {
SYSGATE_UPDATE(rc);
}
}
Controller_Start(1);
}
break;
case MACHINE_CPU_FREQ:
switch (prm.dl.lo)
{
case COREFREQ_TOGGLE_OFF:
Controller_Stop(1);
rc = CoreFreqK_UnRegister_CPU_Freq();
Register_CPU_Freq = -1;
if (rc == RC_SUCCESS) {
SYSGATE_UPDATE(rc);
}
Controller_Start(1);
break;
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
Register_CPU_Freq = 1;
rc = CoreFreqK_Register_CPU_Freq();
if (rc == RC_SUCCESS) {
SYSGATE_UPDATE(rc);
}
Controller_Start(1);
break;
}
break;
case MACHINE_GOVERNOR:
switch (prm.dl.lo)
{
case COREFREQ_TOGGLE_OFF:
Controller_Stop(1);
rc = CoreFreqK_UnRegister_Governor();
Register_Governor = -1;
if (rc == RC_SUCCESS) {
SYSGATE_UPDATE(rc);
}
Controller_Start(1);
break;
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
Register_Governor = 1;
rc = CoreFreqK_Register_Governor();
if (rc == RC_SUCCESS) {
SYSGATE_UPDATE(rc);
}
Controller_Start(1);
break;
}
break;
case MACHINE_CLOCK_SOURCE:
switch (prm.dl.lo)
{
case COREFREQ_TOGGLE_OFF:
Controller_Stop(1);
rc = CoreFreqK_UnRegister_ClockSource();
Register_ClockSource = -1;
Controller_Start(1);
break;
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
Register_ClockSource = 1;
rc=CoreFreqK_Register_ClockSource(PUBLIC(RO(Proc))->Service.Core);
Controller_Start(1);
break;
}
break;
case MACHINE_FORMULA_SCOPE:
switch (prm.dl.lo)
{
case 0:
rc = CoreFreqK_Thermal_Scope(prm.dh.lo);
break;
case 1:
rc = CoreFreqK_Voltage_Scope(prm.dh.lo);
break;
case 2:
rc = CoreFreqK_Power_Scope(prm.dh.lo);
break;
}
break;
}
}
break;
case COREFREQ_IOCTL_TECHNOLOGY:
{
RING_ARG_QWORD prm = {.arg = arg};
switch (prm.dl.hi)
{
case TECHNOLOGY_L1_HW_PREFETCH:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
L1_HW_PREFETCH_Disable = !prm.dl.lo;
Controller_Start(1);
L1_HW_PREFETCH_Disable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_L1_HW_IP_PREFETCH:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
L1_HW_IP_PREFETCH_Disable = !prm.dl.lo;
Controller_Start(1);
L1_HW_IP_PREFETCH_Disable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_L1_NPP_PREFETCH:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
L1_NPP_PREFETCH_Disable = !prm.dl.lo;
Controller_Start(1);
L1_NPP_PREFETCH_Disable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_L1_SCRUBBING:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
L1_Scrubbing_Enable = prm.dl.lo;
Controller_Start(1);
L1_Scrubbing_Enable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_L2_HW_PREFETCH:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
L2_HW_PREFETCH_Disable = !prm.dl.lo;
Controller_Start(1);
L2_HW_PREFETCH_Disable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_L2_HW_CL_PREFETCH:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
L2_HW_CL_PREFETCH_Disable = !prm.dl.lo;
Controller_Start(1);
L2_HW_CL_PREFETCH_Disable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_L2_AMP_PREFETCH:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
L2_AMP_PREFETCH_Disable = !prm.dl.lo;
Controller_Start(1);
L2_AMP_PREFETCH_Disable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_L2_NLP_PREFETCH:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
L2_NLP_PREFETCH_Disable = !prm.dl.lo;
Controller_Start(1);
L2_NLP_PREFETCH_Disable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_L1_STRIDE_PREFETCH:
if (PUBLIC(RO(Proc))->Features.ExtFeature2_EAX.PrefetchCtl_MSR) {
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
L1_STRIDE_PREFETCH_Disable = !prm.dl.lo;
Controller_Start(1);
L1_STRIDE_PREFETCH_Disable = -1;
rc = RC_SUCCESS;
break;
}
} else {
rc = -ENXIO;
}
break;
case TECHNOLOGY_L1_REGION_PREFETCH:
if (PUBLIC(RO(Proc))->Features.ExtFeature2_EAX.PrefetchCtl_MSR) {
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
L1_REGION_PREFETCH_Disable = !prm.dl.lo;
Controller_Start(1);
L1_REGION_PREFETCH_Disable = -1;
rc = RC_SUCCESS;
break;
}
} else {
rc = -ENXIO;
}
break;
case TECHNOLOGY_L1_BURST_PREFETCH:
if (PUBLIC(RO(Proc))->Features.ExtFeature2_EAX.PrefetchCtl_MSR) {
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
L1_BURST_PREFETCH_Disable = !prm.dl.lo;
Controller_Start(1);
L1_BURST_PREFETCH_Disable = -1;
rc = RC_SUCCESS;
break;
}
} else {
rc = -ENXIO;
}
break;
case TECHNOLOGY_L2_STREAM_HW_PREFETCH:
if (PUBLIC(RO(Proc))->Features.ExtFeature2_EAX.PrefetchCtl_MSR) {
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
L2_STREAM_PREFETCH_Disable = !prm.dl.lo;
Controller_Start(1);
L2_STREAM_PREFETCH_Disable = -1;
rc = RC_SUCCESS;
break;
}
} else {
rc = -ENXIO;
}
break;
case TECHNOLOGY_L2_UPDOWN_PREFETCH:
if (PUBLIC(RO(Proc))->Features.ExtFeature2_EAX.PrefetchCtl_MSR) {
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
L2_UPDOWN_PREFETCH_Disable = !prm.dl.lo;
Controller_Start(1);
L2_UPDOWN_PREFETCH_Disable = -1;
rc = RC_SUCCESS;
break;
}
} else {
rc = -ENXIO;
}
break;
case TECHNOLOGY_LLC_STREAMER:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
LLC_Streamer_Disable = !prm.dl.lo;
Controller_Start(1);
LLC_Streamer_Disable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_EIST:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
SpeedStep_Enable = prm.dl.lo;
Controller_Start(1);
SpeedStep_Enable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_C1E:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
C1E_Enable = prm.dl.lo;
Controller_Start(1);
C1E_Enable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_TURBO:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
TurboBoost_Enable[0] = prm.dl.lo;
Controller_Start(1);
TurboBoost_Enable[0] = -1;
#ifdef CONFIG_CPU_FREQ
Policy_Aggregate_Turbo();
#endif /* CONFIG_CPU_FREQ */
rc = RC_OK_COMPUTE;
break;
}
break;
case TECHNOLOGY_C1A:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
C1A_Enable = prm.dl.lo;
Controller_Start(1);
C1A_Enable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_C3A:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
C3A_Enable = prm.dl.lo;
Controller_Start(1);
C3A_Enable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_C1U:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
C1U_Enable = prm.dl.lo;
Controller_Start(1);
C1U_Enable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_C3U:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
C3U_Enable = prm.dl.lo;
Controller_Start(1);
C3U_Enable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_CC6:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
CC6_Enable = prm.dl.lo;
Controller_Start(1);
CC6_Enable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_PC6:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
PC6_Enable = prm.dl.lo;
Controller_Start(1);
PC6_Enable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_PKG_CSTATE:
Controller_Stop(1);
PkgCStateLimit = prm.dl.lo;
Controller_Start(1);
PkgCStateLimit = -1;
rc = RC_SUCCESS;
break;
case TECHNOLOGY_IO_MWAIT:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
IOMWAIT_Enable = prm.dl.lo;
Controller_Start(1);
IOMWAIT_Enable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_IO_MWAIT_REDIR:
Controller_Stop(1);
CStateIORedir = prm.dl.lo;
Controller_Start(1);
CStateIORedir = -1;
rc = RC_SUCCESS;
break;
case TECHNOLOGY_ODCM:
Controller_Stop(1);
ODCM_Enable = prm.dl.lo;
Controller_Start(1);
ODCM_Enable = -1;
rc = RC_SUCCESS;
break;
case TECHNOLOGY_ODCM_DUTYCYCLE:
Controller_Stop(1);
ODCM_DutyCycle = prm.dl.lo;
Controller_Start(1);
ODCM_DutyCycle = -1;
rc = RC_SUCCESS;
break;
case TECHNOLOGY_POWER_POLICY:
Controller_Stop(1);
PowerPolicy = prm.dl.lo;
Controller_Start(1);
PowerPolicy = -1;
rc = RC_SUCCESS;
break;
case TECHNOLOGY_HWP:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
HWP_Enable = prm.dl.lo;
if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL) {
Intel_Hardware_Performance();
} else if ((PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
|| (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON))
{
if (PUBLIC(RO(Proc))->Features.leaf80000008.EBX.CPPC) {
AMD_F17h_CPPC();
} else if (PUBLIC(RO(Proc))->Features.ACPI_CPPC) {
For_All_ACPI_CPPC(Enable_ACPI_CPPC, NULL);
}
}
Controller_Start(1);
HWP_Enable = -1;
rc = RC_SUCCESS;
break;
case COREFREQ_TOGGLE_OFF:
if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL) {
rc = -RC_UNIMPLEMENTED;
} else if ((PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_AMD)
|| (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_HYGON))
{
if (PUBLIC(RO(Proc))->Features.leaf80000008.EBX.CPPC) {
rc = -RC_UNIMPLEMENTED;
} else if (PUBLIC(RO(Proc))->Features.ACPI_CPPC) {
Controller_Stop(1);
For_All_ACPI_CPPC(Disable_ACPI_CPPC, NULL);
Controller_Start(1);
rc = RC_SUCCESS;
}
}
break;
}
break;
case TECHNOLOGY_HWP_EPP:
Controller_Stop(1);
HWP_EPP = prm.dl.lo;
if (PUBLIC(RO(Proc))->Features.OSPM_EPP == 1)
{
For_All_ACPI_CPPC(Set_EPP_ACPI_CPPC, NULL);
}
Controller_Start(1);
HWP_EPP = -1;
rc = RC_SUCCESS;
break;
case TECHNOLOGY_HDC:
Controller_Stop(1);
HDC_Enable = prm.dl.lo;
Intel_Hardware_Performance();
Controller_Start(1);
HDC_Enable = -1;
rc = RC_SUCCESS;
break;
case TECHNOLOGY_EEO:
if (PUBLIC(RO(Proc))->Features.EEO_Capable)
{
EEO_Disable = prm.dl.lo;
Skylake_PowerControl();
EEO_Disable = -1;
rc = RC_SUCCESS;
} else {
rc = -ENXIO;
}
break;
case TECHNOLOGY_R2H:
if (PUBLIC(RO(Proc))->Features.R2H_Capable)
{
R2H_Disable = prm.dl.lo;
Skylake_PowerControl();
R2H_Disable = -1;
rc = RC_SUCCESS;
} else {
rc = -ENXIO;
}
break;
case TECHNOLOGY_CFG_TDP_LVL:
Controller_Stop(1);
Config_TDP_Level = prm.dl.lo;
Controller_Start(1);
Config_TDP_Level = -1;
rc = RC_SUCCESS;
break;
case TECHNOLOGY_TDP_LIMITING:
{
const enum PWR_DOMAIN pw = prm.dh.lo;
const enum PWR_LIMIT pl = prm.dh.hi;
const unsigned int idx = (PWR_LIMIT_SIZE * pw) + pl;
if (idx < PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE))
{
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
TDP_Limiting_Count = PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE);
RESET_ARRAY(Activate_TDP_Limit, TDP_Limiting_Count, -1);
Activate_TDP_Limit[idx] = prm.dl.lo;
Controller_Start(1);
RESET_ARRAY(Activate_TDP_Limit, TDP_Limiting_Count, -1);
TDP_Limiting_Count = 0;
rc = RC_SUCCESS;
break;
}
}
}
break;
case TECHNOLOGY_TDP_CLAMPING:
{
const enum PWR_DOMAIN pw = prm.dh.lo;
if ( !( (pw == PWR_DOMAIN(RAM))
&& !PUBLIC(RO(Proc))->Registration.Experimental) )
{
const enum PWR_LIMIT pl = prm.dh.hi;
const unsigned int idx = (PWR_LIMIT_SIZE * pw) + pl;
if (idx < PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE))
{
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
TDP_Clamping_Count = PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE);
RESET_ARRAY(Activate_TDP_Clamp, TDP_Clamping_Count, -1);
Activate_TDP_Clamp[idx] = prm.dl.lo;
Controller_Start(1);
RESET_ARRAY(Activate_TDP_Clamp, TDP_Clamping_Count, -1);
TDP_Clamping_Count = 0;
rc = RC_SUCCESS;
break;
}
}
} else {
rc = -RC_EXPERIMENTAL;
}
}
break;
case TECHNOLOGY_TDP_OFFSET:
{
const enum PWR_DOMAIN pw = prm.dh.lo;
const enum PWR_LIMIT pl = prm.dh.hi;
/* Offset is capped within [ -127 , +128 ] watt */
const signed short offset = (signed char) prm.dl.lo;
const unsigned int idx = (PWR_LIMIT_SIZE * pw) + pl;
if (idx < PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE))
{
Controller_Stop(1);
Custom_TDP_Count = PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE);
RESET_ARRAY(Custom_TDP_Offset, Custom_TDP_Count, 0);
Custom_TDP_Offset[idx] = offset;
Controller_Start(1);
RESET_ARRAY(Custom_TDP_Offset, Custom_TDP_Count, 0);
Custom_TDP_Count = 0;
rc = RC_SUCCESS;
}
}
break;
case TECHNOLOGY_TDC_LIMITING:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
Activate_TDC_Limit = prm.dl.lo;
Controller_Start(1);
Activate_TDC_Limit = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_TDC_OFFSET:
{
const signed short offset = (signed char) prm.dl.lo;
Controller_Stop(1);
Custom_TDC_Offset = offset;
Controller_Start(1);
Custom_TDC_Offset = 0;
rc = RC_SUCCESS;
}
break;
case TECHNOLOGY_THM_OFFSET:
{
const signed short offset = (signed short) prm.dl.lo;
if (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC == CRC_INTEL)
{
PLATFORM_INFO PfInfo = {.value = 0};
RDMSR(PfInfo, MSR_PLATFORM_INFO);
Controller_Stop(1);
ThermalOffset = offset;
rc = Intel_ThermalOffset(PfInfo.ProgrammableTj);
Controller_Start(1);
ThermalOffset = 0;
} else {
rc = -RC_UNIMPLEMENTED;
}
}
break;
case TECHNOLOGY_TW_POWER:
{
const enum PWR_DOMAIN pw = prm.dh.lo;
const enum PWR_LIMIT pl = prm.dh.hi;
const signed short tw = prm.dl.lo;
const unsigned int idx = (PWR_LIMIT_SIZE * pw) + pl;
if (idx < PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE))
{
Controller_Stop(1);
PowerTimeWindow_Count = PWR_LIMIT_SIZE * PWR_DOMAIN(SIZE);
RESET_ARRAY(PowerTimeWindow, PowerTimeWindow_Count, -1);
PowerTimeWindow[idx] = tw;
Controller_Start(1);
RESET_ARRAY(PowerTimeWindow, PowerTimeWindow_Count, -1);
PowerTimeWindow_Count = 0;
rc = RC_SUCCESS;
}
}
break;
case TECHNOLOGY_WDT:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
Controller_Stop(1);
WDT_Enable = prm.dl.lo;
Controller_Start(1);
WDT_Enable = -1;
rc = RC_SUCCESS;
break;
}
break;
case TECHNOLOGY_HSMP:
switch (prm.dl.lo) {
case COREFREQ_TOGGLE_OFF:
case COREFREQ_TOGGLE_ON:
{
unsigned int rx;
Controller_Stop(1);
PUBLIC(RO(Proc))->Features.HSMP_Capable = prm.dl.lo;
rx = Query_AMD_HSMP_Interface();
Controller_Start(1);
rc = (rx >= HSMP_FAIL_BGN && rx <= HSMP_FAIL_END) ?
-RC_UNIMPLEMENTED : RC_SUCCESS;
}
break;
}
break;
}
break;
}
break;
case COREFREQ_IOCTL_CPU_OFF:
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0)
{
unsigned int cpu = (unsigned int) arg;
if (cpu < PUBLIC(RO(Proc))->CPU.Count) {
if (!cpu_is_hotpluggable(cpu)) {
rc = -EBUSY;
} else {
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)
if ((rc = remove_cpu(cpu)) == 0) {
rc = RC_OK_COMPUTE;
}
#else
if ((rc = cpu_down(cpu)) == 0) {
rc = RC_OK_COMPUTE;
}
#endif
}
}
}
#else
rc = -EINVAL;
#endif
break;
case COREFREQ_IOCTL_CPU_ON:
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0)
{
unsigned int cpu = (unsigned int) arg;
if (cpu < PUBLIC(RO(Proc))->CPU.Count) {
if (!cpu_is_hotpluggable(cpu)) {
rc = -EBUSY;
} else {
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)
if ((rc = add_cpu(cpu)) == 0) {
rc = RC_OK_COMPUTE;
}
#else
if ((rc = cpu_up(cpu)) == 0) {
rc = RC_OK_COMPUTE;
}
#endif
}
}
}
#else
rc = -EINVAL;
#endif
break;
case COREFREQ_IOCTL_TURBO_CLOCK:
if (Arch[PUBLIC(RO(Proc))->ArchID].TurboClock) {
CLOCK_ARG clockMod = {.sllong = arg};
Controller_Stop(1);
rc = Arch[PUBLIC(RO(Proc))->ArchID].TurboClock(&clockMod);
Controller_Start(1);
} else {
rc = -RC_UNIMPLEMENTED;
}
break;
case COREFREQ_IOCTL_RATIO_CLOCK:
if (Arch[PUBLIC(RO(Proc))->ArchID].ClockMod) {
CLOCK_ARG clockMod = {.sllong = arg};
Controller_Stop(1);
rc = Arch[PUBLIC(RO(Proc))->ArchID].ClockMod(&clockMod);
Controller_Start(1);
#ifdef CONFIG_CPU_FREQ
Policy_Aggregate_Turbo();
#endif /* CONFIG_CPU_FREQ */
} else {
rc = -RC_UNIMPLEMENTED;
}
break;
case COREFREQ_IOCTL_CONFIG_TDP: {
CLOCK_ARG clockMod = {.sllong = arg};
const short MaxRatio = MAXCLOCK_TO_RATIO(short, \
PUBLIC(RO(Core,AT(PUBLIC(RO(Proc))->Service.Core))->Clock.Hz));
switch (clockMod.NC) {
case CLOCK_MOD_ACT:
if ((clockMod.Ratio >= 0)&&(clockMod.Ratio <= MaxRatio))
{
Controller_Stop(1);
Turbo_Activation_Ratio = clockMod.Ratio;
Controller_Start(1);
Turbo_Activation_Ratio = -1;
rc = RC_OK_COMPUTE;
}
break;
}
}
break;
case COREFREQ_IOCTL_UNCORE_CLOCK:
if (Arch[PUBLIC(RO(Proc))->ArchID].Uncore.ClockMod) {
CLOCK_ARG clockMod = {.sllong = arg};
Controller_Stop(1);
rc = Arch[PUBLIC(RO(Proc))->ArchID].Uncore.ClockMod(&clockMod);
Controller_Start(1);
} else {
rc = -RC_UNIMPLEMENTED;
}
break;
case COREFREQ_IOCTL_CLEAR_EVENTS:
switch (arg) {
case EVENT_THERMAL_LOG:
case EVENT_PROCHOT_LOG:
case EVENT_CRITIC_LOG:
case EVENT_THOLD1_LOG:
case EVENT_THOLD2_LOG:
case EVENT_POWER_LIMIT:
case EVENT_CURRENT_LIMIT:
case EVENT_CROSS_DOMAIN:
case EVENT_CORE_HOT_LOG:
case EVENT_CORE_THM_LOG:
case EVENT_CORE_RES_LOG:
case EVENT_CORE_AVG_LOG:
case EVENT_CORE_VRT_LOG:
case EVENT_CORE_TDC_LOG:
case EVENT_CORE_PL1_LOG:
case EVENT_CORE_PL2_LOG:
case EVENT_CORE_EDP_LOG:
case EVENT_CORE_BST_LOG:
case EVENT_CORE_ATT_LOG:
case EVENT_CORE_TVB_LOG:
case EVENT_GFX_HOT_LOG:
case EVENT_GFX_THM_LOG:
case EVENT_GFX_AVG_LOG:
case EVENT_GFX_VRT_LOG:
case EVENT_GFX_TDC_LOG:
case EVENT_GFX_PL1_LOG:
case EVENT_GFX_PL2_LOG:
case EVENT_GFX_EDP_LOG:
case EVENT_GFX_EFF_LOG:
case EVENT_RING_HOT_LOG:
case EVENT_RING_THM_LOG:
case EVENT_RING_AVG_LOG:
case EVENT_RING_VRT_LOG:
case EVENT_RING_TDC_LOG:
case EVENT_RING_PL1_LOG:
case EVENT_RING_PL2_LOG:
case EVENT_RING_EDP_LOG:
Controller_Stop(1);
Clear_Events = arg;
Controller_Start(1);
Clear_Events = 0;
rc = RC_OK_COMPUTE;
break;
case EVENT_ALL_OF_THEM:
Controller_Stop(1);
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
Clear_Events = (unsigned long)(-1);
#else
Clear_Events = (unsigned long long)(-1);
#endif
Controller_Start(1);
Clear_Events = 0;
rc = RC_OK_COMPUTE;
break;
}
break;
default:
rc = -EINVAL;
break;
}
return rc;
}
#undef SYSGATE_UPDATE
static int CoreFreqK_mmap(struct file *pfile, struct vm_area_struct *vma)
{
unsigned long reqSize = vma->vm_end - vma->vm_start;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0) \
|| (defined(RHEL_MAJOR) && (RHEL_MAJOR >= 9) && (RHEL_MINOR >= 5))
vm_flags_t vm_ro = VM_READ | VM_DONTEXPAND;
vm_flags_t vm_rw = VM_READ | VM_WRITE | VM_DONTEXPAND;
#endif
int rc = -EIO;
UNUSED(pfile);
if (vma->vm_pgoff == ID_RO_VMA_PROC) {
if (PUBLIC(RO(Proc)) != NULL)
{
const unsigned long secSize = ROUND_TO_PAGES(sizeof(PROC_RO));
if (reqSize != secSize) {
rc = -EAGAIN;
goto EXIT_PAGE;
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0) \
|| (defined(RHEL_MAJOR) && (RHEL_MAJOR >= 9) && (RHEL_MINOR >= 5))
vm_flags_reset_once(vma, vm_ro);
#else
vma->vm_flags = VM_READ | VM_DONTEXPAND;
#endif
vma->vm_page_prot = PAGE_READONLY;
rc = remap_pfn_range( vma,
vma->vm_start,
virt_to_phys((void *) PUBLIC(RO(Proc))) >> PAGE_SHIFT,
reqSize,
vma->vm_page_prot);
}
} else if (vma->vm_pgoff == ID_RW_VMA_PROC) {
if (PUBLIC(RW(Proc)) != NULL)
{
const unsigned long secSize = ROUND_TO_PAGES(sizeof(PROC_RW));
if (reqSize != secSize) {
rc = -EAGAIN;
goto EXIT_PAGE;
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0) \
|| (defined(RHEL_MAJOR) && (RHEL_MAJOR >= 9) && (RHEL_MINOR >= 5))
vm_flags_reset_once(vma, vm_rw);
#else
vma->vm_flags = VM_READ | VM_WRITE | VM_DONTEXPAND;
#endif
vma->vm_page_prot = PAGE_SHARED;
rc = remap_pfn_range( vma,
vma->vm_start,
virt_to_phys((void *) PUBLIC(RW(Proc))) >> PAGE_SHIFT,
reqSize,
vma->vm_page_prot);
}
} else if (vma->vm_pgoff == ID_RO_VMA_GATE) {
if (PUBLIC(RO(Proc)) != NULL)
{
switch (SysGate_OnDemand()) {
default:
case -1:
break;
case 1:
fallthrough;
case 0: {
if (reqSize != PUBLIC(RO(Proc))->Gate.ReqMem.Size) {
return -EAGAIN;
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0) \
|| (defined(RHEL_MAJOR) && (RHEL_MAJOR >= 9) && (RHEL_MINOR >= 5))
vm_flags_reset_once(vma, vm_ro);
#else
vma->vm_flags = VM_READ | VM_DONTEXPAND;
#endif
vma->vm_page_prot = PAGE_READONLY;
rc = remap_pfn_range( vma,
vma->vm_start,
virt_to_phys((void *) PUBLIC(OF(Gate))) >> PAGE_SHIFT,
reqSize,
vma->vm_page_prot);
}
break;
}
}
} else if ((vma->vm_pgoff >= ID_RO_VMA_CORE)
&& (vma->vm_pgoff < ID_RW_VMA_CORE))
{
signed int cpu = vma->vm_pgoff - ID_RO_VMA_CORE;
if (PUBLIC(RO(Proc)) != NULL) {
if ((cpu >= 0) && (cpu < PUBLIC(RO(Proc))->CPU.Count)) {
if (PUBLIC(RO(Core, AT(cpu))) != NULL)
{
const unsigned long secSize = ROUND_TO_PAGES(sizeof(CORE_RO));
if (reqSize != secSize) {
rc = -EAGAIN;
goto EXIT_PAGE;
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0) \
|| (defined(RHEL_MAJOR) && (RHEL_MAJOR >= 9) && (RHEL_MINOR >= 5))
vm_flags_reset_once(vma, vm_ro);
#else
vma->vm_flags = VM_READ | VM_DONTEXPAND;
#endif
vma->vm_page_prot = PAGE_READONLY;
rc = remap_pfn_range( vma,
vma->vm_start,
virt_to_phys((void *) PUBLIC(RO(Core, AT(cpu)))) >> PAGE_SHIFT,
reqSize,
vma->vm_page_prot);
}
}
}
} else if ((vma->vm_pgoff >= ID_RW_VMA_CORE)
&& (vma->vm_pgoff < ID_ANY_VMA_JAIL))
{
signed int cpu = vma->vm_pgoff - ID_RW_VMA_CORE;
if (PUBLIC(RO(Proc)) != NULL) {
if ((cpu >= 0) && (cpu < PUBLIC(RO(Proc))->CPU.Count)) {
if (PUBLIC(RW(Core, AT(cpu))) != NULL)
{
const unsigned long secSize = ROUND_TO_PAGES(sizeof(CORE_RW));
if (reqSize != secSize) {
rc = -EAGAIN;
goto EXIT_PAGE;
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0) \
|| (defined(RHEL_MAJOR) && (RHEL_MAJOR >= 9) && (RHEL_MINOR >= 5))
vm_flags_reset_once(vma, vm_rw);
#else
vma->vm_flags = VM_READ | VM_WRITE | VM_DONTEXPAND;
#endif
vma->vm_page_prot = PAGE_SHARED;
rc = remap_pfn_range( vma,
vma->vm_start,
virt_to_phys((void *) PUBLIC(RW(Core, AT(cpu)))) >> PAGE_SHIFT,
reqSize,
vma->vm_page_prot);
}
}
}
}
EXIT_PAGE:
return rc;
}
static DEFINE_MUTEX(CoreFreqK_mutex); /* Only one driver instance. */
static int CoreFreqK_open(struct inode *inode, struct file *pfile)
{
UNUSED(inode);
UNUSED(pfile);
if (!mutex_trylock(&CoreFreqK_mutex))
return -EBUSY;
else
return 0;
}
static int CoreFreqK_release(struct inode *inode, struct file *pfile)
{
UNUSED(inode);
UNUSED(pfile);
mutex_unlock(&CoreFreqK_mutex);
return 0;
}
static struct file_operations CoreFreqK_fops = {
.open = CoreFreqK_open,
.release = CoreFreqK_release,
.mmap = CoreFreqK_mmap,
.unlocked_ioctl = CoreFreqK_ioctl,
.owner = THIS_MODULE,
};
#ifdef CONFIG_PM_SLEEP
static void Print_SuspendResume(void)
{
pr_notice("CoreFreq: %s(%u:%d:%d)\n",
CoreFreqK.ResumeFromSuspend ? "Suspend" : "Resume",
PUBLIC(RO(Proc))->Service.Core,
PUBLIC(RO(Proc))->Service.Thread,
PUBLIC(RO(Proc))->Service.Hybrid);
}
static int CoreFreqK_Suspend(struct device *dev)
{
UNUSED(dev);
CoreFreqK.ResumeFromSuspend = true;
Controller_Stop(1);
Print_SuspendResume();
return 0;
}
static int CoreFreqK_Resume(struct device *dev)
{ /* Probe Processor again */
UNUSED(dev);
if (Arch[PUBLIC(RO(Proc))->ArchID].Query != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Query(PUBLIC(RO(Proc))->Service.Core);
}
/* Probe PCI again */
if (PUBLIC(RO(Proc))->Registration.PCI) {
PUBLIC(RO(Proc))->Registration.PCI = \
CoreFreqK_ProbePCI(Arch[PUBLIC(RO(Proc))->ArchID].PCI_ids,
CoreFreqK_ResetChip, CoreFreqK_AppendChip) == 0;
}
Controller_Start(1);
#ifdef CONFIG_CPU_FREQ
Policy_Aggregate_Turbo();
#endif /* CONFIG_CPU_FREQ */
BITSET(BUS_LOCK, PUBLIC(RW(Proc))->OS.Signal, NTFY); /* Notify Daemon*/
CoreFreqK.ResumeFromSuspend = false;
Print_SuspendResume();
return 0;
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 17, 0)
static DEFINE_SIMPLE_DEV_PM_OPS(CoreFreqK_pm_ops, \
CoreFreqK_Suspend, \
CoreFreqK_Resume);
#else
static SIMPLE_DEV_PM_OPS(CoreFreqK_pm_ops, CoreFreqK_Suspend, CoreFreqK_Resume);
#endif /* KERNEL_VERSION(5, 17, 0) */
#define COREFREQ_PM_OPS (&CoreFreqK_pm_ops)
#else /* CONFIG_PM_SLEEP */
#define COREFREQ_PM_OPS NULL
#endif /* CONFIG_PM_SLEEP */
#ifdef CONFIG_HOTPLUG_CPU
static int CoreFreqK_HotPlug_CPU_Online(unsigned int cpu)
{
if (cpu < PUBLIC(RO(Proc))->CPU.Count)
{
/* Is this the very first time the processor is online ? */
if (PUBLIC(RO(Core, AT(cpu)))->T.ApicID == -1)
{
if (Core_Topology(cpu) == 0)
{
if (PUBLIC(RO(Core, AT(cpu)))->T.ApicID >= 0)
{
BITCLR(LOCKLESS, PUBLIC(RO(Core, AT(cpu)))->OffLine, HW);
/* Is the BCLK frequency missing ? */
if (PUBLIC(RO(Core, AT(cpu)))->Clock.Hz == 0)
{
if (AutoClock & 0b01)
{
COMPUTE_ARG Compute = {
.TSC = {NULL, NULL},
.Clock = {
.Q = PUBLIC(RO(Core, \
AT(PUBLIC(RO(Proc))->Service.Core)))->Boost[BOOST(MAX)],
.R = 0, .Hz = 0
}
};
if ((Compute.TSC[0] = kmalloc(STRUCT_SIZE, GFP_KERNEL)) != NULL)
{
if ((Compute.TSC[1] = kmalloc(STRUCT_SIZE, GFP_KERNEL)) != NULL)
{
PUBLIC(RO(Core, AT(cpu)))->Clock = Compute_Clock(cpu, &Compute);
kfree(Compute.TSC[1]);
}
kfree(Compute.TSC[0]);
}
} else {
PUBLIC(RO(Core, AT(cpu)))->Clock = \
PUBLIC(RO(Core, AT(PUBLIC(RO(Proc))->Service.Core)))->Clock;
}
}
}
} else {
BITSET(LOCKLESS, PUBLIC(RO(Core, AT(cpu)))->OffLine, HW);
}
}
PUBLIC(RO(Proc))->CPU.OnLine++;
BITCLR(LOCKLESS, PUBLIC(RO(Core, AT(cpu)))->OffLine, OS);
MatchPeerForUpService(&PUBLIC(RO(Proc))->Service, cpu);
if (PUBLIC(RO(Proc))->Features.ACPI_CPPC == 1) {
Read_ACPI_CPPC_Registers(cpu, NULL);
}
/* Start the collect timer dedicated to this CPU iff not STR resuming */
#ifdef CONFIG_PM_SLEEP
if (CoreFreqK.ResumeFromSuspend == false)
#endif /* CONFIG_PM_SLEEP */
{
if (Arch[PUBLIC(RO(Proc))->ArchID].Timer != NULL) {
Arch[PUBLIC(RO(Proc))->ArchID].Timer(cpu);
}
if ((BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED) == 0)
&& (Arch[PUBLIC(RO(Proc))->ArchID].Start != NULL)) {
smp_call_function_single(cpu,
Arch[PUBLIC(RO(Proc))->ArchID].Start,
NULL, 0);
}
}
#if defined(CONFIG_CPU_IDLE) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
if (PUBLIC(RO(Proc))->Registration.Driver.CPUidle & REGISTRATION_ENABLE) {
struct cpuidle_device *device = per_cpu_ptr(CoreFreqK.IdleDevice, cpu);
if (device != NULL) {
if (device->registered == 0) {
device->cpu = cpu;
cpuidle_register_device(device);
}
}
}
#endif /* CONFIG_CPU_IDLE */
#ifdef CONFIG_CPU_FREQ
Policy_Aggregate_Turbo();
#endif /* CONFIG_CPU_FREQ */
return 0;
} else
return -EINVAL;
}
static int CoreFreqK_HotPlug_CPU_Offline(unsigned int cpu)
{
if (cpu < PUBLIC(RO(Proc))->CPU.Count)
{ /* Stop the associated collect timer. */
if ((BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, CREATED) == 1)
&& (BITVAL(PRIVATE(OF(Core, AT(cpu)))->Join.TSM, STARTED) == 1)
&& (Arch[PUBLIC(RO(Proc))->ArchID].Stop != NULL)) {
smp_call_function_single(cpu,
Arch[PUBLIC(RO(Proc))->ArchID].Stop,
NULL, 1);
}
PUBLIC(RO(Proc))->CPU.OnLine--;
BITSET(LOCKLESS, PUBLIC(RO(Core, AT(cpu)))->OffLine, OS);
/* Seek for an alternate Service Processor. */
#ifdef CONFIG_PM_SLEEP
if (CoreFreqK.ResumeFromSuspend == false)
#endif /* CONFIG_PM_SLEEP */
{
if ((cpu == PUBLIC(RO(Proc))->Service.Core)
|| (cpu == PUBLIC(RO(Proc))->Service.Thread))
{
MatchPeerForDownService(&PUBLIC(RO(Proc))->Service, cpu);
if (PUBLIC(RO(Proc))->Service.Core != cpu)
{
const unsigned int alt = PUBLIC(RO(Proc))->Service.Core;
if (BITVAL(PRIVATE(OF(Core, AT(alt)))->Join.TSM, CREATED) == 1)
{
if ((BITVAL(PRIVATE(OF(Core, AT(alt)))->Join.TSM, STARTED) == 1)
&& (Arch[PUBLIC(RO(Proc))->ArchID].Stop != NULL)) {
smp_call_function_single(alt,
Arch[PUBLIC(RO(Proc))->ArchID].Stop,
NULL, 1);
}
if ((BITVAL(PRIVATE(OF(Core, AT(alt)))->Join.TSM, STARTED) == 0)
&& (Arch[PUBLIC(RO(Proc))->ArchID].Start != NULL)) {
smp_call_function_single(alt,
Arch[PUBLIC(RO(Proc))->ArchID].Start,
NULL, 0);
}
}
}
} else if ((cpu == PUBLIC(RO(Proc))->Service.Hybrid)
&& (PUBLIC(RO(Proc))->Features.ExtFeature.EDX.Hybrid))
{
PUBLIC(RO(Proc))->Service.Hybrid = Seek_Topology_Hybrid_Core(cpu);
}
}
#ifdef CONFIG_CPU_FREQ
Policy_Aggregate_Turbo();
#endif /* CONFIG_CPU_FREQ */
return 0;
} else
return -EINVAL;
}
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)
static int CoreFreqK_HotPlug( struct notifier_block *nfb,
unsigned long action,
void *hcpu)
{
unsigned int cpu = (unsigned long) hcpu, rc = 0;
switch (action) {
case CPU_ONLINE:
case CPU_DOWN_FAILED:
/*TODO case CPU_ONLINE_FROZEN: */
rc = CoreFreqK_HotPlug_CPU_Online(cpu);
break;
case CPU_DOWN_PREPARE:
/*TODO case CPU_DOWN_PREPARE_FROZEN: */
rc = CoreFreqK_HotPlug_CPU_Offline(cpu);
break;
default:
break;
}
return NOTIFY_OK;
}
static struct notifier_block CoreFreqK_notifier_block = {
.notifier_call = CoreFreqK_HotPlug,
};
#endif /* KERNEL_VERSION(4, 10, 0) */
#endif /* CONFIG_HOTPLUG_CPU */
static void SMBIOS_Collect(void)
{
#ifdef CONFIG_DMI
struct {
enum dmi_field field;
char *recipient;
} dmi_collect[] = {
{ DMI_BIOS_VENDOR, PUBLIC(RO(Proc))->SMB.BIOS.Vendor },
{ DMI_BIOS_VERSION, PUBLIC(RO(Proc))->SMB.BIOS.Version },
{ DMI_BIOS_DATE, PUBLIC(RO(Proc))->SMB.BIOS.Release },
{ DMI_SYS_VENDOR, PUBLIC(RO(Proc))->SMB.System.Vendor },
{ DMI_PRODUCT_NAME, PUBLIC(RO(Proc))->SMB.Product.Name },
{ DMI_PRODUCT_VERSION, PUBLIC(RO(Proc))->SMB.Product.Version},
{ DMI_PRODUCT_SERIAL, PUBLIC(RO(Proc))->SMB.Product.Serial },
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0)
{ DMI_PRODUCT_SKU, PUBLIC(RO(Proc))->SMB.Product.SKU },
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)
{ DMI_PRODUCT_FAMILY, PUBLIC(RO(Proc))->SMB.Product.Family },
#endif
{ DMI_BOARD_VENDOR, PUBLIC(RO(Proc))->SMB.Board.Vendor },
{ DMI_BOARD_NAME, PUBLIC(RO(Proc))->SMB.Board.Name },
{ DMI_BOARD_VERSION, PUBLIC(RO(Proc))->SMB.Board.Version },
{ DMI_BOARD_SERIAL, PUBLIC(RO(Proc))->SMB.Board.Serial }
};
const size_t count = sizeof(dmi_collect) / sizeof(dmi_collect[0]);
size_t idx;
for (idx = 0; idx < count; idx++) {
const char *pInfo = dmi_get_system_info(dmi_collect[idx].field);
if ((pInfo != NULL) && (strlen(pInfo) > 0)) {
StrCopy(dmi_collect[idx].recipient, pInfo, MAX_UTS_LEN);
}
}
#endif /* CONFIG_DMI */
}
#ifdef CONFIG_DMI
static char *SMBIOS_String(const struct dmi_header *dh, u8 id)
{
char *pStr = (char *) dh;
pStr += dh->length;
while (id > 1 && *pStr) {
pStr += strlen(pStr);
pStr++;
id--;
}
if (!*pStr) {
return NULL;
}
return pStr;
}
#define safe_strim(pStr) (strim(pStr == NULL ? "" : pStr))
static void SMBIOS_Entries(const struct dmi_header *dh, void *priv)
{
size_t *count = (size_t*) priv;
switch (dh->type) {
case DMI_ENTRY_PHYS_MEM_ARRAY:
{
const struct SMBIOS16 *entry = (struct SMBIOS16*) dh;
StrFormat(PUBLIC(RO(Proc))->SMB.Phys.Memory.Array, MAX_UTS_LEN,
"Number Of Devices:%d\\Maximum Capacity:%lld kilobytes",
entry->number_devices,
entry->maximum_capacity >= 0x80000000 ?
entry->extended_capacity : entry->maximum_capacity);
}
break;
case DMI_ENTRY_MEM_DEVICE:
{
const struct SMBIOS17 *entry = (struct SMBIOS17*) dh;
if ((entry->length > 0x1a) && (entry->size > 0))
{
if ((*count) < SMB_MAX_DIMM)
{
const char *locator[2] = {
safe_strim(SMBIOS_String(dh, entry->device_locator_id)),
safe_strim(SMBIOS_String(dh, entry->bank_locator_id))
};
const size_t len[2] = {
strlen(locator[0]) > 0 ? strlen(locator[0]) : 0,
strlen(locator[1]) > 0 ? strlen(locator[1]) : 0
}, prop = (len[0] + len[1]) > 0 ? (len[0] + len[1]) : 1;
const int ratio[2] = {
DIV_ROUND_CLOSEST(len[0] * (MAX_UTS_LEN - (1+1)), prop),
DIV_ROUND_CLOSEST(len[1] * (MAX_UTS_LEN - (1+1)), prop)
};
StrFormat(PUBLIC(RO(Proc))->SMB.Memory.Locator[(*count)],
MAX_UTS_LEN, "%.*s\\%.*s",
ratio[0], locator[0],
ratio[1], locator[1]);
StrCopy(PUBLIC(RO(Proc))->SMB.Memory.Manufacturer[(*count)],
safe_strim(SMBIOS_String(dh, entry->manufacturer_id)),
MAX_UTS_LEN);
StrCopy(PUBLIC(RO(Proc))->SMB.Memory.PartNumber[(*count)],
safe_strim(SMBIOS_String(dh, entry->part_number_id)),
MAX_UTS_LEN);
}
}
(*count) = (*count) + 1;
}
break;
}
}
#undef safe_strim
#endif /* CONFIG_DMI */
static void SMBIOS_Decoder(void)
{
#ifdef CONFIG_DMI
size_t count = 0;
dmi_walk(SMBIOS_Entries, &count);
#endif /* CONFIG_DMI */
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 2, 0) \
|| ((RHEL_MAJOR == 8) && ((RHEL_MINOR < 3) || (RHEL_MINOR > 8))) \
|| ((RHEL_MAJOR >= 9) && (RHEL_MINOR > 0) && (RHEL_MINOR < 99))
static char *CoreFreqK_DevNode(const struct device *dev, umode_t *mode)
#else
static char *CoreFreqK_DevNode(struct device *dev, umode_t *mode)
#endif
{
UNUSED(dev);
if (mode != NULL) {
(*mode) = 0600 ; /* Device access is crw------ */
}
return NULL;
}
static void CoreFreqK_Empty_Func_Level_Down(void)
{
}
static void CoreFreqK_Alloc_Features_Level_Down(void)
{
pr_notice("CoreFreq: Unload\n");
}
static int CoreFreqK_Alloc_Features_Level_Up(INIT_ARG *pArg)
{
pArg->Features = kmalloc(sizeof(FEATURES), GFP_KERNEL);
if (pArg->Features == NULL)
{
return -ENOMEM;
} else {
memset(pArg->Features, 0, sizeof(FEATURES));
}
pArg->Brand = kmalloc(BRAND_SIZE, GFP_KERNEL);
if (pArg->Brand == NULL)
{
return -ENOMEM;
} else {
memset(pArg->Brand, 0x20, BRAND_SIZE);
}
return 0;
}
#define CoreFreqK_Query_Features_Level_Down CoreFreqK_Empty_Func_Level_Down
static int CoreFreqK_Query_Features_Level_Up(INIT_ARG *pArg)
{
int rc = 0;
if ((CPU_Count > 0) && (CPU_Count <= CORE_COUNT)
&& (CPU_Count <= NR_CPUS)
&& ((ServiceProcessor == -1) || (ServiceProcessor >= CPU_Count)))
{ /* Force Service to a Core with allocated memory */
ServiceProcessor = CPU_Count - 1;
}
if (ServiceProcessor == -1)
{ /* Query features on any processor. */
pArg->localProcessor = get_cpu(); /* TODO(preempt_disable) */
Query_Features(pArg);
put_cpu(); /* TODO(preempt_enable) */
rc = pArg->rc;
} else { /* Query features on User selected processor. */
if (ServiceProcessor >= 0)
{
pArg->localProcessor = ServiceProcessor;
if ((rc = smp_call_function_single(pArg->localProcessor,
Query_Features,
pArg, 1)) == 0)
{
rc = pArg->rc;
}
} else {
rc = -ENXIO;
}
}
if (rc == 0)
{
switch (CPU_Count) {
case -1:
{ /* Rely on the operating system's cpu counting. */
unsigned int OS_Count = num_present_cpus();
if (pArg->SMT_Count != OS_Count) {
pArg->SMT_Count = OS_Count;
}
}
break;
default:
if ((CPU_Count > 0) && (CPU_Count <= CORE_COUNT)
&& (CPU_Count <= NR_CPUS))
{ /* Hardware probing unless User override value */
pArg->SMT_Count = (unsigned int) CPU_Count;
}
break;
}
} else {
rc = -ENXIO;
}
return rc;
}
static void CoreFreqK_Alloc_Device_Level_Down(void)
{
unregister_chrdev_region(CoreFreqK.mkdev, 1);
}
static int CoreFreqK_Alloc_Device_Level_Up(INIT_ARG *pArg)
{
UNUSED(pArg);
CoreFreqK.kcdev = cdev_alloc();
CoreFreqK.kcdev->ops = &CoreFreqK_fops;
CoreFreqK.kcdev->owner = THIS_MODULE;
if (alloc_chrdev_region(&CoreFreqK.nmdev, 0, 1, DRV_FILENAME) >= 0)
{
return 0;
} else {
return -EBUSY;
}
}
static void CoreFreqK_Make_Device_Level_Down(void)
{
cdev_del(CoreFreqK.kcdev);
}
static int CoreFreqK_Make_Device_Level_Up(INIT_ARG *pArg)
{
UNUSED(pArg);
CoreFreqK.Major = MAJOR(CoreFreqK.nmdev);
CoreFreqK.mkdev = MKDEV(CoreFreqK.Major, 0);
if (cdev_add(CoreFreqK.kcdev, CoreFreqK.mkdev, 1) >= 0)
{
return 0;
} else {
return -EBUSY;
}
}
static void CoreFreqK_Create_Device_Level_Down(void)
{
device_destroy(CoreFreqK.clsdev, CoreFreqK.mkdev);
class_destroy(CoreFreqK.clsdev);
}
static int CoreFreqK_Create_Device_Level_Up(INIT_ARG *pArg)
{
struct device *tmpDev;
UNUSED(pArg);
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0) \
|| (defined(RHEL_MAJOR) && (RHEL_MAJOR >= 9) \
&& (RHEL_MINOR > 0) && (RHEL_MINOR < 99))
CoreFreqK.clsdev = class_create(DRV_DEVNAME);
#else
CoreFreqK.clsdev = class_create(THIS_MODULE, DRV_DEVNAME);
#endif
CoreFreqK.clsdev->pm = COREFREQ_PM_OPS;
CoreFreqK.clsdev->devnode = CoreFreqK_DevNode;
if ((tmpDev = device_create( CoreFreqK.clsdev, NULL,
CoreFreqK.mkdev, NULL,
DRV_DEVNAME)) != NULL)
{
return 0;
} else {
return -EBUSY;
}
}
static void CoreFreqK_Alloc_Public_Level_Down(void)
{
if (PUBLIC() != NULL)
{
kfree(PUBLIC());
}
}
static int CoreFreqK_Alloc_Public_Level_Up(INIT_ARG *pArg)
{
const size_t coreSizeRO = sizeof(CORE_RO*) * pArg->SMT_Count,
coreSizeRW = sizeof(CORE_RW*) * pArg->SMT_Count,
publicSize = sizeof(KPUBLIC),
alloc_size = publicSize + coreSizeRO + coreSizeRW;
if (((PUBLIC() = kmalloc(alloc_size, GFP_KERNEL)) != NULL))
{
void *addr;
memset(PUBLIC(), 0, alloc_size);
addr = (void*) PUBLIC();
addr = addr + publicSize;
PUBLIC(RO(Core)) = (CORE_RO**) addr;
addr = (void*) PUBLIC(RO(Core));
addr = addr + coreSizeRO;
PUBLIC(RW(Core)) = (CORE_RW**) addr;
return 0;
} else{
return -ENOMEM;
}
}
static void CoreFreqK_Alloc_Private_Level_Down(void)
{
if (PRIVATE() != NULL)
{
kfree(PRIVATE());
}
}
static int CoreFreqK_Alloc_Private_Level_Up(INIT_ARG *pArg)
{
const unsigned long
privCoreSize = sizeof(struct PRIV_CORE_ST *) * pArg->SMT_Count,
privateSize = sizeof(KPRIVATE) + privCoreSize;
if (((PRIVATE() = kmalloc(privateSize, GFP_KERNEL)) != NULL))
{
memset(PRIVATE(), 0, privateSize);
return 0;
} else {
return -ENOMEM;
}
}
static void CoreFreqK_Alloc_Processor_RO_Level_Down(void)
{
if (PUBLIC(RO(Proc)) != NULL)
{
kfree(PUBLIC(RO(Proc)));
}
}
static int CoreFreqK_Alloc_Processor_RO_Level_Up(INIT_ARG *pArg)
{
const unsigned long procSize = ROUND_TO_PAGES(sizeof(PROC_RO));
UNUSED(pArg);
if ( (PUBLIC(RO(Proc)) = kmalloc(procSize, GFP_KERNEL)) != NULL)
{
memset(PUBLIC(RO(Proc)), 0, procSize);
return 0;
} else {
return -ENOMEM;
}
}
static void CoreFreqK_Alloc_Processor_RW_Level_Down(void)
{
if (PUBLIC(RW(Proc)) != NULL)
{
kfree(PUBLIC(RW(Proc)));
}
}
static int CoreFreqK_Alloc_Processor_RW_Level_Up(INIT_ARG *pArg)
{
const unsigned long procSize = ROUND_TO_PAGES(sizeof(PROC_RW));
UNUSED(pArg);
if ( (PUBLIC(RW(Proc)) = kmalloc(procSize, GFP_KERNEL)) != NULL)
{
memset(PUBLIC(RW(Proc)), 0, procSize);
return 0;
} else {
return -ENOMEM;
}
}
#define CoreFreqK_Scale_And_Compute_Level_Down CoreFreqK_Empty_Func_Level_Down
static int CoreFreqK_Scale_And_Compute_Level_Up(INIT_ARG *pArg)
{
SET_FOOTPRINT(PUBLIC(RO(Proc))->FootPrint, \
MAX_FREQ_HZ, \
CORE_COUNT, \
TASK_ORDER, \
COREFREQ_MAJOR, \
COREFREQ_MINOR, \
COREFREQ_REV );
PUBLIC(RO(Proc))->CPU.Count = pArg->SMT_Count;
if (PUBLIC(RO(Proc))->CPU.Count > CORE_COUNT) {
pr_warn("CoreFreq: Detected %u CPUs, but built with CORE_COUNT=%u\n",
PUBLIC(RO(Proc))->CPU.Count, CORE_COUNT);
pr_warn("CoreFreq: Run 'make help' for instructions " \
"on setting CORE_COUNT.\n");
return -ENOMEM;
} else {
/* PreCompute SysGate memory allocation. */
PUBLIC(RO(Proc))->Gate.ReqMem.Size = sizeof(SYSGATE_RO);
PUBLIC(RO(Proc))->Gate.ReqMem.Order = \
get_order(PUBLIC(RO(Proc))->Gate.ReqMem.Size);
PUBLIC(RO(Proc))->Gate.ReqMem.Size = \
PAGE_SIZE << PUBLIC(RO(Proc))->Gate.ReqMem.Order;
PUBLIC(RO(Proc))->Registration.AutoClock = AutoClock;
PUBLIC(RO(Proc))->Registration.Experimental = Experimental;
Compute_Interval();
memcpy(&PUBLIC(RO(Proc))->Features, pArg->Features, sizeof(FEATURES));
/* Initialize default uArch's codename with the CPUID brand. */
Arch[GenuineArch].Architecture[0] = \
PUBLIC(RO(Proc))->Features.Info.Vendor.ID;
return 0;
}
}
static void CoreFreqK_Alloc_Public_Cache_Level_Down(void)
{
if (PUBLIC(OF(Cache)) != NULL)
{
kmem_cache_destroy(PUBLIC(OF(Cache)));
}
}
static int CoreFreqK_Alloc_Public_Cache_Level_Up(INIT_ARG *pArg)
{
const unsigned long cacheSize = KMAX( ROUND_TO_PAGES(sizeof(CORE_RO)),
ROUND_TO_PAGES(sizeof(CORE_RW)) );
UNUSED(pArg);
if ( (PUBLIC(OF(Cache)) = kmem_cache_create( "corefreqk-pub",
cacheSize, 0,
SLAB_HWCACHE_ALIGN,
NULL ) ) != NULL)
{
return 0;
} else {
return -ENOMEM;
}
}
static void CoreFreqK_Alloc_Private_Cache_Level_Down(void)
{
if (PRIVATE(OF(Cache)) != NULL)
{
kmem_cache_destroy(PRIVATE(OF(Cache)));
}
}
static int CoreFreqK_Alloc_Private_Cache_Level_Up(INIT_ARG *pArg)
{
const unsigned long joinSize = \
ROUND_TO_PAGES(sizeof(struct PRIV_CORE_ST));
UNUSED(pArg);
if ( (PRIVATE(OF(Cache)) = kmem_cache_create( "corefreqk-priv",
joinSize, 0,
SLAB_HWCACHE_ALIGN,
NULL ) ) != NULL)
{
return 0;
} else {
return -ENOMEM;
}
}
static void CoreFreqK_Alloc_Per_CPU_Level_Down(void)
{
unsigned int cpu;
for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++)
{
if (PUBLIC(OF(Cache)) != NULL)
{
if (PUBLIC(RO(Core, AT(cpu))) != NULL) {
kmem_cache_free(PUBLIC(OF(Cache)), PUBLIC(RO(Core, AT(cpu))));
}
if (PUBLIC(RW(Core, AT(cpu))) != NULL) {
kmem_cache_free(PUBLIC(OF(Cache)), PUBLIC(RW(Core, AT(cpu))));
}
}
if (PRIVATE(OF(Cache)) != NULL)
{
if (PRIVATE(OF(Core, AT(cpu))) != NULL) {
kmem_cache_free(PRIVATE(OF(Cache)), PRIVATE(OF(Core, AT(cpu))));
}
}
}
}
static int CoreFreqK_Alloc_Per_CPU_Level_Up(INIT_ARG *pArg)
{
const unsigned long cacheSize = KMAX( ROUND_TO_PAGES(sizeof(CORE_RO)),
ROUND_TO_PAGES(sizeof(CORE_RW)) );
const unsigned long joinSize = \
ROUND_TO_PAGES(sizeof(struct PRIV_CORE_ST));
unsigned int cpu;
int rc = 0;
UNUSED(pArg);
for (cpu = 0; cpu < PUBLIC(RO(Proc))->CPU.Count; cpu++)
{
void *kcache = NULL;
kcache = kmem_cache_alloc(PUBLIC(OF(Cache)), GFP_KERNEL);
if (kcache != NULL) {
memset(kcache, 0, cacheSize);
PUBLIC(RO(Core, AT(cpu))) = kcache;
} else {
rc = -ENOMEM;
break;
}
kcache = kmem_cache_alloc(PUBLIC(OF(Cache)), GFP_KERNEL);
if (kcache != NULL) {
memset(kcache, 0, cacheSize);
PUBLIC(RW(Core, AT(cpu))) = kcache;
} else {
rc = -ENOMEM;
break;
}
kcache = kmem_cache_alloc(PRIVATE(OF(Cache)), GFP_KERNEL);
if (kcache != NULL) {
memset(kcache, 0, joinSize);
PRIVATE(OF(Core, AT(cpu))) = kcache;
} else {
rc = -ENOMEM;
break;
}
if (rc == 0) {
BITCLR(LOCKLESS, PUBLIC(RW(Core, AT(cpu)))->Sync.V, NTFY);
PUBLIC(RO(Core, AT(cpu)))->Bind = cpu;
Define_CPUID(PUBLIC(RO(Core, AT(cpu))), CpuIDforVendor);
}
}
return rc;
}
static void CoreFreqK_Ignition_Level_Down(void)
{
CoreFreqK_UnRegister_ClockSource();
CoreFreqK_UnRegister_Governor();
CoreFreqK_UnRegister_CPU_Freq();
CoreFreqK_UnRegister_CPU_Idle();
CoreFreqK_UnRegister_NMI();
#ifdef CONFIG_HOTPLUG_CPU
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)
unregister_hotcpu_notifier(&CoreFreqK_notifier_block);
#else /* KERNEL_VERSION(4, 10, 0) */
if ( !(PUBLIC(RO(Proc))->Registration.HotPlug < 0) )
{
cpuhp_remove_state_nocalls(PUBLIC(RO(Proc))->Registration.HotPlug);
}
#endif /* KERNEL_VERSION(4, 10, 0) */
#endif /* CONFIG_HOTPLUG_CPU */
Controller_Stop(1);
Controller_Exit();
if (PUBLIC(OF(Gate)) != NULL)
{
free_pages_exact(PUBLIC(OF(Gate)), PUBLIC(RO(Proc))->Gate.ReqMem.Size);
}
}
static int CoreFreqK_Ignition_Level_Up(INIT_ARG *pArg)
{
BIT_ATOM_INIT(PRIVATE(OF(Zen)).AMD_HSMP_LOCK, ATOMIC_SEED);
BIT_ATOM_INIT(PRIVATE(OF(Zen)).AMD_SMN_LOCK, ATOMIC_SEED);
BIT_ATOM_INIT(PRIVATE(OF(Zen)).AMD_FCH_LOCK, ATOMIC_SEED);
switch (PUBLIC(RO(Proc))->Features.Info.Vendor.CRC) {
case CRC_INTEL: {
Arch[GenuineArch].Query = Query_GenuineIntel;
Arch[GenuineArch].Update= PerCore_Intel_Query;
Arch[GenuineArch].Start = Start_GenuineIntel;
Arch[GenuineArch].Stop = Stop_GenuineIntel;
Arch[GenuineArch].Timer = InitTimer_GenuineIntel;
Arch[GenuineArch].BaseClock = BaseClock_GenuineIntel;
Arch[GenuineArch].thermalFormula=THERMAL_FORMULA_INTEL;
Arch[GenuineArch].voltageFormula=VOLTAGE_FORMULA_INTEL;
Arch[GenuineArch].powerFormula = POWER_FORMULA_INTEL;
}
break;
case CRC_HYGON:
fallthrough;
case CRC_AMD: {
Arch[GenuineArch].Query = Query_AuthenticAMD;
Arch[GenuineArch].Update= PerCore_AuthenticAMD_Query;
Arch[GenuineArch].Start = Start_AuthenticAMD;
Arch[GenuineArch].Stop = Stop_AuthenticAMD;
Arch[GenuineArch].Timer = InitTimer_AuthenticAMD;
Arch[GenuineArch].BaseClock = BaseClock_AuthenticAMD;
Arch[GenuineArch].thermalFormula = THERMAL_FORMULA_AMD;
Arch[GenuineArch].voltageFormula = VOLTAGE_FORMULA_AMD;
Arch[GenuineArch].powerFormula = POWER_FORMULA_AMD;
}
break;
case CRC_KVM:
case CRC_VBOX:
case CRC_KBOX:
case CRC_VMWARE:
case CRC_HYPERV:
/* Unexpected */
break;
}
/* Is an architecture identifier requested by user ? */
if ( (ArchID != -1) && (ArchID >= 0) && (ArchID < ARCHITECTURES) )
{
PUBLIC(RO(Proc))->ArchID = ArchID;
} else {
PUBLIC(RO(Proc))->ArchID = SearchArchitectureID();
}
/* Set the uArch's name with the first found codename */
StrCopy(PUBLIC(RO(Proc))->Architecture,
Arch[PUBLIC(RO(Proc))->ArchID].Architecture[0],
CODENAME_LEN);
/* Check if the Processor is actually virtualized ? */
#ifdef CONFIG_XEN
if (xen_pv_domain() || xen_hvm_domain())
{
if (PUBLIC(RO(Proc))->Features.Std.ECX.Hyperv == 0) {
PUBLIC(RO(Proc))->Features.Std.ECX.Hyperv = 1;
}
PUBLIC(RO(Proc))->HypervisorID = HYPERV_XEN;
}
#endif /* CONFIG_XEN */
if ((PUBLIC(RO(Proc))->Features.Std.ECX.Hyperv == 1) && (ArchID == -1))
{
VendorFromCPUID(PUBLIC(RO(Proc))->Features.Info.Hypervisor.ID,
&PUBLIC(RO(Proc))->Features.Info.LargestHypFunc,
&PUBLIC(RO(Proc))->Features.Info.Hypervisor.CRC,
&PUBLIC(RO(Proc))->HypervisorID,
0x40000000LU, 0x0);
switch (PUBLIC(RO(Proc))->HypervisorID) {
case HYPERV_NONE:
PUBLIC(RO(Proc))->ArchID = GenuineArch;
Arch[GenuineArch].Query = Query_VirtualMachine;
Arch[GenuineArch].Update= PerCore_VirtualMachine;
Arch[GenuineArch].Start = Start_Safe_VirtualMachine;
Arch[GenuineArch].Stop = Stop_VirtualMachine;
Arch[GenuineArch].Timer = InitTimer_Safe_VirtualMachine;
Arch[GenuineArch].thermalFormula = THERMAL_FORMULA_NONE;
Arch[GenuineArch].voltageFormula = VOLTAGE_FORMULA_NONE;
Arch[GenuineArch].powerFormula = POWER_FORMULA_NONE;
break;
case HYPERV_KVM:
case HYPERV_VBOX:
case HYPERV_KBOX:
case HYPERV_VMWARE:
case HYPERV_HYPERV:
PUBLIC(RO(Proc))->ArchID = GenuineArch;
Arch[GenuineArch].Query = Query_VirtualMachine;
Arch[GenuineArch].Update= PerCore_VirtualMachine;
Arch[GenuineArch].Start = Start_VirtualMachine;
Arch[GenuineArch].Stop = Stop_VirtualMachine;
Arch[GenuineArch].Timer = InitTimer_VirtualMachine;
Arch[GenuineArch].thermalFormula = THERMAL_FORMULA_NONE;
Arch[GenuineArch].voltageFormula = VOLTAGE_FORMULA_NONE;
Arch[GenuineArch].powerFormula = POWER_FORMULA_NONE;
break;
case BARE_METAL:
case HYPERV_XEN:
/* Xen virtualizes better the MSR & PCI registers */
break;
}
}
PUBLIC(RO(Proc))->thermalFormula = \
Arch[PUBLIC(RO(Proc))->ArchID].thermalFormula;
PUBLIC(RO(Proc))->voltageFormula = \
Arch[PUBLIC(RO(Proc))->ArchID].voltageFormula;
PUBLIC(RO(Proc))->powerFormula = \
Arch[PUBLIC(RO(Proc))->ArchID].powerFormula;
CoreFreqK_Thermal_Scope(ThermalScope);
CoreFreqK_Voltage_Scope(VoltageScope);
CoreFreqK_Power_Scope(PowerScope);
/* Copy various SMBIOS data [version 3.2] */
SMBIOS_Collect();
SMBIOS_Decoder();
/* Initialize the CoreFreq controller */
Controller_Init();
/* Seek for an appropriate service processor */
MatchPeerForDefaultService( &PUBLIC(RO(Proc))->Service,
pArg->localProcessor );
/* Register the Idle & Frequency sub-drivers */
CoreFreqK_Register_CPU_Idle();
CoreFreqK_Register_CPU_Freq();
CoreFreqK_Register_Governor();
if (NMI_Disable == 0) {
CoreFreqK_Register_NMI();
}
pr_info("CoreFreq(%u:%d:%d):" \
" Processor [%2X%1X_%1X%1X]" \
" Architecture [%s] %3s [%u/%u]\n",
PUBLIC(RO(Proc))->Service.Core,
PUBLIC(RO(Proc))->Service.Thread,
PUBLIC(RO(Proc))->Service.Hybrid,
PUBLIC(RO(Proc))->Features.Std.EAX.ExtFamily,
PUBLIC(RO(Proc))->Features.Std.EAX.Family,
PUBLIC(RO(Proc))->Features.Std.EAX.ExtModel,
PUBLIC(RO(Proc))->Features.Std.EAX.Model,
PUBLIC(RO(Proc))->Architecture,
PUBLIC(RO(Proc))->Features.HTT_Enable ? "SMT" : "CPU",
PUBLIC(RO(Proc))->CPU.OnLine,
PUBLIC(RO(Proc))->CPU.Count);
PUBLIC(RO(Proc))->Registration.PCI = \
CoreFreqK_ProbePCI(Arch[PUBLIC(RO(Proc))->ArchID].PCI_ids,
CoreFreqK_ResetChip, CoreFreqK_AppendChip) == 0;
Controller_Start(0);
#ifdef CONFIG_HOTPLUG_CPU
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
/* Always returns zero (kernel/notifier.c) */
PUBLIC(RO(Proc))->Registration.HotPlug = \
register_hotcpu_notifier(&CoreFreqK_notifier_block);
#else /* Continue with or without cpu hot-plugging. */
PUBLIC(RO(Proc))->Registration.HotPlug = \
cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
"corefreqk/cpu:online",
CoreFreqK_HotPlug_CPU_Online,
CoreFreqK_HotPlug_CPU_Offline);
#endif /* KERNEL_VERSION(4, 6, 0) */
#endif /* CONFIG_HOTPLUG_CPU */
#ifdef CONFIG_CPU_FREQ
Policy_Aggregate_Turbo();
#endif /* CONFIG_CPU_FREQ */
return 0;
}
#define CoreFreqK_User_Ops_Level_Down CoreFreqK_Empty_Func_Level_Down
static int CoreFreqK_User_Ops_Level_Up(INIT_ARG *pArg)
{
UNUSED(pArg);
if (Register_ClockSource == COREFREQ_TOGGLE_ON)
{
Controller_Stop(1);
Controller_Start(1);
CoreFreqK_Register_ClockSource(PUBLIC(RO(Proc))->Service.Core);
}
if (Ratio_HWP_Count > 0)
{
CLOCK_ARG clockMod;
enum RATIO_BOOST boost;
Controller_Stop(1);
for (boost = 0; boost < Ratio_HWP_Count; boost++)
{
if (Ratio_HWP[boost] >= 0)
{
long rc = RC_SUCCESS;
switch (boost) {
case BOOST(HWP_MIN) - BOOST(HWP_MIN):
if (Arch[PUBLIC(RO(Proc))->ArchID].ClockMod) {
clockMod.Ratio = Ratio_HWP[boost];
clockMod.cpu = -1;
clockMod.NC = CLOCK_MOD_HWP_MIN;
rc = Arch[PUBLIC(RO(Proc))->ArchID].ClockMod(&clockMod);
#ifdef CONFIG_CPU_FREQ
Policy_Aggregate_Turbo();
#endif
}
break;
case BOOST(HWP_MAX) - BOOST(HWP_MIN):
if (Arch[PUBLIC(RO(Proc))->ArchID].ClockMod) {
clockMod.Ratio = Ratio_HWP[boost];
clockMod.cpu = -1;
clockMod.NC = CLOCK_MOD_HWP_MAX;
rc = Arch[PUBLIC(RO(Proc))->ArchID].ClockMod(&clockMod);
#ifdef CONFIG_CPU_FREQ
Policy_Aggregate_Turbo();
#endif
}
break;
case BOOST(HWP_TGT) - BOOST(HWP_MIN):
if (Arch[PUBLIC(RO(Proc))->ArchID].ClockMod) {
clockMod.Ratio = Ratio_HWP[boost];
clockMod.cpu = -1;
clockMod.NC = CLOCK_MOD_HWP_TGT;
rc = Arch[PUBLIC(RO(Proc))->ArchID].ClockMod(&clockMod);
#ifdef CONFIG_CPU_FREQ
Policy_Aggregate_Turbo();
#endif
}
break;
default:
rc = -RC_UNIMPLEMENTED;
break;
};
if (rc < RC_SUCCESS) {
pr_warn("CoreFreq: " \
"'Ratio_HWP' at #%d Execution failure code %ld\n",
boost, rc);
}
}
}
Controller_Start(1);
RESET_ARRAY(Ratio_HWP, Ratio_HWP_Count, -1);
Ratio_HWP_Count = 0;
}
if (Ratio_Boost_Count > 0)
{
CLOCK_ARG clockMod;
enum RATIO_BOOST boost;
Controller_Stop(1);
for (boost = 0; boost < Ratio_Boost_Count; boost++)
{
if (Ratio_Boost[boost] >= 0)
{
long rc = RC_SUCCESS;
switch (boost) {
case BOOST(1C) - BOOST(1C) ... BOOST(1C) - BOOST(18C):
if (Arch[PUBLIC(RO(Proc))->ArchID].TurboClock) {
clockMod.Ratio = Ratio_Boost[boost];
clockMod.cpu = -1;
clockMod.NC = BOOST(SIZE) - BOOST(18C) - boost;
rc = Arch[PUBLIC(RO(Proc))->ArchID].TurboClock(&clockMod);
}
break;
default:
rc = -RC_UNIMPLEMENTED;
break;
};
if (rc < RC_SUCCESS) {
pr_warn("CoreFreq: " \
"'Ratio_Boost' at #%d Execution failure code %ld\n",
boost, rc);
}
}
}
if (TurboBoost_Enable_Count == 2) {
TurboBoost_Enable[0] = TurboBoost_Enable[1];
}
Controller_Start(1);
RESET_ARRAY(Ratio_Boost, Ratio_Boost_Count, -1);
Ratio_Boost_Count = 0;
}
if (PUBLIC(RO(Proc))->ArchID != AMD_Family_0Fh)
{
const unsigned int cpu = PUBLIC(RO(Proc))->Service.Core;
const signed int MinFID = PUBLIC(RO(Core, AT(cpu)))->Boost[BOOST(MIN)],
MaxFID = MAXCLOCK_TO_RATIO( signed int,
PUBLIC(RO(Core, AT(cpu))->Clock.Hz) );
if (PState_FID != -1) {
if ((PState_FID >= MinFID) && (PState_FID <= MaxFID)) {
if (Arch[PUBLIC(RO(Proc))->ArchID].ClockMod != NULL)
{
long rc;
CLOCK_ARG clockMod = {
.NC = CLOCK_MOD_MAX,
.Ratio = PState_FID,
.cpu = -1
};
Controller_Stop(1);
rc = Arch[PUBLIC(RO(Proc))->ArchID].ClockMod(&clockMod);
Controller_Start(1);
if (rc < RC_SUCCESS) {
pr_warn("CoreFreq: " \
"'PState_FID' Execution failure code %ld\n", rc);
}
} else {
pr_warn("CoreFreq: " \
"Unsupported architecture #%d for 'PState_FID'\n",
PUBLIC(RO(Proc))->ArchID);
}
} else {
pr_warn("CoreFreq: " \
"'PState_FID' is out of range [%d, %d]\n", MinFID, MaxFID);
}
PState_FID = -1;
}
} /* else handled by function PerCore_AMD_Family_0Fh_PStates() */
if (Ratio_PPC >= 0)
{
long rc = RC_SUCCESS;
if (Arch[PUBLIC(RO(Proc))->ArchID].ClockMod) {
CLOCK_ARG clockMod={.Ratio = Ratio_PPC, .cpu = -1, .NC = CLOCK_MOD_TGT};
Controller_Stop(1);
rc = Arch[PUBLIC(RO(Proc))->ArchID].ClockMod(&clockMod);
Controller_Start(0);
} else {
rc = -RC_UNIMPLEMENTED;
}
if (rc < RC_SUCCESS) {
pr_warn("CoreFreq: 'Ratio_PPC' Execution failure code %ld\n", rc);
}
Ratio_PPC = -1;
}
return 0;
}
enum RUN_LEVEL {
Alloc_Features_Level,
Query_Features_Level,
Alloc_Device_Level,
Make_Device_Level,
Create_Device_Level,
Alloc_Public_Level,
Alloc_Private_Level,
Alloc_Processor_RO_Level,
Alloc_Processor_RW_Level,
Scale_And_Compute_Level,
Alloc_Public_Cache_Level,
Alloc_Private_Cache_Level,
Alloc_Per_CPU_Level,
Ignition_Level,
User_Ops_Level,
Running_Level
};
static enum RUN_LEVEL RunLevel = Alloc_Features_Level;
#define COREFREQ_RUN( _level, _action ) CoreFreqK_##_level##_##_action
static void CoreFreqK_ShutDown(void)
{
void (*LevelFunc[Running_Level])(void) = {
COREFREQ_RUN(Alloc_Features_Level, Down),
COREFREQ_RUN(Query_Features_Level, Down),
COREFREQ_RUN(Alloc_Device_Level, Down),
COREFREQ_RUN(Make_Device_Level, Down),
COREFREQ_RUN(Create_Device_Level, Down),
COREFREQ_RUN(Alloc_Public_Level, Down),
COREFREQ_RUN(Alloc_Private_Level, Down),
COREFREQ_RUN(Alloc_Processor_RO_Level, Down),
COREFREQ_RUN(Alloc_Processor_RW_Level, Down),
COREFREQ_RUN(Scale_And_Compute_Level, Down),
COREFREQ_RUN(Alloc_Public_Cache_Level, Down),
COREFREQ_RUN(Alloc_Private_Cache_Level, Down),
COREFREQ_RUN(Alloc_Per_CPU_Level, Down),
COREFREQ_RUN(Ignition_Level, Down),
COREFREQ_RUN(User_Ops_Level, Down)
};
do
{
RunLevel--;
LevelFunc[RunLevel]();
} while (RunLevel != Alloc_Features_Level) ;
}
static int CoreFreqK_StartUp(void)
{
int (*LevelFunc[Running_Level])(INIT_ARG *pArg) = {
COREFREQ_RUN(Alloc_Features_Level, Up),
COREFREQ_RUN(Query_Features_Level, Up),
COREFREQ_RUN(Alloc_Device_Level, Up),
COREFREQ_RUN(Make_Device_Level, Up),
COREFREQ_RUN(Create_Device_Level, Up),
COREFREQ_RUN(Alloc_Public_Level, Up),
COREFREQ_RUN(Alloc_Private_Level, Up),
COREFREQ_RUN(Alloc_Processor_RO_Level, Up),
COREFREQ_RUN(Alloc_Processor_RW_Level, Up),
COREFREQ_RUN(Scale_And_Compute_Level, Up),
COREFREQ_RUN(Alloc_Public_Cache_Level, Up),
COREFREQ_RUN(Alloc_Private_Cache_Level, Up),
COREFREQ_RUN(Alloc_Per_CPU_Level, Up),
COREFREQ_RUN(Ignition_Level, Up),
COREFREQ_RUN(User_Ops_Level, Up)
};
INIT_ARG iArg = {
.Features = NULL, .Brand = NULL,
.SMT_Count = 0, .localProcessor = 0, .rc = 0
};
int rc = 0;
do
{
rc = LevelFunc[RunLevel](&iArg);
RunLevel++;
} while (RunLevel != Running_Level && rc == 0) ;
/* Free any initialization memory allocation. */
if (iArg.Features != NULL)
{
kfree(iArg.Features);
}
if (iArg.Brand != NULL)
{
kfree(iArg.Brand);
}
return rc;
}
#undef COREFREQ_RUN
#undef CoreFreqK_Scale_And_Compute_Level_Down
#undef CoreFreqK_Query_Features_Level_Down
#undef COREFREQ_PM_OPS
static int __init CoreFreqK_Init(void)
{
int rc = CoreFreqK_StartUp();
if (rc != 0) {
CoreFreqK_ShutDown();
}
return rc;
}
static void __exit CoreFreqK_Exit(void)
{
CoreFreqK_ShutDown();
}
module_init(CoreFreqK_Init)
module_exit(CoreFreqK_Exit)
Reference in New Issue View Git Blame Copy Permalink