123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450 |
- // SPDX-License-Identifier: GPL-2.0
- /*
- * Intel Uncore Frequency Setting
- * Copyright (c) 2019, Intel Corporation.
- * All rights reserved.
- *
- * Provide interface to set MSR 620 at a granularity of per die. On CPU online,
- * one control CPU is identified per die to read/write limit. This control CPU
- * is changed, if the CPU state is changed to offline. When the last CPU is
- * offline in a die then remove the sysfs object for that die.
- * The majority of actual code is related to sysfs create and read/write
- * attributes.
- *
- * Author: Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>
- */
- #include <linux/cpu.h>
- #include <linux/module.h>
- #include <linux/slab.h>
- #include <linux/suspend.h>
- #include <asm/cpu_device_id.h>
- #include <asm/intel-family.h>
- #define MSR_UNCORE_RATIO_LIMIT 0x620
- #define UNCORE_FREQ_KHZ_MULTIPLIER 100000
- /**
- * struct uncore_data - Encapsulate all uncore data
- * @stored_uncore_data: Last user changed MSR 620 value, which will be restored
- * on system resume.
- * @initial_min_freq_khz: Sampled minimum uncore frequency at driver init
- * @initial_max_freq_khz: Sampled maximum uncore frequency at driver init
- * @control_cpu: Designated CPU for a die to read/write
- * @valid: Mark the data valid/invalid
- *
- * This structure is used to encapsulate all data related to uncore sysfs
- * settings for a die/package.
- */
- struct uncore_data {
- struct kobject kobj;
- struct completion kobj_unregister;
- u64 stored_uncore_data;
- u32 initial_min_freq_khz;
- u32 initial_max_freq_khz;
- int control_cpu;
- bool valid;
- };
- #define to_uncore_data(a) container_of(a, struct uncore_data, kobj)
- /* Max instances for uncore data, one for each die */
- static int uncore_max_entries __read_mostly;
- /* Storage for uncore data for all instances */
- static struct uncore_data *uncore_instances;
- /* Root of the all uncore sysfs kobjs */
- static struct kobject *uncore_root_kobj;
- /* Stores the CPU mask of the target CPUs to use during uncore read/write */
- static cpumask_t uncore_cpu_mask;
- /* CPU online callback register instance */
- static enum cpuhp_state uncore_hp_state __read_mostly;
- /* Mutex to control all mutual exclusions */
- static DEFINE_MUTEX(uncore_lock);
- struct uncore_attr {
- struct attribute attr;
- ssize_t (*show)(struct kobject *kobj,
- struct attribute *attr, char *buf);
- ssize_t (*store)(struct kobject *kobj,
- struct attribute *attr, const char *c, ssize_t count);
- };
- #define define_one_uncore_ro(_name) \
- static struct uncore_attr _name = \
- __ATTR(_name, 0444, show_##_name, NULL)
- #define define_one_uncore_rw(_name) \
- static struct uncore_attr _name = \
- __ATTR(_name, 0644, show_##_name, store_##_name)
- #define show_uncore_data(member_name) \
- static ssize_t show_##member_name(struct kobject *kobj, \
- struct attribute *attr, \
- char *buf) \
- { \
- struct uncore_data *data = to_uncore_data(kobj); \
- return scnprintf(buf, PAGE_SIZE, "%u\n", \
- data->member_name); \
- } \
- define_one_uncore_ro(member_name)
- show_uncore_data(initial_min_freq_khz);
- show_uncore_data(initial_max_freq_khz);
- /* Common function to read MSR 0x620 and read min/max */
- static int uncore_read_ratio(struct uncore_data *data, unsigned int *min,
- unsigned int *max)
- {
- u64 cap;
- int ret;
- if (data->control_cpu < 0)
- return -ENXIO;
- ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
- if (ret)
- return ret;
- *max = (cap & 0x7F) * UNCORE_FREQ_KHZ_MULTIPLIER;
- *min = ((cap & GENMASK(14, 8)) >> 8) * UNCORE_FREQ_KHZ_MULTIPLIER;
- return 0;
- }
- /* Common function to set min/max ratios to be used by sysfs callbacks */
- static int uncore_write_ratio(struct uncore_data *data, unsigned int input,
- int set_max)
- {
- int ret;
- u64 cap;
- mutex_lock(&uncore_lock);
- if (data->control_cpu < 0) {
- ret = -ENXIO;
- goto finish_write;
- }
- input /= UNCORE_FREQ_KHZ_MULTIPLIER;
- if (!input || input > 0x7F) {
- ret = -EINVAL;
- goto finish_write;
- }
- ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
- if (ret)
- goto finish_write;
- if (set_max) {
- cap &= ~0x7F;
- cap |= input;
- } else {
- cap &= ~GENMASK(14, 8);
- cap |= (input << 8);
- }
- ret = wrmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, cap);
- if (ret)
- goto finish_write;
- data->stored_uncore_data = cap;
- finish_write:
- mutex_unlock(&uncore_lock);
- return ret;
- }
- static ssize_t store_min_max_freq_khz(struct kobject *kobj,
- struct attribute *attr,
- const char *buf, ssize_t count,
- int min_max)
- {
- struct uncore_data *data = to_uncore_data(kobj);
- unsigned int input;
- if (kstrtouint(buf, 10, &input))
- return -EINVAL;
- uncore_write_ratio(data, input, min_max);
- return count;
- }
- static ssize_t show_min_max_freq_khz(struct kobject *kobj,
- struct attribute *attr,
- char *buf, int min_max)
- {
- struct uncore_data *data = to_uncore_data(kobj);
- unsigned int min, max;
- int ret;
- mutex_lock(&uncore_lock);
- ret = uncore_read_ratio(data, &min, &max);
- mutex_unlock(&uncore_lock);
- if (ret)
- return ret;
- if (min_max)
- return sprintf(buf, "%u\n", max);
- return sprintf(buf, "%u\n", min);
- }
- #define store_uncore_min_max(name, min_max) \
- static ssize_t store_##name(struct kobject *kobj, \
- struct attribute *attr, \
- const char *buf, ssize_t count) \
- { \
- \
- return store_min_max_freq_khz(kobj, attr, buf, count, \
- min_max); \
- }
- #define show_uncore_min_max(name, min_max) \
- static ssize_t show_##name(struct kobject *kobj, \
- struct attribute *attr, char *buf) \
- { \
- \
- return show_min_max_freq_khz(kobj, attr, buf, min_max); \
- }
- store_uncore_min_max(min_freq_khz, 0);
- store_uncore_min_max(max_freq_khz, 1);
- show_uncore_min_max(min_freq_khz, 0);
- show_uncore_min_max(max_freq_khz, 1);
- define_one_uncore_rw(min_freq_khz);
- define_one_uncore_rw(max_freq_khz);
- static struct attribute *uncore_attrs[] = {
- &initial_min_freq_khz.attr,
- &initial_max_freq_khz.attr,
- &max_freq_khz.attr,
- &min_freq_khz.attr,
- NULL
- };
- static void uncore_sysfs_entry_release(struct kobject *kobj)
- {
- struct uncore_data *data = to_uncore_data(kobj);
- complete(&data->kobj_unregister);
- }
- static struct kobj_type uncore_ktype = {
- .release = uncore_sysfs_entry_release,
- .sysfs_ops = &kobj_sysfs_ops,
- .default_attrs = uncore_attrs,
- };
- /* Caller provides protection */
- static struct uncore_data *uncore_get_instance(unsigned int cpu)
- {
- int id = topology_logical_die_id(cpu);
- if (id >= 0 && id < uncore_max_entries)
- return &uncore_instances[id];
- return NULL;
- }
- static void uncore_add_die_entry(int cpu)
- {
- struct uncore_data *data;
- mutex_lock(&uncore_lock);
- data = uncore_get_instance(cpu);
- if (!data) {
- mutex_unlock(&uncore_lock);
- return;
- }
- if (data->valid) {
- /* control cpu changed */
- data->control_cpu = cpu;
- } else {
- char str[64];
- int ret;
- memset(data, 0, sizeof(*data));
- sprintf(str, "package_%02d_die_%02d",
- topology_physical_package_id(cpu),
- topology_die_id(cpu));
- uncore_read_ratio(data, &data->initial_min_freq_khz,
- &data->initial_max_freq_khz);
- init_completion(&data->kobj_unregister);
- ret = kobject_init_and_add(&data->kobj, &uncore_ktype,
- uncore_root_kobj, str);
- if (!ret) {
- data->control_cpu = cpu;
- data->valid = true;
- }
- }
- mutex_unlock(&uncore_lock);
- }
- /* Last CPU in this die is offline, make control cpu invalid */
- static void uncore_remove_die_entry(int cpu)
- {
- struct uncore_data *data;
- mutex_lock(&uncore_lock);
- data = uncore_get_instance(cpu);
- if (data)
- data->control_cpu = -1;
- mutex_unlock(&uncore_lock);
- }
- static int uncore_event_cpu_online(unsigned int cpu)
- {
- int target;
- /* Check if there is an online cpu in the package for uncore MSR */
- target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu));
- if (target < nr_cpu_ids)
- return 0;
- /* Use this CPU on this die as a control CPU */
- cpumask_set_cpu(cpu, &uncore_cpu_mask);
- uncore_add_die_entry(cpu);
- return 0;
- }
- static int uncore_event_cpu_offline(unsigned int cpu)
- {
- int target;
- /* Check if existing cpu is used for uncore MSRs */
- if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
- return 0;
- /* Find a new cpu to set uncore MSR */
- target = cpumask_any_but(topology_die_cpumask(cpu), cpu);
- if (target < nr_cpu_ids) {
- cpumask_set_cpu(target, &uncore_cpu_mask);
- uncore_add_die_entry(target);
- } else {
- uncore_remove_die_entry(cpu);
- }
- return 0;
- }
- static int uncore_pm_notify(struct notifier_block *nb, unsigned long mode,
- void *_unused)
- {
- int cpu;
- switch (mode) {
- case PM_POST_HIBERNATION:
- case PM_POST_RESTORE:
- case PM_POST_SUSPEND:
- for_each_cpu(cpu, &uncore_cpu_mask) {
- struct uncore_data *data;
- int ret;
- data = uncore_get_instance(cpu);
- if (!data || !data->valid || !data->stored_uncore_data)
- continue;
- ret = wrmsrl_on_cpu(cpu, MSR_UNCORE_RATIO_LIMIT,
- data->stored_uncore_data);
- if (ret)
- return ret;
- }
- break;
- default:
- break;
- }
- return 0;
- }
- static struct notifier_block uncore_pm_nb = {
- .notifier_call = uncore_pm_notify,
- };
- static const struct x86_cpu_id intel_uncore_cpu_ids[] = {
- X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, NULL),
- X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, NULL),
- X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, NULL),
- X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, NULL),
- X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, NULL),
- X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, NULL),
- {}
- };
- static int __init intel_uncore_init(void)
- {
- const struct x86_cpu_id *id;
- int ret;
- id = x86_match_cpu(intel_uncore_cpu_ids);
- if (!id)
- return -ENODEV;
- uncore_max_entries = topology_max_packages() *
- topology_max_die_per_package();
- uncore_instances = kcalloc(uncore_max_entries,
- sizeof(*uncore_instances), GFP_KERNEL);
- if (!uncore_instances)
- return -ENOMEM;
- uncore_root_kobj = kobject_create_and_add("intel_uncore_frequency",
- &cpu_subsys.dev_root->kobj);
- if (!uncore_root_kobj) {
- ret = -ENOMEM;
- goto err_free;
- }
- ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
- "platform/x86/uncore-freq:online",
- uncore_event_cpu_online,
- uncore_event_cpu_offline);
- if (ret < 0)
- goto err_rem_kobj;
- uncore_hp_state = ret;
- ret = register_pm_notifier(&uncore_pm_nb);
- if (ret)
- goto err_rem_state;
- return 0;
- err_rem_state:
- cpuhp_remove_state(uncore_hp_state);
- err_rem_kobj:
- kobject_put(uncore_root_kobj);
- err_free:
- kfree(uncore_instances);
- return ret;
- }
- module_init(intel_uncore_init)
- static void __exit intel_uncore_exit(void)
- {
- int i;
- unregister_pm_notifier(&uncore_pm_nb);
- cpuhp_remove_state(uncore_hp_state);
- for (i = 0; i < uncore_max_entries; ++i) {
- if (uncore_instances[i].valid) {
- kobject_put(&uncore_instances[i].kobj);
- wait_for_completion(&uncore_instances[i].kobj_unregister);
- }
- }
- kobject_put(uncore_root_kobj);
- kfree(uncore_instances);
- }
- module_exit(intel_uncore_exit)
- MODULE_LICENSE("GPL v2");
- MODULE_DESCRIPTION("Intel Uncore Frequency Limits Driver");
|