123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886 |
- // SPDX-License-Identifier: GPL-2.0-only
- /*
- * linux/drivers/cpufreq/cpufreq.c
- *
- * Copyright (C) 2001 Russell King
- * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
- * (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
- *
- * Oct 2005 - Ashok Raj <ashok.raj@intel.com>
- * Added handling for CPU hotplug
- * Feb 2006 - Jacob Shin <jacob.shin@amd.com>
- * Fix handling for CPU hotplug -- affected CPUs
- */
- #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
- #include <linux/cpu.h>
- #include <linux/cpufreq.h>
- #include <linux/cpufreq_times.h>
- #include <linux/cpu_cooling.h>
- #include <linux/delay.h>
- #include <linux/device.h>
- #include <linux/init.h>
- #include <linux/kernel_stat.h>
- #include <linux/module.h>
- #include <linux/mutex.h>
- #include <linux/pm_qos.h>
- #include <linux/slab.h>
- #include <linux/suspend.h>
- #include <linux/syscore_ops.h>
- #include <linux/tick.h>
- #include <trace/events/power.h>
- #include <trace/hooks/cpufreq.h>
- static LIST_HEAD(cpufreq_policy_list);
- /* Macros to iterate over CPU policies */
- #define for_each_suitable_policy(__policy, __active) \
- list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \
- if ((__active) == !policy_is_inactive(__policy))
- #define for_each_active_policy(__policy) \
- for_each_suitable_policy(__policy, true)
- #define for_each_inactive_policy(__policy) \
- for_each_suitable_policy(__policy, false)
- #define for_each_policy(__policy) \
- list_for_each_entry(__policy, &cpufreq_policy_list, policy_list)
- /* Iterate over governors */
- static LIST_HEAD(cpufreq_governor_list);
- #define for_each_governor(__governor) \
- list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
- static char default_governor[CPUFREQ_NAME_LEN];
- /*
- * The "cpufreq driver" - the arch- or hardware-dependent low
- * level driver of CPUFreq support, and its spinlock. This lock
- * also protects the cpufreq_cpu_data array.
- */
- static struct cpufreq_driver *cpufreq_driver;
- static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
- static DEFINE_RWLOCK(cpufreq_driver_lock);
- static DEFINE_STATIC_KEY_FALSE(cpufreq_freq_invariance);
- bool cpufreq_supports_freq_invariance(void)
- {
- return static_branch_likely(&cpufreq_freq_invariance);
- }
- /* Flag to suspend/resume CPUFreq governors */
- static bool cpufreq_suspended;
- static inline bool has_target(void)
- {
- return cpufreq_driver->target_index || cpufreq_driver->target;
- }
- /* internal prototypes */
- static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
- static int cpufreq_init_governor(struct cpufreq_policy *policy);
- static void cpufreq_exit_governor(struct cpufreq_policy *policy);
- static void cpufreq_governor_limits(struct cpufreq_policy *policy);
- static int cpufreq_set_policy(struct cpufreq_policy *policy,
- struct cpufreq_governor *new_gov,
- unsigned int new_pol);
- /*
- * Two notifier lists: the "policy" list is involved in the
- * validation process for a new CPU frequency policy; the
- * "transition" list for kernel code that needs to handle
- * changes to devices when the CPU clock speed changes.
- * The mutex locks both lists.
- */
- static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
- SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list);
- static int off __read_mostly;
- static int cpufreq_disabled(void)
- {
- return off;
- }
- void disable_cpufreq(void)
- {
- off = 1;
- }
- static DEFINE_MUTEX(cpufreq_governor_mutex);
- bool have_governor_per_policy(void)
- {
- return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
- }
- EXPORT_SYMBOL_GPL(have_governor_per_policy);
- static struct kobject *cpufreq_global_kobject;
- struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
- {
- if (have_governor_per_policy())
- return &policy->kobj;
- else
- return cpufreq_global_kobject;
- }
- EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
- static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
- {
- struct kernel_cpustat kcpustat;
- u64 cur_wall_time;
- u64 idle_time;
- u64 busy_time;
- cur_wall_time = jiffies64_to_nsecs(get_jiffies_64());
- kcpustat_cpu_fetch(&kcpustat, cpu);
- busy_time = kcpustat.cpustat[CPUTIME_USER];
- busy_time += kcpustat.cpustat[CPUTIME_SYSTEM];
- busy_time += kcpustat.cpustat[CPUTIME_IRQ];
- busy_time += kcpustat.cpustat[CPUTIME_SOFTIRQ];
- busy_time += kcpustat.cpustat[CPUTIME_STEAL];
- busy_time += kcpustat.cpustat[CPUTIME_NICE];
- idle_time = cur_wall_time - busy_time;
- if (wall)
- *wall = div_u64(cur_wall_time, NSEC_PER_USEC);
- return div_u64(idle_time, NSEC_PER_USEC);
- }
- u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
- {
- u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
- if (idle_time == -1ULL)
- return get_cpu_idle_time_jiffy(cpu, wall);
- else if (!io_busy)
- idle_time += get_cpu_iowait_time_us(cpu, wall);
- return idle_time;
- }
- EXPORT_SYMBOL_GPL(get_cpu_idle_time);
- /*
- * This is a generic cpufreq init() routine which can be used by cpufreq
- * drivers of SMP systems. It will do following:
- * - validate & show freq table passed
- * - set policies transition latency
- * - policy->cpus with all possible CPUs
- */
- void cpufreq_generic_init(struct cpufreq_policy *policy,
- struct cpufreq_frequency_table *table,
- unsigned int transition_latency)
- {
- policy->freq_table = table;
- policy->cpuinfo.transition_latency = transition_latency;
- /*
- * The driver only supports the SMP configuration where all processors
- * share the clock and voltage and clock.
- */
- cpumask_setall(policy->cpus);
- }
- EXPORT_SYMBOL_GPL(cpufreq_generic_init);
- struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
- {
- struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
- return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
- }
- EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
- unsigned int cpufreq_generic_get(unsigned int cpu)
- {
- struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
- if (!policy || IS_ERR(policy->clk)) {
- pr_err("%s: No %s associated to cpu: %d\n",
- __func__, policy ? "clk" : "policy", cpu);
- return 0;
- }
- return clk_get_rate(policy->clk) / 1000;
- }
- EXPORT_SYMBOL_GPL(cpufreq_generic_get);
- /**
- * cpufreq_cpu_get - Return policy for a CPU and mark it as busy.
- * @cpu: CPU to find the policy for.
- *
- * Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment
- * the kobject reference counter of that policy. Return a valid policy on
- * success or NULL on failure.
- *
- * The policy returned by this function has to be released with the help of
- * cpufreq_cpu_put() to balance its kobject reference counter properly.
- */
- struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
- {
- struct cpufreq_policy *policy = NULL;
- unsigned long flags;
- if (WARN_ON(cpu >= nr_cpu_ids))
- return NULL;
- /* get the cpufreq driver */
- read_lock_irqsave(&cpufreq_driver_lock, flags);
- if (cpufreq_driver) {
- /* get the CPU */
- policy = cpufreq_cpu_get_raw(cpu);
- if (policy)
- kobject_get(&policy->kobj);
- }
- read_unlock_irqrestore(&cpufreq_driver_lock, flags);
- return policy;
- }
- EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
- /**
- * cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy.
- * @policy: cpufreq policy returned by cpufreq_cpu_get().
- */
- void cpufreq_cpu_put(struct cpufreq_policy *policy)
- {
- kobject_put(&policy->kobj);
- }
- EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
- /**
- * cpufreq_cpu_release - Unlock a policy and decrement its usage counter.
- * @policy: cpufreq policy returned by cpufreq_cpu_acquire().
- */
- void cpufreq_cpu_release(struct cpufreq_policy *policy)
- {
- if (WARN_ON(!policy))
- return;
- lockdep_assert_held(&policy->rwsem);
- up_write(&policy->rwsem);
- cpufreq_cpu_put(policy);
- }
- /**
- * cpufreq_cpu_acquire - Find policy for a CPU, mark it as busy and lock it.
- * @cpu: CPU to find the policy for.
- *
- * Call cpufreq_cpu_get() to get a reference on the cpufreq policy for @cpu and
- * if the policy returned by it is not NULL, acquire its rwsem for writing.
- * Return the policy if it is active or release it and return NULL otherwise.
- *
- * The policy returned by this function has to be released with the help of
- * cpufreq_cpu_release() in order to release its rwsem and balance its usage
- * counter properly.
- */
- struct cpufreq_policy *cpufreq_cpu_acquire(unsigned int cpu)
- {
- struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
- if (!policy)
- return NULL;
- down_write(&policy->rwsem);
- if (policy_is_inactive(policy)) {
- cpufreq_cpu_release(policy);
- return NULL;
- }
- return policy;
- }
- /*********************************************************************
- * EXTERNALLY AFFECTING FREQUENCY CHANGES *
- *********************************************************************/
- /*
- * adjust_jiffies - adjust the system "loops_per_jiffy"
- *
- * This function alters the system "loops_per_jiffy" for the clock
- * speed change. Note that loops_per_jiffy cannot be updated on SMP
- * systems as each CPU might be scaled differently. So, use the arch
- * per-CPU loops_per_jiffy value wherever possible.
- */
- static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
- {
- #ifndef CONFIG_SMP
- static unsigned long l_p_j_ref;
- static unsigned int l_p_j_ref_freq;
- if (ci->flags & CPUFREQ_CONST_LOOPS)
- return;
- if (!l_p_j_ref_freq) {
- l_p_j_ref = loops_per_jiffy;
- l_p_j_ref_freq = ci->old;
- pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
- l_p_j_ref, l_p_j_ref_freq);
- }
- if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
- loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
- ci->new);
- pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
- loops_per_jiffy, ci->new);
- }
- #endif
- }
- /**
- * cpufreq_notify_transition - Notify frequency transition and adjust_jiffies.
- * @policy: cpufreq policy to enable fast frequency switching for.
- * @freqs: contain details of the frequency update.
- * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
- *
- * This function calls the transition notifiers and the "adjust_jiffies"
- * function. It is called twice on all CPU frequency changes that have
- * external effects.
- */
- static void cpufreq_notify_transition(struct cpufreq_policy *policy,
- struct cpufreq_freqs *freqs,
- unsigned int state)
- {
- int cpu;
- BUG_ON(irqs_disabled());
- if (cpufreq_disabled())
- return;
- freqs->policy = policy;
- freqs->flags = cpufreq_driver->flags;
- pr_debug("notification %u of frequency transition to %u kHz\n",
- state, freqs->new);
- switch (state) {
- case CPUFREQ_PRECHANGE:
- /*
- * Detect if the driver reported a value as "old frequency"
- * which is not equal to what the cpufreq core thinks is
- * "old frequency".
- */
- if (policy->cur && policy->cur != freqs->old) {
- pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
- freqs->old, policy->cur);
- freqs->old = policy->cur;
- }
- srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
- CPUFREQ_PRECHANGE, freqs);
- adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
- break;
- case CPUFREQ_POSTCHANGE:
- adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
- pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new,
- cpumask_pr_args(policy->cpus));
- for_each_cpu(cpu, policy->cpus)
- trace_cpu_frequency(freqs->new, cpu);
- srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
- CPUFREQ_POSTCHANGE, freqs);
- cpufreq_stats_record_transition(policy, freqs->new);
- cpufreq_times_record_transition(policy, freqs->new);
- policy->cur = freqs->new;
- trace_android_rvh_cpufreq_transition(policy);
- }
- }
- /* Do post notifications when there are chances that transition has failed */
- static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
- struct cpufreq_freqs *freqs, int transition_failed)
- {
- cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
- if (!transition_failed)
- return;
- swap(freqs->old, freqs->new);
- cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
- cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
- }
- void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
- struct cpufreq_freqs *freqs)
- {
- /*
- * Catch double invocations of _begin() which lead to self-deadlock.
- * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
- * doesn't invoke _begin() on their behalf, and hence the chances of
- * double invocations are very low. Moreover, there are scenarios
- * where these checks can emit false-positive warnings in these
- * drivers; so we avoid that by skipping them altogether.
- */
- WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
- && current == policy->transition_task);
- wait:
- wait_event(policy->transition_wait, !policy->transition_ongoing);
- spin_lock(&policy->transition_lock);
- if (unlikely(policy->transition_ongoing)) {
- spin_unlock(&policy->transition_lock);
- goto wait;
- }
- policy->transition_ongoing = true;
- policy->transition_task = current;
- spin_unlock(&policy->transition_lock);
- cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
- }
- EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
- void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
- struct cpufreq_freqs *freqs, int transition_failed)
- {
- if (WARN_ON(!policy->transition_ongoing))
- return;
- cpufreq_notify_post_transition(policy, freqs, transition_failed);
- arch_set_freq_scale(policy->related_cpus,
- policy->cur,
- policy->cpuinfo.max_freq);
- policy->transition_ongoing = false;
- policy->transition_task = NULL;
- wake_up(&policy->transition_wait);
- }
- EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
- /*
- * Fast frequency switching status count. Positive means "enabled", negative
- * means "disabled" and 0 means "not decided yet".
- */
- static int cpufreq_fast_switch_count;
- static DEFINE_MUTEX(cpufreq_fast_switch_lock);
- static void cpufreq_list_transition_notifiers(void)
- {
- struct notifier_block *nb;
- pr_info("Registered transition notifiers:\n");
- mutex_lock(&cpufreq_transition_notifier_list.mutex);
- for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next)
- pr_info("%pS\n", nb->notifier_call);
- mutex_unlock(&cpufreq_transition_notifier_list.mutex);
- }
- /**
- * cpufreq_enable_fast_switch - Enable fast frequency switching for policy.
- * @policy: cpufreq policy to enable fast frequency switching for.
- *
- * Try to enable fast frequency switching for @policy.
- *
- * The attempt will fail if there is at least one transition notifier registered
- * at this point, as fast frequency switching is quite fundamentally at odds
- * with transition notifiers. Thus if successful, it will make registration of
- * transition notifiers fail going forward.
- */
- void cpufreq_enable_fast_switch(struct cpufreq_policy *policy)
- {
- lockdep_assert_held(&policy->rwsem);
- if (!policy->fast_switch_possible)
- return;
- mutex_lock(&cpufreq_fast_switch_lock);
- if (cpufreq_fast_switch_count >= 0) {
- cpufreq_fast_switch_count++;
- policy->fast_switch_enabled = true;
- } else {
- pr_warn("CPU%u: Fast frequency switching not enabled\n",
- policy->cpu);
- cpufreq_list_transition_notifiers();
- }
- mutex_unlock(&cpufreq_fast_switch_lock);
- }
- EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch);
- /**
- * cpufreq_disable_fast_switch - Disable fast frequency switching for policy.
- * @policy: cpufreq policy to disable fast frequency switching for.
- */
- void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)
- {
- mutex_lock(&cpufreq_fast_switch_lock);
- if (policy->fast_switch_enabled) {
- policy->fast_switch_enabled = false;
- if (!WARN_ON(cpufreq_fast_switch_count <= 0))
- cpufreq_fast_switch_count--;
- }
- mutex_unlock(&cpufreq_fast_switch_lock);
- }
- EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch);
- /**
- * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported
- * one.
- * @policy: associated policy to interrogate
- * @target_freq: target frequency to resolve.
- *
- * The target to driver frequency mapping is cached in the policy.
- *
- * Return: Lowest driver-supported frequency greater than or equal to the
- * given target_freq, subject to policy (min/max) and driver limitations.
- */
- unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
- unsigned int target_freq)
- {
- unsigned int old_target_freq = target_freq;
- target_freq = clamp_val(target_freq, policy->min, policy->max);
- trace_android_vh_cpufreq_resolve_freq(policy, target_freq, old_target_freq);
- policy->cached_target_freq = target_freq;
- if (cpufreq_driver->target_index) {
- unsigned int idx;
- idx = cpufreq_frequency_table_target(policy, target_freq,
- CPUFREQ_RELATION_L);
- policy->cached_resolved_idx = idx;
- return policy->freq_table[idx].frequency;
- }
- if (cpufreq_driver->resolve_freq)
- return cpufreq_driver->resolve_freq(policy, target_freq);
- return target_freq;
- }
- EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq);
- unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy)
- {
- unsigned int latency;
- if (policy->transition_delay_us)
- return policy->transition_delay_us;
- latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC;
- if (latency) {
- /*
- * For platforms that can change the frequency very fast (< 10
- * us), the above formula gives a decent transition delay. But
- * for platforms where transition_latency is in milliseconds, it
- * ends up giving unrealistic values.
- *
- * Cap the default transition delay to 10 ms, which seems to be
- * a reasonable amount of time after which we should reevaluate
- * the frequency.
- */
- return min(latency * LATENCY_MULTIPLIER, (unsigned int)10000);
- }
- return LATENCY_MULTIPLIER;
- }
- EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us);
- /*********************************************************************
- * SYSFS INTERFACE *
- *********************************************************************/
- static ssize_t show_boost(struct kobject *kobj,
- struct kobj_attribute *attr, char *buf)
- {
- return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
- }
- static ssize_t store_boost(struct kobject *kobj, struct kobj_attribute *attr,
- const char *buf, size_t count)
- {
- int ret, enable;
- ret = sscanf(buf, "%d", &enable);
- if (ret != 1 || enable < 0 || enable > 1)
- return -EINVAL;
- if (cpufreq_boost_trigger_state(enable)) {
- pr_err("%s: Cannot %s BOOST!\n",
- __func__, enable ? "enable" : "disable");
- return -EINVAL;
- }
- pr_debug("%s: cpufreq BOOST %s\n",
- __func__, enable ? "enabled" : "disabled");
- return count;
- }
- define_one_global_rw(boost);
- static struct cpufreq_governor *find_governor(const char *str_governor)
- {
- struct cpufreq_governor *t;
- for_each_governor(t)
- if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
- return t;
- return NULL;
- }
- static struct cpufreq_governor *get_governor(const char *str_governor)
- {
- struct cpufreq_governor *t;
- mutex_lock(&cpufreq_governor_mutex);
- t = find_governor(str_governor);
- if (!t)
- goto unlock;
- if (!try_module_get(t->owner))
- t = NULL;
- unlock:
- mutex_unlock(&cpufreq_governor_mutex);
- return t;
- }
- static unsigned int cpufreq_parse_policy(char *str_governor)
- {
- if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN))
- return CPUFREQ_POLICY_PERFORMANCE;
- if (!strncasecmp(str_governor, "powersave", CPUFREQ_NAME_LEN))
- return CPUFREQ_POLICY_POWERSAVE;
- return CPUFREQ_POLICY_UNKNOWN;
- }
- /**
- * cpufreq_parse_governor - parse a governor string only for has_target()
- * @str_governor: Governor name.
- */
- static struct cpufreq_governor *cpufreq_parse_governor(char *str_governor)
- {
- struct cpufreq_governor *t;
- t = get_governor(str_governor);
- if (t)
- return t;
- if (request_module("cpufreq_%s", str_governor))
- return NULL;
- return get_governor(str_governor);
- }
- /*
- * cpufreq_per_cpu_attr_read() / show_##file_name() -
- * print out cpufreq information
- *
- * Write out information from cpufreq_driver->policy[cpu]; object must be
- * "unsigned int".
- */
- #define show_one(file_name, object) \
- static ssize_t show_##file_name \
- (struct cpufreq_policy *policy, char *buf) \
- { \
- return sprintf(buf, "%u\n", policy->object); \
- }
- static ssize_t show_cpuinfo_max_freq(struct cpufreq_policy *policy, char *buf)
- {
- unsigned int max_freq = policy->cpuinfo.max_freq;
- trace_android_vh_show_max_freq(policy, &max_freq);
- trace_android_rvh_show_max_freq(policy, &max_freq);
- return sprintf(buf, "%u\n", max_freq);
- }
- show_one(cpuinfo_min_freq, cpuinfo.min_freq);
- show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
- show_one(scaling_min_freq, min);
- show_one(scaling_max_freq, max);
- __weak unsigned int arch_freq_get_on_cpu(int cpu)
- {
- return 0;
- }
- static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
- {
- ssize_t ret;
- unsigned int freq;
- freq = arch_freq_get_on_cpu(policy->cpu);
- if (freq)
- ret = sprintf(buf, "%u\n", freq);
- else if (cpufreq_driver->setpolicy && cpufreq_driver->get)
- ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
- else
- ret = sprintf(buf, "%u\n", policy->cur);
- return ret;
- }
- /*
- * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
- */
- #define store_one(file_name, object) \
- static ssize_t store_##file_name \
- (struct cpufreq_policy *policy, const char *buf, size_t count) \
- { \
- unsigned long val; \
- int ret; \
- \
- ret = sscanf(buf, "%lu", &val); \
- if (ret != 1) \
- return -EINVAL; \
- \
- ret = freq_qos_update_request(policy->object##_freq_req, val);\
- return ret >= 0 ? count : ret; \
- }
- store_one(scaling_min_freq, min);
- store_one(scaling_max_freq, max);
- /*
- * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
- */
- static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
- char *buf)
- {
- unsigned int cur_freq = __cpufreq_get(policy);
- if (cur_freq)
- return sprintf(buf, "%u\n", cur_freq);
- return sprintf(buf, "<unknown>\n");
- }
- /*
- * show_scaling_governor - show the current policy for the specified CPU
- */
- static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
- {
- if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
- return sprintf(buf, "powersave\n");
- else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
- return sprintf(buf, "performance\n");
- else if (policy->governor)
- return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
- policy->governor->name);
- return -EINVAL;
- }
- /*
- * store_scaling_governor - store policy for the specified CPU
- */
- static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
- const char *buf, size_t count)
- {
- char str_governor[16];
- int ret;
- ret = sscanf(buf, "%15s", str_governor);
- if (ret != 1)
- return -EINVAL;
- if (cpufreq_driver->setpolicy) {
- unsigned int new_pol;
- new_pol = cpufreq_parse_policy(str_governor);
- if (!new_pol)
- return -EINVAL;
- ret = cpufreq_set_policy(policy, NULL, new_pol);
- } else {
- struct cpufreq_governor *new_gov;
- new_gov = cpufreq_parse_governor(str_governor);
- if (!new_gov)
- return -EINVAL;
- ret = cpufreq_set_policy(policy, new_gov,
- CPUFREQ_POLICY_UNKNOWN);
- module_put(new_gov->owner);
- }
- return ret ? ret : count;
- }
- /*
- * show_scaling_driver - show the cpufreq driver currently loaded
- */
- static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
- {
- return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
- }
- /*
- * show_scaling_available_governors - show the available CPUfreq governors
- */
- static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
- char *buf)
- {
- ssize_t i = 0;
- struct cpufreq_governor *t;
- if (!has_target()) {
- i += sprintf(buf, "performance powersave");
- goto out;
- }
- mutex_lock(&cpufreq_governor_mutex);
- for_each_governor(t) {
- if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
- - (CPUFREQ_NAME_LEN + 2)))
- break;
- i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
- }
- mutex_unlock(&cpufreq_governor_mutex);
- out:
- i += sprintf(&buf[i], "\n");
- return i;
- }
- ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
- {
- ssize_t i = 0;
- unsigned int cpu;
- for_each_cpu(cpu, mask) {
- if (i)
- i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
- i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
- if (i >= (PAGE_SIZE - 5))
- break;
- }
- i += sprintf(&buf[i], "\n");
- return i;
- }
- EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
- /*
- * show_related_cpus - show the CPUs affected by each transition even if
- * hw coordination is in use
- */
- static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
- {
- return cpufreq_show_cpus(policy->related_cpus, buf);
- }
- /*
- * show_affected_cpus - show the CPUs affected by each transition
- */
- static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
- {
- return cpufreq_show_cpus(policy->cpus, buf);
- }
- static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
- const char *buf, size_t count)
- {
- unsigned int freq = 0;
- unsigned int ret;
- if (!policy->governor || !policy->governor->store_setspeed)
- return -EINVAL;
- ret = sscanf(buf, "%u", &freq);
- if (ret != 1)
- return -EINVAL;
- policy->governor->store_setspeed(policy, freq);
- return count;
- }
- static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
- {
- if (!policy->governor || !policy->governor->show_setspeed)
- return sprintf(buf, "<unsupported>\n");
- return policy->governor->show_setspeed(policy, buf);
- }
- /*
- * show_bios_limit - show the current cpufreq HW/BIOS limitation
- */
- static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
- {
- unsigned int limit;
- int ret;
- ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
- if (!ret)
- return sprintf(buf, "%u\n", limit);
- return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
- }
- cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
- cpufreq_freq_attr_ro(cpuinfo_min_freq);
- cpufreq_freq_attr_ro(cpuinfo_max_freq);
- cpufreq_freq_attr_ro(cpuinfo_transition_latency);
- cpufreq_freq_attr_ro(scaling_available_governors);
- cpufreq_freq_attr_ro(scaling_driver);
- cpufreq_freq_attr_ro(scaling_cur_freq);
- cpufreq_freq_attr_ro(bios_limit);
- cpufreq_freq_attr_ro(related_cpus);
- cpufreq_freq_attr_ro(affected_cpus);
- cpufreq_freq_attr_rw(scaling_min_freq);
- cpufreq_freq_attr_rw(scaling_max_freq);
- cpufreq_freq_attr_rw(scaling_governor);
- cpufreq_freq_attr_rw(scaling_setspeed);
- static struct attribute *default_attrs[] = {
- &cpuinfo_min_freq.attr,
- &cpuinfo_max_freq.attr,
- &cpuinfo_transition_latency.attr,
- &scaling_min_freq.attr,
- &scaling_max_freq.attr,
- &affected_cpus.attr,
- &related_cpus.attr,
- &scaling_governor.attr,
- &scaling_driver.attr,
- &scaling_available_governors.attr,
- &scaling_setspeed.attr,
- NULL
- };
- #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
- #define to_attr(a) container_of(a, struct freq_attr, attr)
- static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
- {
- struct cpufreq_policy *policy = to_policy(kobj);
- struct freq_attr *fattr = to_attr(attr);
- ssize_t ret;
- if (!fattr->show)
- return -EIO;
- down_read(&policy->rwsem);
- ret = fattr->show(policy, buf);
- up_read(&policy->rwsem);
- return ret;
- }
- static ssize_t store(struct kobject *kobj, struct attribute *attr,
- const char *buf, size_t count)
- {
- struct cpufreq_policy *policy = to_policy(kobj);
- struct freq_attr *fattr = to_attr(attr);
- ssize_t ret = -EINVAL;
- if (!fattr->store)
- return -EIO;
- /*
- * cpus_read_trylock() is used here to work around a circular lock
- * dependency problem with respect to the cpufreq_register_driver().
- */
- if (!cpus_read_trylock())
- return -EBUSY;
- if (cpu_online(policy->cpu)) {
- down_write(&policy->rwsem);
- ret = fattr->store(policy, buf, count);
- up_write(&policy->rwsem);
- }
- cpus_read_unlock();
- return ret;
- }
- static void cpufreq_sysfs_release(struct kobject *kobj)
- {
- struct cpufreq_policy *policy = to_policy(kobj);
- pr_debug("last reference is dropped\n");
- complete(&policy->kobj_unregister);
- }
- static const struct sysfs_ops sysfs_ops = {
- .show = show,
- .store = store,
- };
- static struct kobj_type ktype_cpufreq = {
- .sysfs_ops = &sysfs_ops,
- .default_attrs = default_attrs,
- .release = cpufreq_sysfs_release,
- };
- static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu,
- struct device *dev)
- {
- if (unlikely(!dev))
- return;
- if (cpumask_test_and_set_cpu(cpu, policy->real_cpus))
- return;
- dev_dbg(dev, "%s: Adding symlink\n", __func__);
- if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq"))
- dev_err(dev, "cpufreq symlink creation failed\n");
- }
- static void remove_cpu_dev_symlink(struct cpufreq_policy *policy,
- struct device *dev)
- {
- dev_dbg(dev, "%s: Removing symlink\n", __func__);
- sysfs_remove_link(&dev->kobj, "cpufreq");
- }
- static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
- {
- struct freq_attr **drv_attr;
- int ret = 0;
- /* set up files for this cpu device */
- drv_attr = cpufreq_driver->attr;
- while (drv_attr && *drv_attr) {
- ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
- if (ret)
- return ret;
- drv_attr++;
- }
- if (cpufreq_driver->get) {
- ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
- if (ret)
- return ret;
- }
- ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
- if (ret)
- return ret;
- if (cpufreq_driver->bios_limit) {
- ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
- if (ret)
- return ret;
- }
- return 0;
- }
- static int cpufreq_init_policy(struct cpufreq_policy *policy)
- {
- struct cpufreq_governor *gov = NULL;
- unsigned int pol = CPUFREQ_POLICY_UNKNOWN;
- int ret;
- if (has_target()) {
- /* Update policy governor to the one used before hotplug. */
- gov = get_governor(policy->last_governor);
- if (gov) {
- pr_debug("Restoring governor %s for cpu %d\n",
- gov->name, policy->cpu);
- } else {
- gov = get_governor(default_governor);
- }
- if (!gov) {
- gov = cpufreq_default_governor();
- __module_get(gov->owner);
- }
- } else {
- /* Use the default policy if there is no last_policy. */
- if (policy->last_policy) {
- pol = policy->last_policy;
- } else {
- pol = cpufreq_parse_policy(default_governor);
- /*
- * In case the default governor is neither "performance"
- * nor "powersave", fall back to the initial policy
- * value set by the driver.
- */
- if (pol == CPUFREQ_POLICY_UNKNOWN)
- pol = policy->policy;
- }
- if (pol != CPUFREQ_POLICY_PERFORMANCE &&
- pol != CPUFREQ_POLICY_POWERSAVE)
- return -ENODATA;
- }
- ret = cpufreq_set_policy(policy, gov, pol);
- if (gov)
- module_put(gov->owner);
- return ret;
- }
- static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
- {
- int ret = 0;
- /* Has this CPU been taken care of already? */
- if (cpumask_test_cpu(cpu, policy->cpus))
- return 0;
- down_write(&policy->rwsem);
- if (has_target())
- cpufreq_stop_governor(policy);
- cpumask_set_cpu(cpu, policy->cpus);
- if (has_target()) {
- ret = cpufreq_start_governor(policy);
- if (ret)
- pr_err("%s: Failed to start governor\n", __func__);
- }
- up_write(&policy->rwsem);
- return ret;
- }
- void refresh_frequency_limits(struct cpufreq_policy *policy)
- {
- if (!policy_is_inactive(policy)) {
- pr_debug("updating policy for CPU %u\n", policy->cpu);
- cpufreq_set_policy(policy, policy->governor, policy->policy);
- }
- }
- EXPORT_SYMBOL(refresh_frequency_limits);
- static void handle_update(struct work_struct *work)
- {
- struct cpufreq_policy *policy =
- container_of(work, struct cpufreq_policy, update);
- pr_debug("handle_update for cpu %u called\n", policy->cpu);
- down_write(&policy->rwsem);
- refresh_frequency_limits(policy);
- up_write(&policy->rwsem);
- }
- static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq,
- void *data)
- {
- struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min);
- schedule_work(&policy->update);
- return 0;
- }
- static int cpufreq_notifier_max(struct notifier_block *nb, unsigned long freq,
- void *data)
- {
- struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max);
- schedule_work(&policy->update);
- return 0;
- }
- static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
- {
- struct kobject *kobj;
- struct completion *cmp;
- down_write(&policy->rwsem);
- cpufreq_stats_free_table(policy);
- kobj = &policy->kobj;
- cmp = &policy->kobj_unregister;
- up_write(&policy->rwsem);
- kobject_put(kobj);
- /*
- * We need to make sure that the underlying kobj is
- * actually not referenced anymore by anybody before we
- * proceed with unloading.
- */
- pr_debug("waiting for dropping of refcount\n");
- wait_for_completion(cmp);
- pr_debug("wait complete\n");
- }
- static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
- {
- struct cpufreq_policy *policy;
- struct device *dev = get_cpu_device(cpu);
- int ret;
- if (!dev)
- return NULL;
- policy = kzalloc(sizeof(*policy), GFP_KERNEL);
- if (!policy)
- return NULL;
- if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
- goto err_free_policy;
- if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
- goto err_free_cpumask;
- if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))
- goto err_free_rcpumask;
- ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
- cpufreq_global_kobject, "policy%u", cpu);
- if (ret) {
- dev_err(dev, "%s: failed to init policy->kobj: %d\n", __func__, ret);
- /*
- * The entire policy object will be freed below, but the extra
- * memory allocated for the kobject name needs to be freed by
- * releasing the kobject.
- */
- kobject_put(&policy->kobj);
- goto err_free_real_cpus;
- }
- freq_constraints_init(&policy->constraints);
- policy->nb_min.notifier_call = cpufreq_notifier_min;
- policy->nb_max.notifier_call = cpufreq_notifier_max;
- ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MIN,
- &policy->nb_min);
- if (ret) {
- dev_err(dev, "Failed to register MIN QoS notifier: %d (%*pbl)\n",
- ret, cpumask_pr_args(policy->cpus));
- goto err_kobj_remove;
- }
- ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MAX,
- &policy->nb_max);
- if (ret) {
- dev_err(dev, "Failed to register MAX QoS notifier: %d (%*pbl)\n",
- ret, cpumask_pr_args(policy->cpus));
- goto err_min_qos_notifier;
- }
- INIT_LIST_HEAD(&policy->policy_list);
- init_rwsem(&policy->rwsem);
- spin_lock_init(&policy->transition_lock);
- init_waitqueue_head(&policy->transition_wait);
- init_completion(&policy->kobj_unregister);
- INIT_WORK(&policy->update, handle_update);
- policy->cpu = cpu;
- return policy;
- err_min_qos_notifier:
- freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
- &policy->nb_min);
- err_kobj_remove:
- cpufreq_policy_put_kobj(policy);
- err_free_real_cpus:
- free_cpumask_var(policy->real_cpus);
- err_free_rcpumask:
- free_cpumask_var(policy->related_cpus);
- err_free_cpumask:
- free_cpumask_var(policy->cpus);
- err_free_policy:
- kfree(policy);
- return NULL;
- }
- static void cpufreq_policy_free(struct cpufreq_policy *policy)
- {
- unsigned long flags;
- int cpu;
- /* Remove policy from list */
- write_lock_irqsave(&cpufreq_driver_lock, flags);
- list_del(&policy->policy_list);
- for_each_cpu(cpu, policy->related_cpus)
- per_cpu(cpufreq_cpu_data, cpu) = NULL;
- write_unlock_irqrestore(&cpufreq_driver_lock, flags);
- freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MAX,
- &policy->nb_max);
- freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
- &policy->nb_min);
- /* Cancel any pending policy->update work before freeing the policy. */
- cancel_work_sync(&policy->update);
- if (policy->max_freq_req) {
- /*
- * CPUFREQ_CREATE_POLICY notification is sent only after
- * successfully adding max_freq_req request.
- */
- blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
- CPUFREQ_REMOVE_POLICY, policy);
- freq_qos_remove_request(policy->max_freq_req);
- }
- freq_qos_remove_request(policy->min_freq_req);
- kfree(policy->min_freq_req);
- cpufreq_policy_put_kobj(policy);
- free_cpumask_var(policy->real_cpus);
- free_cpumask_var(policy->related_cpus);
- free_cpumask_var(policy->cpus);
- kfree(policy);
- }
- static int cpufreq_online(unsigned int cpu)
- {
- struct cpufreq_policy *policy;
- bool new_policy;
- unsigned long flags;
- unsigned int j;
- int ret;
- pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
- /* Check if this CPU already has a policy to manage it */
- policy = per_cpu(cpufreq_cpu_data, cpu);
- if (policy) {
- WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
- if (!policy_is_inactive(policy))
- return cpufreq_add_policy_cpu(policy, cpu);
- /* This is the only online CPU for the policy. Start over. */
- new_policy = false;
- down_write(&policy->rwsem);
- policy->cpu = cpu;
- policy->governor = NULL;
- up_write(&policy->rwsem);
- } else {
- new_policy = true;
- policy = cpufreq_policy_alloc(cpu);
- if (!policy)
- return -ENOMEM;
- }
- if (!new_policy && cpufreq_driver->online) {
- ret = cpufreq_driver->online(policy);
- if (ret) {
- pr_debug("%s: %d: initialization failed\n", __func__,
- __LINE__);
- goto out_exit_policy;
- }
- /* Recover policy->cpus using related_cpus */
- cpumask_copy(policy->cpus, policy->related_cpus);
- } else {
- cpumask_copy(policy->cpus, cpumask_of(cpu));
- /*
- * Call driver. From then on the cpufreq must be able
- * to accept all calls to ->verify and ->setpolicy for this CPU.
- */
- ret = cpufreq_driver->init(policy);
- if (ret) {
- pr_debug("%s: %d: initialization failed\n", __func__,
- __LINE__);
- goto out_free_policy;
- }
- /*
- * The initialization has succeeded and the policy is online.
- * If there is a problem with its frequency table, take it
- * offline and drop it.
- */
- ret = cpufreq_table_validate_and_sort(policy);
- if (ret)
- goto out_offline_policy;
- /* related_cpus should at least include policy->cpus. */
- cpumask_copy(policy->related_cpus, policy->cpus);
- }
- down_write(&policy->rwsem);
- /*
- * affected cpus must always be the one, which are online. We aren't
- * managing offline cpus here.
- */
- cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
- if (new_policy) {
- for_each_cpu(j, policy->related_cpus) {
- per_cpu(cpufreq_cpu_data, j) = policy;
- add_cpu_dev_symlink(policy, j, get_cpu_device(j));
- }
- policy->min_freq_req = kzalloc(2 * sizeof(*policy->min_freq_req),
- GFP_KERNEL);
- if (!policy->min_freq_req)
- goto out_destroy_policy;
- ret = freq_qos_add_request(&policy->constraints,
- policy->min_freq_req, FREQ_QOS_MIN,
- FREQ_QOS_MIN_DEFAULT_VALUE);
- if (ret < 0) {
- /*
- * So we don't call freq_qos_remove_request() for an
- * uninitialized request.
- */
- kfree(policy->min_freq_req);
- policy->min_freq_req = NULL;
- goto out_destroy_policy;
- }
- /*
- * This must be initialized right here to avoid calling
- * freq_qos_remove_request() on uninitialized request in case
- * of errors.
- */
- policy->max_freq_req = policy->min_freq_req + 1;
- ret = freq_qos_add_request(&policy->constraints,
- policy->max_freq_req, FREQ_QOS_MAX,
- FREQ_QOS_MAX_DEFAULT_VALUE);
- if (ret < 0) {
- policy->max_freq_req = NULL;
- goto out_destroy_policy;
- }
- blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
- CPUFREQ_CREATE_POLICY, policy);
- }
- if (cpufreq_driver->get && has_target()) {
- policy->cur = cpufreq_driver->get(policy->cpu);
- if (!policy->cur) {
- pr_err("%s: ->get() failed\n", __func__);
- goto out_destroy_policy;
- }
- }
- /*
- * Sometimes boot loaders set CPU frequency to a value outside of
- * frequency table present with cpufreq core. In such cases CPU might be
- * unstable if it has to run on that frequency for long duration of time
- * and so its better to set it to a frequency which is specified in
- * freq-table. This also makes cpufreq stats inconsistent as
- * cpufreq-stats would fail to register because current frequency of CPU
- * isn't found in freq-table.
- *
- * Because we don't want this change to effect boot process badly, we go
- * for the next freq which is >= policy->cur ('cur' must be set by now,
- * otherwise we will end up setting freq to lowest of the table as 'cur'
- * is initialized to zero).
- *
- * We are passing target-freq as "policy->cur - 1" otherwise
- * __cpufreq_driver_target() would simply fail, as policy->cur will be
- * equal to target-freq.
- */
- if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
- && has_target()) {
- unsigned int old_freq = policy->cur;
- /* Are we running at unknown frequency ? */
- ret = cpufreq_frequency_table_get_index(policy, old_freq);
- if (ret == -EINVAL) {
- ret = __cpufreq_driver_target(policy, old_freq - 1,
- CPUFREQ_RELATION_L);
- /*
- * Reaching here after boot in a few seconds may not
- * mean that system will remain stable at "unknown"
- * frequency for longer duration. Hence, a BUG_ON().
- */
- BUG_ON(ret);
- pr_info("%s: CPU%d: Running at unlisted initial frequency: %u KHz, changing to: %u KHz\n",
- __func__, policy->cpu, old_freq, policy->cur);
- }
- }
- if (new_policy) {
- ret = cpufreq_add_dev_interface(policy);
- if (ret)
- goto out_destroy_policy;
- cpufreq_stats_create_table(policy);
- cpufreq_times_create_policy(policy);
- write_lock_irqsave(&cpufreq_driver_lock, flags);
- list_add(&policy->policy_list, &cpufreq_policy_list);
- write_unlock_irqrestore(&cpufreq_driver_lock, flags);
- }
- ret = cpufreq_init_policy(policy);
- if (ret) {
- pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
- __func__, cpu, ret);
- goto out_destroy_policy;
- }
- up_write(&policy->rwsem);
- kobject_uevent(&policy->kobj, KOBJ_ADD);
- /* Callback for handling stuff after policy is ready */
- if (cpufreq_driver->ready)
- cpufreq_driver->ready(policy);
- if (cpufreq_thermal_control_enabled(cpufreq_driver))
- policy->cdev = of_cpufreq_cooling_register(policy);
- pr_debug("initialization complete\n");
- return 0;
- out_destroy_policy:
- for_each_cpu(j, policy->real_cpus)
- remove_cpu_dev_symlink(policy, get_cpu_device(j));
- up_write(&policy->rwsem);
- out_offline_policy:
- if (cpufreq_driver->offline)
- cpufreq_driver->offline(policy);
- out_exit_policy:
- if (cpufreq_driver->exit)
- cpufreq_driver->exit(policy);
- out_free_policy:
- cpufreq_policy_free(policy);
- return ret;
- }
- /**
- * cpufreq_add_dev - the cpufreq interface for a CPU device.
- * @dev: CPU device.
- * @sif: Subsystem interface structure pointer (not used)
- */
- static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
- {
- struct cpufreq_policy *policy;
- unsigned cpu = dev->id;
- int ret;
- dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
- if (cpu_online(cpu)) {
- ret = cpufreq_online(cpu);
- if (ret)
- return ret;
- }
- /* Create sysfs link on CPU registration */
- policy = per_cpu(cpufreq_cpu_data, cpu);
- if (policy)
- add_cpu_dev_symlink(policy, cpu, dev);
- return 0;
- }
- static int cpufreq_offline(unsigned int cpu)
- {
- struct cpufreq_policy *policy;
- int ret;
- pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
- policy = cpufreq_cpu_get_raw(cpu);
- if (!policy) {
- pr_debug("%s: No cpu_data found\n", __func__);
- return 0;
- }
- down_write(&policy->rwsem);
- if (has_target())
- cpufreq_stop_governor(policy);
- cpumask_clear_cpu(cpu, policy->cpus);
- if (policy_is_inactive(policy)) {
- if (has_target())
- strncpy(policy->last_governor, policy->governor->name,
- CPUFREQ_NAME_LEN);
- else
- policy->last_policy = policy->policy;
- } else if (cpu == policy->cpu) {
- /* Nominate new CPU */
- policy->cpu = cpumask_any(policy->cpus);
- }
- /* Start governor again for active policy */
- if (!policy_is_inactive(policy)) {
- if (has_target()) {
- ret = cpufreq_start_governor(policy);
- if (ret)
- pr_err("%s: Failed to start governor\n", __func__);
- }
- goto unlock;
- }
- if (cpufreq_thermal_control_enabled(cpufreq_driver)) {
- cpufreq_cooling_unregister(policy->cdev);
- policy->cdev = NULL;
- }
- if (cpufreq_driver->stop_cpu)
- cpufreq_driver->stop_cpu(policy);
- if (has_target())
- cpufreq_exit_governor(policy);
- /*
- * Perform the ->offline() during light-weight tear-down, as
- * that allows fast recovery when the CPU comes back.
- */
- if (cpufreq_driver->offline) {
- cpufreq_driver->offline(policy);
- } else if (cpufreq_driver->exit) {
- cpufreq_driver->exit(policy);
- policy->freq_table = NULL;
- }
- unlock:
- up_write(&policy->rwsem);
- return 0;
- }
- /*
- * cpufreq_remove_dev - remove a CPU device
- *
- * Removes the cpufreq interface for a CPU device.
- */
- static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
- {
- unsigned int cpu = dev->id;
- struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
- if (!policy)
- return;
- if (cpu_online(cpu))
- cpufreq_offline(cpu);
- cpumask_clear_cpu(cpu, policy->real_cpus);
- remove_cpu_dev_symlink(policy, dev);
- if (cpumask_empty(policy->real_cpus)) {
- /* We did light-weight exit earlier, do full tear down now */
- if (cpufreq_driver->offline)
- cpufreq_driver->exit(policy);
- cpufreq_policy_free(policy);
- }
- }
- /**
- * cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
- * in deep trouble.
- * @policy: policy managing CPUs
- * @new_freq: CPU frequency the CPU actually runs at
- *
- * We adjust to current frequency first, and need to clean up later.
- * So either call to cpufreq_update_policy() or schedule handle_update()).
- */
- static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
- unsigned int new_freq)
- {
- struct cpufreq_freqs freqs;
- pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
- policy->cur, new_freq);
- freqs.old = policy->cur;
- freqs.new = new_freq;
- cpufreq_freq_transition_begin(policy, &freqs);
- cpufreq_freq_transition_end(policy, &freqs, 0);
- }
- static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, bool update)
- {
- unsigned int new_freq;
- new_freq = cpufreq_driver->get(policy->cpu);
- if (!new_freq)
- return 0;
- /*
- * If fast frequency switching is used with the given policy, the check
- * against policy->cur is pointless, so skip it in that case.
- */
- if (policy->fast_switch_enabled || !has_target())
- return new_freq;
- if (policy->cur != new_freq) {
- cpufreq_out_of_sync(policy, new_freq);
- if (update)
- schedule_work(&policy->update);
- }
- return new_freq;
- }
- /**
- * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
- * @cpu: CPU number
- *
- * This is the last known freq, without actually getting it from the driver.
- * Return value will be same as what is shown in scaling_cur_freq in sysfs.
- */
- unsigned int cpufreq_quick_get(unsigned int cpu)
- {
- struct cpufreq_policy *policy;
- unsigned int ret_freq = 0;
- unsigned long flags;
- read_lock_irqsave(&cpufreq_driver_lock, flags);
- if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) {
- ret_freq = cpufreq_driver->get(cpu);
- read_unlock_irqrestore(&cpufreq_driver_lock, flags);
- return ret_freq;
- }
- read_unlock_irqrestore(&cpufreq_driver_lock, flags);
- policy = cpufreq_cpu_get(cpu);
- if (policy) {
- ret_freq = policy->cur;
- cpufreq_cpu_put(policy);
- }
- return ret_freq;
- }
- EXPORT_SYMBOL(cpufreq_quick_get);
- /**
- * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
- * @cpu: CPU number
- *
- * Just return the max possible frequency for a given CPU.
- */
- unsigned int cpufreq_quick_get_max(unsigned int cpu)
- {
- struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
- unsigned int ret_freq = 0;
- if (policy) {
- ret_freq = policy->max;
- cpufreq_cpu_put(policy);
- }
- return ret_freq;
- }
- EXPORT_SYMBOL(cpufreq_quick_get_max);
- /**
- * cpufreq_get_hw_max_freq - get the max hardware frequency of the CPU
- * @cpu: CPU number
- *
- * The default return value is the max_freq field of cpuinfo.
- */
- __weak unsigned int cpufreq_get_hw_max_freq(unsigned int cpu)
- {
- struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
- unsigned int ret_freq = 0;
- if (policy) {
- ret_freq = policy->cpuinfo.max_freq;
- cpufreq_cpu_put(policy);
- }
- return ret_freq;
- }
- EXPORT_SYMBOL(cpufreq_get_hw_max_freq);
- static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
- {
- if (unlikely(policy_is_inactive(policy)))
- return 0;
- return cpufreq_verify_current_freq(policy, true);
- }
- /**
- * cpufreq_get - get the current CPU frequency (in kHz)
- * @cpu: CPU number
- *
- * Get the CPU current (static) CPU frequency
- */
- unsigned int cpufreq_get(unsigned int cpu)
- {
- struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
- unsigned int ret_freq = 0;
- if (policy) {
- down_read(&policy->rwsem);
- if (cpufreq_driver->get)
- ret_freq = __cpufreq_get(policy);
- up_read(&policy->rwsem);
- cpufreq_cpu_put(policy);
- }
- return ret_freq;
- }
- EXPORT_SYMBOL(cpufreq_get);
- static struct subsys_interface cpufreq_interface = {
- .name = "cpufreq",
- .subsys = &cpu_subsys,
- .add_dev = cpufreq_add_dev,
- .remove_dev = cpufreq_remove_dev,
- };
- /*
- * In case platform wants some specific frequency to be configured
- * during suspend..
- */
- int cpufreq_generic_suspend(struct cpufreq_policy *policy)
- {
- int ret;
- if (!policy->suspend_freq) {
- pr_debug("%s: suspend_freq not defined\n", __func__);
- return 0;
- }
- pr_debug("%s: Setting suspend-freq: %u\n", __func__,
- policy->suspend_freq);
- ret = __cpufreq_driver_target(policy, policy->suspend_freq,
- CPUFREQ_RELATION_H);
- if (ret)
- pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
- __func__, policy->suspend_freq, ret);
- return ret;
- }
- EXPORT_SYMBOL(cpufreq_generic_suspend);
- /**
- * cpufreq_suspend() - Suspend CPUFreq governors
- *
- * Called during system wide Suspend/Hibernate cycles for suspending governors
- * as some platforms can't change frequency after this point in suspend cycle.
- * Because some of the devices (like: i2c, regulators, etc) they use for
- * changing frequency are suspended quickly after this point.
- */
- void cpufreq_suspend(void)
- {
- struct cpufreq_policy *policy;
- if (!cpufreq_driver)
- return;
- if (!has_target() && !cpufreq_driver->suspend)
- goto suspend;
- pr_debug("%s: Suspending Governors\n", __func__);
- for_each_active_policy(policy) {
- if (has_target()) {
- down_write(&policy->rwsem);
- cpufreq_stop_governor(policy);
- up_write(&policy->rwsem);
- }
- if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy))
- pr_err("%s: Failed to suspend driver: %s\n", __func__,
- cpufreq_driver->name);
- }
- suspend:
- cpufreq_suspended = true;
- }
- /**
- * cpufreq_resume() - Resume CPUFreq governors
- *
- * Called during system wide Suspend/Hibernate cycle for resuming governors that
- * are suspended with cpufreq_suspend().
- */
- void cpufreq_resume(void)
- {
- struct cpufreq_policy *policy;
- int ret;
- if (!cpufreq_driver)
- return;
- if (unlikely(!cpufreq_suspended))
- return;
- cpufreq_suspended = false;
- if (!has_target() && !cpufreq_driver->resume)
- return;
- pr_debug("%s: Resuming Governors\n", __func__);
- for_each_active_policy(policy) {
- if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) {
- pr_err("%s: Failed to resume driver: %p\n", __func__,
- policy);
- } else if (has_target()) {
- down_write(&policy->rwsem);
- ret = cpufreq_start_governor(policy);
- up_write(&policy->rwsem);
- if (ret)
- pr_err("%s: Failed to start governor for policy: %p\n",
- __func__, policy);
- }
- }
- }
- /**
- * cpufreq_driver_test_flags - Test cpufreq driver's flags against given ones.
- * @flags: Flags to test against the current cpufreq driver's flags.
- *
- * Assumes that the driver is there, so callers must ensure that this is the
- * case.
- */
- bool cpufreq_driver_test_flags(u16 flags)
- {
- return !!(cpufreq_driver->flags & flags);
- }
- /**
- * cpufreq_get_current_driver - return current driver's name
- *
- * Return the name string of the currently loaded cpufreq driver
- * or NULL, if none.
- */
- const char *cpufreq_get_current_driver(void)
- {
- if (cpufreq_driver)
- return cpufreq_driver->name;
- return NULL;
- }
- EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
- /**
- * cpufreq_get_driver_data - return current driver data
- *
- * Return the private data of the currently loaded cpufreq
- * driver, or NULL if no cpufreq driver is loaded.
- */
- void *cpufreq_get_driver_data(void)
- {
- if (cpufreq_driver)
- return cpufreq_driver->driver_data;
- return NULL;
- }
- EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
- /*********************************************************************
- * NOTIFIER LISTS INTERFACE *
- *********************************************************************/
- /**
- * cpufreq_register_notifier - register a driver with cpufreq
- * @nb: notifier function to register
- * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
- *
- * Add a driver to one of two lists: either a list of drivers that
- * are notified about clock rate changes (once before and once after
- * the transition), or a list of drivers that are notified about
- * changes in cpufreq policy.
- *
- * This function may sleep, and has the same return conditions as
- * blocking_notifier_chain_register.
- */
- int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
- {
- int ret;
- if (cpufreq_disabled())
- return -EINVAL;
- switch (list) {
- case CPUFREQ_TRANSITION_NOTIFIER:
- mutex_lock(&cpufreq_fast_switch_lock);
- if (cpufreq_fast_switch_count > 0) {
- mutex_unlock(&cpufreq_fast_switch_lock);
- return -EBUSY;
- }
- ret = srcu_notifier_chain_register(
- &cpufreq_transition_notifier_list, nb);
- if (!ret)
- cpufreq_fast_switch_count--;
- mutex_unlock(&cpufreq_fast_switch_lock);
- break;
- case CPUFREQ_POLICY_NOTIFIER:
- ret = blocking_notifier_chain_register(
- &cpufreq_policy_notifier_list, nb);
- break;
- default:
- ret = -EINVAL;
- }
- return ret;
- }
- EXPORT_SYMBOL(cpufreq_register_notifier);
- /**
- * cpufreq_unregister_notifier - unregister a driver with cpufreq
- * @nb: notifier block to be unregistered
- * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
- *
- * Remove a driver from the CPU frequency notifier list.
- *
- * This function may sleep, and has the same return conditions as
- * blocking_notifier_chain_unregister.
- */
- int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
- {
- int ret;
- if (cpufreq_disabled())
- return -EINVAL;
- switch (list) {
- case CPUFREQ_TRANSITION_NOTIFIER:
- mutex_lock(&cpufreq_fast_switch_lock);
- ret = srcu_notifier_chain_unregister(
- &cpufreq_transition_notifier_list, nb);
- if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0))
- cpufreq_fast_switch_count++;
- mutex_unlock(&cpufreq_fast_switch_lock);
- break;
- case CPUFREQ_POLICY_NOTIFIER:
- ret = blocking_notifier_chain_unregister(
- &cpufreq_policy_notifier_list, nb);
- break;
- default:
- ret = -EINVAL;
- }
- return ret;
- }
- EXPORT_SYMBOL(cpufreq_unregister_notifier);
- /*********************************************************************
- * GOVERNORS *
- *********************************************************************/
- /**
- * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch.
- * @policy: cpufreq policy to switch the frequency for.
- * @target_freq: New frequency to set (may be approximate).
- *
- * Carry out a fast frequency switch without sleeping.
- *
- * The driver's ->fast_switch() callback invoked by this function must be
- * suitable for being called from within RCU-sched read-side critical sections
- * and it is expected to select the minimum available frequency greater than or
- * equal to @target_freq (CPUFREQ_RELATION_L).
- *
- * This function must not be called if policy->fast_switch_enabled is unset.
- *
- * Governors calling this function must guarantee that it will never be invoked
- * twice in parallel for the same policy and that it will never be called in
- * parallel with either ->target() or ->target_index() for the same policy.
- *
- * Returns the actual frequency set for the CPU.
- *
- * If 0 is returned by the driver's ->fast_switch() callback to indicate an
- * error condition, the hardware configuration must be preserved.
- */
- unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
- unsigned int target_freq)
- {
- unsigned int freq;
- unsigned int old_target_freq = target_freq;
- int cpu;
- target_freq = clamp_val(target_freq, policy->min, policy->max);
- trace_android_vh_cpufreq_fast_switch(policy, target_freq, old_target_freq);
- freq = cpufreq_driver->fast_switch(policy, target_freq);
- if (!freq)
- return 0;
- policy->cur = freq;
- arch_set_freq_scale(policy->related_cpus, freq,
- policy->cpuinfo.max_freq);
- cpufreq_stats_record_transition(policy, freq);
- cpufreq_times_record_transition(policy, freq);
- trace_android_rvh_cpufreq_transition(policy);
- if (trace_cpu_frequency_enabled()) {
- for_each_cpu(cpu, policy->cpus)
- trace_cpu_frequency(freq, cpu);
- }
- return freq;
- }
- EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch);
- /* Must set freqs->new to intermediate frequency */
- static int __target_intermediate(struct cpufreq_policy *policy,
- struct cpufreq_freqs *freqs, int index)
- {
- int ret;
- freqs->new = cpufreq_driver->get_intermediate(policy, index);
- /* We don't need to switch to intermediate freq */
- if (!freqs->new)
- return 0;
- pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
- __func__, policy->cpu, freqs->old, freqs->new);
- cpufreq_freq_transition_begin(policy, freqs);
- ret = cpufreq_driver->target_intermediate(policy, index);
- cpufreq_freq_transition_end(policy, freqs, ret);
- if (ret)
- pr_err("%s: Failed to change to intermediate frequency: %d\n",
- __func__, ret);
- return ret;
- }
- static int __target_index(struct cpufreq_policy *policy, int index)
- {
- struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
- unsigned int intermediate_freq = 0;
- unsigned int newfreq = policy->freq_table[index].frequency;
- int retval = -EINVAL;
- bool notify;
- if (newfreq == policy->cur)
- return 0;
- notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
- if (notify) {
- /* Handle switching to intermediate frequency */
- if (cpufreq_driver->get_intermediate) {
- retval = __target_intermediate(policy, &freqs, index);
- if (retval)
- return retval;
- intermediate_freq = freqs.new;
- /* Set old freq to intermediate */
- if (intermediate_freq)
- freqs.old = freqs.new;
- }
- freqs.new = newfreq;
- pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
- __func__, policy->cpu, freqs.old, freqs.new);
- cpufreq_freq_transition_begin(policy, &freqs);
- }
- retval = cpufreq_driver->target_index(policy, index);
- if (retval)
- pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
- retval);
- if (notify) {
- cpufreq_freq_transition_end(policy, &freqs, retval);
- /*
- * Failed after setting to intermediate freq? Driver should have
- * reverted back to initial frequency and so should we. Check
- * here for intermediate_freq instead of get_intermediate, in
- * case we haven't switched to intermediate freq at all.
- */
- if (unlikely(retval && intermediate_freq)) {
- freqs.old = intermediate_freq;
- freqs.new = policy->restore_freq;
- cpufreq_freq_transition_begin(policy, &freqs);
- cpufreq_freq_transition_end(policy, &freqs, 0);
- }
- }
- return retval;
- }
- int __cpufreq_driver_target(struct cpufreq_policy *policy,
- unsigned int target_freq,
- unsigned int relation)
- {
- unsigned int old_target_freq = target_freq;
- int index;
- if (cpufreq_disabled())
- return -ENODEV;
- /* Make sure that target_freq is within supported range */
- target_freq = clamp_val(target_freq, policy->min, policy->max);
- trace_android_vh_cpufreq_target(policy, target_freq, old_target_freq);
- pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
- policy->cpu, target_freq, relation, old_target_freq);
- /*
- * This might look like a redundant call as we are checking it again
- * after finding index. But it is left intentionally for cases where
- * exactly same freq is called again and so we can save on few function
- * calls.
- */
- if (target_freq == policy->cur &&
- !(cpufreq_driver->flags & CPUFREQ_NEED_UPDATE_LIMITS))
- return 0;
- /* Save last value to restore later on errors */
- policy->restore_freq = policy->cur;
- if (cpufreq_driver->target)
- return cpufreq_driver->target(policy, target_freq, relation);
- if (!cpufreq_driver->target_index)
- return -EINVAL;
- index = cpufreq_frequency_table_target(policy, target_freq, relation);
- return __target_index(policy, index);
- }
- EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
- int cpufreq_driver_target(struct cpufreq_policy *policy,
- unsigned int target_freq,
- unsigned int relation)
- {
- int ret;
- down_write(&policy->rwsem);
- ret = __cpufreq_driver_target(policy, target_freq, relation);
- up_write(&policy->rwsem);
- return ret;
- }
- EXPORT_SYMBOL_GPL(cpufreq_driver_target);
- __weak struct cpufreq_governor *cpufreq_fallback_governor(void)
- {
- return NULL;
- }
- static int cpufreq_init_governor(struct cpufreq_policy *policy)
- {
- int ret;
- /* Don't start any governor operations if we are entering suspend */
- if (cpufreq_suspended)
- return 0;
- /*
- * Governor might not be initiated here if ACPI _PPC changed
- * notification happened, so check it.
- */
- if (!policy->governor)
- return -EINVAL;
- /* Platform doesn't want dynamic frequency switching ? */
- if (policy->governor->flags & CPUFREQ_GOV_DYNAMIC_SWITCHING &&
- cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) {
- struct cpufreq_governor *gov = cpufreq_fallback_governor();
- if (gov) {
- pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n",
- policy->governor->name, gov->name);
- policy->governor = gov;
- } else {
- return -EINVAL;
- }
- }
- if (!try_module_get(policy->governor->owner))
- return -EINVAL;
- pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
- if (policy->governor->init) {
- ret = policy->governor->init(policy);
- if (ret) {
- module_put(policy->governor->owner);
- return ret;
- }
- }
- policy->strict_target = !!(policy->governor->flags & CPUFREQ_GOV_STRICT_TARGET);
- return 0;
- }
- static void cpufreq_exit_governor(struct cpufreq_policy *policy)
- {
- if (cpufreq_suspended || !policy->governor)
- return;
- pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
- if (policy->governor->exit)
- policy->governor->exit(policy);
- module_put(policy->governor->owner);
- }
- int cpufreq_start_governor(struct cpufreq_policy *policy)
- {
- int ret;
- if (cpufreq_suspended)
- return 0;
- if (!policy->governor)
- return -EINVAL;
- pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
- if (cpufreq_driver->get)
- cpufreq_verify_current_freq(policy, false);
- if (policy->governor->start) {
- ret = policy->governor->start(policy);
- if (ret)
- return ret;
- }
- if (policy->governor->limits)
- policy->governor->limits(policy);
- return 0;
- }
- void cpufreq_stop_governor(struct cpufreq_policy *policy)
- {
- if (cpufreq_suspended || !policy->governor)
- return;
- pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
- if (policy->governor->stop)
- policy->governor->stop(policy);
- }
- static void cpufreq_governor_limits(struct cpufreq_policy *policy)
- {
- if (cpufreq_suspended || !policy->governor)
- return;
- pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
- if (policy->governor->limits)
- policy->governor->limits(policy);
- }
- int cpufreq_register_governor(struct cpufreq_governor *governor)
- {
- int err;
- if (!governor)
- return -EINVAL;
- if (cpufreq_disabled())
- return -ENODEV;
- mutex_lock(&cpufreq_governor_mutex);
- err = -EBUSY;
- if (!find_governor(governor->name)) {
- err = 0;
- list_add(&governor->governor_list, &cpufreq_governor_list);
- }
- mutex_unlock(&cpufreq_governor_mutex);
- return err;
- }
- EXPORT_SYMBOL_GPL(cpufreq_register_governor);
- void cpufreq_unregister_governor(struct cpufreq_governor *governor)
- {
- struct cpufreq_policy *policy;
- unsigned long flags;
- if (!governor)
- return;
- if (cpufreq_disabled())
- return;
- /* clear last_governor for all inactive policies */
- read_lock_irqsave(&cpufreq_driver_lock, flags);
- for_each_inactive_policy(policy) {
- if (!strcmp(policy->last_governor, governor->name)) {
- policy->governor = NULL;
- strcpy(policy->last_governor, "\0");
- }
- }
- read_unlock_irqrestore(&cpufreq_driver_lock, flags);
- mutex_lock(&cpufreq_governor_mutex);
- list_del(&governor->governor_list);
- mutex_unlock(&cpufreq_governor_mutex);
- }
- EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
- /*********************************************************************
- * POLICY INTERFACE *
- *********************************************************************/
- /**
- * cpufreq_get_policy - get the current cpufreq_policy
- * @policy: struct cpufreq_policy into which the current cpufreq_policy
- * is written
- * @cpu: CPU to find the policy for
- *
- * Reads the current cpufreq policy.
- */
- int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
- {
- struct cpufreq_policy *cpu_policy;
- if (!policy)
- return -EINVAL;
- cpu_policy = cpufreq_cpu_get(cpu);
- if (!cpu_policy)
- return -EINVAL;
- memcpy(policy, cpu_policy, sizeof(*policy));
- cpufreq_cpu_put(cpu_policy);
- return 0;
- }
- EXPORT_SYMBOL(cpufreq_get_policy);
- /**
- * cpufreq_set_policy - Modify cpufreq policy parameters.
- * @policy: Policy object to modify.
- * @new_gov: Policy governor pointer.
- * @new_pol: Policy value (for drivers with built-in governors).
- *
- * Invoke the cpufreq driver's ->verify() callback to sanity-check the frequency
- * limits to be set for the policy, update @policy with the verified limits
- * values and either invoke the driver's ->setpolicy() callback (if present) or
- * carry out a governor update for @policy. That is, run the current governor's
- * ->limits() callback (if @new_gov points to the same object as the one in
- * @policy) or replace the governor for @policy with @new_gov.
- *
- * The cpuinfo part of @policy is not updated by this function.
- */
- static int cpufreq_set_policy(struct cpufreq_policy *policy,
- struct cpufreq_governor *new_gov,
- unsigned int new_pol)
- {
- struct cpufreq_policy_data new_data;
- struct cpufreq_governor *old_gov;
- int ret;
- memcpy(&new_data.cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
- new_data.freq_table = policy->freq_table;
- new_data.cpu = policy->cpu;
- /*
- * PM QoS framework collects all the requests from users and provide us
- * the final aggregated value here.
- */
- new_data.min = freq_qos_read_value(&policy->constraints, FREQ_QOS_MIN);
- new_data.max = freq_qos_read_value(&policy->constraints, FREQ_QOS_MAX);
- pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
- new_data.cpu, new_data.min, new_data.max);
- /*
- * Verify that the CPU speed can be set within these limits and make sure
- * that min <= max.
- */
- ret = cpufreq_driver->verify(&new_data);
- if (ret)
- return ret;
- policy->min = new_data.min;
- policy->max = new_data.max;
- trace_cpu_frequency_limits(policy);
- policy->cached_target_freq = UINT_MAX;
- pr_debug("new min and max freqs are %u - %u kHz\n",
- policy->min, policy->max);
- if (cpufreq_driver->setpolicy) {
- policy->policy = new_pol;
- pr_debug("setting range\n");
- return cpufreq_driver->setpolicy(policy);
- }
- if (new_gov == policy->governor) {
- pr_debug("governor limits update\n");
- cpufreq_governor_limits(policy);
- return 0;
- }
- pr_debug("governor switch\n");
- /* save old, working values */
- old_gov = policy->governor;
- /* end old governor */
- if (old_gov) {
- cpufreq_stop_governor(policy);
- cpufreq_exit_governor(policy);
- }
- /* start new governor */
- policy->governor = new_gov;
- ret = cpufreq_init_governor(policy);
- if (!ret) {
- ret = cpufreq_start_governor(policy);
- if (!ret) {
- pr_debug("governor change\n");
- return 0;
- }
- cpufreq_exit_governor(policy);
- }
- /* new governor failed, so re-start old one */
- pr_debug("starting governor %s failed\n", policy->governor->name);
- if (old_gov) {
- policy->governor = old_gov;
- if (cpufreq_init_governor(policy))
- policy->governor = NULL;
- else
- cpufreq_start_governor(policy);
- }
- return ret;
- }
- EXPORT_TRACEPOINT_SYMBOL_GPL(cpu_frequency_limits);
- /**
- * cpufreq_update_policy - Re-evaluate an existing cpufreq policy.
- * @cpu: CPU to re-evaluate the policy for.
- *
- * Update the current frequency for the cpufreq policy of @cpu and use
- * cpufreq_set_policy() to re-apply the min and max limits, which triggers the
- * evaluation of policy notifiers and the cpufreq driver's ->verify() callback
- * for the policy in question, among other things.
- */
- void cpufreq_update_policy(unsigned int cpu)
- {
- struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu);
- if (!policy)
- return;
- /*
- * BIOS might change freq behind our back
- * -> ask driver for current freq and notify governors about a change
- */
- if (cpufreq_driver->get && has_target() &&
- (cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false))))
- goto unlock;
- refresh_frequency_limits(policy);
- unlock:
- cpufreq_cpu_release(policy);
- }
- EXPORT_SYMBOL(cpufreq_update_policy);
- /**
- * cpufreq_update_limits - Update policy limits for a given CPU.
- * @cpu: CPU to update the policy limits for.
- *
- * Invoke the driver's ->update_limits callback if present or call
- * cpufreq_update_policy() for @cpu.
- */
- void cpufreq_update_limits(unsigned int cpu)
- {
- if (cpufreq_driver->update_limits)
- cpufreq_driver->update_limits(cpu);
- else
- cpufreq_update_policy(cpu);
- }
- EXPORT_SYMBOL_GPL(cpufreq_update_limits);
- /*********************************************************************
- * BOOST *
- *********************************************************************/
- static int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state)
- {
- int ret;
- if (!policy->freq_table)
- return -ENXIO;
- ret = cpufreq_frequency_table_cpuinfo(policy, policy->freq_table);
- if (ret) {
- pr_err("%s: Policy frequency update failed\n", __func__);
- return ret;
- }
- ret = freq_qos_update_request(policy->max_freq_req, policy->max);
- if (ret < 0)
- return ret;
- return 0;
- }
- int cpufreq_boost_trigger_state(int state)
- {
- struct cpufreq_policy *policy;
- unsigned long flags;
- int ret = 0;
- if (cpufreq_driver->boost_enabled == state)
- return 0;
- write_lock_irqsave(&cpufreq_driver_lock, flags);
- cpufreq_driver->boost_enabled = state;
- write_unlock_irqrestore(&cpufreq_driver_lock, flags);
- get_online_cpus();
- for_each_active_policy(policy) {
- ret = cpufreq_driver->set_boost(policy, state);
- if (ret)
- goto err_reset_state;
- }
- put_online_cpus();
- return 0;
- err_reset_state:
- put_online_cpus();
- write_lock_irqsave(&cpufreq_driver_lock, flags);
- cpufreq_driver->boost_enabled = !state;
- write_unlock_irqrestore(&cpufreq_driver_lock, flags);
- pr_err("%s: Cannot %s BOOST\n",
- __func__, state ? "enable" : "disable");
- return ret;
- }
- static bool cpufreq_boost_supported(void)
- {
- return cpufreq_driver->set_boost;
- }
- static int create_boost_sysfs_file(void)
- {
- int ret;
- ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr);
- if (ret)
- pr_err("%s: cannot register global BOOST sysfs file\n",
- __func__);
- return ret;
- }
- static void remove_boost_sysfs_file(void)
- {
- if (cpufreq_boost_supported())
- sysfs_remove_file(cpufreq_global_kobject, &boost.attr);
- }
- int cpufreq_enable_boost_support(void)
- {
- if (!cpufreq_driver)
- return -EINVAL;
- if (cpufreq_boost_supported())
- return 0;
- cpufreq_driver->set_boost = cpufreq_boost_set_sw;
- /* This will get removed on driver unregister */
- return create_boost_sysfs_file();
- }
- EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support);
- int cpufreq_boost_enabled(void)
- {
- return cpufreq_driver->boost_enabled;
- }
- EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
- /*********************************************************************
- * REGISTER / UNREGISTER CPUFREQ DRIVER *
- *********************************************************************/
- static enum cpuhp_state hp_online;
- static int cpuhp_cpufreq_online(unsigned int cpu)
- {
- cpufreq_online(cpu);
- return 0;
- }
- static int cpuhp_cpufreq_offline(unsigned int cpu)
- {
- cpufreq_offline(cpu);
- return 0;
- }
- /**
- * cpufreq_register_driver - register a CPU Frequency driver
- * @driver_data: A struct cpufreq_driver containing the values#
- * submitted by the CPU Frequency driver.
- *
- * Registers a CPU Frequency driver to this core code. This code
- * returns zero on success, -EEXIST when another driver got here first
- * (and isn't unregistered in the meantime).
- *
- */
- int cpufreq_register_driver(struct cpufreq_driver *driver_data)
- {
- unsigned long flags;
- int ret;
- if (cpufreq_disabled())
- return -ENODEV;
- /*
- * The cpufreq core depends heavily on the availability of device
- * structure, make sure they are available before proceeding further.
- */
- if (!get_cpu_device(0))
- return -EPROBE_DEFER;
- if (!driver_data || !driver_data->verify || !driver_data->init ||
- !(driver_data->setpolicy || driver_data->target_index ||
- driver_data->target) ||
- (driver_data->setpolicy && (driver_data->target_index ||
- driver_data->target)) ||
- (!driver_data->get_intermediate != !driver_data->target_intermediate) ||
- (!driver_data->online != !driver_data->offline))
- return -EINVAL;
- pr_debug("trying to register driver %s\n", driver_data->name);
- /* Protect against concurrent CPU online/offline. */
- cpus_read_lock();
- write_lock_irqsave(&cpufreq_driver_lock, flags);
- if (cpufreq_driver) {
- write_unlock_irqrestore(&cpufreq_driver_lock, flags);
- ret = -EEXIST;
- goto out;
- }
- cpufreq_driver = driver_data;
- write_unlock_irqrestore(&cpufreq_driver_lock, flags);
- /*
- * Mark support for the scheduler's frequency invariance engine for
- * drivers that implement target(), target_index() or fast_switch().
- */
- if (!cpufreq_driver->setpolicy) {
- static_branch_enable_cpuslocked(&cpufreq_freq_invariance);
- pr_debug("supports frequency invariance");
- }
- if (driver_data->setpolicy)
- driver_data->flags |= CPUFREQ_CONST_LOOPS;
- if (cpufreq_boost_supported()) {
- ret = create_boost_sysfs_file();
- if (ret)
- goto err_null_driver;
- }
- ret = subsys_interface_register(&cpufreq_interface);
- if (ret)
- goto err_boost_unreg;
- if (!(cpufreq_driver->flags & CPUFREQ_STICKY) &&
- list_empty(&cpufreq_policy_list)) {
- /* if all ->init() calls failed, unregister */
- ret = -ENODEV;
- pr_debug("%s: No CPU initialized for driver %s\n", __func__,
- driver_data->name);
- goto err_if_unreg;
- }
- ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN,
- "cpufreq:online",
- cpuhp_cpufreq_online,
- cpuhp_cpufreq_offline);
- if (ret < 0)
- goto err_if_unreg;
- hp_online = ret;
- ret = 0;
- pr_debug("driver %s up and running\n", driver_data->name);
- goto out;
- err_if_unreg:
- subsys_interface_unregister(&cpufreq_interface);
- err_boost_unreg:
- remove_boost_sysfs_file();
- err_null_driver:
- write_lock_irqsave(&cpufreq_driver_lock, flags);
- cpufreq_driver = NULL;
- write_unlock_irqrestore(&cpufreq_driver_lock, flags);
- out:
- cpus_read_unlock();
- return ret;
- }
- EXPORT_SYMBOL_GPL(cpufreq_register_driver);
- /*
- * cpufreq_unregister_driver - unregister the current CPUFreq driver
- *
- * Unregister the current CPUFreq driver. Only call this if you have
- * the right to do so, i.e. if you have succeeded in initialising before!
- * Returns zero if successful, and -EINVAL if the cpufreq_driver is
- * currently not initialised.
- */
- int cpufreq_unregister_driver(struct cpufreq_driver *driver)
- {
- unsigned long flags;
- if (!cpufreq_driver || (driver != cpufreq_driver))
- return -EINVAL;
- pr_debug("unregistering driver %s\n", driver->name);
- /* Protect against concurrent cpu hotplug */
- cpus_read_lock();
- subsys_interface_unregister(&cpufreq_interface);
- remove_boost_sysfs_file();
- static_branch_disable_cpuslocked(&cpufreq_freq_invariance);
- cpuhp_remove_state_nocalls_cpuslocked(hp_online);
- write_lock_irqsave(&cpufreq_driver_lock, flags);
- cpufreq_driver = NULL;
- write_unlock_irqrestore(&cpufreq_driver_lock, flags);
- cpus_read_unlock();
- return 0;
- }
- EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
- static int __init cpufreq_core_init(void)
- {
- struct cpufreq_governor *gov = cpufreq_default_governor();
- if (cpufreq_disabled())
- return -ENODEV;
- cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj);
- BUG_ON(!cpufreq_global_kobject);
- if (!strlen(default_governor))
- strncpy(default_governor, gov->name, CPUFREQ_NAME_LEN);
- return 0;
- }
- module_param(off, int, 0444);
- module_param_string(default_governor, default_governor, CPUFREQ_NAME_LEN, 0444);
- core_initcall(cpufreq_core_init);
|