lapic.c 74 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Local APIC virtualization
  4. *
  5. * Copyright (C) 2006 Qumranet, Inc.
  6. * Copyright (C) 2007 Novell
  7. * Copyright (C) 2007 Intel
  8. * Copyright 2009 Red Hat, Inc. and/or its affiliates.
  9. *
  10. * Authors:
  11. * Dor Laor <dor.laor@qumranet.com>
  12. * Gregory Haskins <ghaskins@novell.com>
  13. * Yaozu (Eddie) Dong <eddie.dong@intel.com>
  14. *
  15. * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
  16. */
  17. #include <linux/kvm_host.h>
  18. #include <linux/kvm.h>
  19. #include <linux/mm.h>
  20. #include <linux/highmem.h>
  21. #include <linux/smp.h>
  22. #include <linux/hrtimer.h>
  23. #include <linux/io.h>
  24. #include <linux/export.h>
  25. #include <linux/math64.h>
  26. #include <linux/slab.h>
  27. #include <asm/processor.h>
  28. #include <asm/msr.h>
  29. #include <asm/page.h>
  30. #include <asm/current.h>
  31. #include <asm/apicdef.h>
  32. #include <asm/delay.h>
  33. #include <linux/atomic.h>
  34. #include <linux/jump_label.h>
  35. #include "kvm_cache_regs.h"
  36. #include "irq.h"
  37. #include "ioapic.h"
  38. #include "trace.h"
  39. #include "x86.h"
  40. #include "cpuid.h"
  41. #include "hyperv.h"
  42. #ifndef CONFIG_X86_64
  43. #define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
  44. #else
  45. #define mod_64(x, y) ((x) % (y))
  46. #endif
  47. #define PRId64 "d"
  48. #define PRIx64 "llx"
  49. #define PRIu64 "u"
  50. #define PRIo64 "o"
  51. /* 14 is the version for Xeon and Pentium 8.4.8*/
  52. #define APIC_VERSION (0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16))
  53. #define LAPIC_MMIO_LENGTH (1 << 12)
  54. /* followed define is not in apicdef.h */
  55. #define MAX_APIC_VECTOR 256
  56. #define APIC_VECTORS_PER_REG 32
  57. static bool lapic_timer_advance_dynamic __read_mostly;
  58. #define LAPIC_TIMER_ADVANCE_ADJUST_MIN 100 /* clock cycles */
  59. #define LAPIC_TIMER_ADVANCE_ADJUST_MAX 10000 /* clock cycles */
  60. #define LAPIC_TIMER_ADVANCE_NS_INIT 1000
  61. #define LAPIC_TIMER_ADVANCE_NS_MAX 5000
  62. /* step-by-step approximation to mitigate fluctuation */
  63. #define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8
  64. static inline int apic_test_vector(int vec, void *bitmap)
  65. {
  66. return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  67. }
  68. bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector)
  69. {
  70. struct kvm_lapic *apic = vcpu->arch.apic;
  71. return apic_test_vector(vector, apic->regs + APIC_ISR) ||
  72. apic_test_vector(vector, apic->regs + APIC_IRR);
  73. }
  74. static inline int __apic_test_and_set_vector(int vec, void *bitmap)
  75. {
  76. return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  77. }
  78. static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
  79. {
  80. return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  81. }
  82. struct static_key_deferred apic_hw_disabled __read_mostly;
  83. struct static_key_deferred apic_sw_disabled __read_mostly;
  84. static inline int apic_enabled(struct kvm_lapic *apic)
  85. {
  86. return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic);
  87. }
  88. #define LVT_MASK \
  89. (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)
  90. #define LINT_MASK \
  91. (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
  92. APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
  93. static inline u32 kvm_x2apic_id(struct kvm_lapic *apic)
  94. {
  95. return apic->vcpu->vcpu_id;
  96. }
  97. static bool kvm_can_post_timer_interrupt(struct kvm_vcpu *vcpu)
  98. {
  99. return pi_inject_timer && kvm_vcpu_apicv_active(vcpu);
  100. }
  101. bool kvm_can_use_hv_timer(struct kvm_vcpu *vcpu)
  102. {
  103. return kvm_x86_ops.set_hv_timer
  104. && !(kvm_mwait_in_guest(vcpu->kvm) ||
  105. kvm_can_post_timer_interrupt(vcpu));
  106. }
  107. EXPORT_SYMBOL_GPL(kvm_can_use_hv_timer);
  108. static bool kvm_use_posted_timer_interrupt(struct kvm_vcpu *vcpu)
  109. {
  110. return kvm_can_post_timer_interrupt(vcpu) && vcpu->mode == IN_GUEST_MODE;
  111. }
  112. static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map,
  113. u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) {
  114. switch (map->mode) {
  115. case KVM_APIC_MODE_X2APIC: {
  116. u32 offset = (dest_id >> 16) * 16;
  117. u32 max_apic_id = map->max_apic_id;
  118. if (offset <= max_apic_id) {
  119. u8 cluster_size = min(max_apic_id - offset + 1, 16U);
  120. offset = array_index_nospec(offset, map->max_apic_id + 1);
  121. *cluster = &map->phys_map[offset];
  122. *mask = dest_id & (0xffff >> (16 - cluster_size));
  123. } else {
  124. *mask = 0;
  125. }
  126. return true;
  127. }
  128. case KVM_APIC_MODE_XAPIC_FLAT:
  129. *cluster = map->xapic_flat_map;
  130. *mask = dest_id & 0xff;
  131. return true;
  132. case KVM_APIC_MODE_XAPIC_CLUSTER:
  133. *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf];
  134. *mask = dest_id & 0xf;
  135. return true;
  136. default:
  137. /* Not optimized. */
  138. return false;
  139. }
  140. }
  141. static void kvm_apic_map_free(struct rcu_head *rcu)
  142. {
  143. struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu);
  144. kvfree(map);
  145. }
  146. /*
  147. * CLEAN -> DIRTY and UPDATE_IN_PROGRESS -> DIRTY changes happen without a lock.
  148. *
  149. * DIRTY -> UPDATE_IN_PROGRESS and UPDATE_IN_PROGRESS -> CLEAN happen with
  150. * apic_map_lock_held.
  151. */
  152. enum {
  153. CLEAN,
  154. UPDATE_IN_PROGRESS,
  155. DIRTY
  156. };
  157. void kvm_recalculate_apic_map(struct kvm *kvm)
  158. {
  159. struct kvm_apic_map *new, *old = NULL;
  160. struct kvm_vcpu *vcpu;
  161. int i;
  162. u32 max_id = 255; /* enough space for any xAPIC ID */
  163. /* Read kvm->arch.apic_map_dirty before kvm->arch.apic_map. */
  164. if (atomic_read_acquire(&kvm->arch.apic_map_dirty) == CLEAN)
  165. return;
  166. mutex_lock(&kvm->arch.apic_map_lock);
  167. /*
  168. * Read kvm->arch.apic_map_dirty before kvm->arch.apic_map
  169. * (if clean) or the APIC registers (if dirty).
  170. */
  171. if (atomic_cmpxchg_acquire(&kvm->arch.apic_map_dirty,
  172. DIRTY, UPDATE_IN_PROGRESS) == CLEAN) {
  173. /* Someone else has updated the map. */
  174. mutex_unlock(&kvm->arch.apic_map_lock);
  175. return;
  176. }
  177. kvm_for_each_vcpu(i, vcpu, kvm)
  178. if (kvm_apic_present(vcpu))
  179. max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic));
  180. new = kvzalloc(sizeof(struct kvm_apic_map) +
  181. sizeof(struct kvm_lapic *) * ((u64)max_id + 1),
  182. GFP_KERNEL_ACCOUNT);
  183. if (!new)
  184. goto out;
  185. new->max_apic_id = max_id;
  186. kvm_for_each_vcpu(i, vcpu, kvm) {
  187. struct kvm_lapic *apic = vcpu->arch.apic;
  188. struct kvm_lapic **cluster;
  189. u16 mask;
  190. u32 ldr;
  191. u8 xapic_id;
  192. u32 x2apic_id;
  193. if (!kvm_apic_present(vcpu))
  194. continue;
  195. xapic_id = kvm_xapic_id(apic);
  196. x2apic_id = kvm_x2apic_id(apic);
  197. /* Hotplug hack: see kvm_apic_match_physical_addr(), ... */
  198. if ((apic_x2apic_mode(apic) || x2apic_id > 0xff) &&
  199. x2apic_id <= new->max_apic_id)
  200. new->phys_map[x2apic_id] = apic;
  201. /*
  202. * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around,
  203. * prevent them from masking VCPUs with APIC ID <= 0xff.
  204. */
  205. if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id])
  206. new->phys_map[xapic_id] = apic;
  207. if (!kvm_apic_sw_enabled(apic))
  208. continue;
  209. ldr = kvm_lapic_get_reg(apic, APIC_LDR);
  210. if (apic_x2apic_mode(apic)) {
  211. new->mode |= KVM_APIC_MODE_X2APIC;
  212. } else if (ldr) {
  213. ldr = GET_APIC_LOGICAL_ID(ldr);
  214. if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT)
  215. new->mode |= KVM_APIC_MODE_XAPIC_FLAT;
  216. else
  217. new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER;
  218. }
  219. if (!kvm_apic_map_get_logical_dest(new, ldr, &cluster, &mask))
  220. continue;
  221. if (mask)
  222. cluster[ffs(mask) - 1] = apic;
  223. }
  224. out:
  225. old = rcu_dereference_protected(kvm->arch.apic_map,
  226. lockdep_is_held(&kvm->arch.apic_map_lock));
  227. rcu_assign_pointer(kvm->arch.apic_map, new);
  228. /*
  229. * Write kvm->arch.apic_map before clearing apic->apic_map_dirty.
  230. * If another update has come in, leave it DIRTY.
  231. */
  232. atomic_cmpxchg_release(&kvm->arch.apic_map_dirty,
  233. UPDATE_IN_PROGRESS, CLEAN);
  234. mutex_unlock(&kvm->arch.apic_map_lock);
  235. if (old)
  236. call_rcu(&old->rcu, kvm_apic_map_free);
  237. kvm_make_scan_ioapic_request(kvm);
  238. }
  239. static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
  240. {
  241. bool enabled = val & APIC_SPIV_APIC_ENABLED;
  242. kvm_lapic_set_reg(apic, APIC_SPIV, val);
  243. if (enabled != apic->sw_enabled) {
  244. apic->sw_enabled = enabled;
  245. if (enabled)
  246. static_key_slow_dec_deferred(&apic_sw_disabled);
  247. else
  248. static_key_slow_inc(&apic_sw_disabled.key);
  249. atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
  250. }
  251. }
  252. static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id)
  253. {
  254. kvm_lapic_set_reg(apic, APIC_ID, id << 24);
  255. atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
  256. }
  257. static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id)
  258. {
  259. kvm_lapic_set_reg(apic, APIC_LDR, id);
  260. atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
  261. }
  262. static inline void kvm_apic_set_dfr(struct kvm_lapic *apic, u32 val)
  263. {
  264. kvm_lapic_set_reg(apic, APIC_DFR, val);
  265. atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
  266. }
  267. static inline u32 kvm_apic_calc_x2apic_ldr(u32 id)
  268. {
  269. return ((id >> 4) << 16) | (1 << (id & 0xf));
  270. }
  271. static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id)
  272. {
  273. u32 ldr = kvm_apic_calc_x2apic_ldr(id);
  274. WARN_ON_ONCE(id != apic->vcpu->vcpu_id);
  275. kvm_lapic_set_reg(apic, APIC_ID, id);
  276. kvm_lapic_set_reg(apic, APIC_LDR, ldr);
  277. atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
  278. }
  279. static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
  280. {
  281. return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
  282. }
  283. static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
  284. {
  285. return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT;
  286. }
  287. static inline int apic_lvtt_period(struct kvm_lapic *apic)
  288. {
  289. return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC;
  290. }
  291. static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
  292. {
  293. return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE;
  294. }
  295. static inline int apic_lvt_nmi_mode(u32 lvt_val)
  296. {
  297. return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI;
  298. }
  299. void kvm_apic_set_version(struct kvm_vcpu *vcpu)
  300. {
  301. struct kvm_lapic *apic = vcpu->arch.apic;
  302. u32 v = APIC_VERSION;
  303. if (!lapic_in_kernel(vcpu))
  304. return;
  305. /*
  306. * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation)
  307. * which doesn't have EOI register; Some buggy OSes (e.g. Windows with
  308. * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC
  309. * version first and level-triggered interrupts never get EOIed in
  310. * IOAPIC.
  311. */
  312. if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) &&
  313. !ioapic_in_kernel(vcpu->kvm))
  314. v |= APIC_LVR_DIRECTED_EOI;
  315. kvm_lapic_set_reg(apic, APIC_LVR, v);
  316. }
  317. static const unsigned int apic_lvt_mask[KVM_APIC_LVT_NUM] = {
  318. LVT_MASK , /* part LVTT mask, timer mode mask added at runtime */
  319. LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */
  320. LVT_MASK | APIC_MODE_MASK, /* LVTPC */
  321. LINT_MASK, LINT_MASK, /* LVT0-1 */
  322. LVT_MASK /* LVTERR */
  323. };
  324. static int find_highest_vector(void *bitmap)
  325. {
  326. int vec;
  327. u32 *reg;
  328. for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG;
  329. vec >= 0; vec -= APIC_VECTORS_PER_REG) {
  330. reg = bitmap + REG_POS(vec);
  331. if (*reg)
  332. return __fls(*reg) + vec;
  333. }
  334. return -1;
  335. }
  336. static u8 count_vectors(void *bitmap)
  337. {
  338. int vec;
  339. u32 *reg;
  340. u8 count = 0;
  341. for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) {
  342. reg = bitmap + REG_POS(vec);
  343. count += hweight32(*reg);
  344. }
  345. return count;
  346. }
  347. bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr)
  348. {
  349. u32 i, vec;
  350. u32 pir_val, irr_val, prev_irr_val;
  351. int max_updated_irr;
  352. max_updated_irr = -1;
  353. *max_irr = -1;
  354. for (i = vec = 0; i <= 7; i++, vec += 32) {
  355. pir_val = READ_ONCE(pir[i]);
  356. irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10));
  357. if (pir_val) {
  358. prev_irr_val = irr_val;
  359. irr_val |= xchg(&pir[i], 0);
  360. *((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val;
  361. if (prev_irr_val != irr_val) {
  362. max_updated_irr =
  363. __fls(irr_val ^ prev_irr_val) + vec;
  364. }
  365. }
  366. if (irr_val)
  367. *max_irr = __fls(irr_val) + vec;
  368. }
  369. return ((max_updated_irr != -1) &&
  370. (max_updated_irr == *max_irr));
  371. }
  372. EXPORT_SYMBOL_GPL(__kvm_apic_update_irr);
  373. bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr)
  374. {
  375. struct kvm_lapic *apic = vcpu->arch.apic;
  376. return __kvm_apic_update_irr(pir, apic->regs, max_irr);
  377. }
  378. EXPORT_SYMBOL_GPL(kvm_apic_update_irr);
  379. static inline int apic_search_irr(struct kvm_lapic *apic)
  380. {
  381. return find_highest_vector(apic->regs + APIC_IRR);
  382. }
  383. static inline int apic_find_highest_irr(struct kvm_lapic *apic)
  384. {
  385. int result;
  386. /*
  387. * Note that irr_pending is just a hint. It will be always
  388. * true with virtual interrupt delivery enabled.
  389. */
  390. if (!apic->irr_pending)
  391. return -1;
  392. result = apic_search_irr(apic);
  393. ASSERT(result == -1 || result >= 16);
  394. return result;
  395. }
  396. static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
  397. {
  398. struct kvm_vcpu *vcpu;
  399. vcpu = apic->vcpu;
  400. if (unlikely(vcpu->arch.apicv_active)) {
  401. /* need to update RVI */
  402. kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
  403. kvm_x86_ops.hwapic_irr_update(vcpu,
  404. apic_find_highest_irr(apic));
  405. } else {
  406. apic->irr_pending = false;
  407. kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
  408. if (apic_search_irr(apic) != -1)
  409. apic->irr_pending = true;
  410. }
  411. }
  412. void kvm_apic_clear_irr(struct kvm_vcpu *vcpu, int vec)
  413. {
  414. apic_clear_irr(vec, vcpu->arch.apic);
  415. }
  416. EXPORT_SYMBOL_GPL(kvm_apic_clear_irr);
  417. static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
  418. {
  419. struct kvm_vcpu *vcpu;
  420. if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR))
  421. return;
  422. vcpu = apic->vcpu;
  423. /*
  424. * With APIC virtualization enabled, all caching is disabled
  425. * because the processor can modify ISR under the hood. Instead
  426. * just set SVI.
  427. */
  428. if (unlikely(vcpu->arch.apicv_active))
  429. kvm_x86_ops.hwapic_isr_update(vcpu, vec);
  430. else {
  431. ++apic->isr_count;
  432. BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
  433. /*
  434. * ISR (in service register) bit is set when injecting an interrupt.
  435. * The highest vector is injected. Thus the latest bit set matches
  436. * the highest bit in ISR.
  437. */
  438. apic->highest_isr_cache = vec;
  439. }
  440. }
  441. static inline int apic_find_highest_isr(struct kvm_lapic *apic)
  442. {
  443. int result;
  444. /*
  445. * Note that isr_count is always 1, and highest_isr_cache
  446. * is always -1, with APIC virtualization enabled.
  447. */
  448. if (!apic->isr_count)
  449. return -1;
  450. if (likely(apic->highest_isr_cache != -1))
  451. return apic->highest_isr_cache;
  452. result = find_highest_vector(apic->regs + APIC_ISR);
  453. ASSERT(result == -1 || result >= 16);
  454. return result;
  455. }
  456. static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
  457. {
  458. struct kvm_vcpu *vcpu;
  459. if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR))
  460. return;
  461. vcpu = apic->vcpu;
  462. /*
  463. * We do get here for APIC virtualization enabled if the guest
  464. * uses the Hyper-V APIC enlightenment. In this case we may need
  465. * to trigger a new interrupt delivery by writing the SVI field;
  466. * on the other hand isr_count and highest_isr_cache are unused
  467. * and must be left alone.
  468. */
  469. if (unlikely(vcpu->arch.apicv_active))
  470. kvm_x86_ops.hwapic_isr_update(vcpu,
  471. apic_find_highest_isr(apic));
  472. else {
  473. --apic->isr_count;
  474. BUG_ON(apic->isr_count < 0);
  475. apic->highest_isr_cache = -1;
  476. }
  477. }
  478. int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
  479. {
  480. /* This may race with setting of irr in __apic_accept_irq() and
  481. * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
  482. * will cause vmexit immediately and the value will be recalculated
  483. * on the next vmentry.
  484. */
  485. return apic_find_highest_irr(vcpu->arch.apic);
  486. }
  487. EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr);
  488. static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
  489. int vector, int level, int trig_mode,
  490. struct dest_map *dest_map);
  491. int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
  492. struct dest_map *dest_map)
  493. {
  494. struct kvm_lapic *apic = vcpu->arch.apic;
  495. return __apic_accept_irq(apic, irq->delivery_mode, irq->vector,
  496. irq->level, irq->trig_mode, dest_map);
  497. }
  498. static int __pv_send_ipi(unsigned long *ipi_bitmap, struct kvm_apic_map *map,
  499. struct kvm_lapic_irq *irq, u32 min)
  500. {
  501. int i, count = 0;
  502. struct kvm_vcpu *vcpu;
  503. if (min > map->max_apic_id)
  504. return 0;
  505. for_each_set_bit(i, ipi_bitmap,
  506. min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) {
  507. if (map->phys_map[min + i]) {
  508. vcpu = map->phys_map[min + i]->vcpu;
  509. count += kvm_apic_set_irq(vcpu, irq, NULL);
  510. }
  511. }
  512. return count;
  513. }
  514. int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
  515. unsigned long ipi_bitmap_high, u32 min,
  516. unsigned long icr, int op_64_bit)
  517. {
  518. struct kvm_apic_map *map;
  519. struct kvm_lapic_irq irq = {0};
  520. int cluster_size = op_64_bit ? 64 : 32;
  521. int count;
  522. if (icr & (APIC_DEST_MASK | APIC_SHORT_MASK))
  523. return -KVM_EINVAL;
  524. irq.vector = icr & APIC_VECTOR_MASK;
  525. irq.delivery_mode = icr & APIC_MODE_MASK;
  526. irq.level = (icr & APIC_INT_ASSERT) != 0;
  527. irq.trig_mode = icr & APIC_INT_LEVELTRIG;
  528. rcu_read_lock();
  529. map = rcu_dereference(kvm->arch.apic_map);
  530. count = -EOPNOTSUPP;
  531. if (likely(map)) {
  532. count = __pv_send_ipi(&ipi_bitmap_low, map, &irq, min);
  533. min += cluster_size;
  534. count += __pv_send_ipi(&ipi_bitmap_high, map, &irq, min);
  535. }
  536. rcu_read_unlock();
  537. return count;
  538. }
  539. static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val)
  540. {
  541. return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
  542. sizeof(val));
  543. }
  544. static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val)
  545. {
  546. return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
  547. sizeof(*val));
  548. }
  549. static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
  550. {
  551. return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
  552. }
  553. static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu)
  554. {
  555. u8 val;
  556. if (pv_eoi_get_user(vcpu, &val) < 0) {
  557. printk(KERN_WARNING "Can't read EOI MSR value: 0x%llx\n",
  558. (unsigned long long)vcpu->arch.pv_eoi.msr_val);
  559. return false;
  560. }
  561. return val & 0x1;
  562. }
  563. static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
  564. {
  565. if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) {
  566. printk(KERN_WARNING "Can't set EOI MSR value: 0x%llx\n",
  567. (unsigned long long)vcpu->arch.pv_eoi.msr_val);
  568. return;
  569. }
  570. __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
  571. }
  572. static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
  573. {
  574. if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) {
  575. printk(KERN_WARNING "Can't clear EOI MSR value: 0x%llx\n",
  576. (unsigned long long)vcpu->arch.pv_eoi.msr_val);
  577. return;
  578. }
  579. __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
  580. }
  581. static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr)
  582. {
  583. int highest_irr;
  584. if (apic->vcpu->arch.apicv_active)
  585. highest_irr = kvm_x86_ops.sync_pir_to_irr(apic->vcpu);
  586. else
  587. highest_irr = apic_find_highest_irr(apic);
  588. if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr)
  589. return -1;
  590. return highest_irr;
  591. }
  592. static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr)
  593. {
  594. u32 tpr, isrv, ppr, old_ppr;
  595. int isr;
  596. old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI);
  597. tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI);
  598. isr = apic_find_highest_isr(apic);
  599. isrv = (isr != -1) ? isr : 0;
  600. if ((tpr & 0xf0) >= (isrv & 0xf0))
  601. ppr = tpr & 0xff;
  602. else
  603. ppr = isrv & 0xf0;
  604. *new_ppr = ppr;
  605. if (old_ppr != ppr)
  606. kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr);
  607. return ppr < old_ppr;
  608. }
  609. static void apic_update_ppr(struct kvm_lapic *apic)
  610. {
  611. u32 ppr;
  612. if (__apic_update_ppr(apic, &ppr) &&
  613. apic_has_interrupt_for_ppr(apic, ppr) != -1)
  614. kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
  615. }
  616. void kvm_apic_update_ppr(struct kvm_vcpu *vcpu)
  617. {
  618. apic_update_ppr(vcpu->arch.apic);
  619. }
  620. EXPORT_SYMBOL_GPL(kvm_apic_update_ppr);
  621. static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
  622. {
  623. kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr);
  624. apic_update_ppr(apic);
  625. }
  626. static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda)
  627. {
  628. return mda == (apic_x2apic_mode(apic) ?
  629. X2APIC_BROADCAST : APIC_BROADCAST);
  630. }
  631. static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda)
  632. {
  633. if (kvm_apic_broadcast(apic, mda))
  634. return true;
  635. if (apic_x2apic_mode(apic))
  636. return mda == kvm_x2apic_id(apic);
  637. /*
  638. * Hotplug hack: Make LAPIC in xAPIC mode also accept interrupts as if
  639. * it were in x2APIC mode. Hotplugged VCPUs start in xAPIC mode and
  640. * this allows unique addressing of VCPUs with APIC ID over 0xff.
  641. * The 0xff condition is needed because writeable xAPIC ID.
  642. */
  643. if (kvm_x2apic_id(apic) > 0xff && mda == kvm_x2apic_id(apic))
  644. return true;
  645. return mda == kvm_xapic_id(apic);
  646. }
  647. static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
  648. {
  649. u32 logical_id;
  650. if (kvm_apic_broadcast(apic, mda))
  651. return true;
  652. logical_id = kvm_lapic_get_reg(apic, APIC_LDR);
  653. if (apic_x2apic_mode(apic))
  654. return ((logical_id >> 16) == (mda >> 16))
  655. && (logical_id & mda & 0xffff) != 0;
  656. logical_id = GET_APIC_LOGICAL_ID(logical_id);
  657. switch (kvm_lapic_get_reg(apic, APIC_DFR)) {
  658. case APIC_DFR_FLAT:
  659. return (logical_id & mda) != 0;
  660. case APIC_DFR_CLUSTER:
  661. return ((logical_id >> 4) == (mda >> 4))
  662. && (logical_id & mda & 0xf) != 0;
  663. default:
  664. return false;
  665. }
  666. }
  667. /* The KVM local APIC implementation has two quirks:
  668. *
  669. * - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs
  670. * in xAPIC mode if the "destination & 0xff" matches its xAPIC ID.
  671. * KVM doesn't do that aliasing.
  672. *
  673. * - in-kernel IOAPIC messages have to be delivered directly to
  674. * x2APIC, because the kernel does not support interrupt remapping.
  675. * In order to support broadcast without interrupt remapping, x2APIC
  676. * rewrites the destination of non-IPI messages from APIC_BROADCAST
  677. * to X2APIC_BROADCAST.
  678. *
  679. * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API. This is
  680. * important when userspace wants to use x2APIC-format MSIs, because
  681. * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
  682. */
  683. static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id,
  684. struct kvm_lapic *source, struct kvm_lapic *target)
  685. {
  686. bool ipi = source != NULL;
  687. if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled &&
  688. !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target))
  689. return X2APIC_BROADCAST;
  690. return dest_id;
  691. }
  692. bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
  693. int shorthand, unsigned int dest, int dest_mode)
  694. {
  695. struct kvm_lapic *target = vcpu->arch.apic;
  696. u32 mda = kvm_apic_mda(vcpu, dest, source, target);
  697. ASSERT(target);
  698. switch (shorthand) {
  699. case APIC_DEST_NOSHORT:
  700. if (dest_mode == APIC_DEST_PHYSICAL)
  701. return kvm_apic_match_physical_addr(target, mda);
  702. else
  703. return kvm_apic_match_logical_addr(target, mda);
  704. case APIC_DEST_SELF:
  705. return target == source;
  706. case APIC_DEST_ALLINC:
  707. return true;
  708. case APIC_DEST_ALLBUT:
  709. return target != source;
  710. default:
  711. return false;
  712. }
  713. }
  714. EXPORT_SYMBOL_GPL(kvm_apic_match_dest);
  715. int kvm_vector_to_index(u32 vector, u32 dest_vcpus,
  716. const unsigned long *bitmap, u32 bitmap_size)
  717. {
  718. u32 mod;
  719. int i, idx = -1;
  720. mod = vector % dest_vcpus;
  721. for (i = 0; i <= mod; i++) {
  722. idx = find_next_bit(bitmap, bitmap_size, idx + 1);
  723. BUG_ON(idx == bitmap_size);
  724. }
  725. return idx;
  726. }
  727. static void kvm_apic_disabled_lapic_found(struct kvm *kvm)
  728. {
  729. if (!kvm->arch.disabled_lapic_found) {
  730. kvm->arch.disabled_lapic_found = true;
  731. printk(KERN_INFO
  732. "Disabled LAPIC found during irq injection\n");
  733. }
  734. }
  735. static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src,
  736. struct kvm_lapic_irq *irq, struct kvm_apic_map *map)
  737. {
  738. if (kvm->arch.x2apic_broadcast_quirk_disabled) {
  739. if ((irq->dest_id == APIC_BROADCAST &&
  740. map->mode != KVM_APIC_MODE_X2APIC))
  741. return true;
  742. if (irq->dest_id == X2APIC_BROADCAST)
  743. return true;
  744. } else {
  745. bool x2apic_ipi = src && *src && apic_x2apic_mode(*src);
  746. if (irq->dest_id == (x2apic_ipi ?
  747. X2APIC_BROADCAST : APIC_BROADCAST))
  748. return true;
  749. }
  750. return false;
  751. }
  752. /* Return true if the interrupt can be handled by using *bitmap as index mask
  753. * for valid destinations in *dst array.
  754. * Return false if kvm_apic_map_get_dest_lapic did nothing useful.
  755. * Note: we may have zero kvm_lapic destinations when we return true, which
  756. * means that the interrupt should be dropped. In this case, *bitmap would be
  757. * zero and *dst undefined.
  758. */
  759. static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm,
  760. struct kvm_lapic **src, struct kvm_lapic_irq *irq,
  761. struct kvm_apic_map *map, struct kvm_lapic ***dst,
  762. unsigned long *bitmap)
  763. {
  764. int i, lowest;
  765. if (irq->shorthand == APIC_DEST_SELF && src) {
  766. *dst = src;
  767. *bitmap = 1;
  768. return true;
  769. } else if (irq->shorthand)
  770. return false;
  771. if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map))
  772. return false;
  773. if (irq->dest_mode == APIC_DEST_PHYSICAL) {
  774. if (irq->dest_id > map->max_apic_id) {
  775. *bitmap = 0;
  776. } else {
  777. u32 dest_id = array_index_nospec(irq->dest_id, map->max_apic_id + 1);
  778. *dst = &map->phys_map[dest_id];
  779. *bitmap = 1;
  780. }
  781. return true;
  782. }
  783. *bitmap = 0;
  784. if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst,
  785. (u16 *)bitmap))
  786. return false;
  787. if (!kvm_lowest_prio_delivery(irq))
  788. return true;
  789. if (!kvm_vector_hashing_enabled()) {
  790. lowest = -1;
  791. for_each_set_bit(i, bitmap, 16) {
  792. if (!(*dst)[i])
  793. continue;
  794. if (lowest < 0)
  795. lowest = i;
  796. else if (kvm_apic_compare_prio((*dst)[i]->vcpu,
  797. (*dst)[lowest]->vcpu) < 0)
  798. lowest = i;
  799. }
  800. } else {
  801. if (!*bitmap)
  802. return true;
  803. lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap),
  804. bitmap, 16);
  805. if (!(*dst)[lowest]) {
  806. kvm_apic_disabled_lapic_found(kvm);
  807. *bitmap = 0;
  808. return true;
  809. }
  810. }
  811. *bitmap = (lowest >= 0) ? 1 << lowest : 0;
  812. return true;
  813. }
  814. bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
  815. struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map)
  816. {
  817. struct kvm_apic_map *map;
  818. unsigned long bitmap;
  819. struct kvm_lapic **dst = NULL;
  820. int i;
  821. bool ret;
  822. *r = -1;
  823. if (irq->shorthand == APIC_DEST_SELF) {
  824. *r = kvm_apic_set_irq(src->vcpu, irq, dest_map);
  825. return true;
  826. }
  827. rcu_read_lock();
  828. map = rcu_dereference(kvm->arch.apic_map);
  829. ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap);
  830. if (ret) {
  831. *r = 0;
  832. for_each_set_bit(i, &bitmap, 16) {
  833. if (!dst[i])
  834. continue;
  835. *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map);
  836. }
  837. }
  838. rcu_read_unlock();
  839. return ret;
  840. }
  841. /*
  842. * This routine tries to handle interrupts in posted mode, here is how
  843. * it deals with different cases:
  844. * - For single-destination interrupts, handle it in posted mode
  845. * - Else if vector hashing is enabled and it is a lowest-priority
  846. * interrupt, handle it in posted mode and use the following mechanism
  847. * to find the destination vCPU.
  848. * 1. For lowest-priority interrupts, store all the possible
  849. * destination vCPUs in an array.
  850. * 2. Use "guest vector % max number of destination vCPUs" to find
  851. * the right destination vCPU in the array for the lowest-priority
  852. * interrupt.
  853. * - Otherwise, use remapped mode to inject the interrupt.
  854. */
  855. bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq,
  856. struct kvm_vcpu **dest_vcpu)
  857. {
  858. struct kvm_apic_map *map;
  859. unsigned long bitmap;
  860. struct kvm_lapic **dst = NULL;
  861. bool ret = false;
  862. if (irq->shorthand)
  863. return false;
  864. rcu_read_lock();
  865. map = rcu_dereference(kvm->arch.apic_map);
  866. if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) &&
  867. hweight16(bitmap) == 1) {
  868. unsigned long i = find_first_bit(&bitmap, 16);
  869. if (dst[i]) {
  870. *dest_vcpu = dst[i]->vcpu;
  871. ret = true;
  872. }
  873. }
  874. rcu_read_unlock();
  875. return ret;
  876. }
  877. /*
  878. * Add a pending IRQ into lapic.
  879. * Return 1 if successfully added and 0 if discarded.
  880. */
  881. static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
  882. int vector, int level, int trig_mode,
  883. struct dest_map *dest_map)
  884. {
  885. int result = 0;
  886. struct kvm_vcpu *vcpu = apic->vcpu;
  887. trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode,
  888. trig_mode, vector);
  889. switch (delivery_mode) {
  890. case APIC_DM_LOWEST:
  891. vcpu->arch.apic_arb_prio++;
  892. fallthrough;
  893. case APIC_DM_FIXED:
  894. if (unlikely(trig_mode && !level))
  895. break;
  896. /* FIXME add logic for vcpu on reset */
  897. if (unlikely(!apic_enabled(apic)))
  898. break;
  899. result = 1;
  900. if (dest_map) {
  901. __set_bit(vcpu->vcpu_id, dest_map->map);
  902. dest_map->vectors[vcpu->vcpu_id] = vector;
  903. }
  904. if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) {
  905. if (trig_mode)
  906. kvm_lapic_set_vector(vector,
  907. apic->regs + APIC_TMR);
  908. else
  909. kvm_lapic_clear_vector(vector,
  910. apic->regs + APIC_TMR);
  911. }
  912. if (kvm_x86_ops.deliver_posted_interrupt(vcpu, vector)) {
  913. kvm_lapic_set_irr(vector, apic);
  914. kvm_make_request(KVM_REQ_EVENT, vcpu);
  915. kvm_vcpu_kick(vcpu);
  916. }
  917. break;
  918. case APIC_DM_REMRD:
  919. result = 1;
  920. vcpu->arch.pv.pv_unhalted = 1;
  921. kvm_make_request(KVM_REQ_EVENT, vcpu);
  922. kvm_vcpu_kick(vcpu);
  923. break;
  924. case APIC_DM_SMI:
  925. result = 1;
  926. kvm_make_request(KVM_REQ_SMI, vcpu);
  927. kvm_vcpu_kick(vcpu);
  928. break;
  929. case APIC_DM_NMI:
  930. result = 1;
  931. kvm_inject_nmi(vcpu);
  932. kvm_vcpu_kick(vcpu);
  933. break;
  934. case APIC_DM_INIT:
  935. if (!trig_mode || level) {
  936. result = 1;
  937. /* assumes that there are only KVM_APIC_INIT/SIPI */
  938. apic->pending_events = (1UL << KVM_APIC_INIT);
  939. kvm_make_request(KVM_REQ_EVENT, vcpu);
  940. kvm_vcpu_kick(vcpu);
  941. }
  942. break;
  943. case APIC_DM_STARTUP:
  944. result = 1;
  945. apic->sipi_vector = vector;
  946. /* make sure sipi_vector is visible for the receiver */
  947. smp_wmb();
  948. set_bit(KVM_APIC_SIPI, &apic->pending_events);
  949. kvm_make_request(KVM_REQ_EVENT, vcpu);
  950. kvm_vcpu_kick(vcpu);
  951. break;
  952. case APIC_DM_EXTINT:
  953. /*
  954. * Should only be called by kvm_apic_local_deliver() with LVT0,
  955. * before NMI watchdog was enabled. Already handled by
  956. * kvm_apic_accept_pic_intr().
  957. */
  958. break;
  959. default:
  960. printk(KERN_ERR "TODO: unsupported delivery mode %x\n",
  961. delivery_mode);
  962. break;
  963. }
  964. return result;
  965. }
  966. /*
  967. * This routine identifies the destination vcpus mask meant to receive the
  968. * IOAPIC interrupts. It either uses kvm_apic_map_get_dest_lapic() to find
  969. * out the destination vcpus array and set the bitmap or it traverses to
  970. * each available vcpu to identify the same.
  971. */
  972. void kvm_bitmap_or_dest_vcpus(struct kvm *kvm, struct kvm_lapic_irq *irq,
  973. unsigned long *vcpu_bitmap)
  974. {
  975. struct kvm_lapic **dest_vcpu = NULL;
  976. struct kvm_lapic *src = NULL;
  977. struct kvm_apic_map *map;
  978. struct kvm_vcpu *vcpu;
  979. unsigned long bitmap;
  980. int i, vcpu_idx;
  981. bool ret;
  982. rcu_read_lock();
  983. map = rcu_dereference(kvm->arch.apic_map);
  984. ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dest_vcpu,
  985. &bitmap);
  986. if (ret) {
  987. for_each_set_bit(i, &bitmap, 16) {
  988. if (!dest_vcpu[i])
  989. continue;
  990. vcpu_idx = dest_vcpu[i]->vcpu->vcpu_idx;
  991. __set_bit(vcpu_idx, vcpu_bitmap);
  992. }
  993. } else {
  994. kvm_for_each_vcpu(i, vcpu, kvm) {
  995. if (!kvm_apic_present(vcpu))
  996. continue;
  997. if (!kvm_apic_match_dest(vcpu, NULL,
  998. irq->shorthand,
  999. irq->dest_id,
  1000. irq->dest_mode))
  1001. continue;
  1002. __set_bit(i, vcpu_bitmap);
  1003. }
  1004. }
  1005. rcu_read_unlock();
  1006. }
  1007. int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
  1008. {
  1009. return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
  1010. }
  1011. static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector)
  1012. {
  1013. return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors);
  1014. }
  1015. static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
  1016. {
  1017. int trigger_mode;
  1018. /* Eoi the ioapic only if the ioapic doesn't own the vector. */
  1019. if (!kvm_ioapic_handles_vector(apic, vector))
  1020. return;
  1021. /* Request a KVM exit to inform the userspace IOAPIC. */
  1022. if (irqchip_split(apic->vcpu->kvm)) {
  1023. apic->vcpu->arch.pending_ioapic_eoi = vector;
  1024. kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu);
  1025. return;
  1026. }
  1027. if (apic_test_vector(vector, apic->regs + APIC_TMR))
  1028. trigger_mode = IOAPIC_LEVEL_TRIG;
  1029. else
  1030. trigger_mode = IOAPIC_EDGE_TRIG;
  1031. kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode);
  1032. }
  1033. static int apic_set_eoi(struct kvm_lapic *apic)
  1034. {
  1035. int vector = apic_find_highest_isr(apic);
  1036. trace_kvm_eoi(apic, vector);
  1037. /*
  1038. * Not every write EOI will has corresponding ISR,
  1039. * one example is when Kernel check timer on setup_IO_APIC
  1040. */
  1041. if (vector == -1)
  1042. return vector;
  1043. apic_clear_isr(vector, apic);
  1044. apic_update_ppr(apic);
  1045. if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap))
  1046. kvm_hv_synic_send_eoi(apic->vcpu, vector);
  1047. kvm_ioapic_send_eoi(apic, vector);
  1048. kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
  1049. return vector;
  1050. }
  1051. /*
  1052. * this interface assumes a trap-like exit, which has already finished
  1053. * desired side effect including vISR and vPPR update.
  1054. */
  1055. void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector)
  1056. {
  1057. struct kvm_lapic *apic = vcpu->arch.apic;
  1058. trace_kvm_eoi(apic, vector);
  1059. kvm_ioapic_send_eoi(apic, vector);
  1060. kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
  1061. }
  1062. EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);
  1063. void kvm_apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high)
  1064. {
  1065. struct kvm_lapic_irq irq;
  1066. irq.vector = icr_low & APIC_VECTOR_MASK;
  1067. irq.delivery_mode = icr_low & APIC_MODE_MASK;
  1068. irq.dest_mode = icr_low & APIC_DEST_MASK;
  1069. irq.level = (icr_low & APIC_INT_ASSERT) != 0;
  1070. irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
  1071. irq.shorthand = icr_low & APIC_SHORT_MASK;
  1072. irq.msi_redir_hint = false;
  1073. if (apic_x2apic_mode(apic))
  1074. irq.dest_id = icr_high;
  1075. else
  1076. irq.dest_id = GET_APIC_DEST_FIELD(icr_high);
  1077. trace_kvm_apic_ipi(icr_low, irq.dest_id);
  1078. kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL);
  1079. }
  1080. static u32 apic_get_tmcct(struct kvm_lapic *apic)
  1081. {
  1082. ktime_t remaining, now;
  1083. s64 ns;
  1084. u32 tmcct;
  1085. ASSERT(apic != NULL);
  1086. /* if initial count is 0, current count should also be 0 */
  1087. if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 ||
  1088. apic->lapic_timer.period == 0)
  1089. return 0;
  1090. now = ktime_get();
  1091. remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
  1092. if (ktime_to_ns(remaining) < 0)
  1093. remaining = 0;
  1094. ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period);
  1095. tmcct = div64_u64(ns,
  1096. (APIC_BUS_CYCLE_NS * apic->divide_count));
  1097. return tmcct;
  1098. }
  1099. static void __report_tpr_access(struct kvm_lapic *apic, bool write)
  1100. {
  1101. struct kvm_vcpu *vcpu = apic->vcpu;
  1102. struct kvm_run *run = vcpu->run;
  1103. kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu);
  1104. run->tpr_access.rip = kvm_rip_read(vcpu);
  1105. run->tpr_access.is_write = write;
  1106. }
  1107. static inline void report_tpr_access(struct kvm_lapic *apic, bool write)
  1108. {
  1109. if (apic->vcpu->arch.tpr_access_reporting)
  1110. __report_tpr_access(apic, write);
  1111. }
  1112. static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
  1113. {
  1114. u32 val = 0;
  1115. if (offset >= LAPIC_MMIO_LENGTH)
  1116. return 0;
  1117. switch (offset) {
  1118. case APIC_ARBPRI:
  1119. break;
  1120. case APIC_TMCCT: /* Timer CCR */
  1121. if (apic_lvtt_tscdeadline(apic))
  1122. return 0;
  1123. val = apic_get_tmcct(apic);
  1124. break;
  1125. case APIC_PROCPRI:
  1126. apic_update_ppr(apic);
  1127. val = kvm_lapic_get_reg(apic, offset);
  1128. break;
  1129. case APIC_TASKPRI:
  1130. report_tpr_access(apic, false);
  1131. fallthrough;
  1132. default:
  1133. val = kvm_lapic_get_reg(apic, offset);
  1134. break;
  1135. }
  1136. return val;
  1137. }
  1138. static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev)
  1139. {
  1140. return container_of(dev, struct kvm_lapic, dev);
  1141. }
  1142. #define APIC_REG_MASK(reg) (1ull << ((reg) >> 4))
  1143. #define APIC_REGS_MASK(first, count) \
  1144. (APIC_REG_MASK(first) * ((1ull << (count)) - 1))
  1145. int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
  1146. void *data)
  1147. {
  1148. unsigned char alignment = offset & 0xf;
  1149. u32 result;
  1150. /* this bitmask has a bit cleared for each reserved register */
  1151. u64 valid_reg_mask =
  1152. APIC_REG_MASK(APIC_ID) |
  1153. APIC_REG_MASK(APIC_LVR) |
  1154. APIC_REG_MASK(APIC_TASKPRI) |
  1155. APIC_REG_MASK(APIC_PROCPRI) |
  1156. APIC_REG_MASK(APIC_LDR) |
  1157. APIC_REG_MASK(APIC_DFR) |
  1158. APIC_REG_MASK(APIC_SPIV) |
  1159. APIC_REGS_MASK(APIC_ISR, APIC_ISR_NR) |
  1160. APIC_REGS_MASK(APIC_TMR, APIC_ISR_NR) |
  1161. APIC_REGS_MASK(APIC_IRR, APIC_ISR_NR) |
  1162. APIC_REG_MASK(APIC_ESR) |
  1163. APIC_REG_MASK(APIC_ICR) |
  1164. APIC_REG_MASK(APIC_ICR2) |
  1165. APIC_REG_MASK(APIC_LVTT) |
  1166. APIC_REG_MASK(APIC_LVTTHMR) |
  1167. APIC_REG_MASK(APIC_LVTPC) |
  1168. APIC_REG_MASK(APIC_LVT0) |
  1169. APIC_REG_MASK(APIC_LVT1) |
  1170. APIC_REG_MASK(APIC_LVTERR) |
  1171. APIC_REG_MASK(APIC_TMICT) |
  1172. APIC_REG_MASK(APIC_TMCCT) |
  1173. APIC_REG_MASK(APIC_TDCR);
  1174. /* ARBPRI is not valid on x2APIC */
  1175. if (!apic_x2apic_mode(apic))
  1176. valid_reg_mask |= APIC_REG_MASK(APIC_ARBPRI);
  1177. if (alignment + len > 4)
  1178. return 1;
  1179. if (offset > 0x3f0 || !(valid_reg_mask & APIC_REG_MASK(offset)))
  1180. return 1;
  1181. result = __apic_read(apic, offset & ~0xf);
  1182. trace_kvm_apic_read(offset, result);
  1183. switch (len) {
  1184. case 1:
  1185. case 2:
  1186. case 4:
  1187. memcpy(data, (char *)&result + alignment, len);
  1188. break;
  1189. default:
  1190. printk(KERN_ERR "Local APIC read with len = %x, "
  1191. "should be 1,2, or 4 instead\n", len);
  1192. break;
  1193. }
  1194. return 0;
  1195. }
  1196. EXPORT_SYMBOL_GPL(kvm_lapic_reg_read);
  1197. static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
  1198. {
  1199. return addr >= apic->base_address &&
  1200. addr < apic->base_address + LAPIC_MMIO_LENGTH;
  1201. }
  1202. static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
  1203. gpa_t address, int len, void *data)
  1204. {
  1205. struct kvm_lapic *apic = to_lapic(this);
  1206. u32 offset = address - apic->base_address;
  1207. if (!apic_mmio_in_range(apic, address))
  1208. return -EOPNOTSUPP;
  1209. if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
  1210. if (!kvm_check_has_quirk(vcpu->kvm,
  1211. KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
  1212. return -EOPNOTSUPP;
  1213. memset(data, 0xff, len);
  1214. return 0;
  1215. }
  1216. kvm_lapic_reg_read(apic, offset, len, data);
  1217. return 0;
  1218. }
  1219. static void update_divide_count(struct kvm_lapic *apic)
  1220. {
  1221. u32 tmp1, tmp2, tdcr;
  1222. tdcr = kvm_lapic_get_reg(apic, APIC_TDCR);
  1223. tmp1 = tdcr & 0xf;
  1224. tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
  1225. apic->divide_count = 0x1 << (tmp2 & 0x7);
  1226. }
  1227. static void limit_periodic_timer_frequency(struct kvm_lapic *apic)
  1228. {
  1229. /*
  1230. * Do not allow the guest to program periodic timers with small
  1231. * interval, since the hrtimers are not throttled by the host
  1232. * scheduler.
  1233. */
  1234. if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
  1235. s64 min_period = min_timer_period_us * 1000LL;
  1236. if (apic->lapic_timer.period < min_period) {
  1237. pr_info_ratelimited(
  1238. "kvm: vcpu %i: requested %lld ns "
  1239. "lapic timer period limited to %lld ns\n",
  1240. apic->vcpu->vcpu_id,
  1241. apic->lapic_timer.period, min_period);
  1242. apic->lapic_timer.period = min_period;
  1243. }
  1244. }
  1245. }
  1246. static void cancel_hv_timer(struct kvm_lapic *apic);
  1247. static void apic_update_lvtt(struct kvm_lapic *apic)
  1248. {
  1249. u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) &
  1250. apic->lapic_timer.timer_mode_mask;
  1251. if (apic->lapic_timer.timer_mode != timer_mode) {
  1252. if (apic_lvtt_tscdeadline(apic) != (timer_mode ==
  1253. APIC_LVT_TIMER_TSCDEADLINE)) {
  1254. hrtimer_cancel(&apic->lapic_timer.timer);
  1255. preempt_disable();
  1256. if (apic->lapic_timer.hv_timer_in_use)
  1257. cancel_hv_timer(apic);
  1258. preempt_enable();
  1259. kvm_lapic_set_reg(apic, APIC_TMICT, 0);
  1260. apic->lapic_timer.period = 0;
  1261. apic->lapic_timer.tscdeadline = 0;
  1262. }
  1263. apic->lapic_timer.timer_mode = timer_mode;
  1264. limit_periodic_timer_frequency(apic);
  1265. }
  1266. }
  1267. /*
  1268. * On APICv, this test will cause a busy wait
  1269. * during a higher-priority task.
  1270. */
  1271. static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu)
  1272. {
  1273. struct kvm_lapic *apic = vcpu->arch.apic;
  1274. u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT);
  1275. if (kvm_apic_hw_enabled(apic)) {
  1276. int vec = reg & APIC_VECTOR_MASK;
  1277. void *bitmap = apic->regs + APIC_ISR;
  1278. if (vcpu->arch.apicv_active)
  1279. bitmap = apic->regs + APIC_IRR;
  1280. if (apic_test_vector(vec, bitmap))
  1281. return true;
  1282. }
  1283. return false;
  1284. }
  1285. static inline void __wait_lapic_expire(struct kvm_vcpu *vcpu, u64 guest_cycles)
  1286. {
  1287. u64 timer_advance_ns = vcpu->arch.apic->lapic_timer.timer_advance_ns;
  1288. /*
  1289. * If the guest TSC is running at a different ratio than the host, then
  1290. * convert the delay to nanoseconds to achieve an accurate delay. Note
  1291. * that __delay() uses delay_tsc whenever the hardware has TSC, thus
  1292. * always for VMX enabled hardware.
  1293. */
  1294. if (vcpu->arch.tsc_scaling_ratio == kvm_default_tsc_scaling_ratio) {
  1295. __delay(min(guest_cycles,
  1296. nsec_to_cycles(vcpu, timer_advance_ns)));
  1297. } else {
  1298. u64 delay_ns = guest_cycles * 1000000ULL;
  1299. do_div(delay_ns, vcpu->arch.virtual_tsc_khz);
  1300. ndelay(min_t(u32, delay_ns, timer_advance_ns));
  1301. }
  1302. }
  1303. static inline void adjust_lapic_timer_advance(struct kvm_vcpu *vcpu,
  1304. s64 advance_expire_delta)
  1305. {
  1306. struct kvm_lapic *apic = vcpu->arch.apic;
  1307. u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns;
  1308. u64 ns;
  1309. /* Do not adjust for tiny fluctuations or large random spikes. */
  1310. if (abs(advance_expire_delta) > LAPIC_TIMER_ADVANCE_ADJUST_MAX ||
  1311. abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_MIN)
  1312. return;
  1313. /* too early */
  1314. if (advance_expire_delta < 0) {
  1315. ns = -advance_expire_delta * 1000000ULL;
  1316. do_div(ns, vcpu->arch.virtual_tsc_khz);
  1317. timer_advance_ns -= ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
  1318. } else {
  1319. /* too late */
  1320. ns = advance_expire_delta * 1000000ULL;
  1321. do_div(ns, vcpu->arch.virtual_tsc_khz);
  1322. timer_advance_ns += ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
  1323. }
  1324. if (unlikely(timer_advance_ns > LAPIC_TIMER_ADVANCE_NS_MAX))
  1325. timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT;
  1326. apic->lapic_timer.timer_advance_ns = timer_advance_ns;
  1327. }
  1328. static void __kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
  1329. {
  1330. struct kvm_lapic *apic = vcpu->arch.apic;
  1331. u64 guest_tsc, tsc_deadline;
  1332. tsc_deadline = apic->lapic_timer.expired_tscdeadline;
  1333. apic->lapic_timer.expired_tscdeadline = 0;
  1334. guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
  1335. apic->lapic_timer.advance_expire_delta = guest_tsc - tsc_deadline;
  1336. if (guest_tsc < tsc_deadline)
  1337. __wait_lapic_expire(vcpu, tsc_deadline - guest_tsc);
  1338. if (lapic_timer_advance_dynamic)
  1339. adjust_lapic_timer_advance(vcpu, apic->lapic_timer.advance_expire_delta);
  1340. }
  1341. void kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
  1342. {
  1343. if (lapic_in_kernel(vcpu) &&
  1344. vcpu->arch.apic->lapic_timer.expired_tscdeadline &&
  1345. vcpu->arch.apic->lapic_timer.timer_advance_ns &&
  1346. lapic_timer_int_injected(vcpu))
  1347. __kvm_wait_lapic_expire(vcpu);
  1348. }
  1349. EXPORT_SYMBOL_GPL(kvm_wait_lapic_expire);
  1350. static void kvm_apic_inject_pending_timer_irqs(struct kvm_lapic *apic)
  1351. {
  1352. struct kvm_timer *ktimer = &apic->lapic_timer;
  1353. kvm_apic_local_deliver(apic, APIC_LVTT);
  1354. if (apic_lvtt_tscdeadline(apic)) {
  1355. ktimer->tscdeadline = 0;
  1356. } else if (apic_lvtt_oneshot(apic)) {
  1357. ktimer->tscdeadline = 0;
  1358. ktimer->target_expiration = 0;
  1359. }
  1360. }
  1361. static void apic_timer_expired(struct kvm_lapic *apic, bool from_timer_fn)
  1362. {
  1363. struct kvm_vcpu *vcpu = apic->vcpu;
  1364. struct kvm_timer *ktimer = &apic->lapic_timer;
  1365. if (atomic_read(&apic->lapic_timer.pending))
  1366. return;
  1367. if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use)
  1368. ktimer->expired_tscdeadline = ktimer->tscdeadline;
  1369. if (!from_timer_fn && vcpu->arch.apicv_active) {
  1370. WARN_ON(kvm_get_running_vcpu() != vcpu);
  1371. kvm_apic_inject_pending_timer_irqs(apic);
  1372. return;
  1373. }
  1374. if (kvm_use_posted_timer_interrupt(apic->vcpu)) {
  1375. /*
  1376. * Ensure the guest's timer has truly expired before posting an
  1377. * interrupt. Open code the relevant checks to avoid querying
  1378. * lapic_timer_int_injected(), which will be false since the
  1379. * interrupt isn't yet injected. Waiting until after injecting
  1380. * is not an option since that won't help a posted interrupt.
  1381. */
  1382. if (vcpu->arch.apic->lapic_timer.expired_tscdeadline &&
  1383. vcpu->arch.apic->lapic_timer.timer_advance_ns)
  1384. __kvm_wait_lapic_expire(vcpu);
  1385. kvm_apic_inject_pending_timer_irqs(apic);
  1386. return;
  1387. }
  1388. atomic_inc(&apic->lapic_timer.pending);
  1389. kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
  1390. if (from_timer_fn)
  1391. kvm_vcpu_kick(vcpu);
  1392. }
  1393. static void start_sw_tscdeadline(struct kvm_lapic *apic)
  1394. {
  1395. struct kvm_timer *ktimer = &apic->lapic_timer;
  1396. u64 guest_tsc, tscdeadline = ktimer->tscdeadline;
  1397. u64 ns = 0;
  1398. ktime_t expire;
  1399. struct kvm_vcpu *vcpu = apic->vcpu;
  1400. unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz;
  1401. unsigned long flags;
  1402. ktime_t now;
  1403. if (unlikely(!tscdeadline || !this_tsc_khz))
  1404. return;
  1405. local_irq_save(flags);
  1406. now = ktime_get();
  1407. guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
  1408. ns = (tscdeadline - guest_tsc) * 1000000ULL;
  1409. do_div(ns, this_tsc_khz);
  1410. if (likely(tscdeadline > guest_tsc) &&
  1411. likely(ns > apic->lapic_timer.timer_advance_ns)) {
  1412. expire = ktime_add_ns(now, ns);
  1413. expire = ktime_sub_ns(expire, ktimer->timer_advance_ns);
  1414. hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS_HARD);
  1415. } else
  1416. apic_timer_expired(apic, false);
  1417. local_irq_restore(flags);
  1418. }
  1419. static inline u64 tmict_to_ns(struct kvm_lapic *apic, u32 tmict)
  1420. {
  1421. return (u64)tmict * APIC_BUS_CYCLE_NS * (u64)apic->divide_count;
  1422. }
  1423. static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor)
  1424. {
  1425. ktime_t now, remaining;
  1426. u64 ns_remaining_old, ns_remaining_new;
  1427. apic->lapic_timer.period =
  1428. tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT));
  1429. limit_periodic_timer_frequency(apic);
  1430. now = ktime_get();
  1431. remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
  1432. if (ktime_to_ns(remaining) < 0)
  1433. remaining = 0;
  1434. ns_remaining_old = ktime_to_ns(remaining);
  1435. ns_remaining_new = mul_u64_u32_div(ns_remaining_old,
  1436. apic->divide_count, old_divisor);
  1437. apic->lapic_timer.tscdeadline +=
  1438. nsec_to_cycles(apic->vcpu, ns_remaining_new) -
  1439. nsec_to_cycles(apic->vcpu, ns_remaining_old);
  1440. apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new);
  1441. }
  1442. static bool set_target_expiration(struct kvm_lapic *apic, u32 count_reg)
  1443. {
  1444. ktime_t now;
  1445. u64 tscl = rdtsc();
  1446. s64 deadline;
  1447. now = ktime_get();
  1448. apic->lapic_timer.period =
  1449. tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT));
  1450. if (!apic->lapic_timer.period) {
  1451. apic->lapic_timer.tscdeadline = 0;
  1452. return false;
  1453. }
  1454. limit_periodic_timer_frequency(apic);
  1455. deadline = apic->lapic_timer.period;
  1456. if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) {
  1457. if (unlikely(count_reg != APIC_TMICT)) {
  1458. deadline = tmict_to_ns(apic,
  1459. kvm_lapic_get_reg(apic, count_reg));
  1460. if (unlikely(deadline <= 0))
  1461. deadline = apic->lapic_timer.period;
  1462. else if (unlikely(deadline > apic->lapic_timer.period)) {
  1463. pr_info_ratelimited(
  1464. "kvm: vcpu %i: requested lapic timer restore with "
  1465. "starting count register %#x=%u (%lld ns) > initial count (%lld ns). "
  1466. "Using initial count to start timer.\n",
  1467. apic->vcpu->vcpu_id,
  1468. count_reg,
  1469. kvm_lapic_get_reg(apic, count_reg),
  1470. deadline, apic->lapic_timer.period);
  1471. kvm_lapic_set_reg(apic, count_reg, 0);
  1472. deadline = apic->lapic_timer.period;
  1473. }
  1474. }
  1475. }
  1476. apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
  1477. nsec_to_cycles(apic->vcpu, deadline);
  1478. apic->lapic_timer.target_expiration = ktime_add_ns(now, deadline);
  1479. return true;
  1480. }
  1481. static void advance_periodic_target_expiration(struct kvm_lapic *apic)
  1482. {
  1483. ktime_t now = ktime_get();
  1484. u64 tscl = rdtsc();
  1485. ktime_t delta;
  1486. /*
  1487. * Synchronize both deadlines to the same time source or
  1488. * differences in the periods (caused by differences in the
  1489. * underlying clocks or numerical approximation errors) will
  1490. * cause the two to drift apart over time as the errors
  1491. * accumulate.
  1492. */
  1493. apic->lapic_timer.target_expiration =
  1494. ktime_add_ns(apic->lapic_timer.target_expiration,
  1495. apic->lapic_timer.period);
  1496. delta = ktime_sub(apic->lapic_timer.target_expiration, now);
  1497. apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
  1498. nsec_to_cycles(apic->vcpu, delta);
  1499. }
  1500. static void start_sw_period(struct kvm_lapic *apic)
  1501. {
  1502. if (!apic->lapic_timer.period)
  1503. return;
  1504. if (ktime_after(ktime_get(),
  1505. apic->lapic_timer.target_expiration)) {
  1506. apic_timer_expired(apic, false);
  1507. if (apic_lvtt_oneshot(apic))
  1508. return;
  1509. advance_periodic_target_expiration(apic);
  1510. }
  1511. hrtimer_start(&apic->lapic_timer.timer,
  1512. apic->lapic_timer.target_expiration,
  1513. HRTIMER_MODE_ABS_HARD);
  1514. }
  1515. bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu)
  1516. {
  1517. if (!lapic_in_kernel(vcpu))
  1518. return false;
  1519. return vcpu->arch.apic->lapic_timer.hv_timer_in_use;
  1520. }
  1521. EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use);
  1522. static void cancel_hv_timer(struct kvm_lapic *apic)
  1523. {
  1524. WARN_ON(preemptible());
  1525. WARN_ON(!apic->lapic_timer.hv_timer_in_use);
  1526. kvm_x86_ops.cancel_hv_timer(apic->vcpu);
  1527. apic->lapic_timer.hv_timer_in_use = false;
  1528. }
  1529. static bool start_hv_timer(struct kvm_lapic *apic)
  1530. {
  1531. struct kvm_timer *ktimer = &apic->lapic_timer;
  1532. struct kvm_vcpu *vcpu = apic->vcpu;
  1533. bool expired;
  1534. WARN_ON(preemptible());
  1535. if (!kvm_can_use_hv_timer(vcpu))
  1536. return false;
  1537. if (!ktimer->tscdeadline)
  1538. return false;
  1539. if (kvm_x86_ops.set_hv_timer(vcpu, ktimer->tscdeadline, &expired))
  1540. return false;
  1541. ktimer->hv_timer_in_use = true;
  1542. hrtimer_cancel(&ktimer->timer);
  1543. /*
  1544. * To simplify handling the periodic timer, leave the hv timer running
  1545. * even if the deadline timer has expired, i.e. rely on the resulting
  1546. * VM-Exit to recompute the periodic timer's target expiration.
  1547. */
  1548. if (!apic_lvtt_period(apic)) {
  1549. /*
  1550. * Cancel the hv timer if the sw timer fired while the hv timer
  1551. * was being programmed, or if the hv timer itself expired.
  1552. */
  1553. if (atomic_read(&ktimer->pending)) {
  1554. cancel_hv_timer(apic);
  1555. } else if (expired) {
  1556. apic_timer_expired(apic, false);
  1557. cancel_hv_timer(apic);
  1558. }
  1559. }
  1560. trace_kvm_hv_timer_state(vcpu->vcpu_id, ktimer->hv_timer_in_use);
  1561. return true;
  1562. }
  1563. static void start_sw_timer(struct kvm_lapic *apic)
  1564. {
  1565. struct kvm_timer *ktimer = &apic->lapic_timer;
  1566. WARN_ON(preemptible());
  1567. if (apic->lapic_timer.hv_timer_in_use)
  1568. cancel_hv_timer(apic);
  1569. if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending))
  1570. return;
  1571. if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
  1572. start_sw_period(apic);
  1573. else if (apic_lvtt_tscdeadline(apic))
  1574. start_sw_tscdeadline(apic);
  1575. trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false);
  1576. }
  1577. static void restart_apic_timer(struct kvm_lapic *apic)
  1578. {
  1579. preempt_disable();
  1580. if (!apic_lvtt_period(apic) && atomic_read(&apic->lapic_timer.pending))
  1581. goto out;
  1582. if (!start_hv_timer(apic))
  1583. start_sw_timer(apic);
  1584. out:
  1585. preempt_enable();
  1586. }
  1587. void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu)
  1588. {
  1589. struct kvm_lapic *apic = vcpu->arch.apic;
  1590. preempt_disable();
  1591. /* If the preempt notifier has already run, it also called apic_timer_expired */
  1592. if (!apic->lapic_timer.hv_timer_in_use)
  1593. goto out;
  1594. WARN_ON(rcuwait_active(&vcpu->wait));
  1595. apic_timer_expired(apic, false);
  1596. cancel_hv_timer(apic);
  1597. if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
  1598. advance_periodic_target_expiration(apic);
  1599. restart_apic_timer(apic);
  1600. }
  1601. out:
  1602. preempt_enable();
  1603. }
  1604. EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer);
  1605. void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu)
  1606. {
  1607. restart_apic_timer(vcpu->arch.apic);
  1608. }
  1609. EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer);
  1610. void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu)
  1611. {
  1612. struct kvm_lapic *apic = vcpu->arch.apic;
  1613. preempt_disable();
  1614. /* Possibly the TSC deadline timer is not enabled yet */
  1615. if (apic->lapic_timer.hv_timer_in_use)
  1616. start_sw_timer(apic);
  1617. preempt_enable();
  1618. }
  1619. EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer);
  1620. void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu)
  1621. {
  1622. struct kvm_lapic *apic = vcpu->arch.apic;
  1623. WARN_ON(!apic->lapic_timer.hv_timer_in_use);
  1624. restart_apic_timer(apic);
  1625. }
  1626. static void __start_apic_timer(struct kvm_lapic *apic, u32 count_reg)
  1627. {
  1628. atomic_set(&apic->lapic_timer.pending, 0);
  1629. if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
  1630. && !set_target_expiration(apic, count_reg))
  1631. return;
  1632. restart_apic_timer(apic);
  1633. }
  1634. static void start_apic_timer(struct kvm_lapic *apic)
  1635. {
  1636. __start_apic_timer(apic, APIC_TMICT);
  1637. }
  1638. static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
  1639. {
  1640. bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val);
  1641. if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) {
  1642. apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode;
  1643. if (lvt0_in_nmi_mode) {
  1644. atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
  1645. } else
  1646. atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
  1647. }
  1648. }
  1649. int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
  1650. {
  1651. int ret = 0;
  1652. trace_kvm_apic_write(reg, val);
  1653. switch (reg) {
  1654. case APIC_ID: /* Local APIC ID */
  1655. if (!apic_x2apic_mode(apic))
  1656. kvm_apic_set_xapic_id(apic, val >> 24);
  1657. else
  1658. ret = 1;
  1659. break;
  1660. case APIC_TASKPRI:
  1661. report_tpr_access(apic, true);
  1662. apic_set_tpr(apic, val & 0xff);
  1663. break;
  1664. case APIC_EOI:
  1665. apic_set_eoi(apic);
  1666. break;
  1667. case APIC_LDR:
  1668. if (!apic_x2apic_mode(apic))
  1669. kvm_apic_set_ldr(apic, val & APIC_LDR_MASK);
  1670. else
  1671. ret = 1;
  1672. break;
  1673. case APIC_DFR:
  1674. if (!apic_x2apic_mode(apic))
  1675. kvm_apic_set_dfr(apic, val | 0x0FFFFFFF);
  1676. else
  1677. ret = 1;
  1678. break;
  1679. case APIC_SPIV: {
  1680. u32 mask = 0x3ff;
  1681. if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
  1682. mask |= APIC_SPIV_DIRECTED_EOI;
  1683. apic_set_spiv(apic, val & mask);
  1684. if (!(val & APIC_SPIV_APIC_ENABLED)) {
  1685. int i;
  1686. u32 lvt_val;
  1687. for (i = 0; i < KVM_APIC_LVT_NUM; i++) {
  1688. lvt_val = kvm_lapic_get_reg(apic,
  1689. APIC_LVTT + 0x10 * i);
  1690. kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i,
  1691. lvt_val | APIC_LVT_MASKED);
  1692. }
  1693. apic_update_lvtt(apic);
  1694. atomic_set(&apic->lapic_timer.pending, 0);
  1695. }
  1696. break;
  1697. }
  1698. case APIC_ICR:
  1699. /* No delay here, so we always clear the pending bit */
  1700. val &= ~(1 << 12);
  1701. kvm_apic_send_ipi(apic, val, kvm_lapic_get_reg(apic, APIC_ICR2));
  1702. kvm_lapic_set_reg(apic, APIC_ICR, val);
  1703. break;
  1704. case APIC_ICR2:
  1705. if (!apic_x2apic_mode(apic))
  1706. val &= 0xff000000;
  1707. kvm_lapic_set_reg(apic, APIC_ICR2, val);
  1708. break;
  1709. case APIC_LVT0:
  1710. apic_manage_nmi_watchdog(apic, val);
  1711. fallthrough;
  1712. case APIC_LVTTHMR:
  1713. case APIC_LVTPC:
  1714. case APIC_LVT1:
  1715. case APIC_LVTERR: {
  1716. /* TODO: Check vector */
  1717. size_t size;
  1718. u32 index;
  1719. if (!kvm_apic_sw_enabled(apic))
  1720. val |= APIC_LVT_MASKED;
  1721. size = ARRAY_SIZE(apic_lvt_mask);
  1722. index = array_index_nospec(
  1723. (reg - APIC_LVTT) >> 4, size);
  1724. val &= apic_lvt_mask[index];
  1725. kvm_lapic_set_reg(apic, reg, val);
  1726. break;
  1727. }
  1728. case APIC_LVTT:
  1729. if (!kvm_apic_sw_enabled(apic))
  1730. val |= APIC_LVT_MASKED;
  1731. val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask);
  1732. kvm_lapic_set_reg(apic, APIC_LVTT, val);
  1733. apic_update_lvtt(apic);
  1734. break;
  1735. case APIC_TMICT:
  1736. if (apic_lvtt_tscdeadline(apic))
  1737. break;
  1738. hrtimer_cancel(&apic->lapic_timer.timer);
  1739. kvm_lapic_set_reg(apic, APIC_TMICT, val);
  1740. start_apic_timer(apic);
  1741. break;
  1742. case APIC_TDCR: {
  1743. uint32_t old_divisor = apic->divide_count;
  1744. kvm_lapic_set_reg(apic, APIC_TDCR, val & 0xb);
  1745. update_divide_count(apic);
  1746. if (apic->divide_count != old_divisor &&
  1747. apic->lapic_timer.period) {
  1748. hrtimer_cancel(&apic->lapic_timer.timer);
  1749. update_target_expiration(apic, old_divisor);
  1750. restart_apic_timer(apic);
  1751. }
  1752. break;
  1753. }
  1754. case APIC_ESR:
  1755. if (apic_x2apic_mode(apic) && val != 0)
  1756. ret = 1;
  1757. break;
  1758. case APIC_SELF_IPI:
  1759. if (apic_x2apic_mode(apic)) {
  1760. kvm_lapic_reg_write(apic, APIC_ICR,
  1761. APIC_DEST_SELF | (val & APIC_VECTOR_MASK));
  1762. } else
  1763. ret = 1;
  1764. break;
  1765. default:
  1766. ret = 1;
  1767. break;
  1768. }
  1769. kvm_recalculate_apic_map(apic->vcpu->kvm);
  1770. return ret;
  1771. }
  1772. EXPORT_SYMBOL_GPL(kvm_lapic_reg_write);
  1773. static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
  1774. gpa_t address, int len, const void *data)
  1775. {
  1776. struct kvm_lapic *apic = to_lapic(this);
  1777. unsigned int offset = address - apic->base_address;
  1778. u32 val;
  1779. if (!apic_mmio_in_range(apic, address))
  1780. return -EOPNOTSUPP;
  1781. if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
  1782. if (!kvm_check_has_quirk(vcpu->kvm,
  1783. KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
  1784. return -EOPNOTSUPP;
  1785. return 0;
  1786. }
  1787. /*
  1788. * APIC register must be aligned on 128-bits boundary.
  1789. * 32/64/128 bits registers must be accessed thru 32 bits.
  1790. * Refer SDM 8.4.1
  1791. */
  1792. if (len != 4 || (offset & 0xf))
  1793. return 0;
  1794. val = *(u32*)data;
  1795. kvm_lapic_reg_write(apic, offset & 0xff0, val);
  1796. return 0;
  1797. }
  1798. void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
  1799. {
  1800. kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0);
  1801. }
  1802. EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);
  1803. /* emulate APIC access in a trap manner */
  1804. void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset)
  1805. {
  1806. u32 val = 0;
  1807. /* hw has done the conditional check and inst decode */
  1808. offset &= 0xff0;
  1809. kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val);
  1810. /* TODO: optimize to just emulate side effect w/o one more write */
  1811. kvm_lapic_reg_write(vcpu->arch.apic, offset, val);
  1812. }
  1813. EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode);
  1814. void kvm_free_lapic(struct kvm_vcpu *vcpu)
  1815. {
  1816. struct kvm_lapic *apic = vcpu->arch.apic;
  1817. if (!vcpu->arch.apic)
  1818. return;
  1819. hrtimer_cancel(&apic->lapic_timer.timer);
  1820. if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
  1821. static_key_slow_dec_deferred(&apic_hw_disabled);
  1822. if (!apic->sw_enabled)
  1823. static_key_slow_dec_deferred(&apic_sw_disabled);
  1824. if (apic->regs)
  1825. free_page((unsigned long)apic->regs);
  1826. kfree(apic);
  1827. }
  1828. /*
  1829. *----------------------------------------------------------------------
  1830. * LAPIC interface
  1831. *----------------------------------------------------------------------
  1832. */
  1833. u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
  1834. {
  1835. struct kvm_lapic *apic = vcpu->arch.apic;
  1836. if (!kvm_apic_present(vcpu) || !apic_lvtt_tscdeadline(apic))
  1837. return 0;
  1838. return apic->lapic_timer.tscdeadline;
  1839. }
  1840. void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
  1841. {
  1842. struct kvm_lapic *apic = vcpu->arch.apic;
  1843. if (!kvm_apic_present(vcpu) || !apic_lvtt_tscdeadline(apic))
  1844. return;
  1845. hrtimer_cancel(&apic->lapic_timer.timer);
  1846. apic->lapic_timer.tscdeadline = data;
  1847. start_apic_timer(apic);
  1848. }
  1849. void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
  1850. {
  1851. apic_set_tpr(vcpu->arch.apic, (cr8 & 0x0f) << 4);
  1852. }
  1853. u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
  1854. {
  1855. u64 tpr;
  1856. tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI);
  1857. return (tpr & 0xf0) >> 4;
  1858. }
  1859. void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
  1860. {
  1861. u64 old_value = vcpu->arch.apic_base;
  1862. struct kvm_lapic *apic = vcpu->arch.apic;
  1863. if (!apic)
  1864. value |= MSR_IA32_APICBASE_BSP;
  1865. vcpu->arch.apic_base = value;
  1866. if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE)
  1867. kvm_update_cpuid_runtime(vcpu);
  1868. if (!apic)
  1869. return;
  1870. /* update jump label if enable bit changes */
  1871. if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) {
  1872. if (value & MSR_IA32_APICBASE_ENABLE) {
  1873. kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
  1874. static_key_slow_dec_deferred(&apic_hw_disabled);
  1875. } else {
  1876. static_key_slow_inc(&apic_hw_disabled.key);
  1877. atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
  1878. }
  1879. }
  1880. if (((old_value ^ value) & X2APIC_ENABLE) && (value & X2APIC_ENABLE))
  1881. kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
  1882. if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE))
  1883. kvm_x86_ops.set_virtual_apic_mode(vcpu);
  1884. apic->base_address = apic->vcpu->arch.apic_base &
  1885. MSR_IA32_APICBASE_BASE;
  1886. if ((value & MSR_IA32_APICBASE_ENABLE) &&
  1887. apic->base_address != APIC_DEFAULT_PHYS_BASE)
  1888. pr_warn_once("APIC base relocation is unsupported by KVM");
  1889. }
  1890. void kvm_apic_update_apicv(struct kvm_vcpu *vcpu)
  1891. {
  1892. struct kvm_lapic *apic = vcpu->arch.apic;
  1893. if (vcpu->arch.apicv_active) {
  1894. /* irr_pending is always true when apicv is activated. */
  1895. apic->irr_pending = true;
  1896. apic->isr_count = 1;
  1897. } else {
  1898. apic->irr_pending = (apic_search_irr(apic) != -1);
  1899. apic->isr_count = count_vectors(apic->regs + APIC_ISR);
  1900. }
  1901. }
  1902. EXPORT_SYMBOL_GPL(kvm_apic_update_apicv);
  1903. void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
  1904. {
  1905. struct kvm_lapic *apic = vcpu->arch.apic;
  1906. int i;
  1907. if (!apic)
  1908. return;
  1909. /* Stop the timer in case it's a reset to an active apic */
  1910. hrtimer_cancel(&apic->lapic_timer.timer);
  1911. if (!init_event) {
  1912. kvm_lapic_set_base(vcpu, APIC_DEFAULT_PHYS_BASE |
  1913. MSR_IA32_APICBASE_ENABLE);
  1914. kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
  1915. }
  1916. kvm_apic_set_version(apic->vcpu);
  1917. for (i = 0; i < KVM_APIC_LVT_NUM; i++)
  1918. kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED);
  1919. apic_update_lvtt(apic);
  1920. if (kvm_vcpu_is_reset_bsp(vcpu) &&
  1921. kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED))
  1922. kvm_lapic_set_reg(apic, APIC_LVT0,
  1923. SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
  1924. apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
  1925. kvm_apic_set_dfr(apic, 0xffffffffU);
  1926. apic_set_spiv(apic, 0xff);
  1927. kvm_lapic_set_reg(apic, APIC_TASKPRI, 0);
  1928. if (!apic_x2apic_mode(apic))
  1929. kvm_apic_set_ldr(apic, 0);
  1930. kvm_lapic_set_reg(apic, APIC_ESR, 0);
  1931. kvm_lapic_set_reg(apic, APIC_ICR, 0);
  1932. kvm_lapic_set_reg(apic, APIC_ICR2, 0);
  1933. kvm_lapic_set_reg(apic, APIC_TDCR, 0);
  1934. kvm_lapic_set_reg(apic, APIC_TMICT, 0);
  1935. for (i = 0; i < 8; i++) {
  1936. kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0);
  1937. kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
  1938. kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
  1939. }
  1940. kvm_apic_update_apicv(vcpu);
  1941. apic->highest_isr_cache = -1;
  1942. update_divide_count(apic);
  1943. atomic_set(&apic->lapic_timer.pending, 0);
  1944. if (kvm_vcpu_is_bsp(vcpu))
  1945. kvm_lapic_set_base(vcpu,
  1946. vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP);
  1947. vcpu->arch.pv_eoi.msr_val = 0;
  1948. apic_update_ppr(apic);
  1949. if (vcpu->arch.apicv_active) {
  1950. kvm_x86_ops.apicv_post_state_restore(vcpu);
  1951. kvm_x86_ops.hwapic_irr_update(vcpu, -1);
  1952. kvm_x86_ops.hwapic_isr_update(vcpu, -1);
  1953. }
  1954. vcpu->arch.apic_arb_prio = 0;
  1955. vcpu->arch.apic_attention = 0;
  1956. kvm_recalculate_apic_map(vcpu->kvm);
  1957. }
  1958. /*
  1959. *----------------------------------------------------------------------
  1960. * timer interface
  1961. *----------------------------------------------------------------------
  1962. */
  1963. static bool lapic_is_periodic(struct kvm_lapic *apic)
  1964. {
  1965. return apic_lvtt_period(apic);
  1966. }
  1967. int apic_has_pending_timer(struct kvm_vcpu *vcpu)
  1968. {
  1969. struct kvm_lapic *apic = vcpu->arch.apic;
  1970. if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT))
  1971. return atomic_read(&apic->lapic_timer.pending);
  1972. return 0;
  1973. }
  1974. int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type)
  1975. {
  1976. u32 reg = kvm_lapic_get_reg(apic, lvt_type);
  1977. int vector, mode, trig_mode;
  1978. if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
  1979. vector = reg & APIC_VECTOR_MASK;
  1980. mode = reg & APIC_MODE_MASK;
  1981. trig_mode = reg & APIC_LVT_LEVEL_TRIGGER;
  1982. return __apic_accept_irq(apic, mode, vector, 1, trig_mode,
  1983. NULL);
  1984. }
  1985. return 0;
  1986. }
  1987. void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
  1988. {
  1989. struct kvm_lapic *apic = vcpu->arch.apic;
  1990. if (apic)
  1991. kvm_apic_local_deliver(apic, APIC_LVT0);
  1992. }
  1993. static const struct kvm_io_device_ops apic_mmio_ops = {
  1994. .read = apic_mmio_read,
  1995. .write = apic_mmio_write,
  1996. };
  1997. static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
  1998. {
  1999. struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
  2000. struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
  2001. apic_timer_expired(apic, true);
  2002. if (lapic_is_periodic(apic)) {
  2003. advance_periodic_target_expiration(apic);
  2004. hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
  2005. return HRTIMER_RESTART;
  2006. } else
  2007. return HRTIMER_NORESTART;
  2008. }
  2009. int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns)
  2010. {
  2011. struct kvm_lapic *apic;
  2012. ASSERT(vcpu != NULL);
  2013. apic = kzalloc(sizeof(*apic), GFP_KERNEL_ACCOUNT);
  2014. if (!apic)
  2015. goto nomem;
  2016. vcpu->arch.apic = apic;
  2017. apic->regs = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
  2018. if (!apic->regs) {
  2019. printk(KERN_ERR "malloc apic regs error for vcpu %x\n",
  2020. vcpu->vcpu_id);
  2021. goto nomem_free_apic;
  2022. }
  2023. apic->vcpu = vcpu;
  2024. hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
  2025. HRTIMER_MODE_ABS_HARD);
  2026. apic->lapic_timer.timer.function = apic_timer_fn;
  2027. if (timer_advance_ns == -1) {
  2028. apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT;
  2029. lapic_timer_advance_dynamic = true;
  2030. } else {
  2031. apic->lapic_timer.timer_advance_ns = timer_advance_ns;
  2032. lapic_timer_advance_dynamic = false;
  2033. }
  2034. /*
  2035. * APIC is created enabled. This will prevent kvm_lapic_set_base from
  2036. * thinking that APIC state has changed.
  2037. */
  2038. vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
  2039. static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
  2040. kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
  2041. return 0;
  2042. nomem_free_apic:
  2043. kfree(apic);
  2044. vcpu->arch.apic = NULL;
  2045. nomem:
  2046. return -ENOMEM;
  2047. }
  2048. int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
  2049. {
  2050. struct kvm_lapic *apic = vcpu->arch.apic;
  2051. u32 ppr;
  2052. if (!kvm_apic_present(vcpu))
  2053. return -1;
  2054. __apic_update_ppr(apic, &ppr);
  2055. return apic_has_interrupt_for_ppr(apic, ppr);
  2056. }
  2057. EXPORT_SYMBOL_GPL(kvm_apic_has_interrupt);
  2058. int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
  2059. {
  2060. u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0);
  2061. if (!kvm_apic_hw_enabled(vcpu->arch.apic))
  2062. return 1;
  2063. if ((lvt0 & APIC_LVT_MASKED) == 0 &&
  2064. GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
  2065. return 1;
  2066. return 0;
  2067. }
  2068. void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
  2069. {
  2070. struct kvm_lapic *apic = vcpu->arch.apic;
  2071. if (atomic_read(&apic->lapic_timer.pending) > 0) {
  2072. kvm_apic_inject_pending_timer_irqs(apic);
  2073. atomic_set(&apic->lapic_timer.pending, 0);
  2074. }
  2075. }
  2076. int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
  2077. {
  2078. int vector = kvm_apic_has_interrupt(vcpu);
  2079. struct kvm_lapic *apic = vcpu->arch.apic;
  2080. u32 ppr;
  2081. if (vector == -1)
  2082. return -1;
  2083. /*
  2084. * We get here even with APIC virtualization enabled, if doing
  2085. * nested virtualization and L1 runs with the "acknowledge interrupt
  2086. * on exit" mode. Then we cannot inject the interrupt via RVI,
  2087. * because the process would deliver it through the IDT.
  2088. */
  2089. apic_clear_irr(vector, apic);
  2090. if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) {
  2091. /*
  2092. * For auto-EOI interrupts, there might be another pending
  2093. * interrupt above PPR, so check whether to raise another
  2094. * KVM_REQ_EVENT.
  2095. */
  2096. apic_update_ppr(apic);
  2097. } else {
  2098. /*
  2099. * For normal interrupts, PPR has been raised and there cannot
  2100. * be a higher-priority pending interrupt---except if there was
  2101. * a concurrent interrupt injection, but that would have
  2102. * triggered KVM_REQ_EVENT already.
  2103. */
  2104. apic_set_isr(vector, apic);
  2105. __apic_update_ppr(apic, &ppr);
  2106. }
  2107. return vector;
  2108. }
  2109. static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu,
  2110. struct kvm_lapic_state *s, bool set)
  2111. {
  2112. if (apic_x2apic_mode(vcpu->arch.apic)) {
  2113. u32 *id = (u32 *)(s->regs + APIC_ID);
  2114. u32 *ldr = (u32 *)(s->regs + APIC_LDR);
  2115. if (vcpu->kvm->arch.x2apic_format) {
  2116. if (*id != vcpu->vcpu_id)
  2117. return -EINVAL;
  2118. } else {
  2119. if (set)
  2120. *id >>= 24;
  2121. else
  2122. *id <<= 24;
  2123. }
  2124. /* In x2APIC mode, the LDR is fixed and based on the id */
  2125. if (set)
  2126. *ldr = kvm_apic_calc_x2apic_ldr(*id);
  2127. }
  2128. return 0;
  2129. }
  2130. int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
  2131. {
  2132. memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s));
  2133. /*
  2134. * Get calculated timer current count for remaining timer period (if
  2135. * any) and store it in the returned register set.
  2136. */
  2137. __kvm_lapic_set_reg(s->regs, APIC_TMCCT,
  2138. __apic_read(vcpu->arch.apic, APIC_TMCCT));
  2139. return kvm_apic_state_fixup(vcpu, s, false);
  2140. }
  2141. int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
  2142. {
  2143. struct kvm_lapic *apic = vcpu->arch.apic;
  2144. int r;
  2145. kvm_lapic_set_base(vcpu, vcpu->arch.apic_base);
  2146. /* set SPIV separately to get count of SW disabled APICs right */
  2147. apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV)));
  2148. r = kvm_apic_state_fixup(vcpu, s, true);
  2149. if (r) {
  2150. kvm_recalculate_apic_map(vcpu->kvm);
  2151. return r;
  2152. }
  2153. memcpy(vcpu->arch.apic->regs, s->regs, sizeof(*s));
  2154. atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
  2155. kvm_recalculate_apic_map(vcpu->kvm);
  2156. kvm_apic_set_version(vcpu);
  2157. apic_update_ppr(apic);
  2158. hrtimer_cancel(&apic->lapic_timer.timer);
  2159. apic_update_lvtt(apic);
  2160. apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
  2161. update_divide_count(apic);
  2162. __start_apic_timer(apic, APIC_TMCCT);
  2163. kvm_apic_update_apicv(vcpu);
  2164. apic->highest_isr_cache = -1;
  2165. if (vcpu->arch.apicv_active) {
  2166. kvm_x86_ops.apicv_post_state_restore(vcpu);
  2167. kvm_x86_ops.hwapic_irr_update(vcpu,
  2168. apic_find_highest_irr(apic));
  2169. kvm_x86_ops.hwapic_isr_update(vcpu,
  2170. apic_find_highest_isr(apic));
  2171. }
  2172. kvm_make_request(KVM_REQ_EVENT, vcpu);
  2173. if (ioapic_in_kernel(vcpu->kvm))
  2174. kvm_rtc_eoi_tracking_restore_one(vcpu);
  2175. vcpu->arch.apic_arb_prio = 0;
  2176. return 0;
  2177. }
  2178. void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
  2179. {
  2180. struct hrtimer *timer;
  2181. if (!lapic_in_kernel(vcpu) ||
  2182. kvm_can_post_timer_interrupt(vcpu))
  2183. return;
  2184. timer = &vcpu->arch.apic->lapic_timer.timer;
  2185. if (hrtimer_cancel(timer))
  2186. hrtimer_start_expires(timer, HRTIMER_MODE_ABS_HARD);
  2187. }
  2188. /*
  2189. * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
  2190. *
  2191. * Detect whether guest triggered PV EOI since the
  2192. * last entry. If yes, set EOI on guests's behalf.
  2193. * Clear PV EOI in guest memory in any case.
  2194. */
  2195. static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
  2196. struct kvm_lapic *apic)
  2197. {
  2198. bool pending;
  2199. int vector;
  2200. /*
  2201. * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host
  2202. * and KVM_PV_EOI_ENABLED in guest memory as follows:
  2203. *
  2204. * KVM_APIC_PV_EOI_PENDING is unset:
  2205. * -> host disabled PV EOI.
  2206. * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set:
  2207. * -> host enabled PV EOI, guest did not execute EOI yet.
  2208. * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset:
  2209. * -> host enabled PV EOI, guest executed EOI.
  2210. */
  2211. BUG_ON(!pv_eoi_enabled(vcpu));
  2212. pending = pv_eoi_get_pending(vcpu);
  2213. /*
  2214. * Clear pending bit in any case: it will be set again on vmentry.
  2215. * While this might not be ideal from performance point of view,
  2216. * this makes sure pv eoi is only enabled when we know it's safe.
  2217. */
  2218. pv_eoi_clr_pending(vcpu);
  2219. if (pending)
  2220. return;
  2221. vector = apic_set_eoi(apic);
  2222. trace_kvm_pv_eoi(apic, vector);
  2223. }
  2224. void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
  2225. {
  2226. u32 data;
  2227. if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention))
  2228. apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic);
  2229. if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
  2230. return;
  2231. if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
  2232. sizeof(u32)))
  2233. return;
  2234. apic_set_tpr(vcpu->arch.apic, data & 0xff);
  2235. }
  2236. /*
  2237. * apic_sync_pv_eoi_to_guest - called before vmentry
  2238. *
  2239. * Detect whether it's safe to enable PV EOI and
  2240. * if yes do so.
  2241. */
  2242. static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu,
  2243. struct kvm_lapic *apic)
  2244. {
  2245. if (!pv_eoi_enabled(vcpu) ||
  2246. /* IRR set or many bits in ISR: could be nested. */
  2247. apic->irr_pending ||
  2248. /* Cache not set: could be safe but we don't bother. */
  2249. apic->highest_isr_cache == -1 ||
  2250. /* Need EOI to update ioapic. */
  2251. kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) {
  2252. /*
  2253. * PV EOI was disabled by apic_sync_pv_eoi_from_guest
  2254. * so we need not do anything here.
  2255. */
  2256. return;
  2257. }
  2258. pv_eoi_set_pending(apic->vcpu);
  2259. }
  2260. void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
  2261. {
  2262. u32 data, tpr;
  2263. int max_irr, max_isr;
  2264. struct kvm_lapic *apic = vcpu->arch.apic;
  2265. apic_sync_pv_eoi_to_guest(vcpu, apic);
  2266. if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
  2267. return;
  2268. tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff;
  2269. max_irr = apic_find_highest_irr(apic);
  2270. if (max_irr < 0)
  2271. max_irr = 0;
  2272. max_isr = apic_find_highest_isr(apic);
  2273. if (max_isr < 0)
  2274. max_isr = 0;
  2275. data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24);
  2276. kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
  2277. sizeof(u32));
  2278. }
  2279. int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
  2280. {
  2281. if (vapic_addr) {
  2282. if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
  2283. &vcpu->arch.apic->vapic_cache,
  2284. vapic_addr, sizeof(u32)))
  2285. return -EINVAL;
  2286. __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
  2287. } else {
  2288. __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
  2289. }
  2290. vcpu->arch.apic->vapic_addr = vapic_addr;
  2291. return 0;
  2292. }
  2293. int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  2294. {
  2295. struct kvm_lapic *apic = vcpu->arch.apic;
  2296. u32 reg = (msr - APIC_BASE_MSR) << 4;
  2297. if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
  2298. return 1;
  2299. if (reg == APIC_ICR2)
  2300. return 1;
  2301. /* if this is ICR write vector before command */
  2302. if (reg == APIC_ICR)
  2303. kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
  2304. return kvm_lapic_reg_write(apic, reg, (u32)data);
  2305. }
  2306. int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data)
  2307. {
  2308. struct kvm_lapic *apic = vcpu->arch.apic;
  2309. u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0;
  2310. if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
  2311. return 1;
  2312. if (reg == APIC_DFR || reg == APIC_ICR2)
  2313. return 1;
  2314. if (kvm_lapic_reg_read(apic, reg, 4, &low))
  2315. return 1;
  2316. if (reg == APIC_ICR)
  2317. kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
  2318. *data = (((u64)high) << 32) | low;
  2319. return 0;
  2320. }
  2321. int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data)
  2322. {
  2323. struct kvm_lapic *apic = vcpu->arch.apic;
  2324. if (!lapic_in_kernel(vcpu))
  2325. return 1;
  2326. /* if this is ICR write vector before command */
  2327. if (reg == APIC_ICR)
  2328. kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
  2329. return kvm_lapic_reg_write(apic, reg, (u32)data);
  2330. }
  2331. int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
  2332. {
  2333. struct kvm_lapic *apic = vcpu->arch.apic;
  2334. u32 low, high = 0;
  2335. if (!lapic_in_kernel(vcpu))
  2336. return 1;
  2337. if (kvm_lapic_reg_read(apic, reg, 4, &low))
  2338. return 1;
  2339. if (reg == APIC_ICR)
  2340. kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
  2341. *data = (((u64)high) << 32) | low;
  2342. return 0;
  2343. }
  2344. int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len)
  2345. {
  2346. u64 addr = data & ~KVM_MSR_ENABLED;
  2347. struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_eoi.data;
  2348. unsigned long new_len;
  2349. if (!IS_ALIGNED(addr, 4))
  2350. return 1;
  2351. vcpu->arch.pv_eoi.msr_val = data;
  2352. if (!pv_eoi_enabled(vcpu))
  2353. return 0;
  2354. if (addr == ghc->gpa && len <= ghc->len)
  2355. new_len = ghc->len;
  2356. else
  2357. new_len = len;
  2358. return kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len);
  2359. }
  2360. void kvm_apic_accept_events(struct kvm_vcpu *vcpu)
  2361. {
  2362. struct kvm_lapic *apic = vcpu->arch.apic;
  2363. u8 sipi_vector;
  2364. unsigned long pe;
  2365. if (!lapic_in_kernel(vcpu) || !apic->pending_events)
  2366. return;
  2367. /*
  2368. * INITs are latched while CPU is in specific states
  2369. * (SMM, VMX non-root mode, SVM with GIF=0).
  2370. * Because a CPU cannot be in these states immediately
  2371. * after it has processed an INIT signal (and thus in
  2372. * KVM_MP_STATE_INIT_RECEIVED state), just eat SIPIs
  2373. * and leave the INIT pending.
  2374. */
  2375. if (kvm_vcpu_latch_init(vcpu)) {
  2376. WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED);
  2377. if (test_bit(KVM_APIC_SIPI, &apic->pending_events))
  2378. clear_bit(KVM_APIC_SIPI, &apic->pending_events);
  2379. return;
  2380. }
  2381. pe = xchg(&apic->pending_events, 0);
  2382. if (test_bit(KVM_APIC_INIT, &pe)) {
  2383. kvm_vcpu_reset(vcpu, true);
  2384. if (kvm_vcpu_is_bsp(apic->vcpu))
  2385. vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
  2386. else
  2387. vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
  2388. }
  2389. if (test_bit(KVM_APIC_SIPI, &pe) &&
  2390. vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
  2391. /* evaluate pending_events before reading the vector */
  2392. smp_rmb();
  2393. sipi_vector = apic->sipi_vector;
  2394. kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector);
  2395. vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
  2396. }
  2397. }
  2398. void kvm_lapic_init(void)
  2399. {
  2400. /* do not patch jump label more than once per second */
  2401. jump_label_rate_limit(&apic_hw_disabled, HZ);
  2402. jump_label_rate_limit(&apic_sw_disabled, HZ);
  2403. }
  2404. void kvm_lapic_exit(void)
  2405. {
  2406. static_key_deferred_flush(&apic_hw_disabled);
  2407. static_key_deferred_flush(&apic_sw_disabled);
  2408. }