CVE-2023-4039.patch 117 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093
  1. From: Richard Sandiford <richard.sandiford@arm.com>
  2. Subject: [PATCH 00/19] aarch64: Fix -fstack-protector issue
  3. Date: Tue, 12 Sep 2023 16:25:10 +0100
  4. This series of patches fixes deficiencies in GCC's -fstack-protector
  5. implementation for AArch64 when using dynamically allocated stack space.
  6. This is CVE-2023-4039. See:
  7. https://developer.arm.com/Arm%20Security%20Center/GCC%20Stack%20Protector%20Vulnerability%20AArch64
  8. https://github.com/metaredteam/external-disclosures/security/advisories/GHSA-x7ch-h5rf-w2mf
  9. for more details.
  10. The fix is to put the saved registers above the locals area when
  11. -fstack-protector is used.
  12. The series also fixes a stack-clash problem that I found while working
  13. on the CVE. In unpatched sources, the stack-clash problem would only
  14. trigger for unrealistic numbers of arguments (8K 64-bit arguments, or an
  15. equivalent). But it would be a more significant issue with the new
  16. -fstack-protector frame layout. It's therefore important that both
  17. problems are fixed together.
  18. Some reorganisation of the code seemed necessary to fix the problems in a
  19. cleanish way. The series is therefore quite long, but only a handful of
  20. patches should have any effect on code generation.
  21. See the individual patches for a detailed description.
  22. Tested on aarch64-linux-gnu. Pushed to trunk and to all active branches.
  23. I've also pushed backports to GCC 7+ to vendors/ARM/heads/CVE-2023-4039.
  24. CVE: CVE-2023-4039
  25. Upstream-Status: Backport
  26. Signed-off-by: Ross Burton <ross.burton@arm.com>
  27. From 71a2aa2127283f450c623d3604dbcabe0e14a8d4 Mon Sep 17 00:00:00 2001
  28. From: Richard Sandiford <richard.sandiford@arm.com>
  29. Date: Tue, 12 Sep 2023 16:07:12 +0100
  30. Subject: [PATCH 01/19] aarch64: Use local frame vars in shrink-wrapping code
  31. aarch64_layout_frame uses a shorthand for referring to
  32. cfun->machine->frame:
  33. aarch64_frame &frame = cfun->machine->frame;
  34. This patch does the same for some other heavy users of the structure.
  35. No functional change intended.
  36. gcc/
  37. * config/aarch64/aarch64.cc (aarch64_save_callee_saves): Use
  38. a local shorthand for cfun->machine->frame.
  39. (aarch64_restore_callee_saves, aarch64_get_separate_components):
  40. (aarch64_process_components): Likewise.
  41. (aarch64_allocate_and_probe_stack_space): Likewise.
  42. (aarch64_expand_prologue, aarch64_expand_epilogue): Likewise.
  43. (aarch64_layout_frame): Use existing shorthand for one more case.
  44. ---
  45. gcc/config/aarch64/aarch64.cc | 123 ++++++++++++++++++----------------
  46. 1 file changed, 64 insertions(+), 59 deletions(-)
  47. diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
  48. index 822a2b49a46..5d473d161d9 100644
  49. --- a/gcc/config/aarch64/aarch64.cc
  50. +++ b/gcc/config/aarch64/aarch64.cc
  51. @@ -8612,7 +8612,7 @@ aarch64_layout_frame (void)
  52. frame.is_scs_enabled
  53. = (!crtl->calls_eh_return
  54. && sanitize_flags_p (SANITIZE_SHADOW_CALL_STACK)
  55. - && known_ge (cfun->machine->frame.reg_offset[LR_REGNUM], 0));
  56. + && known_ge (frame.reg_offset[LR_REGNUM], 0));
  57. /* When shadow call stack is enabled, the scs_pop in the epilogue will
  58. restore x30, and we don't need to pop x30 again in the traditional
  59. @@ -9078,6 +9078,7 @@ aarch64_save_callee_saves (poly_int64 start_offset,
  60. unsigned start, unsigned limit, bool skip_wb,
  61. bool hard_fp_valid_p)
  62. {
  63. + aarch64_frame &frame = cfun->machine->frame;
  64. rtx_insn *insn;
  65. unsigned regno;
  66. unsigned regno2;
  67. @@ -9092,8 +9093,8 @@ aarch64_save_callee_saves (poly_int64 start_offset,
  68. bool frame_related_p = aarch64_emit_cfi_for_reg_p (regno);
  69. if (skip_wb
  70. - && (regno == cfun->machine->frame.wb_push_candidate1
  71. - || regno == cfun->machine->frame.wb_push_candidate2))
  72. + && (regno == frame.wb_push_candidate1
  73. + || regno == frame.wb_push_candidate2))
  74. continue;
  75. if (cfun->machine->reg_is_wrapped_separately[regno])
  76. @@ -9101,7 +9102,7 @@ aarch64_save_callee_saves (poly_int64 start_offset,
  77. machine_mode mode = aarch64_reg_save_mode (regno);
  78. reg = gen_rtx_REG (mode, regno);
  79. - offset = start_offset + cfun->machine->frame.reg_offset[regno];
  80. + offset = start_offset + frame.reg_offset[regno];
  81. rtx base_rtx = stack_pointer_rtx;
  82. poly_int64 sp_offset = offset;
  83. @@ -9114,7 +9115,7 @@ aarch64_save_callee_saves (poly_int64 start_offset,
  84. {
  85. gcc_assert (known_eq (start_offset, 0));
  86. poly_int64 fp_offset
  87. - = cfun->machine->frame.below_hard_fp_saved_regs_size;
  88. + = frame.below_hard_fp_saved_regs_size;
  89. if (hard_fp_valid_p)
  90. base_rtx = hard_frame_pointer_rtx;
  91. else
  92. @@ -9136,8 +9137,7 @@ aarch64_save_callee_saves (poly_int64 start_offset,
  93. && (regno2 = aarch64_next_callee_save (regno + 1, limit)) <= limit
  94. && !cfun->machine->reg_is_wrapped_separately[regno2]
  95. && known_eq (GET_MODE_SIZE (mode),
  96. - cfun->machine->frame.reg_offset[regno2]
  97. - - cfun->machine->frame.reg_offset[regno]))
  98. + frame.reg_offset[regno2] - frame.reg_offset[regno]))
  99. {
  100. rtx reg2 = gen_rtx_REG (mode, regno2);
  101. rtx mem2;
  102. @@ -9187,6 +9187,7 @@ static void
  103. aarch64_restore_callee_saves (poly_int64 start_offset, unsigned start,
  104. unsigned limit, bool skip_wb, rtx *cfi_ops)
  105. {
  106. + aarch64_frame &frame = cfun->machine->frame;
  107. unsigned regno;
  108. unsigned regno2;
  109. poly_int64 offset;
  110. @@ -9203,13 +9204,13 @@ aarch64_restore_callee_saves (poly_int64 start_offset, unsigned start,
  111. rtx reg, mem;
  112. if (skip_wb
  113. - && (regno == cfun->machine->frame.wb_pop_candidate1
  114. - || regno == cfun->machine->frame.wb_pop_candidate2))
  115. + && (regno == frame.wb_pop_candidate1
  116. + || regno == frame.wb_pop_candidate2))
  117. continue;
  118. machine_mode mode = aarch64_reg_save_mode (regno);
  119. reg = gen_rtx_REG (mode, regno);
  120. - offset = start_offset + cfun->machine->frame.reg_offset[regno];
  121. + offset = start_offset + frame.reg_offset[regno];
  122. rtx base_rtx = stack_pointer_rtx;
  123. if (mode == VNx2DImode && BYTES_BIG_ENDIAN)
  124. aarch64_adjust_sve_callee_save_base (mode, base_rtx, anchor_reg,
  125. @@ -9220,8 +9221,7 @@ aarch64_restore_callee_saves (poly_int64 start_offset, unsigned start,
  126. && (regno2 = aarch64_next_callee_save (regno + 1, limit)) <= limit
  127. && !cfun->machine->reg_is_wrapped_separately[regno2]
  128. && known_eq (GET_MODE_SIZE (mode),
  129. - cfun->machine->frame.reg_offset[regno2]
  130. - - cfun->machine->frame.reg_offset[regno]))
  131. + frame.reg_offset[regno2] - frame.reg_offset[regno]))
  132. {
  133. rtx reg2 = gen_rtx_REG (mode, regno2);
  134. rtx mem2;
  135. @@ -9326,6 +9326,7 @@ offset_12bit_unsigned_scaled_p (machine_mode mode, poly_int64 offset)
  136. static sbitmap
  137. aarch64_get_separate_components (void)
  138. {
  139. + aarch64_frame &frame = cfun->machine->frame;
  140. sbitmap components = sbitmap_alloc (LAST_SAVED_REGNUM + 1);
  141. bitmap_clear (components);
  142. @@ -9342,18 +9343,18 @@ aarch64_get_separate_components (void)
  143. if (mode == VNx2DImode && BYTES_BIG_ENDIAN)
  144. continue;
  145. - poly_int64 offset = cfun->machine->frame.reg_offset[regno];
  146. + poly_int64 offset = frame.reg_offset[regno];
  147. /* If the register is saved in the first SVE save slot, we use
  148. it as a stack probe for -fstack-clash-protection. */
  149. if (flag_stack_clash_protection
  150. - && maybe_ne (cfun->machine->frame.below_hard_fp_saved_regs_size, 0)
  151. + && maybe_ne (frame.below_hard_fp_saved_regs_size, 0)
  152. && known_eq (offset, 0))
  153. continue;
  154. /* Get the offset relative to the register we'll use. */
  155. if (frame_pointer_needed)
  156. - offset -= cfun->machine->frame.below_hard_fp_saved_regs_size;
  157. + offset -= frame.below_hard_fp_saved_regs_size;
  158. else
  159. offset += crtl->outgoing_args_size;
  160. @@ -9372,11 +9373,11 @@ aarch64_get_separate_components (void)
  161. /* If the spare predicate register used by big-endian SVE code
  162. is call-preserved, it must be saved in the main prologue
  163. before any saves that use it. */
  164. - if (cfun->machine->frame.spare_pred_reg != INVALID_REGNUM)
  165. - bitmap_clear_bit (components, cfun->machine->frame.spare_pred_reg);
  166. + if (frame.spare_pred_reg != INVALID_REGNUM)
  167. + bitmap_clear_bit (components, frame.spare_pred_reg);
  168. - unsigned reg1 = cfun->machine->frame.wb_push_candidate1;
  169. - unsigned reg2 = cfun->machine->frame.wb_push_candidate2;
  170. + unsigned reg1 = frame.wb_push_candidate1;
  171. + unsigned reg2 = frame.wb_push_candidate2;
  172. /* If registers have been chosen to be stored/restored with
  173. writeback don't interfere with them to avoid having to output explicit
  174. stack adjustment instructions. */
  175. @@ -9485,6 +9486,7 @@ aarch64_get_next_set_bit (sbitmap bmp, unsigned int start)
  176. static void
  177. aarch64_process_components (sbitmap components, bool prologue_p)
  178. {
  179. + aarch64_frame &frame = cfun->machine->frame;
  180. rtx ptr_reg = gen_rtx_REG (Pmode, frame_pointer_needed
  181. ? HARD_FRAME_POINTER_REGNUM
  182. : STACK_POINTER_REGNUM);
  183. @@ -9499,9 +9501,9 @@ aarch64_process_components (sbitmap components, bool prologue_p)
  184. machine_mode mode = aarch64_reg_save_mode (regno);
  185. rtx reg = gen_rtx_REG (mode, regno);
  186. - poly_int64 offset = cfun->machine->frame.reg_offset[regno];
  187. + poly_int64 offset = frame.reg_offset[regno];
  188. if (frame_pointer_needed)
  189. - offset -= cfun->machine->frame.below_hard_fp_saved_regs_size;
  190. + offset -= frame.below_hard_fp_saved_regs_size;
  191. else
  192. offset += crtl->outgoing_args_size;
  193. @@ -9526,14 +9528,14 @@ aarch64_process_components (sbitmap components, bool prologue_p)
  194. break;
  195. }
  196. - poly_int64 offset2 = cfun->machine->frame.reg_offset[regno2];
  197. + poly_int64 offset2 = frame.reg_offset[regno2];
  198. /* The next register is not of the same class or its offset is not
  199. mergeable with the current one into a pair. */
  200. if (aarch64_sve_mode_p (mode)
  201. || !satisfies_constraint_Ump (mem)
  202. || GP_REGNUM_P (regno) != GP_REGNUM_P (regno2)
  203. || (crtl->abi->id () == ARM_PCS_SIMD && FP_REGNUM_P (regno))
  204. - || maybe_ne ((offset2 - cfun->machine->frame.reg_offset[regno]),
  205. + || maybe_ne ((offset2 - frame.reg_offset[regno]),
  206. GET_MODE_SIZE (mode)))
  207. {
  208. insn = emit_insn (set);
  209. @@ -9555,7 +9557,7 @@ aarch64_process_components (sbitmap components, bool prologue_p)
  210. /* REGNO2 can be saved/restored in a pair with REGNO. */
  211. rtx reg2 = gen_rtx_REG (mode, regno2);
  212. if (frame_pointer_needed)
  213. - offset2 -= cfun->machine->frame.below_hard_fp_saved_regs_size;
  214. + offset2 -= frame.below_hard_fp_saved_regs_size;
  215. else
  216. offset2 += crtl->outgoing_args_size;
  217. rtx addr2 = plus_constant (Pmode, ptr_reg, offset2);
  218. @@ -9650,6 +9652,7 @@ aarch64_allocate_and_probe_stack_space (rtx temp1, rtx temp2,
  219. bool frame_related_p,
  220. bool final_adjustment_p)
  221. {
  222. + aarch64_frame &frame = cfun->machine->frame;
  223. HOST_WIDE_INT guard_size
  224. = 1 << param_stack_clash_protection_guard_size;
  225. HOST_WIDE_INT guard_used_by_caller = STACK_CLASH_CALLER_GUARD;
  226. @@ -9670,25 +9673,25 @@ aarch64_allocate_and_probe_stack_space (rtx temp1, rtx temp2,
  227. register as a probe. We can't assume that LR was saved at position 0
  228. though, so treat any space below it as unprobed. */
  229. if (final_adjustment_p
  230. - && known_eq (cfun->machine->frame.below_hard_fp_saved_regs_size, 0))
  231. + && known_eq (frame.below_hard_fp_saved_regs_size, 0))
  232. {
  233. - poly_int64 lr_offset = cfun->machine->frame.reg_offset[LR_REGNUM];
  234. + poly_int64 lr_offset = frame.reg_offset[LR_REGNUM];
  235. if (known_ge (lr_offset, 0))
  236. min_probe_threshold -= lr_offset.to_constant ();
  237. else
  238. gcc_assert (!flag_stack_clash_protection || known_eq (poly_size, 0));
  239. }
  240. - poly_int64 frame_size = cfun->machine->frame.frame_size;
  241. + poly_int64 frame_size = frame.frame_size;
  242. /* We should always have a positive probe threshold. */
  243. gcc_assert (min_probe_threshold > 0);
  244. if (flag_stack_clash_protection && !final_adjustment_p)
  245. {
  246. - poly_int64 initial_adjust = cfun->machine->frame.initial_adjust;
  247. - poly_int64 sve_callee_adjust = cfun->machine->frame.sve_callee_adjust;
  248. - poly_int64 final_adjust = cfun->machine->frame.final_adjust;
  249. + poly_int64 initial_adjust = frame.initial_adjust;
  250. + poly_int64 sve_callee_adjust = frame.sve_callee_adjust;
  251. + poly_int64 final_adjust = frame.final_adjust;
  252. if (known_eq (frame_size, 0))
  253. {
  254. @@ -9977,17 +9980,18 @@ aarch64_epilogue_uses (int regno)
  255. void
  256. aarch64_expand_prologue (void)
  257. {
  258. - poly_int64 frame_size = cfun->machine->frame.frame_size;
  259. - poly_int64 initial_adjust = cfun->machine->frame.initial_adjust;
  260. - HOST_WIDE_INT callee_adjust = cfun->machine->frame.callee_adjust;
  261. - poly_int64 final_adjust = cfun->machine->frame.final_adjust;
  262. - poly_int64 callee_offset = cfun->machine->frame.callee_offset;
  263. - poly_int64 sve_callee_adjust = cfun->machine->frame.sve_callee_adjust;
  264. + aarch64_frame &frame = cfun->machine->frame;
  265. + poly_int64 frame_size = frame.frame_size;
  266. + poly_int64 initial_adjust = frame.initial_adjust;
  267. + HOST_WIDE_INT callee_adjust = frame.callee_adjust;
  268. + poly_int64 final_adjust = frame.final_adjust;
  269. + poly_int64 callee_offset = frame.callee_offset;
  270. + poly_int64 sve_callee_adjust = frame.sve_callee_adjust;
  271. poly_int64 below_hard_fp_saved_regs_size
  272. - = cfun->machine->frame.below_hard_fp_saved_regs_size;
  273. - unsigned reg1 = cfun->machine->frame.wb_push_candidate1;
  274. - unsigned reg2 = cfun->machine->frame.wb_push_candidate2;
  275. - bool emit_frame_chain = cfun->machine->frame.emit_frame_chain;
  276. + = frame.below_hard_fp_saved_regs_size;
  277. + unsigned reg1 = frame.wb_push_candidate1;
  278. + unsigned reg2 = frame.wb_push_candidate2;
  279. + bool emit_frame_chain = frame.emit_frame_chain;
  280. rtx_insn *insn;
  281. if (flag_stack_clash_protection && known_eq (callee_adjust, 0))
  282. @@ -10018,7 +10022,7 @@ aarch64_expand_prologue (void)
  283. }
  284. /* Push return address to shadow call stack. */
  285. - if (cfun->machine->frame.is_scs_enabled)
  286. + if (frame.is_scs_enabled)
  287. emit_insn (gen_scs_push ());
  288. if (flag_stack_usage_info)
  289. @@ -10057,7 +10061,7 @@ aarch64_expand_prologue (void)
  290. /* The offset of the frame chain record (if any) from the current SP. */
  291. poly_int64 chain_offset = (initial_adjust + callee_adjust
  292. - - cfun->machine->frame.hard_fp_offset);
  293. + - frame.hard_fp_offset);
  294. gcc_assert (known_ge (chain_offset, 0));
  295. /* The offset of the bottom of the save area from the current SP. */
  296. @@ -10160,16 +10164,17 @@ aarch64_use_return_insn_p (void)
  297. void
  298. aarch64_expand_epilogue (bool for_sibcall)
  299. {
  300. - poly_int64 initial_adjust = cfun->machine->frame.initial_adjust;
  301. - HOST_WIDE_INT callee_adjust = cfun->machine->frame.callee_adjust;
  302. - poly_int64 final_adjust = cfun->machine->frame.final_adjust;
  303. - poly_int64 callee_offset = cfun->machine->frame.callee_offset;
  304. - poly_int64 sve_callee_adjust = cfun->machine->frame.sve_callee_adjust;
  305. + aarch64_frame &frame = cfun->machine->frame;
  306. + poly_int64 initial_adjust = frame.initial_adjust;
  307. + HOST_WIDE_INT callee_adjust = frame.callee_adjust;
  308. + poly_int64 final_adjust = frame.final_adjust;
  309. + poly_int64 callee_offset = frame.callee_offset;
  310. + poly_int64 sve_callee_adjust = frame.sve_callee_adjust;
  311. poly_int64 below_hard_fp_saved_regs_size
  312. - = cfun->machine->frame.below_hard_fp_saved_regs_size;
  313. - unsigned reg1 = cfun->machine->frame.wb_pop_candidate1;
  314. - unsigned reg2 = cfun->machine->frame.wb_pop_candidate2;
  315. - unsigned int last_gpr = (cfun->machine->frame.is_scs_enabled
  316. + = frame.below_hard_fp_saved_regs_size;
  317. + unsigned reg1 = frame.wb_pop_candidate1;
  318. + unsigned reg2 = frame.wb_pop_candidate2;
  319. + unsigned int last_gpr = (frame.is_scs_enabled
  320. ? R29_REGNUM : R30_REGNUM);
  321. rtx cfi_ops = NULL;
  322. rtx_insn *insn;
  323. @@ -10203,7 +10208,7 @@ aarch64_expand_epilogue (bool for_sibcall)
  324. /* We need to add memory barrier to prevent read from deallocated stack. */
  325. bool need_barrier_p
  326. = maybe_ne (get_frame_size ()
  327. - + cfun->machine->frame.saved_varargs_size, 0);
  328. + + frame.saved_varargs_size, 0);
  329. /* Emit a barrier to prevent loads from a deallocated stack. */
  330. if (maybe_gt (final_adjust, crtl->outgoing_args_size)
  331. @@ -10284,7 +10289,7 @@ aarch64_expand_epilogue (bool for_sibcall)
  332. }
  333. /* Pop return address from shadow call stack. */
  334. - if (cfun->machine->frame.is_scs_enabled)
  335. + if (frame.is_scs_enabled)
  336. {
  337. machine_mode mode = aarch64_reg_save_mode (R30_REGNUM);
  338. rtx reg = gen_rtx_REG (mode, R30_REGNUM);
  339. @@ -12740,24 +12745,24 @@ aarch64_can_eliminate (const int from ATTRIBUTE_UNUSED, const int to)
  340. poly_int64
  341. aarch64_initial_elimination_offset (unsigned from, unsigned to)
  342. {
  343. + aarch64_frame &frame = cfun->machine->frame;
  344. +
  345. if (to == HARD_FRAME_POINTER_REGNUM)
  346. {
  347. if (from == ARG_POINTER_REGNUM)
  348. - return cfun->machine->frame.hard_fp_offset;
  349. + return frame.hard_fp_offset;
  350. if (from == FRAME_POINTER_REGNUM)
  351. - return cfun->machine->frame.hard_fp_offset
  352. - - cfun->machine->frame.locals_offset;
  353. + return frame.hard_fp_offset - frame.locals_offset;
  354. }
  355. if (to == STACK_POINTER_REGNUM)
  356. {
  357. if (from == FRAME_POINTER_REGNUM)
  358. - return cfun->machine->frame.frame_size
  359. - - cfun->machine->frame.locals_offset;
  360. + return frame.frame_size - frame.locals_offset;
  361. }
  362. - return cfun->machine->frame.frame_size;
  363. + return frame.frame_size;
  364. }
  365. --
  366. 2.34.1
  367. From 89a9fa287706c5011f61926eaf65e7b996b963a3 Mon Sep 17 00:00:00 2001
  368. From: Richard Sandiford <richard.sandiford@arm.com>
  369. Date: Tue, 12 Sep 2023 16:07:12 +0100
  370. Subject: [PATCH 02/19] aarch64: Avoid a use of callee_offset
  371. When we emit the frame chain, i.e. when we reach Here in this statement
  372. of aarch64_expand_prologue:
  373. if (emit_frame_chain)
  374. {
  375. // Here
  376. ...
  377. }
  378. the stack is in one of two states:
  379. - We've allocated up to the frame chain, but no more.
  380. - We've allocated the whole frame, and the frame chain is within easy
  381. reach of the new SP.
  382. The offset of the frame chain from the current SP is available
  383. in aarch64_frame as callee_offset. It is also available as the
  384. chain_offset local variable, where the latter is calculated from other
  385. data. (However, chain_offset is not always equal to callee_offset when
  386. !emit_frame_chain, so chain_offset isn't redundant.)
  387. In c600df9a4060da3c6121ff4d0b93f179eafd69d1 I switched to using
  388. chain_offset for the initialisation of the hard frame pointer:
  389. aarch64_add_offset (Pmode, hard_frame_pointer_rtx,
  390. - stack_pointer_rtx, callee_offset,
  391. + stack_pointer_rtx, chain_offset,
  392. tmp1_rtx, tmp0_rtx, frame_pointer_needed);
  393. But the later REG_CFA_ADJUST_CFA handling still used callee_offset.
  394. I think the difference is harmless, but it's more logical for the
  395. CFA note to be in sync, and it's more convenient for later patches
  396. if it uses chain_offset.
  397. gcc/
  398. * config/aarch64/aarch64.cc (aarch64_expand_prologue): Use
  399. chain_offset rather than callee_offset.
  400. ---
  401. gcc/config/aarch64/aarch64.cc | 4 +---
  402. 1 file changed, 1 insertion(+), 3 deletions(-)
  403. diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
  404. index 5d473d161d9..4f233c95140 100644
  405. --- a/gcc/config/aarch64/aarch64.cc
  406. +++ b/gcc/config/aarch64/aarch64.cc
  407. @@ -9985,7 +9985,6 @@ aarch64_expand_prologue (void)
  408. poly_int64 initial_adjust = frame.initial_adjust;
  409. HOST_WIDE_INT callee_adjust = frame.callee_adjust;
  410. poly_int64 final_adjust = frame.final_adjust;
  411. - poly_int64 callee_offset = frame.callee_offset;
  412. poly_int64 sve_callee_adjust = frame.sve_callee_adjust;
  413. poly_int64 below_hard_fp_saved_regs_size
  414. = frame.below_hard_fp_saved_regs_size;
  415. @@ -10098,8 +10097,7 @@ aarch64_expand_prologue (void)
  416. implicit. */
  417. if (!find_reg_note (insn, REG_CFA_ADJUST_CFA, NULL_RTX))
  418. {
  419. - rtx src = plus_constant (Pmode, stack_pointer_rtx,
  420. - callee_offset);
  421. + rtx src = plus_constant (Pmode, stack_pointer_rtx, chain_offset);
  422. add_reg_note (insn, REG_CFA_ADJUST_CFA,
  423. gen_rtx_SET (hard_frame_pointer_rtx, src));
  424. }
  425. --
  426. 2.34.1
  427. From b36a2a78040722dab6124366c5d6baf8eaf80aef Mon Sep 17 00:00:00 2001
  428. From: Richard Sandiford <richard.sandiford@arm.com>
  429. Date: Tue, 12 Sep 2023 16:07:13 +0100
  430. Subject: [PATCH 03/19] aarch64: Explicitly handle frames with no saved
  431. registers
  432. If a frame has no saved registers, it can be allocated in one go.
  433. There is no need to treat the areas below and above the saved
  434. registers as separate.
  435. And if we allocate the frame in one go, it should be allocated
  436. as the initial_adjust rather than the final_adjust. This allows the
  437. frame size to grow to guard_size - guard_used_by_caller before a stack
  438. probe is needed. (A frame with no register saves is necessarily a
  439. leaf frame.)
  440. This is a no-op as thing stand, since a leaf function will have
  441. no outgoing arguments, and so all the frame will be above where
  442. the saved registers normally go.
  443. gcc/
  444. * config/aarch64/aarch64.cc (aarch64_layout_frame): Explicitly
  445. allocate the frame in one go if there are no saved registers.
  446. ---
  447. gcc/config/aarch64/aarch64.cc | 8 +++++---
  448. 1 file changed, 5 insertions(+), 3 deletions(-)
  449. diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
  450. index 4f233c95140..37643041ffb 100644
  451. --- a/gcc/config/aarch64/aarch64.cc
  452. +++ b/gcc/config/aarch64/aarch64.cc
  453. @@ -8639,9 +8639,11 @@ aarch64_layout_frame (void)
  454. HOST_WIDE_INT const_size, const_outgoing_args_size, const_fp_offset;
  455. HOST_WIDE_INT const_saved_regs_size;
  456. - if (frame.frame_size.is_constant (&const_size)
  457. - && const_size < max_push_offset
  458. - && known_eq (frame.hard_fp_offset, const_size))
  459. + if (known_eq (frame.saved_regs_size, 0))
  460. + frame.initial_adjust = frame.frame_size;
  461. + else if (frame.frame_size.is_constant (&const_size)
  462. + && const_size < max_push_offset
  463. + && known_eq (frame.hard_fp_offset, const_size))
  464. {
  465. /* Simple, small frame with no outgoing arguments:
  466. --
  467. 2.34.1
  468. From ada2ab0093596be707f23a3466ac82cff59fcffe Mon Sep 17 00:00:00 2001
  469. From: Richard Sandiford <richard.sandiford@arm.com>
  470. Date: Tue, 12 Sep 2023 16:07:13 +0100
  471. Subject: [PATCH 04/19] aarch64: Add bytes_below_saved_regs to frame info
  472. The frame layout code currently hard-codes the assumption that
  473. the number of bytes below the saved registers is equal to the
  474. size of the outgoing arguments. This patch abstracts that
  475. value into a new field of aarch64_frame.
  476. gcc/
  477. * config/aarch64/aarch64.h (aarch64_frame::bytes_below_saved_regs): New
  478. field.
  479. * config/aarch64/aarch64.cc (aarch64_layout_frame): Initialize it,
  480. and use it instead of crtl->outgoing_args_size.
  481. (aarch64_get_separate_components): Use bytes_below_saved_regs instead
  482. of outgoing_args_size.
  483. (aarch64_process_components): Likewise.
  484. ---
  485. gcc/config/aarch64/aarch64.cc | 71 ++++++++++++++++++-----------------
  486. gcc/config/aarch64/aarch64.h | 5 +++
  487. 2 files changed, 41 insertions(+), 35 deletions(-)
  488. diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
  489. index 37643041ffb..dacc2b0e4dd 100644
  490. --- a/gcc/config/aarch64/aarch64.cc
  491. +++ b/gcc/config/aarch64/aarch64.cc
  492. @@ -8478,6 +8478,8 @@ aarch64_layout_frame (void)
  493. gcc_assert (crtl->is_leaf
  494. || maybe_ne (frame.reg_offset[R30_REGNUM], SLOT_NOT_REQUIRED));
  495. + frame.bytes_below_saved_regs = crtl->outgoing_args_size;
  496. +
  497. /* Now assign stack slots for the registers. Start with the predicate
  498. registers, since predicate LDR and STR have a relatively small
  499. offset range. These saves happen below the hard frame pointer. */
  500. @@ -8582,18 +8584,18 @@ aarch64_layout_frame (void)
  501. poly_int64 varargs_and_saved_regs_size = offset + frame.saved_varargs_size;
  502. - poly_int64 above_outgoing_args
  503. + poly_int64 saved_regs_and_above
  504. = aligned_upper_bound (varargs_and_saved_regs_size
  505. + get_frame_size (),
  506. STACK_BOUNDARY / BITS_PER_UNIT);
  507. frame.hard_fp_offset
  508. - = above_outgoing_args - frame.below_hard_fp_saved_regs_size;
  509. + = saved_regs_and_above - frame.below_hard_fp_saved_regs_size;
  510. /* Both these values are already aligned. */
  511. - gcc_assert (multiple_p (crtl->outgoing_args_size,
  512. + gcc_assert (multiple_p (frame.bytes_below_saved_regs,
  513. STACK_BOUNDARY / BITS_PER_UNIT));
  514. - frame.frame_size = above_outgoing_args + crtl->outgoing_args_size;
  515. + frame.frame_size = saved_regs_and_above + frame.bytes_below_saved_regs;
  516. frame.locals_offset = frame.saved_varargs_size;
  517. @@ -8637,7 +8639,7 @@ aarch64_layout_frame (void)
  518. else if (frame.wb_pop_candidate1 != INVALID_REGNUM)
  519. max_push_offset = 256;
  520. - HOST_WIDE_INT const_size, const_outgoing_args_size, const_fp_offset;
  521. + HOST_WIDE_INT const_size, const_below_saved_regs, const_fp_offset;
  522. HOST_WIDE_INT const_saved_regs_size;
  523. if (known_eq (frame.saved_regs_size, 0))
  524. frame.initial_adjust = frame.frame_size;
  525. @@ -8645,31 +8647,31 @@ aarch64_layout_frame (void)
  526. && const_size < max_push_offset
  527. && known_eq (frame.hard_fp_offset, const_size))
  528. {
  529. - /* Simple, small frame with no outgoing arguments:
  530. + /* Simple, small frame with no data below the saved registers.
  531. stp reg1, reg2, [sp, -frame_size]!
  532. stp reg3, reg4, [sp, 16] */
  533. frame.callee_adjust = const_size;
  534. }
  535. - else if (crtl->outgoing_args_size.is_constant (&const_outgoing_args_size)
  536. + else if (frame.bytes_below_saved_regs.is_constant (&const_below_saved_regs)
  537. && frame.saved_regs_size.is_constant (&const_saved_regs_size)
  538. - && const_outgoing_args_size + const_saved_regs_size < 512
  539. - /* We could handle this case even with outgoing args, provided
  540. - that the number of args left us with valid offsets for all
  541. - predicate and vector save slots. It's such a rare case that
  542. - it hardly seems worth the effort though. */
  543. - && (!saves_below_hard_fp_p || const_outgoing_args_size == 0)
  544. + && const_below_saved_regs + const_saved_regs_size < 512
  545. + /* We could handle this case even with data below the saved
  546. + registers, provided that that data left us with valid offsets
  547. + for all predicate and vector save slots. It's such a rare
  548. + case that it hardly seems worth the effort though. */
  549. + && (!saves_below_hard_fp_p || const_below_saved_regs == 0)
  550. && !(cfun->calls_alloca
  551. && frame.hard_fp_offset.is_constant (&const_fp_offset)
  552. && const_fp_offset < max_push_offset))
  553. {
  554. - /* Frame with small outgoing arguments:
  555. + /* Frame with small area below the saved registers:
  556. sub sp, sp, frame_size
  557. - stp reg1, reg2, [sp, outgoing_args_size]
  558. - stp reg3, reg4, [sp, outgoing_args_size + 16] */
  559. + stp reg1, reg2, [sp, bytes_below_saved_regs]
  560. + stp reg3, reg4, [sp, bytes_below_saved_regs + 16] */
  561. frame.initial_adjust = frame.frame_size;
  562. - frame.callee_offset = const_outgoing_args_size;
  563. + frame.callee_offset = const_below_saved_regs;
  564. }
  565. else if (saves_below_hard_fp_p
  566. && known_eq (frame.saved_regs_size,
  567. @@ -8679,30 +8681,29 @@ aarch64_layout_frame (void)
  568. sub sp, sp, hard_fp_offset + below_hard_fp_saved_regs_size
  569. save SVE registers relative to SP
  570. - sub sp, sp, outgoing_args_size */
  571. + sub sp, sp, bytes_below_saved_regs */
  572. frame.initial_adjust = (frame.hard_fp_offset
  573. + frame.below_hard_fp_saved_regs_size);
  574. - frame.final_adjust = crtl->outgoing_args_size;
  575. + frame.final_adjust = frame.bytes_below_saved_regs;
  576. }
  577. else if (frame.hard_fp_offset.is_constant (&const_fp_offset)
  578. && const_fp_offset < max_push_offset)
  579. {
  580. - /* Frame with large outgoing arguments or SVE saves, but with
  581. - a small local area:
  582. + /* Frame with large area below the saved registers, or with SVE saves,
  583. + but with a small area above:
  584. stp reg1, reg2, [sp, -hard_fp_offset]!
  585. stp reg3, reg4, [sp, 16]
  586. [sub sp, sp, below_hard_fp_saved_regs_size]
  587. [save SVE registers relative to SP]
  588. - sub sp, sp, outgoing_args_size */
  589. + sub sp, sp, bytes_below_saved_regs */
  590. frame.callee_adjust = const_fp_offset;
  591. frame.sve_callee_adjust = frame.below_hard_fp_saved_regs_size;
  592. - frame.final_adjust = crtl->outgoing_args_size;
  593. + frame.final_adjust = frame.bytes_below_saved_regs;
  594. }
  595. else
  596. {
  597. - /* Frame with large local area and outgoing arguments or SVE saves,
  598. - using frame pointer:
  599. + /* General case:
  600. sub sp, sp, hard_fp_offset
  601. stp x29, x30, [sp, 0]
  602. @@ -8710,10 +8711,10 @@ aarch64_layout_frame (void)
  603. stp reg3, reg4, [sp, 16]
  604. [sub sp, sp, below_hard_fp_saved_regs_size]
  605. [save SVE registers relative to SP]
  606. - sub sp, sp, outgoing_args_size */
  607. + sub sp, sp, bytes_below_saved_regs */
  608. frame.initial_adjust = frame.hard_fp_offset;
  609. frame.sve_callee_adjust = frame.below_hard_fp_saved_regs_size;
  610. - frame.final_adjust = crtl->outgoing_args_size;
  611. + frame.final_adjust = frame.bytes_below_saved_regs;
  612. }
  613. /* Make sure the individual adjustments add up to the full frame size. */
  614. @@ -9358,7 +9359,7 @@ aarch64_get_separate_components (void)
  615. if (frame_pointer_needed)
  616. offset -= frame.below_hard_fp_saved_regs_size;
  617. else
  618. - offset += crtl->outgoing_args_size;
  619. + offset += frame.bytes_below_saved_regs;
  620. /* Check that we can access the stack slot of the register with one
  621. direct load with no adjustments needed. */
  622. @@ -9507,7 +9508,7 @@ aarch64_process_components (sbitmap components, bool prologue_p)
  623. if (frame_pointer_needed)
  624. offset -= frame.below_hard_fp_saved_regs_size;
  625. else
  626. - offset += crtl->outgoing_args_size;
  627. + offset += frame.bytes_below_saved_regs;
  628. rtx addr = plus_constant (Pmode, ptr_reg, offset);
  629. rtx mem = gen_frame_mem (mode, addr);
  630. @@ -9561,7 +9562,7 @@ aarch64_process_components (sbitmap components, bool prologue_p)
  631. if (frame_pointer_needed)
  632. offset2 -= frame.below_hard_fp_saved_regs_size;
  633. else
  634. - offset2 += crtl->outgoing_args_size;
  635. + offset2 += frame.bytes_below_saved_regs;
  636. rtx addr2 = plus_constant (Pmode, ptr_reg, offset2);
  637. rtx mem2 = gen_frame_mem (mode, addr2);
  638. rtx set2 = prologue_p ? gen_rtx_SET (mem2, reg2)
  639. @@ -9635,10 +9636,10 @@ aarch64_stack_clash_protection_alloca_probe_range (void)
  640. registers. If POLY_SIZE is not large enough to require a probe this function
  641. will only adjust the stack. When allocating the stack space
  642. FRAME_RELATED_P is then used to indicate if the allocation is frame related.
  643. - FINAL_ADJUSTMENT_P indicates whether we are allocating the outgoing
  644. - arguments. If we are then we ensure that any allocation larger than the ABI
  645. - defined buffer needs a probe so that the invariant of having a 1KB buffer is
  646. - maintained.
  647. + FINAL_ADJUSTMENT_P indicates whether we are allocating the area below
  648. + the saved registers. If we are then we ensure that any allocation
  649. + larger than the ABI defined buffer needs a probe so that the
  650. + invariant of having a 1KB buffer is maintained.
  651. We emit barriers after each stack adjustment to prevent optimizations from
  652. breaking the invariant that we never drop the stack more than a page. This
  653. @@ -9847,7 +9848,7 @@ aarch64_allocate_and_probe_stack_space (rtx temp1, rtx temp2,
  654. /* Handle any residuals. Residuals of at least MIN_PROBE_THRESHOLD have to
  655. be probed. This maintains the requirement that each page is probed at
  656. least once. For initial probing we probe only if the allocation is
  657. - more than GUARD_SIZE - buffer, and for the outgoing arguments we probe
  658. + more than GUARD_SIZE - buffer, and below the saved registers we probe
  659. if the amount is larger than buffer. GUARD_SIZE - buffer + buffer ==
  660. GUARD_SIZE. This works that for any allocation that is large enough to
  661. trigger a probe here, we'll have at least one, and if they're not large
  662. diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
  663. index 73b09e20508..0b6faa3ddf1 100644
  664. --- a/gcc/config/aarch64/aarch64.h
  665. +++ b/gcc/config/aarch64/aarch64.h
  666. @@ -777,6 +777,11 @@ struct GTY (()) aarch64_frame
  667. /* The size of the callee-save registers with a slot in REG_OFFSET. */
  668. poly_int64 saved_regs_size;
  669. + /* The number of bytes between the bottom of the static frame (the bottom
  670. + of the outgoing arguments) and the bottom of the register save area.
  671. + This value is always a multiple of STACK_BOUNDARY. */
  672. + poly_int64 bytes_below_saved_regs;
  673. +
  674. /* The size of the callee-save registers with a slot in REG_OFFSET that
  675. are saved below the hard frame pointer. */
  676. poly_int64 below_hard_fp_saved_regs_size;
  677. --
  678. 2.34.1
  679. From 82f6b3e1b596ef0f4e3ac3bb9c6e88fb4458f402 Mon Sep 17 00:00:00 2001
  680. From: Richard Sandiford <richard.sandiford@arm.com>
  681. Date: Tue, 12 Sep 2023 16:07:14 +0100
  682. Subject: [PATCH 05/19] aarch64: Add bytes_below_hard_fp to frame info
  683. Following on from the previous bytes_below_saved_regs patch, this one
  684. records the number of bytes that are below the hard frame pointer.
  685. This eventually replaces below_hard_fp_saved_regs_size.
  686. If a frame pointer is not needed, the epilogue adds final_adjust
  687. to the stack pointer before restoring registers:
  688. aarch64_add_sp (tmp1_rtx, tmp0_rtx, final_adjust, true);
  689. Therefore, if the epilogue needs to restore the stack pointer from
  690. the hard frame pointer, the directly corresponding offset is:
  691. -bytes_below_hard_fp + final_adjust
  692. i.e. go from the hard frame pointer to the bottom of the frame,
  693. then add the same amount as if we were using the stack pointer
  694. from the outset.
  695. gcc/
  696. * config/aarch64/aarch64.h (aarch64_frame::bytes_below_hard_fp): New
  697. field.
  698. * config/aarch64/aarch64.cc (aarch64_layout_frame): Initialize it.
  699. (aarch64_expand_epilogue): Use it instead of
  700. below_hard_fp_saved_regs_size.
  701. ---
  702. gcc/config/aarch64/aarch64.cc | 6 +++---
  703. gcc/config/aarch64/aarch64.h | 5 +++++
  704. 2 files changed, 8 insertions(+), 3 deletions(-)
  705. diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
  706. index dacc2b0e4dd..a3f7aabcc59 100644
  707. --- a/gcc/config/aarch64/aarch64.cc
  708. +++ b/gcc/config/aarch64/aarch64.cc
  709. @@ -8530,6 +8530,7 @@ aarch64_layout_frame (void)
  710. of the callee save area. */
  711. bool saves_below_hard_fp_p = maybe_ne (offset, 0);
  712. frame.below_hard_fp_saved_regs_size = offset;
  713. + frame.bytes_below_hard_fp = offset + frame.bytes_below_saved_regs;
  714. if (frame.emit_frame_chain)
  715. {
  716. /* FP and LR are placed in the linkage record. */
  717. @@ -10171,8 +10172,7 @@ aarch64_expand_epilogue (bool for_sibcall)
  718. poly_int64 final_adjust = frame.final_adjust;
  719. poly_int64 callee_offset = frame.callee_offset;
  720. poly_int64 sve_callee_adjust = frame.sve_callee_adjust;
  721. - poly_int64 below_hard_fp_saved_regs_size
  722. - = frame.below_hard_fp_saved_regs_size;
  723. + poly_int64 bytes_below_hard_fp = frame.bytes_below_hard_fp;
  724. unsigned reg1 = frame.wb_pop_candidate1;
  725. unsigned reg2 = frame.wb_pop_candidate2;
  726. unsigned int last_gpr = (frame.is_scs_enabled
  727. @@ -10230,7 +10230,7 @@ aarch64_expand_epilogue (bool for_sibcall)
  728. is restored on the instruction doing the writeback. */
  729. aarch64_add_offset (Pmode, stack_pointer_rtx,
  730. hard_frame_pointer_rtx,
  731. - -callee_offset - below_hard_fp_saved_regs_size,
  732. + -bytes_below_hard_fp + final_adjust,
  733. tmp1_rtx, tmp0_rtx, callee_adjust == 0);
  734. else
  735. /* The case where we need to re-use the register here is very rare, so
  736. diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
  737. index 0b6faa3ddf1..4263d29d29d 100644
  738. --- a/gcc/config/aarch64/aarch64.h
  739. +++ b/gcc/config/aarch64/aarch64.h
  740. @@ -786,6 +786,11 @@ struct GTY (()) aarch64_frame
  741. are saved below the hard frame pointer. */
  742. poly_int64 below_hard_fp_saved_regs_size;
  743. + /* The number of bytes between the bottom of the static frame (the bottom
  744. + of the outgoing arguments) and the hard frame pointer. This value is
  745. + always a multiple of STACK_BOUNDARY. */
  746. + poly_int64 bytes_below_hard_fp;
  747. +
  748. /* Offset from the base of the frame (incomming SP) to the
  749. top of the locals area. This value is always a multiple of
  750. STACK_BOUNDARY. */
  751. --
  752. 2.34.1
  753. From 86fa43e9fe4a8bf954f2919f07cbe3646d1d1df3 Mon Sep 17 00:00:00 2001
  754. From: Richard Sandiford <richard.sandiford@arm.com>
  755. Date: Tue, 12 Sep 2023 16:07:14 +0100
  756. Subject: [PATCH 06/19] aarch64: Tweak aarch64_save/restore_callee_saves
  757. aarch64_save_callee_saves and aarch64_restore_callee_saves took
  758. a parameter called start_offset that gives the offset of the
  759. bottom of the saved register area from the current stack pointer.
  760. However, it's more convenient for later patches if we use the
  761. bottom of the entire frame as the reference point, rather than
  762. the bottom of the saved registers.
  763. Doing that removes the need for the callee_offset field.
  764. Other than that, this is not a win on its own. It only really
  765. makes sense in combination with the follow-on patches.
  766. gcc/
  767. * config/aarch64/aarch64.h (aarch64_frame::callee_offset): Delete.
  768. * config/aarch64/aarch64.cc (aarch64_layout_frame): Remove
  769. callee_offset handling.
  770. (aarch64_save_callee_saves): Replace the start_offset parameter
  771. with a bytes_below_sp parameter.
  772. (aarch64_restore_callee_saves): Likewise.
  773. (aarch64_expand_prologue): Update accordingly.
  774. (aarch64_expand_epilogue): Likewise.
  775. ---
  776. gcc/config/aarch64/aarch64.cc | 56 +++++++++++++++++------------------
  777. gcc/config/aarch64/aarch64.h | 4 ---
  778. 2 files changed, 28 insertions(+), 32 deletions(-)
  779. diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
  780. index a3f7aabcc59..46ae5cf7673 100644
  781. --- a/gcc/config/aarch64/aarch64.cc
  782. +++ b/gcc/config/aarch64/aarch64.cc
  783. @@ -8604,7 +8604,6 @@ aarch64_layout_frame (void)
  784. frame.final_adjust = 0;
  785. frame.callee_adjust = 0;
  786. frame.sve_callee_adjust = 0;
  787. - frame.callee_offset = 0;
  788. frame.wb_pop_candidate1 = frame.wb_push_candidate1;
  789. frame.wb_pop_candidate2 = frame.wb_push_candidate2;
  790. @@ -8672,7 +8671,6 @@ aarch64_layout_frame (void)
  791. stp reg1, reg2, [sp, bytes_below_saved_regs]
  792. stp reg3, reg4, [sp, bytes_below_saved_regs + 16] */
  793. frame.initial_adjust = frame.frame_size;
  794. - frame.callee_offset = const_below_saved_regs;
  795. }
  796. else if (saves_below_hard_fp_p
  797. && known_eq (frame.saved_regs_size,
  798. @@ -9073,12 +9071,13 @@ aarch64_add_cfa_expression (rtx_insn *insn, rtx reg,
  799. }
  800. /* Emit code to save the callee-saved registers from register number START
  801. - to LIMIT to the stack at the location starting at offset START_OFFSET,
  802. - skipping any write-back candidates if SKIP_WB is true. HARD_FP_VALID_P
  803. - is true if the hard frame pointer has been set up. */
  804. + to LIMIT to the stack. The stack pointer is currently BYTES_BELOW_SP
  805. + bytes above the bottom of the static frame. Skip any write-back
  806. + candidates if SKIP_WB is true. HARD_FP_VALID_P is true if the hard
  807. + frame pointer has been set up. */
  808. static void
  809. -aarch64_save_callee_saves (poly_int64 start_offset,
  810. +aarch64_save_callee_saves (poly_int64 bytes_below_sp,
  811. unsigned start, unsigned limit, bool skip_wb,
  812. bool hard_fp_valid_p)
  813. {
  814. @@ -9106,7 +9105,9 @@ aarch64_save_callee_saves (poly_int64 start_offset,
  815. machine_mode mode = aarch64_reg_save_mode (regno);
  816. reg = gen_rtx_REG (mode, regno);
  817. - offset = start_offset + frame.reg_offset[regno];
  818. + offset = (frame.reg_offset[regno]
  819. + + frame.bytes_below_saved_regs
  820. + - bytes_below_sp);
  821. rtx base_rtx = stack_pointer_rtx;
  822. poly_int64 sp_offset = offset;
  823. @@ -9117,9 +9118,7 @@ aarch64_save_callee_saves (poly_int64 start_offset,
  824. else if (GP_REGNUM_P (regno)
  825. && (!offset.is_constant (&const_offset) || const_offset >= 512))
  826. {
  827. - gcc_assert (known_eq (start_offset, 0));
  828. - poly_int64 fp_offset
  829. - = frame.below_hard_fp_saved_regs_size;
  830. + poly_int64 fp_offset = frame.bytes_below_hard_fp - bytes_below_sp;
  831. if (hard_fp_valid_p)
  832. base_rtx = hard_frame_pointer_rtx;
  833. else
  834. @@ -9183,12 +9182,13 @@ aarch64_save_callee_saves (poly_int64 start_offset,
  835. }
  836. /* Emit code to restore the callee registers from register number START
  837. - up to and including LIMIT. Restore from the stack offset START_OFFSET,
  838. - skipping any write-back candidates if SKIP_WB is true. Write the
  839. - appropriate REG_CFA_RESTORE notes into CFI_OPS. */
  840. + up to and including LIMIT. The stack pointer is currently BYTES_BELOW_SP
  841. + bytes above the bottom of the static frame. Skip any write-back
  842. + candidates if SKIP_WB is true. Write the appropriate REG_CFA_RESTORE
  843. + notes into CFI_OPS. */
  844. static void
  845. -aarch64_restore_callee_saves (poly_int64 start_offset, unsigned start,
  846. +aarch64_restore_callee_saves (poly_int64 bytes_below_sp, unsigned start,
  847. unsigned limit, bool skip_wb, rtx *cfi_ops)
  848. {
  849. aarch64_frame &frame = cfun->machine->frame;
  850. @@ -9214,7 +9214,9 @@ aarch64_restore_callee_saves (poly_int64 start_offset, unsigned start,
  851. machine_mode mode = aarch64_reg_save_mode (regno);
  852. reg = gen_rtx_REG (mode, regno);
  853. - offset = start_offset + frame.reg_offset[regno];
  854. + offset = (frame.reg_offset[regno]
  855. + + frame.bytes_below_saved_regs
  856. + - bytes_below_sp);
  857. rtx base_rtx = stack_pointer_rtx;
  858. if (mode == VNx2DImode && BYTES_BIG_ENDIAN)
  859. aarch64_adjust_sve_callee_save_base (mode, base_rtx, anchor_reg,
  860. @@ -9990,8 +9992,6 @@ aarch64_expand_prologue (void)
  861. HOST_WIDE_INT callee_adjust = frame.callee_adjust;
  862. poly_int64 final_adjust = frame.final_adjust;
  863. poly_int64 sve_callee_adjust = frame.sve_callee_adjust;
  864. - poly_int64 below_hard_fp_saved_regs_size
  865. - = frame.below_hard_fp_saved_regs_size;
  866. unsigned reg1 = frame.wb_push_candidate1;
  867. unsigned reg2 = frame.wb_push_candidate2;
  868. bool emit_frame_chain = frame.emit_frame_chain;
  869. @@ -10067,8 +10067,8 @@ aarch64_expand_prologue (void)
  870. - frame.hard_fp_offset);
  871. gcc_assert (known_ge (chain_offset, 0));
  872. - /* The offset of the bottom of the save area from the current SP. */
  873. - poly_int64 saved_regs_offset = chain_offset - below_hard_fp_saved_regs_size;
  874. + /* The offset of the current SP from the bottom of the static frame. */
  875. + poly_int64 bytes_below_sp = frame_size - initial_adjust - callee_adjust;
  876. if (emit_frame_chain)
  877. {
  878. @@ -10076,7 +10076,7 @@ aarch64_expand_prologue (void)
  879. {
  880. reg1 = R29_REGNUM;
  881. reg2 = R30_REGNUM;
  882. - aarch64_save_callee_saves (saved_regs_offset, reg1, reg2,
  883. + aarch64_save_callee_saves (bytes_below_sp, reg1, reg2,
  884. false, false);
  885. }
  886. else
  887. @@ -10116,7 +10116,7 @@ aarch64_expand_prologue (void)
  888. emit_insn (gen_stack_tie (stack_pointer_rtx, hard_frame_pointer_rtx));
  889. }
  890. - aarch64_save_callee_saves (saved_regs_offset, R0_REGNUM, R30_REGNUM,
  891. + aarch64_save_callee_saves (bytes_below_sp, R0_REGNUM, R30_REGNUM,
  892. callee_adjust != 0 || emit_frame_chain,
  893. emit_frame_chain);
  894. if (maybe_ne (sve_callee_adjust, 0))
  895. @@ -10126,16 +10126,17 @@ aarch64_expand_prologue (void)
  896. aarch64_allocate_and_probe_stack_space (tmp1_rtx, tmp0_rtx,
  897. sve_callee_adjust,
  898. !frame_pointer_needed, false);
  899. - saved_regs_offset += sve_callee_adjust;
  900. + bytes_below_sp -= sve_callee_adjust;
  901. }
  902. - aarch64_save_callee_saves (saved_regs_offset, P0_REGNUM, P15_REGNUM,
  903. + aarch64_save_callee_saves (bytes_below_sp, P0_REGNUM, P15_REGNUM,
  904. false, emit_frame_chain);
  905. - aarch64_save_callee_saves (saved_regs_offset, V0_REGNUM, V31_REGNUM,
  906. + aarch64_save_callee_saves (bytes_below_sp, V0_REGNUM, V31_REGNUM,
  907. callee_adjust != 0 || emit_frame_chain,
  908. emit_frame_chain);
  909. /* We may need to probe the final adjustment if it is larger than the guard
  910. that is assumed by the called. */
  911. + gcc_assert (known_eq (bytes_below_sp, final_adjust));
  912. aarch64_allocate_and_probe_stack_space (tmp1_rtx, tmp0_rtx, final_adjust,
  913. !frame_pointer_needed, true);
  914. }
  915. @@ -10170,7 +10171,6 @@ aarch64_expand_epilogue (bool for_sibcall)
  916. poly_int64 initial_adjust = frame.initial_adjust;
  917. HOST_WIDE_INT callee_adjust = frame.callee_adjust;
  918. poly_int64 final_adjust = frame.final_adjust;
  919. - poly_int64 callee_offset = frame.callee_offset;
  920. poly_int64 sve_callee_adjust = frame.sve_callee_adjust;
  921. poly_int64 bytes_below_hard_fp = frame.bytes_below_hard_fp;
  922. unsigned reg1 = frame.wb_pop_candidate1;
  923. @@ -10240,9 +10240,9 @@ aarch64_expand_epilogue (bool for_sibcall)
  924. /* Restore the vector registers before the predicate registers,
  925. so that we can use P4 as a temporary for big-endian SVE frames. */
  926. - aarch64_restore_callee_saves (callee_offset, V0_REGNUM, V31_REGNUM,
  927. + aarch64_restore_callee_saves (final_adjust, V0_REGNUM, V31_REGNUM,
  928. callee_adjust != 0, &cfi_ops);
  929. - aarch64_restore_callee_saves (callee_offset, P0_REGNUM, P15_REGNUM,
  930. + aarch64_restore_callee_saves (final_adjust, P0_REGNUM, P15_REGNUM,
  931. false, &cfi_ops);
  932. if (maybe_ne (sve_callee_adjust, 0))
  933. aarch64_add_sp (NULL_RTX, NULL_RTX, sve_callee_adjust, true);
  934. @@ -10250,7 +10250,7 @@ aarch64_expand_epilogue (bool for_sibcall)
  935. /* When shadow call stack is enabled, the scs_pop in the epilogue will
  936. restore x30, we don't need to restore x30 again in the traditional
  937. way. */
  938. - aarch64_restore_callee_saves (callee_offset - sve_callee_adjust,
  939. + aarch64_restore_callee_saves (final_adjust + sve_callee_adjust,
  940. R0_REGNUM, last_gpr,
  941. callee_adjust != 0, &cfi_ops);
  942. diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
  943. index 4263d29d29d..fd820b1be4e 100644
  944. --- a/gcc/config/aarch64/aarch64.h
  945. +++ b/gcc/config/aarch64/aarch64.h
  946. @@ -813,10 +813,6 @@ struct GTY (()) aarch64_frame
  947. It is zero when no push is used. */
  948. HOST_WIDE_INT callee_adjust;
  949. - /* The offset from SP to the callee-save registers after initial_adjust.
  950. - It may be non-zero if no push is used (ie. callee_adjust == 0). */
  951. - poly_int64 callee_offset;
  952. -
  953. /* The size of the stack adjustment before saving or after restoring
  954. SVE registers. */
  955. poly_int64 sve_callee_adjust;
  956. --
  957. 2.34.1
  958. From 8ae9181426f2700c2e5a2909487fa630e6fa406b Mon Sep 17 00:00:00 2001
  959. From: Richard Sandiford <richard.sandiford@arm.com>
  960. Date: Tue, 12 Sep 2023 16:07:15 +0100
  961. Subject: [PATCH 07/19] aarch64: Only calculate chain_offset if there is a
  962. chain
  963. After previous patches, it is no longer necessary to calculate
  964. a chain_offset in cases where there is no chain record.
  965. gcc/
  966. * config/aarch64/aarch64.cc (aarch64_expand_prologue): Move the
  967. calculation of chain_offset into the emit_frame_chain block.
  968. ---
  969. gcc/config/aarch64/aarch64.cc | 10 +++++-----
  970. 1 file changed, 5 insertions(+), 5 deletions(-)
  971. diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
  972. index 46ae5cf7673..0e9b9717c08 100644
  973. --- a/gcc/config/aarch64/aarch64.cc
  974. +++ b/gcc/config/aarch64/aarch64.cc
  975. @@ -10062,16 +10062,16 @@ aarch64_expand_prologue (void)
  976. if (callee_adjust != 0)
  977. aarch64_push_regs (reg1, reg2, callee_adjust);
  978. - /* The offset of the frame chain record (if any) from the current SP. */
  979. - poly_int64 chain_offset = (initial_adjust + callee_adjust
  980. - - frame.hard_fp_offset);
  981. - gcc_assert (known_ge (chain_offset, 0));
  982. -
  983. /* The offset of the current SP from the bottom of the static frame. */
  984. poly_int64 bytes_below_sp = frame_size - initial_adjust - callee_adjust;
  985. if (emit_frame_chain)
  986. {
  987. + /* The offset of the frame chain record (if any) from the current SP. */
  988. + poly_int64 chain_offset = (initial_adjust + callee_adjust
  989. + - frame.hard_fp_offset);
  990. + gcc_assert (known_ge (chain_offset, 0));
  991. +
  992. if (callee_adjust == 0)
  993. {
  994. reg1 = R29_REGNUM;
  995. --
  996. 2.34.1
  997. From 375794feb614cee1f41b710b9cc1b6f25da6c1cb Mon Sep 17 00:00:00 2001
  998. From: Richard Sandiford <richard.sandiford@arm.com>
  999. Date: Tue, 12 Sep 2023 16:07:15 +0100
  1000. Subject: [PATCH 08/19] aarch64: Rename locals_offset to bytes_above_locals
  1001. MIME-Version: 1.0
  1002. Content-Type: text/plain; charset=UTF-8
  1003. Content-Transfer-Encoding: 8bit
  1004. locals_offset was described as:
  1005. /* Offset from the base of the frame (incomming SP) to the
  1006. top of the locals area. This value is always a multiple of
  1007. STACK_BOUNDARY. */
  1008. This is implicitly an “upside down” view of the frame: the incoming
  1009. SP is at offset 0, and anything N bytes below the incoming SP is at
  1010. offset N (rather than -N).
  1011. However, reg_offset instead uses a “right way up” view; that is,
  1012. it views offsets in address terms. Something above X is at a
  1013. positive offset from X and something below X is at a negative
  1014. offset from X.
  1015. Also, even on FRAME_GROWS_DOWNWARD targets like AArch64,
  1016. target-independent code views offsets in address terms too:
  1017. locals are allocated at negative offsets to virtual_stack_vars.
  1018. It seems confusing to have *_offset fields of the same structure
  1019. using different polarities like this. This patch tries to avoid
  1020. that by renaming locals_offset to bytes_above_locals.
  1021. gcc/
  1022. * config/aarch64/aarch64.h (aarch64_frame::locals_offset): Rename to...
  1023. (aarch64_frame::bytes_above_locals): ...this.
  1024. * config/aarch64/aarch64.cc (aarch64_layout_frame)
  1025. (aarch64_initial_elimination_offset): Update accordingly.
  1026. ---
  1027. gcc/config/aarch64/aarch64.cc | 6 +++---
  1028. gcc/config/aarch64/aarch64.h | 6 +++---
  1029. 2 files changed, 6 insertions(+), 6 deletions(-)
  1030. diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
  1031. index 0e9b9717c08..0a22f91520e 100644
  1032. --- a/gcc/config/aarch64/aarch64.cc
  1033. +++ b/gcc/config/aarch64/aarch64.cc
  1034. @@ -8598,7 +8598,7 @@ aarch64_layout_frame (void)
  1035. STACK_BOUNDARY / BITS_PER_UNIT));
  1036. frame.frame_size = saved_regs_and_above + frame.bytes_below_saved_regs;
  1037. - frame.locals_offset = frame.saved_varargs_size;
  1038. + frame.bytes_above_locals = frame.saved_varargs_size;
  1039. frame.initial_adjust = 0;
  1040. frame.final_adjust = 0;
  1041. @@ -12754,13 +12754,13 @@ aarch64_initial_elimination_offset (unsigned from, unsigned to)
  1042. return frame.hard_fp_offset;
  1043. if (from == FRAME_POINTER_REGNUM)
  1044. - return frame.hard_fp_offset - frame.locals_offset;
  1045. + return frame.hard_fp_offset - frame.bytes_above_locals;
  1046. }
  1047. if (to == STACK_POINTER_REGNUM)
  1048. {
  1049. if (from == FRAME_POINTER_REGNUM)
  1050. - return frame.frame_size - frame.locals_offset;
  1051. + return frame.frame_size - frame.bytes_above_locals;
  1052. }
  1053. return frame.frame_size;
  1054. diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
  1055. index fd820b1be4e..7ae12d13e2b 100644
  1056. --- a/gcc/config/aarch64/aarch64.h
  1057. +++ b/gcc/config/aarch64/aarch64.h
  1058. @@ -791,10 +791,10 @@ struct GTY (()) aarch64_frame
  1059. always a multiple of STACK_BOUNDARY. */
  1060. poly_int64 bytes_below_hard_fp;
  1061. - /* Offset from the base of the frame (incomming SP) to the
  1062. - top of the locals area. This value is always a multiple of
  1063. + /* The number of bytes between the top of the locals area and the top
  1064. + of the frame (the incomming SP). This value is always a multiple of
  1065. STACK_BOUNDARY. */
  1066. - poly_int64 locals_offset;
  1067. + poly_int64 bytes_above_locals;
  1068. /* Offset from the base of the frame (incomming SP) to the
  1069. hard_frame_pointer. This value is always a multiple of
  1070. --
  1071. 2.34.1
  1072. From 1a9ea1c45c75615ffbfabe652b3598a1d7be2168 Mon Sep 17 00:00:00 2001
  1073. From: Richard Sandiford <richard.sandiford@arm.com>
  1074. Date: Tue, 12 Sep 2023 16:07:16 +0100
  1075. Subject: [PATCH 09/19] aarch64: Rename hard_fp_offset to bytes_above_hard_fp
  1076. MIME-Version: 1.0
  1077. Content-Type: text/plain; charset=UTF-8
  1078. Content-Transfer-Encoding: 8bit
  1079. Similarly to the previous locals_offset patch, hard_fp_offset
  1080. was described as:
  1081. /* Offset from the base of the frame (incomming SP) to the
  1082. hard_frame_pointer. This value is always a multiple of
  1083. STACK_BOUNDARY. */
  1084. poly_int64 hard_fp_offset;
  1085. which again took an “upside-down” view: higher offsets meant lower
  1086. addresses. This patch renames the field to bytes_above_hard_fp instead.
  1087. gcc/
  1088. * config/aarch64/aarch64.h (aarch64_frame::hard_fp_offset): Rename
  1089. to...
  1090. (aarch64_frame::bytes_above_hard_fp): ...this.
  1091. * config/aarch64/aarch64.cc (aarch64_layout_frame)
  1092. (aarch64_expand_prologue): Update accordingly.
  1093. (aarch64_initial_elimination_offset): Likewise.
  1094. ---
  1095. gcc/config/aarch64/aarch64.cc | 26 +++++++++++++-------------
  1096. gcc/config/aarch64/aarch64.h | 6 +++---
  1097. 2 files changed, 16 insertions(+), 16 deletions(-)
  1098. diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
  1099. index 0a22f91520e..95499ae49ba 100644
  1100. --- a/gcc/config/aarch64/aarch64.cc
  1101. +++ b/gcc/config/aarch64/aarch64.cc
  1102. @@ -8590,7 +8590,7 @@ aarch64_layout_frame (void)
  1103. + get_frame_size (),
  1104. STACK_BOUNDARY / BITS_PER_UNIT);
  1105. - frame.hard_fp_offset
  1106. + frame.bytes_above_hard_fp
  1107. = saved_regs_and_above - frame.below_hard_fp_saved_regs_size;
  1108. /* Both these values are already aligned. */
  1109. @@ -8639,13 +8639,13 @@ aarch64_layout_frame (void)
  1110. else if (frame.wb_pop_candidate1 != INVALID_REGNUM)
  1111. max_push_offset = 256;
  1112. - HOST_WIDE_INT const_size, const_below_saved_regs, const_fp_offset;
  1113. + HOST_WIDE_INT const_size, const_below_saved_regs, const_above_fp;
  1114. HOST_WIDE_INT const_saved_regs_size;
  1115. if (known_eq (frame.saved_regs_size, 0))
  1116. frame.initial_adjust = frame.frame_size;
  1117. else if (frame.frame_size.is_constant (&const_size)
  1118. && const_size < max_push_offset
  1119. - && known_eq (frame.hard_fp_offset, const_size))
  1120. + && known_eq (frame.bytes_above_hard_fp, const_size))
  1121. {
  1122. /* Simple, small frame with no data below the saved registers.
  1123. @@ -8662,8 +8662,8 @@ aarch64_layout_frame (void)
  1124. case that it hardly seems worth the effort though. */
  1125. && (!saves_below_hard_fp_p || const_below_saved_regs == 0)
  1126. && !(cfun->calls_alloca
  1127. - && frame.hard_fp_offset.is_constant (&const_fp_offset)
  1128. - && const_fp_offset < max_push_offset))
  1129. + && frame.bytes_above_hard_fp.is_constant (&const_above_fp)
  1130. + && const_above_fp < max_push_offset))
  1131. {
  1132. /* Frame with small area below the saved registers:
  1133. @@ -8681,12 +8681,12 @@ aarch64_layout_frame (void)
  1134. sub sp, sp, hard_fp_offset + below_hard_fp_saved_regs_size
  1135. save SVE registers relative to SP
  1136. sub sp, sp, bytes_below_saved_regs */
  1137. - frame.initial_adjust = (frame.hard_fp_offset
  1138. + frame.initial_adjust = (frame.bytes_above_hard_fp
  1139. + frame.below_hard_fp_saved_regs_size);
  1140. frame.final_adjust = frame.bytes_below_saved_regs;
  1141. }
  1142. - else if (frame.hard_fp_offset.is_constant (&const_fp_offset)
  1143. - && const_fp_offset < max_push_offset)
  1144. + else if (frame.bytes_above_hard_fp.is_constant (&const_above_fp)
  1145. + && const_above_fp < max_push_offset)
  1146. {
  1147. /* Frame with large area below the saved registers, or with SVE saves,
  1148. but with a small area above:
  1149. @@ -8696,7 +8696,7 @@ aarch64_layout_frame (void)
  1150. [sub sp, sp, below_hard_fp_saved_regs_size]
  1151. [save SVE registers relative to SP]
  1152. sub sp, sp, bytes_below_saved_regs */
  1153. - frame.callee_adjust = const_fp_offset;
  1154. + frame.callee_adjust = const_above_fp;
  1155. frame.sve_callee_adjust = frame.below_hard_fp_saved_regs_size;
  1156. frame.final_adjust = frame.bytes_below_saved_regs;
  1157. }
  1158. @@ -8711,7 +8711,7 @@ aarch64_layout_frame (void)
  1159. [sub sp, sp, below_hard_fp_saved_regs_size]
  1160. [save SVE registers relative to SP]
  1161. sub sp, sp, bytes_below_saved_regs */
  1162. - frame.initial_adjust = frame.hard_fp_offset;
  1163. + frame.initial_adjust = frame.bytes_above_hard_fp;
  1164. frame.sve_callee_adjust = frame.below_hard_fp_saved_regs_size;
  1165. frame.final_adjust = frame.bytes_below_saved_regs;
  1166. }
  1167. @@ -10069,7 +10069,7 @@ aarch64_expand_prologue (void)
  1168. {
  1169. /* The offset of the frame chain record (if any) from the current SP. */
  1170. poly_int64 chain_offset = (initial_adjust + callee_adjust
  1171. - - frame.hard_fp_offset);
  1172. + - frame.bytes_above_hard_fp);
  1173. gcc_assert (known_ge (chain_offset, 0));
  1174. if (callee_adjust == 0)
  1175. @@ -12751,10 +12751,10 @@ aarch64_initial_elimination_offset (unsigned from, unsigned to)
  1176. if (to == HARD_FRAME_POINTER_REGNUM)
  1177. {
  1178. if (from == ARG_POINTER_REGNUM)
  1179. - return frame.hard_fp_offset;
  1180. + return frame.bytes_above_hard_fp;
  1181. if (from == FRAME_POINTER_REGNUM)
  1182. - return frame.hard_fp_offset - frame.bytes_above_locals;
  1183. + return frame.bytes_above_hard_fp - frame.bytes_above_locals;
  1184. }
  1185. if (to == STACK_POINTER_REGNUM)
  1186. diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
  1187. index 7ae12d13e2b..3808f49e9ca 100644
  1188. --- a/gcc/config/aarch64/aarch64.h
  1189. +++ b/gcc/config/aarch64/aarch64.h
  1190. @@ -796,10 +796,10 @@ struct GTY (()) aarch64_frame
  1191. STACK_BOUNDARY. */
  1192. poly_int64 bytes_above_locals;
  1193. - /* Offset from the base of the frame (incomming SP) to the
  1194. - hard_frame_pointer. This value is always a multiple of
  1195. + /* The number of bytes between the hard_frame_pointer and the top of
  1196. + the frame (the incomming SP). This value is always a multiple of
  1197. STACK_BOUNDARY. */
  1198. - poly_int64 hard_fp_offset;
  1199. + poly_int64 bytes_above_hard_fp;
  1200. /* The size of the frame. This value is the offset from base of the
  1201. frame (incomming SP) to the stack_pointer. This value is always
  1202. --
  1203. 2.34.1
  1204. From d202ce1ecf60a36a3e1009917dd76109248ce9be Mon Sep 17 00:00:00 2001
  1205. From: Richard Sandiford <richard.sandiford@arm.com>
  1206. Date: Tue, 12 Sep 2023 16:07:16 +0100
  1207. Subject: [PATCH 10/19] aarch64: Tweak frame_size comment
  1208. MIME-Version: 1.0
  1209. Content-Type: text/plain; charset=UTF-8
  1210. Content-Transfer-Encoding: 8bit
  1211. This patch fixes another case in which a value was described with
  1212. an “upside-down” view.
  1213. gcc/
  1214. * config/aarch64/aarch64.h (aarch64_frame::frame_size): Tweak comment.
  1215. ---
  1216. gcc/config/aarch64/aarch64.h | 4 ++--
  1217. 1 file changed, 2 insertions(+), 2 deletions(-)
  1218. diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
  1219. index 3808f49e9ca..108a5731b0d 100644
  1220. --- a/gcc/config/aarch64/aarch64.h
  1221. +++ b/gcc/config/aarch64/aarch64.h
  1222. @@ -801,8 +801,8 @@ struct GTY (()) aarch64_frame
  1223. STACK_BOUNDARY. */
  1224. poly_int64 bytes_above_hard_fp;
  1225. - /* The size of the frame. This value is the offset from base of the
  1226. - frame (incomming SP) to the stack_pointer. This value is always
  1227. + /* The size of the frame, i.e. the number of bytes between the bottom
  1228. + of the outgoing arguments and the incoming SP. This value is always
  1229. a multiple of STACK_BOUNDARY. */
  1230. poly_int64 frame_size;
  1231. --
  1232. 2.34.1
  1233. From f2b585375205b0a1802d79c682ba33766ecd1f0f Mon Sep 17 00:00:00 2001
  1234. From: Richard Sandiford <richard.sandiford@arm.com>
  1235. Date: Tue, 12 Sep 2023 16:07:17 +0100
  1236. Subject: [PATCH 11/19] aarch64: Measure reg_offset from the bottom of the
  1237. frame
  1238. reg_offset was measured from the bottom of the saved register area.
  1239. This made perfect sense with the original layout, since the bottom
  1240. of the saved register area was also the hard frame pointer address.
  1241. It became slightly less obvious with SVE, since we save SVE
  1242. registers below the hard frame pointer, but it still made sense.
  1243. However, if we want to allow different frame layouts, it's more
  1244. convenient and obvious to measure reg_offset from the bottom of
  1245. the frame. After previous patches, it's also a slight simplification
  1246. in its own right.
  1247. gcc/
  1248. * config/aarch64/aarch64.h (aarch64_frame): Add comment above
  1249. reg_offset.
  1250. * config/aarch64/aarch64.cc (aarch64_layout_frame): Walk offsets
  1251. from the bottom of the frame, rather than the bottom of the saved
  1252. register area. Measure reg_offset from the bottom of the frame
  1253. rather than the bottom of the saved register area.
  1254. (aarch64_save_callee_saves): Update accordingly.
  1255. (aarch64_restore_callee_saves): Likewise.
  1256. (aarch64_get_separate_components): Likewise.
  1257. (aarch64_process_components): Likewise.
  1258. ---
  1259. gcc/config/aarch64/aarch64.cc | 53 ++++++++++++++++-------------------
  1260. gcc/config/aarch64/aarch64.h | 3 ++
  1261. 2 files changed, 27 insertions(+), 29 deletions(-)
  1262. diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
  1263. index 95499ae49ba..af99807ef8a 100644
  1264. --- a/gcc/config/aarch64/aarch64.cc
  1265. +++ b/gcc/config/aarch64/aarch64.cc
  1266. @@ -8400,7 +8400,6 @@ aarch64_needs_frame_chain (void)
  1267. static void
  1268. aarch64_layout_frame (void)
  1269. {
  1270. - poly_int64 offset = 0;
  1271. int regno, last_fp_reg = INVALID_REGNUM;
  1272. machine_mode vector_save_mode = aarch64_reg_save_mode (V8_REGNUM);
  1273. poly_int64 vector_save_size = GET_MODE_SIZE (vector_save_mode);
  1274. @@ -8478,7 +8477,9 @@ aarch64_layout_frame (void)
  1275. gcc_assert (crtl->is_leaf
  1276. || maybe_ne (frame.reg_offset[R30_REGNUM], SLOT_NOT_REQUIRED));
  1277. - frame.bytes_below_saved_regs = crtl->outgoing_args_size;
  1278. + poly_int64 offset = crtl->outgoing_args_size;
  1279. + gcc_assert (multiple_p (offset, STACK_BOUNDARY / BITS_PER_UNIT));
  1280. + frame.bytes_below_saved_regs = offset;
  1281. /* Now assign stack slots for the registers. Start with the predicate
  1282. registers, since predicate LDR and STR have a relatively small
  1283. @@ -8490,7 +8491,8 @@ aarch64_layout_frame (void)
  1284. offset += BYTES_PER_SVE_PRED;
  1285. }
  1286. - if (maybe_ne (offset, 0))
  1287. + poly_int64 saved_prs_size = offset - frame.bytes_below_saved_regs;
  1288. + if (maybe_ne (saved_prs_size, 0))
  1289. {
  1290. /* If we have any vector registers to save above the predicate registers,
  1291. the offset of the vector register save slots need to be a multiple
  1292. @@ -8508,10 +8510,10 @@ aarch64_layout_frame (void)
  1293. offset = aligned_upper_bound (offset, STACK_BOUNDARY / BITS_PER_UNIT);
  1294. else
  1295. {
  1296. - if (known_le (offset, vector_save_size))
  1297. - offset = vector_save_size;
  1298. - else if (known_le (offset, vector_save_size * 2))
  1299. - offset = vector_save_size * 2;
  1300. + if (known_le (saved_prs_size, vector_save_size))
  1301. + offset = frame.bytes_below_saved_regs + vector_save_size;
  1302. + else if (known_le (saved_prs_size, vector_save_size * 2))
  1303. + offset = frame.bytes_below_saved_regs + vector_save_size * 2;
  1304. else
  1305. gcc_unreachable ();
  1306. }
  1307. @@ -8528,9 +8530,10 @@ aarch64_layout_frame (void)
  1308. /* OFFSET is now the offset of the hard frame pointer from the bottom
  1309. of the callee save area. */
  1310. - bool saves_below_hard_fp_p = maybe_ne (offset, 0);
  1311. - frame.below_hard_fp_saved_regs_size = offset;
  1312. - frame.bytes_below_hard_fp = offset + frame.bytes_below_saved_regs;
  1313. + frame.below_hard_fp_saved_regs_size = offset - frame.bytes_below_saved_regs;
  1314. + bool saves_below_hard_fp_p
  1315. + = maybe_ne (frame.below_hard_fp_saved_regs_size, 0);
  1316. + frame.bytes_below_hard_fp = offset;
  1317. if (frame.emit_frame_chain)
  1318. {
  1319. /* FP and LR are placed in the linkage record. */
  1320. @@ -8581,9 +8584,10 @@ aarch64_layout_frame (void)
  1321. offset = aligned_upper_bound (offset, STACK_BOUNDARY / BITS_PER_UNIT);
  1322. - frame.saved_regs_size = offset;
  1323. + frame.saved_regs_size = offset - frame.bytes_below_saved_regs;
  1324. - poly_int64 varargs_and_saved_regs_size = offset + frame.saved_varargs_size;
  1325. + poly_int64 varargs_and_saved_regs_size
  1326. + = frame.saved_regs_size + frame.saved_varargs_size;
  1327. poly_int64 saved_regs_and_above
  1328. = aligned_upper_bound (varargs_and_saved_regs_size
  1329. @@ -9105,9 +9109,7 @@ aarch64_save_callee_saves (poly_int64 bytes_below_sp,
  1330. machine_mode mode = aarch64_reg_save_mode (regno);
  1331. reg = gen_rtx_REG (mode, regno);
  1332. - offset = (frame.reg_offset[regno]
  1333. - + frame.bytes_below_saved_regs
  1334. - - bytes_below_sp);
  1335. + offset = frame.reg_offset[regno] - bytes_below_sp;
  1336. rtx base_rtx = stack_pointer_rtx;
  1337. poly_int64 sp_offset = offset;
  1338. @@ -9214,9 +9216,7 @@ aarch64_restore_callee_saves (poly_int64 bytes_below_sp, unsigned start,
  1339. machine_mode mode = aarch64_reg_save_mode (regno);
  1340. reg = gen_rtx_REG (mode, regno);
  1341. - offset = (frame.reg_offset[regno]
  1342. - + frame.bytes_below_saved_regs
  1343. - - bytes_below_sp);
  1344. + offset = frame.reg_offset[regno] - bytes_below_sp;
  1345. rtx base_rtx = stack_pointer_rtx;
  1346. if (mode == VNx2DImode && BYTES_BIG_ENDIAN)
  1347. aarch64_adjust_sve_callee_save_base (mode, base_rtx, anchor_reg,
  1348. @@ -9355,14 +9355,12 @@ aarch64_get_separate_components (void)
  1349. it as a stack probe for -fstack-clash-protection. */
  1350. if (flag_stack_clash_protection
  1351. && maybe_ne (frame.below_hard_fp_saved_regs_size, 0)
  1352. - && known_eq (offset, 0))
  1353. + && known_eq (offset, frame.bytes_below_saved_regs))
  1354. continue;
  1355. /* Get the offset relative to the register we'll use. */
  1356. if (frame_pointer_needed)
  1357. - offset -= frame.below_hard_fp_saved_regs_size;
  1358. - else
  1359. - offset += frame.bytes_below_saved_regs;
  1360. + offset -= frame.bytes_below_hard_fp;
  1361. /* Check that we can access the stack slot of the register with one
  1362. direct load with no adjustments needed. */
  1363. @@ -9509,9 +9507,7 @@ aarch64_process_components (sbitmap components, bool prologue_p)
  1364. rtx reg = gen_rtx_REG (mode, regno);
  1365. poly_int64 offset = frame.reg_offset[regno];
  1366. if (frame_pointer_needed)
  1367. - offset -= frame.below_hard_fp_saved_regs_size;
  1368. - else
  1369. - offset += frame.bytes_below_saved_regs;
  1370. + offset -= frame.bytes_below_hard_fp;
  1371. rtx addr = plus_constant (Pmode, ptr_reg, offset);
  1372. rtx mem = gen_frame_mem (mode, addr);
  1373. @@ -9563,9 +9559,7 @@ aarch64_process_components (sbitmap components, bool prologue_p)
  1374. /* REGNO2 can be saved/restored in a pair with REGNO. */
  1375. rtx reg2 = gen_rtx_REG (mode, regno2);
  1376. if (frame_pointer_needed)
  1377. - offset2 -= frame.below_hard_fp_saved_regs_size;
  1378. - else
  1379. - offset2 += frame.bytes_below_saved_regs;
  1380. + offset2 -= frame.bytes_below_hard_fp;
  1381. rtx addr2 = plus_constant (Pmode, ptr_reg, offset2);
  1382. rtx mem2 = gen_frame_mem (mode, addr2);
  1383. rtx set2 = prologue_p ? gen_rtx_SET (mem2, reg2)
  1384. @@ -9681,7 +9675,8 @@ aarch64_allocate_and_probe_stack_space (rtx temp1, rtx temp2,
  1385. if (final_adjustment_p
  1386. && known_eq (frame.below_hard_fp_saved_regs_size, 0))
  1387. {
  1388. - poly_int64 lr_offset = frame.reg_offset[LR_REGNUM];
  1389. + poly_int64 lr_offset = (frame.reg_offset[LR_REGNUM]
  1390. + - frame.bytes_below_saved_regs);
  1391. if (known_ge (lr_offset, 0))
  1392. min_probe_threshold -= lr_offset.to_constant ();
  1393. else
  1394. diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
  1395. index 108a5731b0d..c8becb098c8 100644
  1396. --- a/gcc/config/aarch64/aarch64.h
  1397. +++ b/gcc/config/aarch64/aarch64.h
  1398. @@ -766,6 +766,9 @@ extern enum aarch64_processor aarch64_tune;
  1399. #ifdef HAVE_POLY_INT_H
  1400. struct GTY (()) aarch64_frame
  1401. {
  1402. + /* The offset from the bottom of the static frame (the bottom of the
  1403. + outgoing arguments) of each register save slot, or -2 if no save is
  1404. + needed. */
  1405. poly_int64 reg_offset[LAST_SAVED_REGNUM + 1];
  1406. /* The number of extra stack bytes taken up by register varargs.
  1407. --
  1408. 2.34.1
  1409. From 79faabda181d0d9fd29a3cf5726ba65bdee945b5 Mon Sep 17 00:00:00 2001
  1410. From: Richard Sandiford <richard.sandiford@arm.com>
  1411. Date: Tue, 12 Sep 2023 16:07:17 +0100
  1412. Subject: [PATCH 12/19] aarch64: Simplify top of frame allocation
  1413. After previous patches, it no longer really makes sense to allocate
  1414. the top of the frame in terms of varargs_and_saved_regs_size and
  1415. saved_regs_and_above.
  1416. gcc/
  1417. * config/aarch64/aarch64.cc (aarch64_layout_frame): Simplify
  1418. the allocation of the top of the frame.
  1419. ---
  1420. gcc/config/aarch64/aarch64.cc | 23 ++++++++---------------
  1421. 1 file changed, 8 insertions(+), 15 deletions(-)
  1422. diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
  1423. index af99807ef8a..31b00094c2a 100644
  1424. --- a/gcc/config/aarch64/aarch64.cc
  1425. +++ b/gcc/config/aarch64/aarch64.cc
  1426. @@ -8586,23 +8586,16 @@ aarch64_layout_frame (void)
  1427. frame.saved_regs_size = offset - frame.bytes_below_saved_regs;
  1428. - poly_int64 varargs_and_saved_regs_size
  1429. - = frame.saved_regs_size + frame.saved_varargs_size;
  1430. -
  1431. - poly_int64 saved_regs_and_above
  1432. - = aligned_upper_bound (varargs_and_saved_regs_size
  1433. - + get_frame_size (),
  1434. - STACK_BOUNDARY / BITS_PER_UNIT);
  1435. -
  1436. - frame.bytes_above_hard_fp
  1437. - = saved_regs_and_above - frame.below_hard_fp_saved_regs_size;
  1438. + offset += get_frame_size ();
  1439. + offset = aligned_upper_bound (offset, STACK_BOUNDARY / BITS_PER_UNIT);
  1440. + auto top_of_locals = offset;
  1441. - /* Both these values are already aligned. */
  1442. - gcc_assert (multiple_p (frame.bytes_below_saved_regs,
  1443. - STACK_BOUNDARY / BITS_PER_UNIT));
  1444. - frame.frame_size = saved_regs_and_above + frame.bytes_below_saved_regs;
  1445. + offset += frame.saved_varargs_size;
  1446. + gcc_assert (multiple_p (offset, STACK_BOUNDARY / BITS_PER_UNIT));
  1447. + frame.frame_size = offset;
  1448. - frame.bytes_above_locals = frame.saved_varargs_size;
  1449. + frame.bytes_above_hard_fp = frame.frame_size - frame.bytes_below_hard_fp;
  1450. + frame.bytes_above_locals = frame.frame_size - top_of_locals;
  1451. frame.initial_adjust = 0;
  1452. frame.final_adjust = 0;
  1453. --
  1454. 2.34.1
  1455. From 4e62049e403b141e6f916176160dac8cbd65fe47 Mon Sep 17 00:00:00 2001
  1456. From: Richard Sandiford <richard.sandiford@arm.com>
  1457. Date: Tue, 12 Sep 2023 16:07:18 +0100
  1458. Subject: [PATCH 13/19] aarch64: Minor initial adjustment tweak
  1459. This patch just changes a calculation of initial_adjust
  1460. to one that makes it slightly more obvious that the total
  1461. adjustment is frame.frame_size.
  1462. gcc/
  1463. * config/aarch64/aarch64.cc (aarch64_layout_frame): Tweak
  1464. calculation of initial_adjust for frames in which all saves
  1465. are SVE saves.
  1466. ---
  1467. gcc/config/aarch64/aarch64.cc | 5 ++---
  1468. 1 file changed, 2 insertions(+), 3 deletions(-)
  1469. diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
  1470. index 31b00094c2a..1aa79da0673 100644
  1471. --- a/gcc/config/aarch64/aarch64.cc
  1472. +++ b/gcc/config/aarch64/aarch64.cc
  1473. @@ -8675,11 +8675,10 @@ aarch64_layout_frame (void)
  1474. {
  1475. /* Frame in which all saves are SVE saves:
  1476. - sub sp, sp, hard_fp_offset + below_hard_fp_saved_regs_size
  1477. + sub sp, sp, frame_size - bytes_below_saved_regs
  1478. save SVE registers relative to SP
  1479. sub sp, sp, bytes_below_saved_regs */
  1480. - frame.initial_adjust = (frame.bytes_above_hard_fp
  1481. - + frame.below_hard_fp_saved_regs_size);
  1482. + frame.initial_adjust = frame.frame_size - frame.bytes_below_saved_regs;
  1483. frame.final_adjust = frame.bytes_below_saved_regs;
  1484. }
  1485. else if (frame.bytes_above_hard_fp.is_constant (&const_above_fp)
  1486. --
  1487. 2.34.1
  1488. From aaa1a0a5912d9e5d571e5f1c6f09ceac99544ab5 Mon Sep 17 00:00:00 2001
  1489. From: Richard Sandiford <richard.sandiford@arm.com>
  1490. Date: Tue, 12 Sep 2023 16:07:18 +0100
  1491. Subject: [PATCH 14/19] aarch64: Tweak stack clash boundary condition
  1492. The AArch64 ABI says that, when stack clash protection is used,
  1493. there can be a maximum of 1KiB of unprobed space at sp on entry
  1494. to a function. Therefore, we need to probe when allocating
  1495. >= guard_size - 1KiB of data (>= rather than >). This is what
  1496. GCC does.
  1497. If an allocation is exactly guard_size bytes, it is enough to allocate
  1498. those bytes and probe once at offset 1024. It isn't possible to use a
  1499. single probe at any other offset: higher would conmplicate later code,
  1500. by leaving more unprobed space than usual, while lower would risk
  1501. leaving an entire page unprobed. For simplicity, the code probes all
  1502. allocations at offset 1024.
  1503. Some register saves also act as probes. If we need to allocate
  1504. more space below the last such register save probe, we need to
  1505. probe the allocation if it is > 1KiB. Again, this allocation is
  1506. then sometimes (but not always) probed at offset 1024. This sort of
  1507. allocation is currently only used for outgoing arguments, which are
  1508. rarely this big.
  1509. However, the code also probed if this final outgoing-arguments
  1510. allocation was == 1KiB, rather than just > 1KiB. This isn't
  1511. necessary, since the register save then probes at offset 1024
  1512. as required. Continuing to probe allocations of exactly 1KiB
  1513. would complicate later patches.
  1514. gcc/
  1515. * config/aarch64/aarch64.cc (aarch64_allocate_and_probe_stack_space):
  1516. Don't probe final allocations that are exactly 1KiB in size (after
  1517. unprobed space above the final allocation has been deducted).
  1518. gcc/testsuite/
  1519. * gcc.target/aarch64/stack-check-prologue-17.c: New test.
  1520. ---
  1521. gcc/config/aarch64/aarch64.cc | 4 +-
  1522. .../aarch64/stack-check-prologue-17.c | 55 +++++++++++++++++++
  1523. 2 files changed, 58 insertions(+), 1 deletion(-)
  1524. create mode 100644 gcc/testsuite/gcc.target/aarch64/stack-check-prologue-17.c
  1525. diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
  1526. index 1aa79da0673..5cad847977a 100644
  1527. --- a/gcc/config/aarch64/aarch64.cc
  1528. +++ b/gcc/config/aarch64/aarch64.cc
  1529. @@ -9648,9 +9648,11 @@ aarch64_allocate_and_probe_stack_space (rtx temp1, rtx temp2,
  1530. HOST_WIDE_INT guard_size
  1531. = 1 << param_stack_clash_protection_guard_size;
  1532. HOST_WIDE_INT guard_used_by_caller = STACK_CLASH_CALLER_GUARD;
  1533. + HOST_WIDE_INT byte_sp_alignment = STACK_BOUNDARY / BITS_PER_UNIT;
  1534. + gcc_assert (multiple_p (poly_size, byte_sp_alignment));
  1535. HOST_WIDE_INT min_probe_threshold
  1536. = (final_adjustment_p
  1537. - ? guard_used_by_caller
  1538. + ? guard_used_by_caller + byte_sp_alignment
  1539. : guard_size - guard_used_by_caller);
  1540. /* When doing the final adjustment for the outgoing arguments, take into
  1541. account any unprobed space there is above the current SP. There are
  1542. diff --git a/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-17.c b/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-17.c
  1543. new file mode 100644
  1544. index 00000000000..0d8a25d73a2
  1545. --- /dev/null
  1546. +++ b/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-17.c
  1547. @@ -0,0 +1,55 @@
  1548. +/* { dg-options "-O2 -fstack-clash-protection -fomit-frame-pointer --param stack-clash-protection-guard-size=12" } */
  1549. +/* { dg-final { check-function-bodies "**" "" } } */
  1550. +
  1551. +void f(int, ...);
  1552. +void g();
  1553. +
  1554. +/*
  1555. +** test1:
  1556. +** ...
  1557. +** str x30, \[sp\]
  1558. +** sub sp, sp, #1024
  1559. +** cbnz w0, .*
  1560. +** bl g
  1561. +** ...
  1562. +*/
  1563. +int test1(int z) {
  1564. + __uint128_t x = 0;
  1565. + int y[0x400];
  1566. + if (z)
  1567. + {
  1568. + f(0, 0, 0, 0, 0, 0, 0, &y,
  1569. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1570. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1571. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1572. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x);
  1573. + }
  1574. + g();
  1575. + return 1;
  1576. +}
  1577. +
  1578. +/*
  1579. +** test2:
  1580. +** ...
  1581. +** str x30, \[sp\]
  1582. +** sub sp, sp, #1040
  1583. +** str xzr, \[sp\]
  1584. +** cbnz w0, .*
  1585. +** bl g
  1586. +** ...
  1587. +*/
  1588. +int test2(int z) {
  1589. + __uint128_t x = 0;
  1590. + int y[0x400];
  1591. + if (z)
  1592. + {
  1593. + f(0, 0, 0, 0, 0, 0, 0, &y,
  1594. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1595. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1596. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1597. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1598. + x);
  1599. + }
  1600. + g();
  1601. + return 1;
  1602. +}
  1603. --
  1604. 2.34.1
  1605. From 8433953434a7b58c0923140d39eb3c5988c1d097 Mon Sep 17 00:00:00 2001
  1606. From: Richard Sandiford <richard.sandiford@arm.com>
  1607. Date: Tue, 12 Sep 2023 16:07:19 +0100
  1608. Subject: [PATCH 15/19] aarch64: Put LR save probe in first 16 bytes
  1609. -fstack-clash-protection uses the save of LR as a probe for the next
  1610. allocation. The next allocation could be:
  1611. * another part of the static frame, e.g. when allocating SVE save slots
  1612. or outgoing arguments
  1613. * an alloca in the same function
  1614. * an allocation made by a callee function
  1615. However, when -fomit-frame-pointer is used, the LR save slot is placed
  1616. above the other GPR save slots. It could therefore be up to 80 bytes
  1617. above the base of the GPR save area (which is also the hard fp address).
  1618. aarch64_allocate_and_probe_stack_space took this into account when
  1619. deciding how much subsequent space could be allocated without needing
  1620. a probe. However, it interacted badly with:
  1621. /* If doing a small final adjustment, we always probe at offset 0.
  1622. This is done to avoid issues when LR is not at position 0 or when
  1623. the final adjustment is smaller than the probing offset. */
  1624. else if (final_adjustment_p && rounded_size == 0)
  1625. residual_probe_offset = 0;
  1626. which forces any allocation that is smaller than the guard page size
  1627. to be probed at offset 0 rather than the usual offset 1024. It was
  1628. therefore possible to construct cases in which we had:
  1629. * a probe using LR at SP + 80 bytes (or some other value >= 16)
  1630. * an allocation of the guard page size - 16 bytes
  1631. * a probe at SP + 0
  1632. which allocates guard page size + 64 consecutive unprobed bytes.
  1633. This patch requires the LR probe to be in the first 16 bytes of the
  1634. save area when stack clash protection is active. Doing it
  1635. unconditionally would cause code-quality regressions.
  1636. Putting LR before other registers prevents push/pop allocation
  1637. when shadow call stacks are enabled, since LR is restored
  1638. separately from the other callee-saved registers.
  1639. The new comment doesn't say that the probe register is required
  1640. to be LR, since a later patch removes that restriction.
  1641. gcc/
  1642. * config/aarch64/aarch64.cc (aarch64_layout_frame): Ensure that
  1643. the LR save slot is in the first 16 bytes of the register save area.
  1644. Only form STP/LDP push/pop candidates if both registers are valid.
  1645. (aarch64_allocate_and_probe_stack_space): Remove workaround for
  1646. when LR was not in the first 16 bytes.
  1647. gcc/testsuite/
  1648. * gcc.target/aarch64/stack-check-prologue-18.c: New test.
  1649. * gcc.target/aarch64/stack-check-prologue-19.c: Likewise.
  1650. * gcc.target/aarch64/stack-check-prologue-20.c: Likewise.
  1651. ---
  1652. gcc/config/aarch64/aarch64.cc | 72 ++++++-------
  1653. .../aarch64/stack-check-prologue-18.c | 100 ++++++++++++++++++
  1654. .../aarch64/stack-check-prologue-19.c | 100 ++++++++++++++++++
  1655. .../aarch64/stack-check-prologue-20.c | 3 +
  1656. 4 files changed, 233 insertions(+), 42 deletions(-)
  1657. create mode 100644 gcc/testsuite/gcc.target/aarch64/stack-check-prologue-18.c
  1658. create mode 100644 gcc/testsuite/gcc.target/aarch64/stack-check-prologue-19.c
  1659. create mode 100644 gcc/testsuite/gcc.target/aarch64/stack-check-prologue-20.c
  1660. diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
  1661. index 5cad847977a..a765f92329d 100644
  1662. --- a/gcc/config/aarch64/aarch64.cc
  1663. +++ b/gcc/config/aarch64/aarch64.cc
  1664. @@ -8534,26 +8534,34 @@ aarch64_layout_frame (void)
  1665. bool saves_below_hard_fp_p
  1666. = maybe_ne (frame.below_hard_fp_saved_regs_size, 0);
  1667. frame.bytes_below_hard_fp = offset;
  1668. +
  1669. + auto allocate_gpr_slot = [&](unsigned int regno)
  1670. + {
  1671. + frame.reg_offset[regno] = offset;
  1672. + if (frame.wb_push_candidate1 == INVALID_REGNUM)
  1673. + frame.wb_push_candidate1 = regno;
  1674. + else if (frame.wb_push_candidate2 == INVALID_REGNUM)
  1675. + frame.wb_push_candidate2 = regno;
  1676. + offset += UNITS_PER_WORD;
  1677. + };
  1678. +
  1679. if (frame.emit_frame_chain)
  1680. {
  1681. /* FP and LR are placed in the linkage record. */
  1682. - frame.reg_offset[R29_REGNUM] = offset;
  1683. - frame.wb_push_candidate1 = R29_REGNUM;
  1684. - frame.reg_offset[R30_REGNUM] = offset + UNITS_PER_WORD;
  1685. - frame.wb_push_candidate2 = R30_REGNUM;
  1686. - offset += 2 * UNITS_PER_WORD;
  1687. + allocate_gpr_slot (R29_REGNUM);
  1688. + allocate_gpr_slot (R30_REGNUM);
  1689. }
  1690. + else if (flag_stack_clash_protection
  1691. + && known_eq (frame.reg_offset[R30_REGNUM], SLOT_REQUIRED))
  1692. + /* Put the LR save slot first, since it makes a good choice of probe
  1693. + for stack clash purposes. The idea is that the link register usually
  1694. + has to be saved before a call anyway, and so we lose little by
  1695. + stopping it from being individually shrink-wrapped. */
  1696. + allocate_gpr_slot (R30_REGNUM);
  1697. for (regno = R0_REGNUM; regno <= R30_REGNUM; regno++)
  1698. if (known_eq (frame.reg_offset[regno], SLOT_REQUIRED))
  1699. - {
  1700. - frame.reg_offset[regno] = offset;
  1701. - if (frame.wb_push_candidate1 == INVALID_REGNUM)
  1702. - frame.wb_push_candidate1 = regno;
  1703. - else if (frame.wb_push_candidate2 == INVALID_REGNUM)
  1704. - frame.wb_push_candidate2 = regno;
  1705. - offset += UNITS_PER_WORD;
  1706. - }
  1707. + allocate_gpr_slot (regno);
  1708. poly_int64 max_int_offset = offset;
  1709. offset = aligned_upper_bound (offset, STACK_BOUNDARY / BITS_PER_UNIT);
  1710. @@ -8631,10 +8639,13 @@ aarch64_layout_frame (void)
  1711. max_push_offset to 0, because no registers are popped at this time,
  1712. so callee_adjust cannot be adjusted. */
  1713. HOST_WIDE_INT max_push_offset = 0;
  1714. - if (frame.wb_pop_candidate2 != INVALID_REGNUM)
  1715. - max_push_offset = 512;
  1716. - else if (frame.wb_pop_candidate1 != INVALID_REGNUM)
  1717. - max_push_offset = 256;
  1718. + if (frame.wb_pop_candidate1 != INVALID_REGNUM)
  1719. + {
  1720. + if (frame.wb_pop_candidate2 != INVALID_REGNUM)
  1721. + max_push_offset = 512;
  1722. + else
  1723. + max_push_offset = 256;
  1724. + }
  1725. HOST_WIDE_INT const_size, const_below_saved_regs, const_above_fp;
  1726. HOST_WIDE_INT const_saved_regs_size;
  1727. @@ -9654,29 +9665,6 @@ aarch64_allocate_and_probe_stack_space (rtx temp1, rtx temp2,
  1728. = (final_adjustment_p
  1729. ? guard_used_by_caller + byte_sp_alignment
  1730. : guard_size - guard_used_by_caller);
  1731. - /* When doing the final adjustment for the outgoing arguments, take into
  1732. - account any unprobed space there is above the current SP. There are
  1733. - two cases:
  1734. -
  1735. - - When saving SVE registers below the hard frame pointer, we force
  1736. - the lowest save to take place in the prologue before doing the final
  1737. - adjustment (i.e. we don't allow the save to be shrink-wrapped).
  1738. - This acts as a probe at SP, so there is no unprobed space.
  1739. -
  1740. - - When there are no SVE register saves, we use the store of the link
  1741. - register as a probe. We can't assume that LR was saved at position 0
  1742. - though, so treat any space below it as unprobed. */
  1743. - if (final_adjustment_p
  1744. - && known_eq (frame.below_hard_fp_saved_regs_size, 0))
  1745. - {
  1746. - poly_int64 lr_offset = (frame.reg_offset[LR_REGNUM]
  1747. - - frame.bytes_below_saved_regs);
  1748. - if (known_ge (lr_offset, 0))
  1749. - min_probe_threshold -= lr_offset.to_constant ();
  1750. - else
  1751. - gcc_assert (!flag_stack_clash_protection || known_eq (poly_size, 0));
  1752. - }
  1753. -
  1754. poly_int64 frame_size = frame.frame_size;
  1755. /* We should always have a positive probe threshold. */
  1756. @@ -9856,8 +9844,8 @@ aarch64_allocate_and_probe_stack_space (rtx temp1, rtx temp2,
  1757. if (final_adjustment_p && rounded_size != 0)
  1758. min_probe_threshold = 0;
  1759. /* If doing a small final adjustment, we always probe at offset 0.
  1760. - This is done to avoid issues when LR is not at position 0 or when
  1761. - the final adjustment is smaller than the probing offset. */
  1762. + This is done to avoid issues when the final adjustment is smaller
  1763. + than the probing offset. */
  1764. else if (final_adjustment_p && rounded_size == 0)
  1765. residual_probe_offset = 0;
  1766. diff --git a/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-18.c b/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-18.c
  1767. new file mode 100644
  1768. index 00000000000..82447d20fff
  1769. --- /dev/null
  1770. +++ b/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-18.c
  1771. @@ -0,0 +1,100 @@
  1772. +/* { dg-options "-O2 -fstack-clash-protection -fomit-frame-pointer --param stack-clash-protection-guard-size=12" } */
  1773. +/* { dg-final { check-function-bodies "**" "" } } */
  1774. +
  1775. +void f(int, ...);
  1776. +void g();
  1777. +
  1778. +/*
  1779. +** test1:
  1780. +** ...
  1781. +** str x30, \[sp\]
  1782. +** sub sp, sp, #4064
  1783. +** str xzr, \[sp\]
  1784. +** cbnz w0, .*
  1785. +** bl g
  1786. +** ...
  1787. +** str x26, \[sp, #?4128\]
  1788. +** ...
  1789. +*/
  1790. +int test1(int z) {
  1791. + __uint128_t x = 0;
  1792. + int y[0x400];
  1793. + if (z)
  1794. + {
  1795. + asm volatile ("" :::
  1796. + "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26");
  1797. + f(0, 0, 0, 0, 0, 0, 0, &y,
  1798. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1799. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1800. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1801. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1802. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1803. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1804. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1805. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1806. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1807. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1808. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1809. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1810. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1811. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1812. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1813. + x, x, x, x, x, x, x, x, x, x, x, x, x, x);
  1814. + }
  1815. + g();
  1816. + return 1;
  1817. +}
  1818. +
  1819. +/*
  1820. +** test2:
  1821. +** ...
  1822. +** str x30, \[sp\]
  1823. +** sub sp, sp, #1040
  1824. +** str xzr, \[sp\]
  1825. +** cbnz w0, .*
  1826. +** bl g
  1827. +** ...
  1828. +*/
  1829. +int test2(int z) {
  1830. + __uint128_t x = 0;
  1831. + int y[0x400];
  1832. + if (z)
  1833. + {
  1834. + asm volatile ("" :::
  1835. + "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26");
  1836. + f(0, 0, 0, 0, 0, 0, 0, &y,
  1837. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1838. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1839. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1840. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1841. + x);
  1842. + }
  1843. + g();
  1844. + return 1;
  1845. +}
  1846. +
  1847. +/*
  1848. +** test3:
  1849. +** ...
  1850. +** str x30, \[sp\]
  1851. +** sub sp, sp, #1024
  1852. +** cbnz w0, .*
  1853. +** bl g
  1854. +** ...
  1855. +*/
  1856. +int test3(int z) {
  1857. + __uint128_t x = 0;
  1858. + int y[0x400];
  1859. + if (z)
  1860. + {
  1861. + asm volatile ("" :::
  1862. + "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26");
  1863. + f(0, 0, 0, 0, 0, 0, 0, &y,
  1864. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1865. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1866. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1867. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x);
  1868. + }
  1869. + g();
  1870. + return 1;
  1871. +}
  1872. diff --git a/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-19.c b/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-19.c
  1873. new file mode 100644
  1874. index 00000000000..73ac3e4e4eb
  1875. --- /dev/null
  1876. +++ b/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-19.c
  1877. @@ -0,0 +1,100 @@
  1878. +/* { dg-options "-O2 -fstack-clash-protection -fomit-frame-pointer --param stack-clash-protection-guard-size=12 -fsanitize=shadow-call-stack -ffixed-x18" } */
  1879. +/* { dg-final { check-function-bodies "**" "" } } */
  1880. +
  1881. +void f(int, ...);
  1882. +void g();
  1883. +
  1884. +/*
  1885. +** test1:
  1886. +** ...
  1887. +** str x30, \[sp\]
  1888. +** sub sp, sp, #4064
  1889. +** str xzr, \[sp\]
  1890. +** cbnz w0, .*
  1891. +** bl g
  1892. +** ...
  1893. +** str x26, \[sp, #?4128\]
  1894. +** ...
  1895. +*/
  1896. +int test1(int z) {
  1897. + __uint128_t x = 0;
  1898. + int y[0x400];
  1899. + if (z)
  1900. + {
  1901. + asm volatile ("" :::
  1902. + "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26");
  1903. + f(0, 0, 0, 0, 0, 0, 0, &y,
  1904. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1905. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1906. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1907. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1908. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1909. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1910. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1911. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1912. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1913. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1914. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1915. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1916. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1917. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1918. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1919. + x, x, x, x, x, x, x, x, x, x, x, x, x, x);
  1920. + }
  1921. + g();
  1922. + return 1;
  1923. +}
  1924. +
  1925. +/*
  1926. +** test2:
  1927. +** ...
  1928. +** str x30, \[sp\]
  1929. +** sub sp, sp, #1040
  1930. +** str xzr, \[sp\]
  1931. +** cbnz w0, .*
  1932. +** bl g
  1933. +** ...
  1934. +*/
  1935. +int test2(int z) {
  1936. + __uint128_t x = 0;
  1937. + int y[0x400];
  1938. + if (z)
  1939. + {
  1940. + asm volatile ("" :::
  1941. + "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26");
  1942. + f(0, 0, 0, 0, 0, 0, 0, &y,
  1943. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1944. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1945. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1946. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1947. + x);
  1948. + }
  1949. + g();
  1950. + return 1;
  1951. +}
  1952. +
  1953. +/*
  1954. +** test3:
  1955. +** ...
  1956. +** str x30, \[sp\]
  1957. +** sub sp, sp, #1024
  1958. +** cbnz w0, .*
  1959. +** bl g
  1960. +** ...
  1961. +*/
  1962. +int test3(int z) {
  1963. + __uint128_t x = 0;
  1964. + int y[0x400];
  1965. + if (z)
  1966. + {
  1967. + asm volatile ("" :::
  1968. + "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26");
  1969. + f(0, 0, 0, 0, 0, 0, 0, &y,
  1970. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1971. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1972. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
  1973. + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x);
  1974. + }
  1975. + g();
  1976. + return 1;
  1977. +}
  1978. diff --git a/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-20.c b/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-20.c
  1979. new file mode 100644
  1980. index 00000000000..690aae8dfd5
  1981. --- /dev/null
  1982. +++ b/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-20.c
  1983. @@ -0,0 +1,3 @@
  1984. +/* { dg-options "-O2 -fstack-protector-all -fstack-clash-protection -fomit-frame-pointer --param stack-clash-protection-guard-size=12 -fsanitize=shadow-call-stack -ffixed-x18" } */
  1985. +
  1986. +#include "stack-check-prologue-19.c"
  1987. --
  1988. 2.34.1
  1989. From eea1759073e09dd1aefbc9a881601ab1eebfdd18 Mon Sep 17 00:00:00 2001
  1990. From: Richard Sandiford <richard.sandiford@arm.com>
  1991. Date: Tue, 12 Sep 2023 16:07:19 +0100
  1992. Subject: [PATCH 16/19] aarch64: Simplify probe of final frame allocation
  1993. Previous patches ensured that the final frame allocation only needs
  1994. a probe when the size is strictly greater than 1KiB. It's therefore
  1995. safe to use the normal 1024 probe offset in all cases.
  1996. The main motivation for doing this is to simplify the code and
  1997. remove the number of special cases.
  1998. gcc/
  1999. * config/aarch64/aarch64.cc (aarch64_allocate_and_probe_stack_space):
  2000. Always probe the residual allocation at offset 1024, asserting
  2001. that that is in range.
  2002. gcc/testsuite/
  2003. * gcc.target/aarch64/stack-check-prologue-17.c: Expect the probe
  2004. to be at offset 1024 rather than offset 0.
  2005. * gcc.target/aarch64/stack-check-prologue-18.c: Likewise.
  2006. * gcc.target/aarch64/stack-check-prologue-19.c: Likewise.
  2007. ---
  2008. gcc/config/aarch64/aarch64.cc | 12 ++++--------
  2009. .../gcc.target/aarch64/stack-check-prologue-17.c | 2 +-
  2010. .../gcc.target/aarch64/stack-check-prologue-18.c | 4 ++--
  2011. .../gcc.target/aarch64/stack-check-prologue-19.c | 4 ++--
  2012. 4 files changed, 9 insertions(+), 13 deletions(-)
  2013. diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
  2014. index a765f92329d..37809a306f7 100644
  2015. --- a/gcc/config/aarch64/aarch64.cc
  2016. +++ b/gcc/config/aarch64/aarch64.cc
  2017. @@ -9838,16 +9838,12 @@ aarch64_allocate_and_probe_stack_space (rtx temp1, rtx temp2,
  2018. are still safe. */
  2019. if (residual)
  2020. {
  2021. - HOST_WIDE_INT residual_probe_offset = guard_used_by_caller;
  2022. + gcc_assert (guard_used_by_caller + byte_sp_alignment <= size);
  2023. +
  2024. /* If we're doing final adjustments, and we've done any full page
  2025. allocations then any residual needs to be probed. */
  2026. if (final_adjustment_p && rounded_size != 0)
  2027. min_probe_threshold = 0;
  2028. - /* If doing a small final adjustment, we always probe at offset 0.
  2029. - This is done to avoid issues when the final adjustment is smaller
  2030. - than the probing offset. */
  2031. - else if (final_adjustment_p && rounded_size == 0)
  2032. - residual_probe_offset = 0;
  2033. aarch64_sub_sp (temp1, temp2, residual, frame_related_p);
  2034. if (residual >= min_probe_threshold)
  2035. @@ -9858,8 +9854,8 @@ aarch64_allocate_and_probe_stack_space (rtx temp1, rtx temp2,
  2036. HOST_WIDE_INT_PRINT_DEC " bytes, probing will be required."
  2037. "\n", residual);
  2038. - emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
  2039. - residual_probe_offset));
  2040. + emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
  2041. + guard_used_by_caller));
  2042. emit_insn (gen_blockage ());
  2043. }
  2044. }
  2045. diff --git a/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-17.c b/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-17.c
  2046. index 0d8a25d73a2..f0ec1389771 100644
  2047. --- a/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-17.c
  2048. +++ b/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-17.c
  2049. @@ -33,7 +33,7 @@ int test1(int z) {
  2050. ** ...
  2051. ** str x30, \[sp\]
  2052. ** sub sp, sp, #1040
  2053. -** str xzr, \[sp\]
  2054. +** str xzr, \[sp, #?1024\]
  2055. ** cbnz w0, .*
  2056. ** bl g
  2057. ** ...
  2058. diff --git a/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-18.c b/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-18.c
  2059. index 82447d20fff..6383bec5ebc 100644
  2060. --- a/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-18.c
  2061. +++ b/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-18.c
  2062. @@ -9,7 +9,7 @@ void g();
  2063. ** ...
  2064. ** str x30, \[sp\]
  2065. ** sub sp, sp, #4064
  2066. -** str xzr, \[sp\]
  2067. +** str xzr, \[sp, #?1024\]
  2068. ** cbnz w0, .*
  2069. ** bl g
  2070. ** ...
  2071. @@ -50,7 +50,7 @@ int test1(int z) {
  2072. ** ...
  2073. ** str x30, \[sp\]
  2074. ** sub sp, sp, #1040
  2075. -** str xzr, \[sp\]
  2076. +** str xzr, \[sp, #?1024\]
  2077. ** cbnz w0, .*
  2078. ** bl g
  2079. ** ...
  2080. diff --git a/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-19.c b/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-19.c
  2081. index 73ac3e4e4eb..562039b5e9b 100644
  2082. --- a/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-19.c
  2083. +++ b/gcc/testsuite/gcc.target/aarch64/stack-check-prologue-19.c
  2084. @@ -9,7 +9,7 @@ void g();
  2085. ** ...
  2086. ** str x30, \[sp\]
  2087. ** sub sp, sp, #4064
  2088. -** str xzr, \[sp\]
  2089. +** str xzr, \[sp, #?1024\]
  2090. ** cbnz w0, .*
  2091. ** bl g
  2092. ** ...
  2093. @@ -50,7 +50,7 @@ int test1(int z) {
  2094. ** ...
  2095. ** str x30, \[sp\]
  2096. ** sub sp, sp, #1040
  2097. -** str xzr, \[sp\]
  2098. +** str xzr, \[sp, #?1024\]
  2099. ** cbnz w0, .*
  2100. ** bl g
  2101. ** ...
  2102. --
  2103. 2.34.1
  2104. From 96d85187c3b9c9a7efc2fd698c3d452e80d8aa47 Mon Sep 17 00:00:00 2001
  2105. From: Richard Sandiford <richard.sandiford@arm.com>
  2106. Date: Tue, 12 Sep 2023 16:07:20 +0100
  2107. Subject: [PATCH 17/19] aarch64: Explicitly record probe registers in frame
  2108. info
  2109. The stack frame is currently divided into three areas:
  2110. A: the area above the hard frame pointer
  2111. B: the SVE saves below the hard frame pointer
  2112. C: the outgoing arguments
  2113. If the stack frame is allocated in one chunk, the allocation needs a
  2114. probe if the frame size is >= guard_size - 1KiB. In addition, if the
  2115. function is not a leaf function, it must probe an address no more than
  2116. 1KiB above the outgoing SP. We ensured the second condition by
  2117. (1) using single-chunk allocations for non-leaf functions only if
  2118. the link register save slot is within 512 bytes of the bottom
  2119. of the frame; and
  2120. (2) using the link register save as a probe (meaning, for instance,
  2121. that it can't be individually shrink wrapped)
  2122. If instead the stack is allocated in multiple chunks, then:
  2123. * an allocation involving only the outgoing arguments (C above) requires
  2124. a probe if the allocation size is > 1KiB
  2125. * any other allocation requires a probe if the allocation size
  2126. is >= guard_size - 1KiB
  2127. * second and subsequent allocations require the previous allocation
  2128. to probe at the bottom of the allocated area, regardless of the size
  2129. of that previous allocation
  2130. The final point means that, unlike for single allocations,
  2131. it can be necessary to have both a non-SVE register probe and
  2132. an SVE register probe. For example:
  2133. * allocate A, probe using a non-SVE register save
  2134. * allocate B, probe using an SVE register save
  2135. * allocate C
  2136. The non-SVE register used in this case was again the link register.
  2137. It was previously used even if the link register save slot was some
  2138. bytes above the bottom of the non-SVE register saves, but an earlier
  2139. patch avoided that by putting the link register save slot first.
  2140. As a belt-and-braces fix, this patch explicitly records which
  2141. probe registers we're using and allows the non-SVE probe to be
  2142. whichever register comes first (as for SVE).
  2143. The patch also avoids unnecessary probes in sve/pcs/stack_clash_3.c.
  2144. gcc/
  2145. * config/aarch64/aarch64.h (aarch64_frame::sve_save_and_probe)
  2146. (aarch64_frame::hard_fp_save_and_probe): New fields.
  2147. * config/aarch64/aarch64.cc (aarch64_layout_frame): Initialize them.
  2148. Rather than asserting that a leaf function saves LR, instead assert
  2149. that a leaf function saves something.
  2150. (aarch64_get_separate_components): Prevent the chosen probe
  2151. registers from being individually shrink-wrapped.
  2152. (aarch64_allocate_and_probe_stack_space): Remove workaround for
  2153. probe registers that aren't at the bottom of the previous allocation.
  2154. gcc/testsuite/
  2155. * gcc.target/aarch64/sve/pcs/stack_clash_3.c: Avoid redundant probes.
  2156. ---
  2157. gcc/config/aarch64/aarch64.cc | 68 +++++++++++++++----
  2158. gcc/config/aarch64/aarch64.h | 8 +++
  2159. .../aarch64/sve/pcs/stack_clash_3.c | 6 +-
  2160. 3 files changed, 64 insertions(+), 18 deletions(-)
  2161. diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
  2162. index 37809a306f7..6c59c39a639 100644
  2163. --- a/gcc/config/aarch64/aarch64.cc
  2164. +++ b/gcc/config/aarch64/aarch64.cc
  2165. @@ -8471,15 +8471,11 @@ aarch64_layout_frame (void)
  2166. && !crtl->abi->clobbers_full_reg_p (regno))
  2167. frame.reg_offset[regno] = SLOT_REQUIRED;
  2168. - /* With stack-clash, LR must be saved in non-leaf functions. The saving of
  2169. - LR counts as an implicit probe which allows us to maintain the invariant
  2170. - described in the comment at expand_prologue. */
  2171. - gcc_assert (crtl->is_leaf
  2172. - || maybe_ne (frame.reg_offset[R30_REGNUM], SLOT_NOT_REQUIRED));
  2173. poly_int64 offset = crtl->outgoing_args_size;
  2174. gcc_assert (multiple_p (offset, STACK_BOUNDARY / BITS_PER_UNIT));
  2175. frame.bytes_below_saved_regs = offset;
  2176. + frame.sve_save_and_probe = INVALID_REGNUM;
  2177. /* Now assign stack slots for the registers. Start with the predicate
  2178. registers, since predicate LDR and STR have a relatively small
  2179. @@ -8487,6 +8483,8 @@ aarch64_layout_frame (void)
  2180. for (regno = P0_REGNUM; regno <= P15_REGNUM; regno++)
  2181. if (known_eq (frame.reg_offset[regno], SLOT_REQUIRED))
  2182. {
  2183. + if (frame.sve_save_and_probe == INVALID_REGNUM)
  2184. + frame.sve_save_and_probe = regno;
  2185. frame.reg_offset[regno] = offset;
  2186. offset += BYTES_PER_SVE_PRED;
  2187. }
  2188. @@ -8524,6 +8522,8 @@ aarch64_layout_frame (void)
  2189. for (regno = V0_REGNUM; regno <= V31_REGNUM; regno++)
  2190. if (known_eq (frame.reg_offset[regno], SLOT_REQUIRED))
  2191. {
  2192. + if (frame.sve_save_and_probe == INVALID_REGNUM)
  2193. + frame.sve_save_and_probe = regno;
  2194. frame.reg_offset[regno] = offset;
  2195. offset += vector_save_size;
  2196. }
  2197. @@ -8533,10 +8533,18 @@ aarch64_layout_frame (void)
  2198. frame.below_hard_fp_saved_regs_size = offset - frame.bytes_below_saved_regs;
  2199. bool saves_below_hard_fp_p
  2200. = maybe_ne (frame.below_hard_fp_saved_regs_size, 0);
  2201. + gcc_assert (!saves_below_hard_fp_p
  2202. + || (frame.sve_save_and_probe != INVALID_REGNUM
  2203. + && known_eq (frame.reg_offset[frame.sve_save_and_probe],
  2204. + frame.bytes_below_saved_regs)));
  2205. +
  2206. frame.bytes_below_hard_fp = offset;
  2207. + frame.hard_fp_save_and_probe = INVALID_REGNUM;
  2208. auto allocate_gpr_slot = [&](unsigned int regno)
  2209. {
  2210. + if (frame.hard_fp_save_and_probe == INVALID_REGNUM)
  2211. + frame.hard_fp_save_and_probe = regno;
  2212. frame.reg_offset[regno] = offset;
  2213. if (frame.wb_push_candidate1 == INVALID_REGNUM)
  2214. frame.wb_push_candidate1 = regno;
  2215. @@ -8570,6 +8578,8 @@ aarch64_layout_frame (void)
  2216. for (regno = V0_REGNUM; regno <= V31_REGNUM; regno++)
  2217. if (known_eq (frame.reg_offset[regno], SLOT_REQUIRED))
  2218. {
  2219. + if (frame.hard_fp_save_and_probe == INVALID_REGNUM)
  2220. + frame.hard_fp_save_and_probe = regno;
  2221. /* If there is an alignment gap between integer and fp callee-saves,
  2222. allocate the last fp register to it if possible. */
  2223. if (regno == last_fp_reg
  2224. @@ -8593,6 +8603,17 @@ aarch64_layout_frame (void)
  2225. offset = aligned_upper_bound (offset, STACK_BOUNDARY / BITS_PER_UNIT);
  2226. frame.saved_regs_size = offset - frame.bytes_below_saved_regs;
  2227. + gcc_assert (known_eq (frame.saved_regs_size,
  2228. + frame.below_hard_fp_saved_regs_size)
  2229. + || (frame.hard_fp_save_and_probe != INVALID_REGNUM
  2230. + && known_eq (frame.reg_offset[frame.hard_fp_save_and_probe],
  2231. + frame.bytes_below_hard_fp)));
  2232. +
  2233. + /* With stack-clash, a register must be saved in non-leaf functions.
  2234. + The saving of the bottommost register counts as an implicit probe,
  2235. + which allows us to maintain the invariant described in the comment
  2236. + at expand_prologue. */
  2237. + gcc_assert (crtl->is_leaf || maybe_ne (frame.saved_regs_size, 0));
  2238. offset += get_frame_size ();
  2239. offset = aligned_upper_bound (offset, STACK_BOUNDARY / BITS_PER_UNIT);
  2240. @@ -8723,6 +8744,25 @@ aarch64_layout_frame (void)
  2241. frame.final_adjust = frame.bytes_below_saved_regs;
  2242. }
  2243. + /* The frame is allocated in pieces, with each non-final piece
  2244. + including a register save at offset 0 that acts as a probe for
  2245. + the following piece. In addition, the save of the bottommost register
  2246. + acts as a probe for callees and allocas. Roll back any probes that
  2247. + aren't needed.
  2248. +
  2249. + A probe isn't needed if it is associated with the final allocation
  2250. + (including callees and allocas) that happens before the epilogue is
  2251. + executed. */
  2252. + if (crtl->is_leaf
  2253. + && !cfun->calls_alloca
  2254. + && known_eq (frame.final_adjust, 0))
  2255. + {
  2256. + if (maybe_ne (frame.sve_callee_adjust, 0))
  2257. + frame.sve_save_and_probe = INVALID_REGNUM;
  2258. + else
  2259. + frame.hard_fp_save_and_probe = INVALID_REGNUM;
  2260. + }
  2261. +
  2262. /* Make sure the individual adjustments add up to the full frame size. */
  2263. gcc_assert (known_eq (frame.initial_adjust
  2264. + frame.callee_adjust
  2265. @@ -9354,13 +9394,6 @@ aarch64_get_separate_components (void)
  2266. poly_int64 offset = frame.reg_offset[regno];
  2267. - /* If the register is saved in the first SVE save slot, we use
  2268. - it as a stack probe for -fstack-clash-protection. */
  2269. - if (flag_stack_clash_protection
  2270. - && maybe_ne (frame.below_hard_fp_saved_regs_size, 0)
  2271. - && known_eq (offset, frame.bytes_below_saved_regs))
  2272. - continue;
  2273. -
  2274. /* Get the offset relative to the register we'll use. */
  2275. if (frame_pointer_needed)
  2276. offset -= frame.bytes_below_hard_fp;
  2277. @@ -9395,6 +9428,13 @@ aarch64_get_separate_components (void)
  2278. bitmap_clear_bit (components, LR_REGNUM);
  2279. bitmap_clear_bit (components, SP_REGNUM);
  2280. + if (flag_stack_clash_protection)
  2281. + {
  2282. + if (frame.sve_save_and_probe != INVALID_REGNUM)
  2283. + bitmap_clear_bit (components, frame.sve_save_and_probe);
  2284. + if (frame.hard_fp_save_and_probe != INVALID_REGNUM)
  2285. + bitmap_clear_bit (components, frame.hard_fp_save_and_probe);
  2286. + }
  2287. return components;
  2288. }
  2289. @@ -9931,8 +9971,8 @@ aarch64_epilogue_uses (int regno)
  2290. When probing is needed, we emit a probe at the start of the prologue
  2291. and every PARAM_STACK_CLASH_PROTECTION_GUARD_SIZE bytes thereafter.
  2292. - We have to track how much space has been allocated and the only stores
  2293. - to the stack we track as implicit probes are the FP/LR stores.
  2294. + We can also use register saves as probes. These are stored in
  2295. + sve_save_and_probe and hard_fp_save_and_probe.
  2296. For outgoing arguments we probe if the size is larger than 1KB, such that
  2297. the ABI specified buffer is maintained for the next callee.
  2298. diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
  2299. index c8becb098c8..fbfb73545ba 100644
  2300. --- a/gcc/config/aarch64/aarch64.h
  2301. +++ b/gcc/config/aarch64/aarch64.h
  2302. @@ -863,6 +863,14 @@ struct GTY (()) aarch64_frame
  2303. This is the register they should use. */
  2304. unsigned spare_pred_reg;
  2305. + /* An SVE register that is saved below the hard frame pointer and that acts
  2306. + as a probe for later allocations, or INVALID_REGNUM if none. */
  2307. + unsigned sve_save_and_probe;
  2308. +
  2309. + /* A register that is saved at the hard frame pointer and that acts
  2310. + as a probe for later allocations, or INVALID_REGNUM if none. */
  2311. + unsigned hard_fp_save_and_probe;
  2312. +
  2313. bool laid_out;
  2314. /* True if shadow call stack should be enabled for the current function. */
  2315. diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pcs/stack_clash_3.c b/gcc/testsuite/gcc.target/aarch64/sve/pcs/stack_clash_3.c
  2316. index 3e01ec36c3a..3530a0d504b 100644
  2317. --- a/gcc/testsuite/gcc.target/aarch64/sve/pcs/stack_clash_3.c
  2318. +++ b/gcc/testsuite/gcc.target/aarch64/sve/pcs/stack_clash_3.c
  2319. @@ -11,11 +11,10 @@
  2320. ** mov x11, sp
  2321. ** ...
  2322. ** sub sp, sp, x13
  2323. -** str p4, \[sp\]
  2324. ** cbz w0, [^\n]*
  2325. +** str p4, \[sp\]
  2326. ** ...
  2327. ** ptrue p0\.b, all
  2328. -** ldr p4, \[sp\]
  2329. ** addvl sp, sp, #1
  2330. ** ldr x24, \[sp\], 32
  2331. ** ret
  2332. @@ -39,13 +38,12 @@ test_1 (int n)
  2333. ** mov x11, sp
  2334. ** ...
  2335. ** sub sp, sp, x13
  2336. -** str p4, \[sp\]
  2337. ** cbz w0, [^\n]*
  2338. +** str p4, \[sp\]
  2339. ** str p5, \[sp, #1, mul vl\]
  2340. ** str p6, \[sp, #2, mul vl\]
  2341. ** ...
  2342. ** ptrue p0\.b, all
  2343. -** ldr p4, \[sp\]
  2344. ** addvl sp, sp, #1
  2345. ** ldr x24, \[sp\], 32
  2346. ** ret
  2347. --
  2348. 2.34.1
  2349. From 56df065080950bb30dda9c260f71be54269bdda5 Mon Sep 17 00:00:00 2001
  2350. From: Richard Sandiford <richard.sandiford@arm.com>
  2351. Date: Tue, 12 Sep 2023 16:07:20 +0100
  2352. Subject: [PATCH 18/19] aarch64: Remove below_hard_fp_saved_regs_size
  2353. After previous patches, it's no longer necessary to store
  2354. saved_regs_size and below_hard_fp_saved_regs_size in the frame info.
  2355. All measurements instead use the top or bottom of the frame as
  2356. reference points.
  2357. gcc/
  2358. * config/aarch64/aarch64.h (aarch64_frame::saved_regs_size)
  2359. (aarch64_frame::below_hard_fp_saved_regs_size): Delete.
  2360. * config/aarch64/aarch64.cc (aarch64_layout_frame): Update accordingly.
  2361. ---
  2362. gcc/config/aarch64/aarch64.cc | 45 ++++++++++++++++-------------------
  2363. gcc/config/aarch64/aarch64.h | 7 ------
  2364. 2 files changed, 21 insertions(+), 31 deletions(-)
  2365. diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
  2366. index 6c59c39a639..b95e805a8cc 100644
  2367. --- a/gcc/config/aarch64/aarch64.cc
  2368. +++ b/gcc/config/aarch64/aarch64.cc
  2369. @@ -8530,9 +8530,8 @@ aarch64_layout_frame (void)
  2370. /* OFFSET is now the offset of the hard frame pointer from the bottom
  2371. of the callee save area. */
  2372. - frame.below_hard_fp_saved_regs_size = offset - frame.bytes_below_saved_regs;
  2373. - bool saves_below_hard_fp_p
  2374. - = maybe_ne (frame.below_hard_fp_saved_regs_size, 0);
  2375. + auto below_hard_fp_saved_regs_size = offset - frame.bytes_below_saved_regs;
  2376. + bool saves_below_hard_fp_p = maybe_ne (below_hard_fp_saved_regs_size, 0);
  2377. gcc_assert (!saves_below_hard_fp_p
  2378. || (frame.sve_save_and_probe != INVALID_REGNUM
  2379. && known_eq (frame.reg_offset[frame.sve_save_and_probe],
  2380. @@ -8602,9 +8601,8 @@ aarch64_layout_frame (void)
  2381. offset = aligned_upper_bound (offset, STACK_BOUNDARY / BITS_PER_UNIT);
  2382. - frame.saved_regs_size = offset - frame.bytes_below_saved_regs;
  2383. - gcc_assert (known_eq (frame.saved_regs_size,
  2384. - frame.below_hard_fp_saved_regs_size)
  2385. + auto saved_regs_size = offset - frame.bytes_below_saved_regs;
  2386. + gcc_assert (known_eq (saved_regs_size, below_hard_fp_saved_regs_size)
  2387. || (frame.hard_fp_save_and_probe != INVALID_REGNUM
  2388. && known_eq (frame.reg_offset[frame.hard_fp_save_and_probe],
  2389. frame.bytes_below_hard_fp)));
  2390. @@ -8613,7 +8611,7 @@ aarch64_layout_frame (void)
  2391. The saving of the bottommost register counts as an implicit probe,
  2392. which allows us to maintain the invariant described in the comment
  2393. at expand_prologue. */
  2394. - gcc_assert (crtl->is_leaf || maybe_ne (frame.saved_regs_size, 0));
  2395. + gcc_assert (crtl->is_leaf || maybe_ne (saved_regs_size, 0));
  2396. offset += get_frame_size ();
  2397. offset = aligned_upper_bound (offset, STACK_BOUNDARY / BITS_PER_UNIT);
  2398. @@ -8670,7 +8668,7 @@ aarch64_layout_frame (void)
  2399. HOST_WIDE_INT const_size, const_below_saved_regs, const_above_fp;
  2400. HOST_WIDE_INT const_saved_regs_size;
  2401. - if (known_eq (frame.saved_regs_size, 0))
  2402. + if (known_eq (saved_regs_size, 0))
  2403. frame.initial_adjust = frame.frame_size;
  2404. else if (frame.frame_size.is_constant (&const_size)
  2405. && const_size < max_push_offset
  2406. @@ -8683,7 +8681,7 @@ aarch64_layout_frame (void)
  2407. frame.callee_adjust = const_size;
  2408. }
  2409. else if (frame.bytes_below_saved_regs.is_constant (&const_below_saved_regs)
  2410. - && frame.saved_regs_size.is_constant (&const_saved_regs_size)
  2411. + && saved_regs_size.is_constant (&const_saved_regs_size)
  2412. && const_below_saved_regs + const_saved_regs_size < 512
  2413. /* We could handle this case even with data below the saved
  2414. registers, provided that that data left us with valid offsets
  2415. @@ -8702,8 +8700,7 @@ aarch64_layout_frame (void)
  2416. frame.initial_adjust = frame.frame_size;
  2417. }
  2418. else if (saves_below_hard_fp_p
  2419. - && known_eq (frame.saved_regs_size,
  2420. - frame.below_hard_fp_saved_regs_size))
  2421. + && known_eq (saved_regs_size, below_hard_fp_saved_regs_size))
  2422. {
  2423. /* Frame in which all saves are SVE saves:
  2424. @@ -8725,7 +8722,7 @@ aarch64_layout_frame (void)
  2425. [save SVE registers relative to SP]
  2426. sub sp, sp, bytes_below_saved_regs */
  2427. frame.callee_adjust = const_above_fp;
  2428. - frame.sve_callee_adjust = frame.below_hard_fp_saved_regs_size;
  2429. + frame.sve_callee_adjust = below_hard_fp_saved_regs_size;
  2430. frame.final_adjust = frame.bytes_below_saved_regs;
  2431. }
  2432. else
  2433. @@ -8740,7 +8737,7 @@ aarch64_layout_frame (void)
  2434. [save SVE registers relative to SP]
  2435. sub sp, sp, bytes_below_saved_regs */
  2436. frame.initial_adjust = frame.bytes_above_hard_fp;
  2437. - frame.sve_callee_adjust = frame.below_hard_fp_saved_regs_size;
  2438. + frame.sve_callee_adjust = below_hard_fp_saved_regs_size;
  2439. frame.final_adjust = frame.bytes_below_saved_regs;
  2440. }
  2441. @@ -9936,17 +9933,17 @@ aarch64_epilogue_uses (int regno)
  2442. | local variables | <-- frame_pointer_rtx
  2443. | |
  2444. +-------------------------------+
  2445. - | padding | \
  2446. - +-------------------------------+ |
  2447. - | callee-saved registers | | frame.saved_regs_size
  2448. - +-------------------------------+ |
  2449. - | LR' | |
  2450. - +-------------------------------+ |
  2451. - | FP' | |
  2452. - +-------------------------------+ |<- hard_frame_pointer_rtx (aligned)
  2453. - | SVE vector registers | | \
  2454. - +-------------------------------+ | | below_hard_fp_saved_regs_size
  2455. - | SVE predicate registers | / /
  2456. + | padding |
  2457. + +-------------------------------+
  2458. + | callee-saved registers |
  2459. + +-------------------------------+
  2460. + | LR' |
  2461. + +-------------------------------+
  2462. + | FP' |
  2463. + +-------------------------------+ <-- hard_frame_pointer_rtx (aligned)
  2464. + | SVE vector registers |
  2465. + +-------------------------------+
  2466. + | SVE predicate registers |
  2467. +-------------------------------+
  2468. | dynamic allocation |
  2469. +-------------------------------+
  2470. diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
  2471. index fbfb73545ba..cfeaf4657ab 100644
  2472. --- a/gcc/config/aarch64/aarch64.h
  2473. +++ b/gcc/config/aarch64/aarch64.h
  2474. @@ -777,18 +777,11 @@ struct GTY (()) aarch64_frame
  2475. STACK_BOUNDARY. */
  2476. HOST_WIDE_INT saved_varargs_size;
  2477. - /* The size of the callee-save registers with a slot in REG_OFFSET. */
  2478. - poly_int64 saved_regs_size;
  2479. -
  2480. /* The number of bytes between the bottom of the static frame (the bottom
  2481. of the outgoing arguments) and the bottom of the register save area.
  2482. This value is always a multiple of STACK_BOUNDARY. */
  2483. poly_int64 bytes_below_saved_regs;
  2484. - /* The size of the callee-save registers with a slot in REG_OFFSET that
  2485. - are saved below the hard frame pointer. */
  2486. - poly_int64 below_hard_fp_saved_regs_size;
  2487. -
  2488. /* The number of bytes between the bottom of the static frame (the bottom
  2489. of the outgoing arguments) and the hard frame pointer. This value is
  2490. always a multiple of STACK_BOUNDARY. */
  2491. --
  2492. 2.34.1
  2493. From b96e66fd4ef3e36983969fb8cdd1956f551a074b Mon Sep 17 00:00:00 2001
  2494. From: Richard Sandiford <richard.sandiford@arm.com>
  2495. Date: Tue, 12 Sep 2023 16:07:21 +0100
  2496. Subject: [PATCH 19/19] aarch64: Make stack smash canary protect saved
  2497. registers
  2498. AArch64 normally puts the saved registers near the bottom of the frame,
  2499. immediately above any dynamic allocations. But this means that a
  2500. stack-smash attack on those dynamic allocations could overwrite the
  2501. saved registers without needing to reach as far as the stack smash
  2502. canary.
  2503. The same thing could also happen for variable-sized arguments that are
  2504. passed by value, since those are allocated before a call and popped on
  2505. return.
  2506. This patch avoids that by putting the locals (and thus the canary) below
  2507. the saved registers when stack smash protection is active.
  2508. The patch fixes CVE-2023-4039.
  2509. gcc/
  2510. * config/aarch64/aarch64.cc (aarch64_save_regs_above_locals_p):
  2511. New function.
  2512. (aarch64_layout_frame): Use it to decide whether locals should
  2513. go above or below the saved registers.
  2514. (aarch64_expand_prologue): Update stack layout comment.
  2515. Emit a stack tie after the final adjustment.
  2516. gcc/testsuite/
  2517. * gcc.target/aarch64/stack-protector-8.c: New test.
  2518. * gcc.target/aarch64/stack-protector-9.c: Likewise.
  2519. ---
  2520. gcc/config/aarch64/aarch64.cc | 46 +++++++--
  2521. .../gcc.target/aarch64/stack-protector-8.c | 95 +++++++++++++++++++
  2522. .../gcc.target/aarch64/stack-protector-9.c | 33 +++++++
  2523. 3 files changed, 168 insertions(+), 6 deletions(-)
  2524. create mode 100644 gcc/testsuite/gcc.target/aarch64/stack-protector-8.c
  2525. create mode 100644 gcc/testsuite/gcc.target/aarch64/stack-protector-9.c
  2526. diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
  2527. index b95e805a8cc..389c0e29353 100644
  2528. --- a/gcc/config/aarch64/aarch64.cc
  2529. +++ b/gcc/config/aarch64/aarch64.cc
  2530. @@ -8394,6 +8394,20 @@ aarch64_needs_frame_chain (void)
  2531. return aarch64_use_frame_pointer;
  2532. }
  2533. +/* Return true if the current function should save registers above
  2534. + the locals area, rather than below it. */
  2535. +
  2536. +static bool
  2537. +aarch64_save_regs_above_locals_p ()
  2538. +{
  2539. + /* When using stack smash protection, make sure that the canary slot
  2540. + comes between the locals and the saved registers. Otherwise,
  2541. + it would be possible for a carefully sized smash attack to change
  2542. + the saved registers (particularly LR and FP) without reaching the
  2543. + canary. */
  2544. + return crtl->stack_protect_guard;
  2545. +}
  2546. +
  2547. /* Mark the registers that need to be saved by the callee and calculate
  2548. the size of the callee-saved registers area and frame record (both FP
  2549. and LR may be omitted). */
  2550. @@ -8405,6 +8419,7 @@ aarch64_layout_frame (void)
  2551. poly_int64 vector_save_size = GET_MODE_SIZE (vector_save_mode);
  2552. bool frame_related_fp_reg_p = false;
  2553. aarch64_frame &frame = cfun->machine->frame;
  2554. + poly_int64 top_of_locals = -1;
  2555. frame.emit_frame_chain = aarch64_needs_frame_chain ();
  2556. @@ -8471,9 +8486,16 @@ aarch64_layout_frame (void)
  2557. && !crtl->abi->clobbers_full_reg_p (regno))
  2558. frame.reg_offset[regno] = SLOT_REQUIRED;
  2559. + bool regs_at_top_p = aarch64_save_regs_above_locals_p ();
  2560. poly_int64 offset = crtl->outgoing_args_size;
  2561. gcc_assert (multiple_p (offset, STACK_BOUNDARY / BITS_PER_UNIT));
  2562. + if (regs_at_top_p)
  2563. + {
  2564. + offset += get_frame_size ();
  2565. + offset = aligned_upper_bound (offset, STACK_BOUNDARY / BITS_PER_UNIT);
  2566. + top_of_locals = offset;
  2567. + }
  2568. frame.bytes_below_saved_regs = offset;
  2569. frame.sve_save_and_probe = INVALID_REGNUM;
  2570. @@ -8613,15 +8635,18 @@ aarch64_layout_frame (void)
  2571. at expand_prologue. */
  2572. gcc_assert (crtl->is_leaf || maybe_ne (saved_regs_size, 0));
  2573. - offset += get_frame_size ();
  2574. - offset = aligned_upper_bound (offset, STACK_BOUNDARY / BITS_PER_UNIT);
  2575. - auto top_of_locals = offset;
  2576. -
  2577. + if (!regs_at_top_p)
  2578. + {
  2579. + offset += get_frame_size ();
  2580. + offset = aligned_upper_bound (offset, STACK_BOUNDARY / BITS_PER_UNIT);
  2581. + top_of_locals = offset;
  2582. + }
  2583. offset += frame.saved_varargs_size;
  2584. gcc_assert (multiple_p (offset, STACK_BOUNDARY / BITS_PER_UNIT));
  2585. frame.frame_size = offset;
  2586. frame.bytes_above_hard_fp = frame.frame_size - frame.bytes_below_hard_fp;
  2587. + gcc_assert (known_ge (top_of_locals, 0));
  2588. frame.bytes_above_locals = frame.frame_size - top_of_locals;
  2589. frame.initial_adjust = 0;
  2590. @@ -9930,10 +9955,10 @@ aarch64_epilogue_uses (int regno)
  2591. | for register varargs |
  2592. | |
  2593. +-------------------------------+
  2594. - | local variables | <-- frame_pointer_rtx
  2595. + | local variables (1) | <-- frame_pointer_rtx
  2596. | |
  2597. +-------------------------------+
  2598. - | padding |
  2599. + | padding (1) |
  2600. +-------------------------------+
  2601. | callee-saved registers |
  2602. +-------------------------------+
  2603. @@ -9945,6 +9970,10 @@ aarch64_epilogue_uses (int regno)
  2604. +-------------------------------+
  2605. | SVE predicate registers |
  2606. +-------------------------------+
  2607. + | local variables (2) |
  2608. + +-------------------------------+
  2609. + | padding (2) |
  2610. + +-------------------------------+
  2611. | dynamic allocation |
  2612. +-------------------------------+
  2613. | padding |
  2614. @@ -9954,6 +9983,9 @@ aarch64_epilogue_uses (int regno)
  2615. +-------------------------------+
  2616. | | <-- stack_pointer_rtx (aligned)
  2617. + The regions marked (1) and (2) are mutually exclusive. (2) is used
  2618. + when aarch64_save_regs_above_locals_p is true.
  2619. +
  2620. Dynamic stack allocations via alloca() decrease stack_pointer_rtx
  2621. but leave frame_pointer_rtx and hard_frame_pointer_rtx
  2622. unchanged.
  2623. @@ -10149,6 +10181,8 @@ aarch64_expand_prologue (void)
  2624. gcc_assert (known_eq (bytes_below_sp, final_adjust));
  2625. aarch64_allocate_and_probe_stack_space (tmp1_rtx, tmp0_rtx, final_adjust,
  2626. !frame_pointer_needed, true);
  2627. + if (emit_frame_chain && maybe_ne (final_adjust, 0))
  2628. + emit_insn (gen_stack_tie (stack_pointer_rtx, hard_frame_pointer_rtx));
  2629. }
  2630. /* Return TRUE if we can use a simple_return insn.
  2631. diff --git a/gcc/testsuite/gcc.target/aarch64/stack-protector-8.c b/gcc/testsuite/gcc.target/aarch64/stack-protector-8.c
  2632. new file mode 100644
  2633. index 00000000000..e71d820e365
  2634. --- /dev/null
  2635. +++ b/gcc/testsuite/gcc.target/aarch64/stack-protector-8.c
  2636. @@ -0,0 +1,95 @@
  2637. +/* { dg-options " -O -fstack-protector-strong -mstack-protector-guard=sysreg -mstack-protector-guard-reg=tpidr2_el0 -mstack-protector-guard-offset=16" } */
  2638. +/* { dg-final { check-function-bodies "**" "" } } */
  2639. +
  2640. +void g(void *);
  2641. +__SVBool_t *h(void *);
  2642. +
  2643. +/*
  2644. +** test1:
  2645. +** sub sp, sp, #288
  2646. +** stp x29, x30, \[sp, #?272\]
  2647. +** add x29, sp, #?272
  2648. +** mrs (x[0-9]+), tpidr2_el0
  2649. +** ldr (x[0-9]+), \[\1, #?16\]
  2650. +** str \2, \[sp, #?264\]
  2651. +** mov \2, #?0
  2652. +** add x0, sp, #?8
  2653. +** bl g
  2654. +** ...
  2655. +** mrs .*
  2656. +** ...
  2657. +** bne .*
  2658. +** ...
  2659. +** ldp x29, x30, \[sp, #?272\]
  2660. +** add sp, sp, #?288
  2661. +** ret
  2662. +** bl __stack_chk_fail
  2663. +*/
  2664. +int test1() {
  2665. + int y[0x40];
  2666. + g(y);
  2667. + return 1;
  2668. +}
  2669. +
  2670. +/*
  2671. +** test2:
  2672. +** stp x29, x30, \[sp, #?-16\]!
  2673. +** mov x29, sp
  2674. +** sub sp, sp, #1040
  2675. +** mrs (x[0-9]+), tpidr2_el0
  2676. +** ldr (x[0-9]+), \[\1, #?16\]
  2677. +** str \2, \[sp, #?1032\]
  2678. +** mov \2, #?0
  2679. +** add x0, sp, #?8
  2680. +** bl g
  2681. +** ...
  2682. +** mrs .*
  2683. +** ...
  2684. +** bne .*
  2685. +** ...
  2686. +** add sp, sp, #?1040
  2687. +** ldp x29, x30, \[sp\], #?16
  2688. +** ret
  2689. +** bl __stack_chk_fail
  2690. +*/
  2691. +int test2() {
  2692. + int y[0x100];
  2693. + g(y);
  2694. + return 1;
  2695. +}
  2696. +
  2697. +#pragma GCC target "+sve"
  2698. +
  2699. +/*
  2700. +** test3:
  2701. +** stp x29, x30, \[sp, #?-16\]!
  2702. +** mov x29, sp
  2703. +** addvl sp, sp, #-18
  2704. +** ...
  2705. +** str p4, \[sp\]
  2706. +** ...
  2707. +** sub sp, sp, #272
  2708. +** mrs (x[0-9]+), tpidr2_el0
  2709. +** ldr (x[0-9]+), \[\1, #?16\]
  2710. +** str \2, \[sp, #?264\]
  2711. +** mov \2, #?0
  2712. +** add x0, sp, #?8
  2713. +** bl h
  2714. +** ...
  2715. +** mrs .*
  2716. +** ...
  2717. +** bne .*
  2718. +** ...
  2719. +** add sp, sp, #?272
  2720. +** ...
  2721. +** ldr p4, \[sp\]
  2722. +** ...
  2723. +** addvl sp, sp, #18
  2724. +** ldp x29, x30, \[sp\], #?16
  2725. +** ret
  2726. +** bl __stack_chk_fail
  2727. +*/
  2728. +__SVBool_t test3() {
  2729. + int y[0x40];
  2730. + return *h(y);
  2731. +}
  2732. diff --git a/gcc/testsuite/gcc.target/aarch64/stack-protector-9.c b/gcc/testsuite/gcc.target/aarch64/stack-protector-9.c
  2733. new file mode 100644
  2734. index 00000000000..58f322aa480
  2735. --- /dev/null
  2736. +++ b/gcc/testsuite/gcc.target/aarch64/stack-protector-9.c
  2737. @@ -0,0 +1,33 @@
  2738. +/* { dg-options "-O2 -mcpu=neoverse-v1 -fstack-protector-all" } */
  2739. +/* { dg-final { check-function-bodies "**" "" } } */
  2740. +
  2741. +/*
  2742. +** main:
  2743. +** ...
  2744. +** stp x29, x30, \[sp, #?-[0-9]+\]!
  2745. +** ...
  2746. +** sub sp, sp, #[0-9]+
  2747. +** ...
  2748. +** str x[0-9]+, \[x29, #?-8\]
  2749. +** ...
  2750. +*/
  2751. +int f(const char *);
  2752. +void g(void *);
  2753. +int main(int argc, char* argv[])
  2754. +{
  2755. + int a;
  2756. + int b;
  2757. + char c[2+f(argv[1])];
  2758. + int d[0x100];
  2759. + char y;
  2760. +
  2761. + y=42; a=4; b=10;
  2762. + c[0] = 'h'; c[1] = '\0';
  2763. +
  2764. + c[f(argv[2])] = '\0';
  2765. +
  2766. + __builtin_printf("%d %d\n%s\n", a, b, c);
  2767. + g(d);
  2768. +
  2769. + return 0;
  2770. +}
  2771. --
  2772. 2.34.1