dm-crypt.c 92 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606
  1. /*
  2. * Copyright (C) 2003 Jana Saout <jana@saout.de>
  3. * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
  4. * Copyright (C) 2006-2020 Red Hat, Inc. All rights reserved.
  5. * Copyright (C) 2013-2020 Milan Broz <gmazyland@gmail.com>
  6. *
  7. * This file is released under the GPL.
  8. */
  9. #include <linux/completion.h>
  10. #include <linux/err.h>
  11. #include <linux/module.h>
  12. #include <linux/init.h>
  13. #include <linux/kernel.h>
  14. #include <linux/key.h>
  15. #include <linux/bio.h>
  16. #include <linux/blkdev.h>
  17. #include <linux/mempool.h>
  18. #include <linux/slab.h>
  19. #include <linux/crypto.h>
  20. #include <linux/workqueue.h>
  21. #include <linux/kthread.h>
  22. #include <linux/backing-dev.h>
  23. #include <linux/atomic.h>
  24. #include <linux/scatterlist.h>
  25. #include <linux/rbtree.h>
  26. #include <linux/ctype.h>
  27. #include <asm/page.h>
  28. #include <asm/unaligned.h>
  29. #include <crypto/hash.h>
  30. #include <crypto/md5.h>
  31. #include <crypto/algapi.h>
  32. #include <crypto/skcipher.h>
  33. #include <crypto/aead.h>
  34. #include <crypto/authenc.h>
  35. #include <linux/rtnetlink.h> /* for struct rtattr and RTA macros only */
  36. #include <linux/key-type.h>
  37. #include <keys/user-type.h>
  38. #include <keys/encrypted-type.h>
  39. #include <linux/device-mapper.h>
  40. #define DM_MSG_PREFIX "crypt"
  41. /*
  42. * context holding the current state of a multi-part conversion
  43. */
  44. struct convert_context {
  45. struct completion restart;
  46. struct bio *bio_in;
  47. struct bio *bio_out;
  48. struct bvec_iter iter_in;
  49. struct bvec_iter iter_out;
  50. u64 cc_sector;
  51. atomic_t cc_pending;
  52. union {
  53. struct skcipher_request *req;
  54. struct aead_request *req_aead;
  55. } r;
  56. };
  57. /*
  58. * per bio private data
  59. */
  60. struct dm_crypt_io {
  61. struct crypt_config *cc;
  62. struct bio *base_bio;
  63. u8 *integrity_metadata;
  64. bool integrity_metadata_from_pool;
  65. struct work_struct work;
  66. struct tasklet_struct tasklet;
  67. struct convert_context ctx;
  68. atomic_t io_pending;
  69. blk_status_t error;
  70. sector_t sector;
  71. struct rb_node rb_node;
  72. } CRYPTO_MINALIGN_ATTR;
  73. struct dm_crypt_request {
  74. struct convert_context *ctx;
  75. struct scatterlist sg_in[4];
  76. struct scatterlist sg_out[4];
  77. u64 iv_sector;
  78. };
  79. struct crypt_config;
  80. struct crypt_iv_operations {
  81. int (*ctr)(struct crypt_config *cc, struct dm_target *ti,
  82. const char *opts);
  83. void (*dtr)(struct crypt_config *cc);
  84. int (*init)(struct crypt_config *cc);
  85. int (*wipe)(struct crypt_config *cc);
  86. int (*generator)(struct crypt_config *cc, u8 *iv,
  87. struct dm_crypt_request *dmreq);
  88. int (*post)(struct crypt_config *cc, u8 *iv,
  89. struct dm_crypt_request *dmreq);
  90. };
  91. struct iv_benbi_private {
  92. int shift;
  93. };
  94. #define LMK_SEED_SIZE 64 /* hash + 0 */
  95. struct iv_lmk_private {
  96. struct crypto_shash *hash_tfm;
  97. u8 *seed;
  98. };
  99. #define TCW_WHITENING_SIZE 16
  100. struct iv_tcw_private {
  101. struct crypto_shash *crc32_tfm;
  102. u8 *iv_seed;
  103. u8 *whitening;
  104. };
  105. #define ELEPHANT_MAX_KEY_SIZE 32
  106. struct iv_elephant_private {
  107. struct crypto_skcipher *tfm;
  108. };
  109. /*
  110. * Crypt: maps a linear range of a block device
  111. * and encrypts / decrypts at the same time.
  112. */
  113. enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID,
  114. DM_CRYPT_SAME_CPU, DM_CRYPT_NO_OFFLOAD,
  115. DM_CRYPT_NO_READ_WORKQUEUE, DM_CRYPT_NO_WRITE_WORKQUEUE,
  116. DM_CRYPT_WRITE_INLINE };
  117. enum cipher_flags {
  118. CRYPT_MODE_INTEGRITY_AEAD, /* Use authenticated mode for cihper */
  119. CRYPT_IV_LARGE_SECTORS, /* Calculate IV from sector_size, not 512B sectors */
  120. CRYPT_ENCRYPT_PREPROCESS, /* Must preprocess data for encryption (elephant) */
  121. };
  122. /*
  123. * The fields in here must be read only after initialization.
  124. */
  125. struct crypt_config {
  126. struct dm_dev *dev;
  127. sector_t start;
  128. struct percpu_counter n_allocated_pages;
  129. struct workqueue_struct *io_queue;
  130. struct workqueue_struct *crypt_queue;
  131. spinlock_t write_thread_lock;
  132. struct task_struct *write_thread;
  133. struct rb_root write_tree;
  134. char *cipher_string;
  135. char *cipher_auth;
  136. char *key_string;
  137. const struct crypt_iv_operations *iv_gen_ops;
  138. union {
  139. struct iv_benbi_private benbi;
  140. struct iv_lmk_private lmk;
  141. struct iv_tcw_private tcw;
  142. struct iv_elephant_private elephant;
  143. } iv_gen_private;
  144. u64 iv_offset;
  145. unsigned int iv_size;
  146. unsigned short int sector_size;
  147. unsigned char sector_shift;
  148. union {
  149. struct crypto_skcipher **tfms;
  150. struct crypto_aead **tfms_aead;
  151. } cipher_tfm;
  152. unsigned tfms_count;
  153. unsigned long cipher_flags;
  154. /*
  155. * Layout of each crypto request:
  156. *
  157. * struct skcipher_request
  158. * context
  159. * padding
  160. * struct dm_crypt_request
  161. * padding
  162. * IV
  163. *
  164. * The padding is added so that dm_crypt_request and the IV are
  165. * correctly aligned.
  166. */
  167. unsigned int dmreq_start;
  168. unsigned int per_bio_data_size;
  169. unsigned long flags;
  170. unsigned int key_size;
  171. unsigned int key_parts; /* independent parts in key buffer */
  172. unsigned int key_extra_size; /* additional keys length */
  173. unsigned int key_mac_size; /* MAC key size for authenc(...) */
  174. unsigned int integrity_tag_size;
  175. unsigned int integrity_iv_size;
  176. unsigned int on_disk_tag_size;
  177. /*
  178. * pool for per bio private data, crypto requests,
  179. * encryption requeusts/buffer pages and integrity tags
  180. */
  181. unsigned tag_pool_max_sectors;
  182. mempool_t tag_pool;
  183. mempool_t req_pool;
  184. mempool_t page_pool;
  185. struct bio_set bs;
  186. struct mutex bio_alloc_lock;
  187. u8 *authenc_key; /* space for keys in authenc() format (if used) */
  188. u8 key[];
  189. };
  190. #define MIN_IOS 64
  191. #define MAX_TAG_SIZE 480
  192. #define POOL_ENTRY_SIZE 512
  193. static DEFINE_SPINLOCK(dm_crypt_clients_lock);
  194. static unsigned dm_crypt_clients_n = 0;
  195. static volatile unsigned long dm_crypt_pages_per_client;
  196. #define DM_CRYPT_MEMORY_PERCENT 2
  197. #define DM_CRYPT_MIN_PAGES_PER_CLIENT (BIO_MAX_PAGES * 16)
  198. static void clone_init(struct dm_crypt_io *, struct bio *);
  199. static void kcryptd_queue_crypt(struct dm_crypt_io *io);
  200. static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc,
  201. struct scatterlist *sg);
  202. static bool crypt_integrity_aead(struct crypt_config *cc);
  203. /*
  204. * Use this to access cipher attributes that are independent of the key.
  205. */
  206. static struct crypto_skcipher *any_tfm(struct crypt_config *cc)
  207. {
  208. return cc->cipher_tfm.tfms[0];
  209. }
  210. static struct crypto_aead *any_tfm_aead(struct crypt_config *cc)
  211. {
  212. return cc->cipher_tfm.tfms_aead[0];
  213. }
  214. /*
  215. * Different IV generation algorithms:
  216. *
  217. * plain: the initial vector is the 32-bit little-endian version of the sector
  218. * number, padded with zeros if necessary.
  219. *
  220. * plain64: the initial vector is the 64-bit little-endian version of the sector
  221. * number, padded with zeros if necessary.
  222. *
  223. * plain64be: the initial vector is the 64-bit big-endian version of the sector
  224. * number, padded with zeros if necessary.
  225. *
  226. * essiv: "encrypted sector|salt initial vector", the sector number is
  227. * encrypted with the bulk cipher using a salt as key. The salt
  228. * should be derived from the bulk cipher's key via hashing.
  229. *
  230. * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1
  231. * (needed for LRW-32-AES and possible other narrow block modes)
  232. *
  233. * null: the initial vector is always zero. Provides compatibility with
  234. * obsolete loop_fish2 devices. Do not use for new devices.
  235. *
  236. * lmk: Compatible implementation of the block chaining mode used
  237. * by the Loop-AES block device encryption system
  238. * designed by Jari Ruusu. See http://loop-aes.sourceforge.net/
  239. * It operates on full 512 byte sectors and uses CBC
  240. * with an IV derived from the sector number, the data and
  241. * optionally extra IV seed.
  242. * This means that after decryption the first block
  243. * of sector must be tweaked according to decrypted data.
  244. * Loop-AES can use three encryption schemes:
  245. * version 1: is plain aes-cbc mode
  246. * version 2: uses 64 multikey scheme with lmk IV generator
  247. * version 3: the same as version 2 with additional IV seed
  248. * (it uses 65 keys, last key is used as IV seed)
  249. *
  250. * tcw: Compatible implementation of the block chaining mode used
  251. * by the TrueCrypt device encryption system (prior to version 4.1).
  252. * For more info see: https://gitlab.com/cryptsetup/cryptsetup/wikis/TrueCryptOnDiskFormat
  253. * It operates on full 512 byte sectors and uses CBC
  254. * with an IV derived from initial key and the sector number.
  255. * In addition, whitening value is applied on every sector, whitening
  256. * is calculated from initial key, sector number and mixed using CRC32.
  257. * Note that this encryption scheme is vulnerable to watermarking attacks
  258. * and should be used for old compatible containers access only.
  259. *
  260. * eboiv: Encrypted byte-offset IV (used in Bitlocker in CBC mode)
  261. * The IV is encrypted little-endian byte-offset (with the same key
  262. * and cipher as the volume).
  263. *
  264. * elephant: The extended version of eboiv with additional Elephant diffuser
  265. * used with Bitlocker CBC mode.
  266. * This mode was used in older Windows systems
  267. * https://download.microsoft.com/download/0/2/3/0238acaf-d3bf-4a6d-b3d6-0a0be4bbb36e/bitlockercipher200608.pdf
  268. */
  269. static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv,
  270. struct dm_crypt_request *dmreq)
  271. {
  272. memset(iv, 0, cc->iv_size);
  273. *(__le32 *)iv = cpu_to_le32(dmreq->iv_sector & 0xffffffff);
  274. return 0;
  275. }
  276. static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv,
  277. struct dm_crypt_request *dmreq)
  278. {
  279. memset(iv, 0, cc->iv_size);
  280. *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
  281. return 0;
  282. }
  283. static int crypt_iv_plain64be_gen(struct crypt_config *cc, u8 *iv,
  284. struct dm_crypt_request *dmreq)
  285. {
  286. memset(iv, 0, cc->iv_size);
  287. /* iv_size is at least of size u64; usually it is 16 bytes */
  288. *(__be64 *)&iv[cc->iv_size - sizeof(u64)] = cpu_to_be64(dmreq->iv_sector);
  289. return 0;
  290. }
  291. static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv,
  292. struct dm_crypt_request *dmreq)
  293. {
  294. /*
  295. * ESSIV encryption of the IV is now handled by the crypto API,
  296. * so just pass the plain sector number here.
  297. */
  298. memset(iv, 0, cc->iv_size);
  299. *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
  300. return 0;
  301. }
  302. static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti,
  303. const char *opts)
  304. {
  305. unsigned bs;
  306. int log;
  307. if (crypt_integrity_aead(cc))
  308. bs = crypto_aead_blocksize(any_tfm_aead(cc));
  309. else
  310. bs = crypto_skcipher_blocksize(any_tfm(cc));
  311. log = ilog2(bs);
  312. /* we need to calculate how far we must shift the sector count
  313. * to get the cipher block count, we use this shift in _gen */
  314. if (1 << log != bs) {
  315. ti->error = "cypher blocksize is not a power of 2";
  316. return -EINVAL;
  317. }
  318. if (log > 9) {
  319. ti->error = "cypher blocksize is > 512";
  320. return -EINVAL;
  321. }
  322. cc->iv_gen_private.benbi.shift = 9 - log;
  323. return 0;
  324. }
  325. static void crypt_iv_benbi_dtr(struct crypt_config *cc)
  326. {
  327. }
  328. static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv,
  329. struct dm_crypt_request *dmreq)
  330. {
  331. __be64 val;
  332. memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */
  333. val = cpu_to_be64(((u64)dmreq->iv_sector << cc->iv_gen_private.benbi.shift) + 1);
  334. put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64)));
  335. return 0;
  336. }
  337. static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv,
  338. struct dm_crypt_request *dmreq)
  339. {
  340. memset(iv, 0, cc->iv_size);
  341. return 0;
  342. }
  343. static void crypt_iv_lmk_dtr(struct crypt_config *cc)
  344. {
  345. struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
  346. if (lmk->hash_tfm && !IS_ERR(lmk->hash_tfm))
  347. crypto_free_shash(lmk->hash_tfm);
  348. lmk->hash_tfm = NULL;
  349. kfree_sensitive(lmk->seed);
  350. lmk->seed = NULL;
  351. }
  352. static int crypt_iv_lmk_ctr(struct crypt_config *cc, struct dm_target *ti,
  353. const char *opts)
  354. {
  355. struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
  356. if (cc->sector_size != (1 << SECTOR_SHIFT)) {
  357. ti->error = "Unsupported sector size for LMK";
  358. return -EINVAL;
  359. }
  360. lmk->hash_tfm = crypto_alloc_shash("md5", 0,
  361. CRYPTO_ALG_ALLOCATES_MEMORY);
  362. if (IS_ERR(lmk->hash_tfm)) {
  363. ti->error = "Error initializing LMK hash";
  364. return PTR_ERR(lmk->hash_tfm);
  365. }
  366. /* No seed in LMK version 2 */
  367. if (cc->key_parts == cc->tfms_count) {
  368. lmk->seed = NULL;
  369. return 0;
  370. }
  371. lmk->seed = kzalloc(LMK_SEED_SIZE, GFP_KERNEL);
  372. if (!lmk->seed) {
  373. crypt_iv_lmk_dtr(cc);
  374. ti->error = "Error kmallocing seed storage in LMK";
  375. return -ENOMEM;
  376. }
  377. return 0;
  378. }
  379. static int crypt_iv_lmk_init(struct crypt_config *cc)
  380. {
  381. struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
  382. int subkey_size = cc->key_size / cc->key_parts;
  383. /* LMK seed is on the position of LMK_KEYS + 1 key */
  384. if (lmk->seed)
  385. memcpy(lmk->seed, cc->key + (cc->tfms_count * subkey_size),
  386. crypto_shash_digestsize(lmk->hash_tfm));
  387. return 0;
  388. }
  389. static int crypt_iv_lmk_wipe(struct crypt_config *cc)
  390. {
  391. struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
  392. if (lmk->seed)
  393. memset(lmk->seed, 0, LMK_SEED_SIZE);
  394. return 0;
  395. }
  396. static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv,
  397. struct dm_crypt_request *dmreq,
  398. u8 *data)
  399. {
  400. struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
  401. SHASH_DESC_ON_STACK(desc, lmk->hash_tfm);
  402. struct md5_state md5state;
  403. __le32 buf[4];
  404. int i, r;
  405. desc->tfm = lmk->hash_tfm;
  406. r = crypto_shash_init(desc);
  407. if (r)
  408. return r;
  409. if (lmk->seed) {
  410. r = crypto_shash_update(desc, lmk->seed, LMK_SEED_SIZE);
  411. if (r)
  412. return r;
  413. }
  414. /* Sector is always 512B, block size 16, add data of blocks 1-31 */
  415. r = crypto_shash_update(desc, data + 16, 16 * 31);
  416. if (r)
  417. return r;
  418. /* Sector is cropped to 56 bits here */
  419. buf[0] = cpu_to_le32(dmreq->iv_sector & 0xFFFFFFFF);
  420. buf[1] = cpu_to_le32((((u64)dmreq->iv_sector >> 32) & 0x00FFFFFF) | 0x80000000);
  421. buf[2] = cpu_to_le32(4024);
  422. buf[3] = 0;
  423. r = crypto_shash_update(desc, (u8 *)buf, sizeof(buf));
  424. if (r)
  425. return r;
  426. /* No MD5 padding here */
  427. r = crypto_shash_export(desc, &md5state);
  428. if (r)
  429. return r;
  430. for (i = 0; i < MD5_HASH_WORDS; i++)
  431. __cpu_to_le32s(&md5state.hash[i]);
  432. memcpy(iv, &md5state.hash, cc->iv_size);
  433. return 0;
  434. }
  435. static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv,
  436. struct dm_crypt_request *dmreq)
  437. {
  438. struct scatterlist *sg;
  439. u8 *src;
  440. int r = 0;
  441. if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
  442. sg = crypt_get_sg_data(cc, dmreq->sg_in);
  443. src = kmap_atomic(sg_page(sg));
  444. r = crypt_iv_lmk_one(cc, iv, dmreq, src + sg->offset);
  445. kunmap_atomic(src);
  446. } else
  447. memset(iv, 0, cc->iv_size);
  448. return r;
  449. }
  450. static int crypt_iv_lmk_post(struct crypt_config *cc, u8 *iv,
  451. struct dm_crypt_request *dmreq)
  452. {
  453. struct scatterlist *sg;
  454. u8 *dst;
  455. int r;
  456. if (bio_data_dir(dmreq->ctx->bio_in) == WRITE)
  457. return 0;
  458. sg = crypt_get_sg_data(cc, dmreq->sg_out);
  459. dst = kmap_atomic(sg_page(sg));
  460. r = crypt_iv_lmk_one(cc, iv, dmreq, dst + sg->offset);
  461. /* Tweak the first block of plaintext sector */
  462. if (!r)
  463. crypto_xor(dst + sg->offset, iv, cc->iv_size);
  464. kunmap_atomic(dst);
  465. return r;
  466. }
  467. static void crypt_iv_tcw_dtr(struct crypt_config *cc)
  468. {
  469. struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
  470. kfree_sensitive(tcw->iv_seed);
  471. tcw->iv_seed = NULL;
  472. kfree_sensitive(tcw->whitening);
  473. tcw->whitening = NULL;
  474. if (tcw->crc32_tfm && !IS_ERR(tcw->crc32_tfm))
  475. crypto_free_shash(tcw->crc32_tfm);
  476. tcw->crc32_tfm = NULL;
  477. }
  478. static int crypt_iv_tcw_ctr(struct crypt_config *cc, struct dm_target *ti,
  479. const char *opts)
  480. {
  481. struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
  482. if (cc->sector_size != (1 << SECTOR_SHIFT)) {
  483. ti->error = "Unsupported sector size for TCW";
  484. return -EINVAL;
  485. }
  486. if (cc->key_size <= (cc->iv_size + TCW_WHITENING_SIZE)) {
  487. ti->error = "Wrong key size for TCW";
  488. return -EINVAL;
  489. }
  490. tcw->crc32_tfm = crypto_alloc_shash("crc32", 0,
  491. CRYPTO_ALG_ALLOCATES_MEMORY);
  492. if (IS_ERR(tcw->crc32_tfm)) {
  493. ti->error = "Error initializing CRC32 in TCW";
  494. return PTR_ERR(tcw->crc32_tfm);
  495. }
  496. tcw->iv_seed = kzalloc(cc->iv_size, GFP_KERNEL);
  497. tcw->whitening = kzalloc(TCW_WHITENING_SIZE, GFP_KERNEL);
  498. if (!tcw->iv_seed || !tcw->whitening) {
  499. crypt_iv_tcw_dtr(cc);
  500. ti->error = "Error allocating seed storage in TCW";
  501. return -ENOMEM;
  502. }
  503. return 0;
  504. }
  505. static int crypt_iv_tcw_init(struct crypt_config *cc)
  506. {
  507. struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
  508. int key_offset = cc->key_size - cc->iv_size - TCW_WHITENING_SIZE;
  509. memcpy(tcw->iv_seed, &cc->key[key_offset], cc->iv_size);
  510. memcpy(tcw->whitening, &cc->key[key_offset + cc->iv_size],
  511. TCW_WHITENING_SIZE);
  512. return 0;
  513. }
  514. static int crypt_iv_tcw_wipe(struct crypt_config *cc)
  515. {
  516. struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
  517. memset(tcw->iv_seed, 0, cc->iv_size);
  518. memset(tcw->whitening, 0, TCW_WHITENING_SIZE);
  519. return 0;
  520. }
  521. static int crypt_iv_tcw_whitening(struct crypt_config *cc,
  522. struct dm_crypt_request *dmreq,
  523. u8 *data)
  524. {
  525. struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
  526. __le64 sector = cpu_to_le64(dmreq->iv_sector);
  527. u8 buf[TCW_WHITENING_SIZE];
  528. SHASH_DESC_ON_STACK(desc, tcw->crc32_tfm);
  529. int i, r;
  530. /* xor whitening with sector number */
  531. crypto_xor_cpy(buf, tcw->whitening, (u8 *)&sector, 8);
  532. crypto_xor_cpy(&buf[8], tcw->whitening + 8, (u8 *)&sector, 8);
  533. /* calculate crc32 for every 32bit part and xor it */
  534. desc->tfm = tcw->crc32_tfm;
  535. for (i = 0; i < 4; i++) {
  536. r = crypto_shash_init(desc);
  537. if (r)
  538. goto out;
  539. r = crypto_shash_update(desc, &buf[i * 4], 4);
  540. if (r)
  541. goto out;
  542. r = crypto_shash_final(desc, &buf[i * 4]);
  543. if (r)
  544. goto out;
  545. }
  546. crypto_xor(&buf[0], &buf[12], 4);
  547. crypto_xor(&buf[4], &buf[8], 4);
  548. /* apply whitening (8 bytes) to whole sector */
  549. for (i = 0; i < ((1 << SECTOR_SHIFT) / 8); i++)
  550. crypto_xor(data + i * 8, buf, 8);
  551. out:
  552. memzero_explicit(buf, sizeof(buf));
  553. return r;
  554. }
  555. static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv,
  556. struct dm_crypt_request *dmreq)
  557. {
  558. struct scatterlist *sg;
  559. struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
  560. __le64 sector = cpu_to_le64(dmreq->iv_sector);
  561. u8 *src;
  562. int r = 0;
  563. /* Remove whitening from ciphertext */
  564. if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {
  565. sg = crypt_get_sg_data(cc, dmreq->sg_in);
  566. src = kmap_atomic(sg_page(sg));
  567. r = crypt_iv_tcw_whitening(cc, dmreq, src + sg->offset);
  568. kunmap_atomic(src);
  569. }
  570. /* Calculate IV */
  571. crypto_xor_cpy(iv, tcw->iv_seed, (u8 *)&sector, 8);
  572. if (cc->iv_size > 8)
  573. crypto_xor_cpy(&iv[8], tcw->iv_seed + 8, (u8 *)&sector,
  574. cc->iv_size - 8);
  575. return r;
  576. }
  577. static int crypt_iv_tcw_post(struct crypt_config *cc, u8 *iv,
  578. struct dm_crypt_request *dmreq)
  579. {
  580. struct scatterlist *sg;
  581. u8 *dst;
  582. int r;
  583. if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)
  584. return 0;
  585. /* Apply whitening on ciphertext */
  586. sg = crypt_get_sg_data(cc, dmreq->sg_out);
  587. dst = kmap_atomic(sg_page(sg));
  588. r = crypt_iv_tcw_whitening(cc, dmreq, dst + sg->offset);
  589. kunmap_atomic(dst);
  590. return r;
  591. }
  592. static int crypt_iv_random_gen(struct crypt_config *cc, u8 *iv,
  593. struct dm_crypt_request *dmreq)
  594. {
  595. /* Used only for writes, there must be an additional space to store IV */
  596. get_random_bytes(iv, cc->iv_size);
  597. return 0;
  598. }
  599. static int crypt_iv_eboiv_ctr(struct crypt_config *cc, struct dm_target *ti,
  600. const char *opts)
  601. {
  602. if (crypt_integrity_aead(cc)) {
  603. ti->error = "AEAD transforms not supported for EBOIV";
  604. return -EINVAL;
  605. }
  606. if (crypto_skcipher_blocksize(any_tfm(cc)) != cc->iv_size) {
  607. ti->error = "Block size of EBOIV cipher does "
  608. "not match IV size of block cipher";
  609. return -EINVAL;
  610. }
  611. return 0;
  612. }
  613. static int crypt_iv_eboiv_gen(struct crypt_config *cc, u8 *iv,
  614. struct dm_crypt_request *dmreq)
  615. {
  616. u8 buf[MAX_CIPHER_BLOCKSIZE] __aligned(__alignof__(__le64));
  617. struct skcipher_request *req;
  618. struct scatterlist src, dst;
  619. DECLARE_CRYPTO_WAIT(wait);
  620. int err;
  621. req = skcipher_request_alloc(any_tfm(cc), GFP_NOIO);
  622. if (!req)
  623. return -ENOMEM;
  624. memset(buf, 0, cc->iv_size);
  625. *(__le64 *)buf = cpu_to_le64(dmreq->iv_sector * cc->sector_size);
  626. sg_init_one(&src, page_address(ZERO_PAGE(0)), cc->iv_size);
  627. sg_init_one(&dst, iv, cc->iv_size);
  628. skcipher_request_set_crypt(req, &src, &dst, cc->iv_size, buf);
  629. skcipher_request_set_callback(req, 0, crypto_req_done, &wait);
  630. err = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
  631. skcipher_request_free(req);
  632. return err;
  633. }
  634. static void crypt_iv_elephant_dtr(struct crypt_config *cc)
  635. {
  636. struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
  637. crypto_free_skcipher(elephant->tfm);
  638. elephant->tfm = NULL;
  639. }
  640. static int crypt_iv_elephant_ctr(struct crypt_config *cc, struct dm_target *ti,
  641. const char *opts)
  642. {
  643. struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
  644. int r;
  645. elephant->tfm = crypto_alloc_skcipher("ecb(aes)", 0,
  646. CRYPTO_ALG_ALLOCATES_MEMORY);
  647. if (IS_ERR(elephant->tfm)) {
  648. r = PTR_ERR(elephant->tfm);
  649. elephant->tfm = NULL;
  650. return r;
  651. }
  652. r = crypt_iv_eboiv_ctr(cc, ti, NULL);
  653. if (r)
  654. crypt_iv_elephant_dtr(cc);
  655. return r;
  656. }
  657. static void diffuser_disk_to_cpu(u32 *d, size_t n)
  658. {
  659. #ifndef __LITTLE_ENDIAN
  660. int i;
  661. for (i = 0; i < n; i++)
  662. d[i] = le32_to_cpu((__le32)d[i]);
  663. #endif
  664. }
  665. static void diffuser_cpu_to_disk(__le32 *d, size_t n)
  666. {
  667. #ifndef __LITTLE_ENDIAN
  668. int i;
  669. for (i = 0; i < n; i++)
  670. d[i] = cpu_to_le32((u32)d[i]);
  671. #endif
  672. }
  673. static void diffuser_a_decrypt(u32 *d, size_t n)
  674. {
  675. int i, i1, i2, i3;
  676. for (i = 0; i < 5; i++) {
  677. i1 = 0;
  678. i2 = n - 2;
  679. i3 = n - 5;
  680. while (i1 < (n - 1)) {
  681. d[i1] += d[i2] ^ (d[i3] << 9 | d[i3] >> 23);
  682. i1++; i2++; i3++;
  683. if (i3 >= n)
  684. i3 -= n;
  685. d[i1] += d[i2] ^ d[i3];
  686. i1++; i2++; i3++;
  687. if (i2 >= n)
  688. i2 -= n;
  689. d[i1] += d[i2] ^ (d[i3] << 13 | d[i3] >> 19);
  690. i1++; i2++; i3++;
  691. d[i1] += d[i2] ^ d[i3];
  692. i1++; i2++; i3++;
  693. }
  694. }
  695. }
  696. static void diffuser_a_encrypt(u32 *d, size_t n)
  697. {
  698. int i, i1, i2, i3;
  699. for (i = 0; i < 5; i++) {
  700. i1 = n - 1;
  701. i2 = n - 2 - 1;
  702. i3 = n - 5 - 1;
  703. while (i1 > 0) {
  704. d[i1] -= d[i2] ^ d[i3];
  705. i1--; i2--; i3--;
  706. d[i1] -= d[i2] ^ (d[i3] << 13 | d[i3] >> 19);
  707. i1--; i2--; i3--;
  708. if (i2 < 0)
  709. i2 += n;
  710. d[i1] -= d[i2] ^ d[i3];
  711. i1--; i2--; i3--;
  712. if (i3 < 0)
  713. i3 += n;
  714. d[i1] -= d[i2] ^ (d[i3] << 9 | d[i3] >> 23);
  715. i1--; i2--; i3--;
  716. }
  717. }
  718. }
  719. static void diffuser_b_decrypt(u32 *d, size_t n)
  720. {
  721. int i, i1, i2, i3;
  722. for (i = 0; i < 3; i++) {
  723. i1 = 0;
  724. i2 = 2;
  725. i3 = 5;
  726. while (i1 < (n - 1)) {
  727. d[i1] += d[i2] ^ d[i3];
  728. i1++; i2++; i3++;
  729. d[i1] += d[i2] ^ (d[i3] << 10 | d[i3] >> 22);
  730. i1++; i2++; i3++;
  731. if (i2 >= n)
  732. i2 -= n;
  733. d[i1] += d[i2] ^ d[i3];
  734. i1++; i2++; i3++;
  735. if (i3 >= n)
  736. i3 -= n;
  737. d[i1] += d[i2] ^ (d[i3] << 25 | d[i3] >> 7);
  738. i1++; i2++; i3++;
  739. }
  740. }
  741. }
  742. static void diffuser_b_encrypt(u32 *d, size_t n)
  743. {
  744. int i, i1, i2, i3;
  745. for (i = 0; i < 3; i++) {
  746. i1 = n - 1;
  747. i2 = 2 - 1;
  748. i3 = 5 - 1;
  749. while (i1 > 0) {
  750. d[i1] -= d[i2] ^ (d[i3] << 25 | d[i3] >> 7);
  751. i1--; i2--; i3--;
  752. if (i3 < 0)
  753. i3 += n;
  754. d[i1] -= d[i2] ^ d[i3];
  755. i1--; i2--; i3--;
  756. if (i2 < 0)
  757. i2 += n;
  758. d[i1] -= d[i2] ^ (d[i3] << 10 | d[i3] >> 22);
  759. i1--; i2--; i3--;
  760. d[i1] -= d[i2] ^ d[i3];
  761. i1--; i2--; i3--;
  762. }
  763. }
  764. }
  765. static int crypt_iv_elephant(struct crypt_config *cc, struct dm_crypt_request *dmreq)
  766. {
  767. struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
  768. u8 *es, *ks, *data, *data2, *data_offset;
  769. struct skcipher_request *req;
  770. struct scatterlist *sg, *sg2, src, dst;
  771. DECLARE_CRYPTO_WAIT(wait);
  772. int i, r;
  773. req = skcipher_request_alloc(elephant->tfm, GFP_NOIO);
  774. es = kzalloc(16, GFP_NOIO); /* Key for AES */
  775. ks = kzalloc(32, GFP_NOIO); /* Elephant sector key */
  776. if (!req || !es || !ks) {
  777. r = -ENOMEM;
  778. goto out;
  779. }
  780. *(__le64 *)es = cpu_to_le64(dmreq->iv_sector * cc->sector_size);
  781. /* E(Ks, e(s)) */
  782. sg_init_one(&src, es, 16);
  783. sg_init_one(&dst, ks, 16);
  784. skcipher_request_set_crypt(req, &src, &dst, 16, NULL);
  785. skcipher_request_set_callback(req, 0, crypto_req_done, &wait);
  786. r = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
  787. if (r)
  788. goto out;
  789. /* E(Ks, e'(s)) */
  790. es[15] = 0x80;
  791. sg_init_one(&dst, &ks[16], 16);
  792. r = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
  793. if (r)
  794. goto out;
  795. sg = crypt_get_sg_data(cc, dmreq->sg_out);
  796. data = kmap_atomic(sg_page(sg));
  797. data_offset = data + sg->offset;
  798. /* Cannot modify original bio, copy to sg_out and apply Elephant to it */
  799. if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
  800. sg2 = crypt_get_sg_data(cc, dmreq->sg_in);
  801. data2 = kmap_atomic(sg_page(sg2));
  802. memcpy(data_offset, data2 + sg2->offset, cc->sector_size);
  803. kunmap_atomic(data2);
  804. }
  805. if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {
  806. diffuser_disk_to_cpu((u32*)data_offset, cc->sector_size / sizeof(u32));
  807. diffuser_b_decrypt((u32*)data_offset, cc->sector_size / sizeof(u32));
  808. diffuser_a_decrypt((u32*)data_offset, cc->sector_size / sizeof(u32));
  809. diffuser_cpu_to_disk((__le32*)data_offset, cc->sector_size / sizeof(u32));
  810. }
  811. for (i = 0; i < (cc->sector_size / 32); i++)
  812. crypto_xor(data_offset + i * 32, ks, 32);
  813. if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
  814. diffuser_disk_to_cpu((u32*)data_offset, cc->sector_size / sizeof(u32));
  815. diffuser_a_encrypt((u32*)data_offset, cc->sector_size / sizeof(u32));
  816. diffuser_b_encrypt((u32*)data_offset, cc->sector_size / sizeof(u32));
  817. diffuser_cpu_to_disk((__le32*)data_offset, cc->sector_size / sizeof(u32));
  818. }
  819. kunmap_atomic(data);
  820. out:
  821. kfree_sensitive(ks);
  822. kfree_sensitive(es);
  823. skcipher_request_free(req);
  824. return r;
  825. }
  826. static int crypt_iv_elephant_gen(struct crypt_config *cc, u8 *iv,
  827. struct dm_crypt_request *dmreq)
  828. {
  829. int r;
  830. if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
  831. r = crypt_iv_elephant(cc, dmreq);
  832. if (r)
  833. return r;
  834. }
  835. return crypt_iv_eboiv_gen(cc, iv, dmreq);
  836. }
  837. static int crypt_iv_elephant_post(struct crypt_config *cc, u8 *iv,
  838. struct dm_crypt_request *dmreq)
  839. {
  840. if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)
  841. return crypt_iv_elephant(cc, dmreq);
  842. return 0;
  843. }
  844. static int crypt_iv_elephant_init(struct crypt_config *cc)
  845. {
  846. struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
  847. int key_offset = cc->key_size - cc->key_extra_size;
  848. return crypto_skcipher_setkey(elephant->tfm, &cc->key[key_offset], cc->key_extra_size);
  849. }
  850. static int crypt_iv_elephant_wipe(struct crypt_config *cc)
  851. {
  852. struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
  853. u8 key[ELEPHANT_MAX_KEY_SIZE];
  854. memset(key, 0, cc->key_extra_size);
  855. return crypto_skcipher_setkey(elephant->tfm, key, cc->key_extra_size);
  856. }
  857. static const struct crypt_iv_operations crypt_iv_plain_ops = {
  858. .generator = crypt_iv_plain_gen
  859. };
  860. static const struct crypt_iv_operations crypt_iv_plain64_ops = {
  861. .generator = crypt_iv_plain64_gen
  862. };
  863. static const struct crypt_iv_operations crypt_iv_plain64be_ops = {
  864. .generator = crypt_iv_plain64be_gen
  865. };
  866. static const struct crypt_iv_operations crypt_iv_essiv_ops = {
  867. .generator = crypt_iv_essiv_gen
  868. };
  869. static const struct crypt_iv_operations crypt_iv_benbi_ops = {
  870. .ctr = crypt_iv_benbi_ctr,
  871. .dtr = crypt_iv_benbi_dtr,
  872. .generator = crypt_iv_benbi_gen
  873. };
  874. static const struct crypt_iv_operations crypt_iv_null_ops = {
  875. .generator = crypt_iv_null_gen
  876. };
  877. static const struct crypt_iv_operations crypt_iv_lmk_ops = {
  878. .ctr = crypt_iv_lmk_ctr,
  879. .dtr = crypt_iv_lmk_dtr,
  880. .init = crypt_iv_lmk_init,
  881. .wipe = crypt_iv_lmk_wipe,
  882. .generator = crypt_iv_lmk_gen,
  883. .post = crypt_iv_lmk_post
  884. };
  885. static const struct crypt_iv_operations crypt_iv_tcw_ops = {
  886. .ctr = crypt_iv_tcw_ctr,
  887. .dtr = crypt_iv_tcw_dtr,
  888. .init = crypt_iv_tcw_init,
  889. .wipe = crypt_iv_tcw_wipe,
  890. .generator = crypt_iv_tcw_gen,
  891. .post = crypt_iv_tcw_post
  892. };
  893. static struct crypt_iv_operations crypt_iv_random_ops = {
  894. .generator = crypt_iv_random_gen
  895. };
  896. static struct crypt_iv_operations crypt_iv_eboiv_ops = {
  897. .ctr = crypt_iv_eboiv_ctr,
  898. .generator = crypt_iv_eboiv_gen
  899. };
  900. static struct crypt_iv_operations crypt_iv_elephant_ops = {
  901. .ctr = crypt_iv_elephant_ctr,
  902. .dtr = crypt_iv_elephant_dtr,
  903. .init = crypt_iv_elephant_init,
  904. .wipe = crypt_iv_elephant_wipe,
  905. .generator = crypt_iv_elephant_gen,
  906. .post = crypt_iv_elephant_post
  907. };
  908. /*
  909. * Integrity extensions
  910. */
  911. static bool crypt_integrity_aead(struct crypt_config *cc)
  912. {
  913. return test_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags);
  914. }
  915. static bool crypt_integrity_hmac(struct crypt_config *cc)
  916. {
  917. return crypt_integrity_aead(cc) && cc->key_mac_size;
  918. }
  919. /* Get sg containing data */
  920. static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc,
  921. struct scatterlist *sg)
  922. {
  923. if (unlikely(crypt_integrity_aead(cc)))
  924. return &sg[2];
  925. return sg;
  926. }
  927. static int dm_crypt_integrity_io_alloc(struct dm_crypt_io *io, struct bio *bio)
  928. {
  929. struct bio_integrity_payload *bip;
  930. unsigned int tag_len;
  931. int ret;
  932. if (!bio_sectors(bio) || !io->cc->on_disk_tag_size)
  933. return 0;
  934. bip = bio_integrity_alloc(bio, GFP_NOIO, 1);
  935. if (IS_ERR(bip))
  936. return PTR_ERR(bip);
  937. tag_len = io->cc->on_disk_tag_size * (bio_sectors(bio) >> io->cc->sector_shift);
  938. bip->bip_iter.bi_size = tag_len;
  939. bip->bip_iter.bi_sector = io->cc->start + io->sector;
  940. ret = bio_integrity_add_page(bio, virt_to_page(io->integrity_metadata),
  941. tag_len, offset_in_page(io->integrity_metadata));
  942. if (unlikely(ret != tag_len))
  943. return -ENOMEM;
  944. return 0;
  945. }
  946. static int crypt_integrity_ctr(struct crypt_config *cc, struct dm_target *ti)
  947. {
  948. #ifdef CONFIG_BLK_DEV_INTEGRITY
  949. struct blk_integrity *bi = blk_get_integrity(cc->dev->bdev->bd_disk);
  950. struct mapped_device *md = dm_table_get_md(ti->table);
  951. /* From now we require underlying device with our integrity profile */
  952. if (!bi || strcasecmp(bi->profile->name, "DM-DIF-EXT-TAG")) {
  953. ti->error = "Integrity profile not supported.";
  954. return -EINVAL;
  955. }
  956. if (bi->tag_size != cc->on_disk_tag_size ||
  957. bi->tuple_size != cc->on_disk_tag_size) {
  958. ti->error = "Integrity profile tag size mismatch.";
  959. return -EINVAL;
  960. }
  961. if (1 << bi->interval_exp != cc->sector_size) {
  962. ti->error = "Integrity profile sector size mismatch.";
  963. return -EINVAL;
  964. }
  965. if (crypt_integrity_aead(cc)) {
  966. cc->integrity_tag_size = cc->on_disk_tag_size - cc->integrity_iv_size;
  967. DMDEBUG("%s: Integrity AEAD, tag size %u, IV size %u.", dm_device_name(md),
  968. cc->integrity_tag_size, cc->integrity_iv_size);
  969. if (crypto_aead_setauthsize(any_tfm_aead(cc), cc->integrity_tag_size)) {
  970. ti->error = "Integrity AEAD auth tag size is not supported.";
  971. return -EINVAL;
  972. }
  973. } else if (cc->integrity_iv_size)
  974. DMDEBUG("%s: Additional per-sector space %u bytes for IV.", dm_device_name(md),
  975. cc->integrity_iv_size);
  976. if ((cc->integrity_tag_size + cc->integrity_iv_size) != bi->tag_size) {
  977. ti->error = "Not enough space for integrity tag in the profile.";
  978. return -EINVAL;
  979. }
  980. return 0;
  981. #else
  982. ti->error = "Integrity profile not supported.";
  983. return -EINVAL;
  984. #endif
  985. }
  986. static void crypt_convert_init(struct crypt_config *cc,
  987. struct convert_context *ctx,
  988. struct bio *bio_out, struct bio *bio_in,
  989. sector_t sector)
  990. {
  991. ctx->bio_in = bio_in;
  992. ctx->bio_out = bio_out;
  993. if (bio_in)
  994. ctx->iter_in = bio_in->bi_iter;
  995. if (bio_out)
  996. ctx->iter_out = bio_out->bi_iter;
  997. ctx->cc_sector = sector + cc->iv_offset;
  998. init_completion(&ctx->restart);
  999. }
  1000. static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc,
  1001. void *req)
  1002. {
  1003. return (struct dm_crypt_request *)((char *)req + cc->dmreq_start);
  1004. }
  1005. static void *req_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq)
  1006. {
  1007. return (void *)((char *)dmreq - cc->dmreq_start);
  1008. }
  1009. static u8 *iv_of_dmreq(struct crypt_config *cc,
  1010. struct dm_crypt_request *dmreq)
  1011. {
  1012. if (crypt_integrity_aead(cc))
  1013. return (u8 *)ALIGN((unsigned long)(dmreq + 1),
  1014. crypto_aead_alignmask(any_tfm_aead(cc)) + 1);
  1015. else
  1016. return (u8 *)ALIGN((unsigned long)(dmreq + 1),
  1017. crypto_skcipher_alignmask(any_tfm(cc)) + 1);
  1018. }
  1019. static u8 *org_iv_of_dmreq(struct crypt_config *cc,
  1020. struct dm_crypt_request *dmreq)
  1021. {
  1022. return iv_of_dmreq(cc, dmreq) + cc->iv_size;
  1023. }
  1024. static __le64 *org_sector_of_dmreq(struct crypt_config *cc,
  1025. struct dm_crypt_request *dmreq)
  1026. {
  1027. u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size + cc->iv_size;
  1028. return (__le64 *) ptr;
  1029. }
  1030. static unsigned int *org_tag_of_dmreq(struct crypt_config *cc,
  1031. struct dm_crypt_request *dmreq)
  1032. {
  1033. u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size +
  1034. cc->iv_size + sizeof(uint64_t);
  1035. return (unsigned int*)ptr;
  1036. }
  1037. static void *tag_from_dmreq(struct crypt_config *cc,
  1038. struct dm_crypt_request *dmreq)
  1039. {
  1040. struct convert_context *ctx = dmreq->ctx;
  1041. struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);
  1042. return &io->integrity_metadata[*org_tag_of_dmreq(cc, dmreq) *
  1043. cc->on_disk_tag_size];
  1044. }
  1045. static void *iv_tag_from_dmreq(struct crypt_config *cc,
  1046. struct dm_crypt_request *dmreq)
  1047. {
  1048. return tag_from_dmreq(cc, dmreq) + cc->integrity_tag_size;
  1049. }
  1050. static int crypt_convert_block_aead(struct crypt_config *cc,
  1051. struct convert_context *ctx,
  1052. struct aead_request *req,
  1053. unsigned int tag_offset)
  1054. {
  1055. struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
  1056. struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
  1057. struct dm_crypt_request *dmreq;
  1058. u8 *iv, *org_iv, *tag_iv, *tag;
  1059. __le64 *sector;
  1060. int r = 0;
  1061. BUG_ON(cc->integrity_iv_size && cc->integrity_iv_size != cc->iv_size);
  1062. /* Reject unexpected unaligned bio. */
  1063. if (unlikely(bv_in.bv_len & (cc->sector_size - 1)))
  1064. return -EIO;
  1065. dmreq = dmreq_of_req(cc, req);
  1066. dmreq->iv_sector = ctx->cc_sector;
  1067. if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
  1068. dmreq->iv_sector >>= cc->sector_shift;
  1069. dmreq->ctx = ctx;
  1070. *org_tag_of_dmreq(cc, dmreq) = tag_offset;
  1071. sector = org_sector_of_dmreq(cc, dmreq);
  1072. *sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset);
  1073. iv = iv_of_dmreq(cc, dmreq);
  1074. org_iv = org_iv_of_dmreq(cc, dmreq);
  1075. tag = tag_from_dmreq(cc, dmreq);
  1076. tag_iv = iv_tag_from_dmreq(cc, dmreq);
  1077. /* AEAD request:
  1078. * |----- AAD -------|------ DATA -------|-- AUTH TAG --|
  1079. * | (authenticated) | (auth+encryption) | |
  1080. * | sector_LE | IV | sector in/out | tag in/out |
  1081. */
  1082. sg_init_table(dmreq->sg_in, 4);
  1083. sg_set_buf(&dmreq->sg_in[0], sector, sizeof(uint64_t));
  1084. sg_set_buf(&dmreq->sg_in[1], org_iv, cc->iv_size);
  1085. sg_set_page(&dmreq->sg_in[2], bv_in.bv_page, cc->sector_size, bv_in.bv_offset);
  1086. sg_set_buf(&dmreq->sg_in[3], tag, cc->integrity_tag_size);
  1087. sg_init_table(dmreq->sg_out, 4);
  1088. sg_set_buf(&dmreq->sg_out[0], sector, sizeof(uint64_t));
  1089. sg_set_buf(&dmreq->sg_out[1], org_iv, cc->iv_size);
  1090. sg_set_page(&dmreq->sg_out[2], bv_out.bv_page, cc->sector_size, bv_out.bv_offset);
  1091. sg_set_buf(&dmreq->sg_out[3], tag, cc->integrity_tag_size);
  1092. if (cc->iv_gen_ops) {
  1093. /* For READs use IV stored in integrity metadata */
  1094. if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) {
  1095. memcpy(org_iv, tag_iv, cc->iv_size);
  1096. } else {
  1097. r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);
  1098. if (r < 0)
  1099. return r;
  1100. /* Store generated IV in integrity metadata */
  1101. if (cc->integrity_iv_size)
  1102. memcpy(tag_iv, org_iv, cc->iv_size);
  1103. }
  1104. /* Working copy of IV, to be modified in crypto API */
  1105. memcpy(iv, org_iv, cc->iv_size);
  1106. }
  1107. aead_request_set_ad(req, sizeof(uint64_t) + cc->iv_size);
  1108. if (bio_data_dir(ctx->bio_in) == WRITE) {
  1109. aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out,
  1110. cc->sector_size, iv);
  1111. r = crypto_aead_encrypt(req);
  1112. if (cc->integrity_tag_size + cc->integrity_iv_size != cc->on_disk_tag_size)
  1113. memset(tag + cc->integrity_tag_size + cc->integrity_iv_size, 0,
  1114. cc->on_disk_tag_size - (cc->integrity_tag_size + cc->integrity_iv_size));
  1115. } else {
  1116. aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out,
  1117. cc->sector_size + cc->integrity_tag_size, iv);
  1118. r = crypto_aead_decrypt(req);
  1119. }
  1120. if (r == -EBADMSG) {
  1121. char b[BDEVNAME_SIZE];
  1122. DMERR_LIMIT("%s: INTEGRITY AEAD ERROR, sector %llu", bio_devname(ctx->bio_in, b),
  1123. (unsigned long long)le64_to_cpu(*sector));
  1124. }
  1125. if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
  1126. r = cc->iv_gen_ops->post(cc, org_iv, dmreq);
  1127. bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size);
  1128. bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size);
  1129. return r;
  1130. }
  1131. static int crypt_convert_block_skcipher(struct crypt_config *cc,
  1132. struct convert_context *ctx,
  1133. struct skcipher_request *req,
  1134. unsigned int tag_offset)
  1135. {
  1136. struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
  1137. struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
  1138. struct scatterlist *sg_in, *sg_out;
  1139. struct dm_crypt_request *dmreq;
  1140. u8 *iv, *org_iv, *tag_iv;
  1141. __le64 *sector;
  1142. int r = 0;
  1143. /* Reject unexpected unaligned bio. */
  1144. if (unlikely(bv_in.bv_len & (cc->sector_size - 1)))
  1145. return -EIO;
  1146. dmreq = dmreq_of_req(cc, req);
  1147. dmreq->iv_sector = ctx->cc_sector;
  1148. if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
  1149. dmreq->iv_sector >>= cc->sector_shift;
  1150. dmreq->ctx = ctx;
  1151. *org_tag_of_dmreq(cc, dmreq) = tag_offset;
  1152. iv = iv_of_dmreq(cc, dmreq);
  1153. org_iv = org_iv_of_dmreq(cc, dmreq);
  1154. tag_iv = iv_tag_from_dmreq(cc, dmreq);
  1155. sector = org_sector_of_dmreq(cc, dmreq);
  1156. *sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset);
  1157. /* For skcipher we use only the first sg item */
  1158. sg_in = &dmreq->sg_in[0];
  1159. sg_out = &dmreq->sg_out[0];
  1160. sg_init_table(sg_in, 1);
  1161. sg_set_page(sg_in, bv_in.bv_page, cc->sector_size, bv_in.bv_offset);
  1162. sg_init_table(sg_out, 1);
  1163. sg_set_page(sg_out, bv_out.bv_page, cc->sector_size, bv_out.bv_offset);
  1164. if (cc->iv_gen_ops) {
  1165. /* For READs use IV stored in integrity metadata */
  1166. if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) {
  1167. memcpy(org_iv, tag_iv, cc->integrity_iv_size);
  1168. } else {
  1169. r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);
  1170. if (r < 0)
  1171. return r;
  1172. /* Data can be already preprocessed in generator */
  1173. if (test_bit(CRYPT_ENCRYPT_PREPROCESS, &cc->cipher_flags))
  1174. sg_in = sg_out;
  1175. /* Store generated IV in integrity metadata */
  1176. if (cc->integrity_iv_size)
  1177. memcpy(tag_iv, org_iv, cc->integrity_iv_size);
  1178. }
  1179. /* Working copy of IV, to be modified in crypto API */
  1180. memcpy(iv, org_iv, cc->iv_size);
  1181. }
  1182. skcipher_request_set_crypt(req, sg_in, sg_out, cc->sector_size, iv);
  1183. if (bio_data_dir(ctx->bio_in) == WRITE)
  1184. r = crypto_skcipher_encrypt(req);
  1185. else
  1186. r = crypto_skcipher_decrypt(req);
  1187. if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
  1188. r = cc->iv_gen_ops->post(cc, org_iv, dmreq);
  1189. bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size);
  1190. bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size);
  1191. return r;
  1192. }
  1193. static void kcryptd_async_done(struct crypto_async_request *async_req,
  1194. int error);
  1195. static int crypt_alloc_req_skcipher(struct crypt_config *cc,
  1196. struct convert_context *ctx)
  1197. {
  1198. unsigned key_index = ctx->cc_sector & (cc->tfms_count - 1);
  1199. if (!ctx->r.req) {
  1200. ctx->r.req = mempool_alloc(&cc->req_pool, in_interrupt() ? GFP_ATOMIC : GFP_NOIO);
  1201. if (!ctx->r.req)
  1202. return -ENOMEM;
  1203. }
  1204. skcipher_request_set_tfm(ctx->r.req, cc->cipher_tfm.tfms[key_index]);
  1205. /*
  1206. * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs
  1207. * requests if driver request queue is full.
  1208. */
  1209. skcipher_request_set_callback(ctx->r.req,
  1210. CRYPTO_TFM_REQ_MAY_BACKLOG,
  1211. kcryptd_async_done, dmreq_of_req(cc, ctx->r.req));
  1212. return 0;
  1213. }
  1214. static int crypt_alloc_req_aead(struct crypt_config *cc,
  1215. struct convert_context *ctx)
  1216. {
  1217. if (!ctx->r.req_aead) {
  1218. ctx->r.req_aead = mempool_alloc(&cc->req_pool, in_interrupt() ? GFP_ATOMIC : GFP_NOIO);
  1219. if (!ctx->r.req_aead)
  1220. return -ENOMEM;
  1221. }
  1222. aead_request_set_tfm(ctx->r.req_aead, cc->cipher_tfm.tfms_aead[0]);
  1223. /*
  1224. * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs
  1225. * requests if driver request queue is full.
  1226. */
  1227. aead_request_set_callback(ctx->r.req_aead,
  1228. CRYPTO_TFM_REQ_MAY_BACKLOG,
  1229. kcryptd_async_done, dmreq_of_req(cc, ctx->r.req_aead));
  1230. return 0;
  1231. }
  1232. static int crypt_alloc_req(struct crypt_config *cc,
  1233. struct convert_context *ctx)
  1234. {
  1235. if (crypt_integrity_aead(cc))
  1236. return crypt_alloc_req_aead(cc, ctx);
  1237. else
  1238. return crypt_alloc_req_skcipher(cc, ctx);
  1239. }
  1240. static void crypt_free_req_skcipher(struct crypt_config *cc,
  1241. struct skcipher_request *req, struct bio *base_bio)
  1242. {
  1243. struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size);
  1244. if ((struct skcipher_request *)(io + 1) != req)
  1245. mempool_free(req, &cc->req_pool);
  1246. }
  1247. static void crypt_free_req_aead(struct crypt_config *cc,
  1248. struct aead_request *req, struct bio *base_bio)
  1249. {
  1250. struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size);
  1251. if ((struct aead_request *)(io + 1) != req)
  1252. mempool_free(req, &cc->req_pool);
  1253. }
  1254. static void crypt_free_req(struct crypt_config *cc, void *req, struct bio *base_bio)
  1255. {
  1256. if (crypt_integrity_aead(cc))
  1257. crypt_free_req_aead(cc, req, base_bio);
  1258. else
  1259. crypt_free_req_skcipher(cc, req, base_bio);
  1260. }
  1261. /*
  1262. * Encrypt / decrypt data from one bio to another one (can be the same one)
  1263. */
  1264. static blk_status_t crypt_convert(struct crypt_config *cc,
  1265. struct convert_context *ctx, bool atomic, bool reset_pending)
  1266. {
  1267. unsigned int tag_offset = 0;
  1268. unsigned int sector_step = cc->sector_size >> SECTOR_SHIFT;
  1269. int r;
  1270. /*
  1271. * if reset_pending is set we are dealing with the bio for the first time,
  1272. * else we're continuing to work on the previous bio, so don't mess with
  1273. * the cc_pending counter
  1274. */
  1275. if (reset_pending)
  1276. atomic_set(&ctx->cc_pending, 1);
  1277. while (ctx->iter_in.bi_size && ctx->iter_out.bi_size) {
  1278. r = crypt_alloc_req(cc, ctx);
  1279. if (r) {
  1280. complete(&ctx->restart);
  1281. return BLK_STS_DEV_RESOURCE;
  1282. }
  1283. atomic_inc(&ctx->cc_pending);
  1284. if (crypt_integrity_aead(cc))
  1285. r = crypt_convert_block_aead(cc, ctx, ctx->r.req_aead, tag_offset);
  1286. else
  1287. r = crypt_convert_block_skcipher(cc, ctx, ctx->r.req, tag_offset);
  1288. switch (r) {
  1289. /*
  1290. * The request was queued by a crypto driver
  1291. * but the driver request queue is full, let's wait.
  1292. */
  1293. case -EBUSY:
  1294. if (in_interrupt()) {
  1295. if (try_wait_for_completion(&ctx->restart)) {
  1296. /*
  1297. * we don't have to block to wait for completion,
  1298. * so proceed
  1299. */
  1300. } else {
  1301. /*
  1302. * we can't wait for completion without blocking
  1303. * exit and continue processing in a workqueue
  1304. */
  1305. ctx->r.req = NULL;
  1306. ctx->cc_sector += sector_step;
  1307. tag_offset++;
  1308. return BLK_STS_DEV_RESOURCE;
  1309. }
  1310. } else {
  1311. wait_for_completion(&ctx->restart);
  1312. }
  1313. reinit_completion(&ctx->restart);
  1314. fallthrough;
  1315. /*
  1316. * The request is queued and processed asynchronously,
  1317. * completion function kcryptd_async_done() will be called.
  1318. */
  1319. case -EINPROGRESS:
  1320. ctx->r.req = NULL;
  1321. ctx->cc_sector += sector_step;
  1322. tag_offset++;
  1323. continue;
  1324. /*
  1325. * The request was already processed (synchronously).
  1326. */
  1327. case 0:
  1328. atomic_dec(&ctx->cc_pending);
  1329. ctx->cc_sector += sector_step;
  1330. tag_offset++;
  1331. if (!atomic)
  1332. cond_resched();
  1333. continue;
  1334. /*
  1335. * There was a data integrity error.
  1336. */
  1337. case -EBADMSG:
  1338. atomic_dec(&ctx->cc_pending);
  1339. return BLK_STS_PROTECTION;
  1340. /*
  1341. * There was an error while processing the request.
  1342. */
  1343. default:
  1344. atomic_dec(&ctx->cc_pending);
  1345. return BLK_STS_IOERR;
  1346. }
  1347. }
  1348. return 0;
  1349. }
  1350. static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone);
  1351. /*
  1352. * Generate a new unfragmented bio with the given size
  1353. * This should never violate the device limitations (but only because
  1354. * max_segment_size is being constrained to PAGE_SIZE).
  1355. *
  1356. * This function may be called concurrently. If we allocate from the mempool
  1357. * concurrently, there is a possibility of deadlock. For example, if we have
  1358. * mempool of 256 pages, two processes, each wanting 256, pages allocate from
  1359. * the mempool concurrently, it may deadlock in a situation where both processes
  1360. * have allocated 128 pages and the mempool is exhausted.
  1361. *
  1362. * In order to avoid this scenario we allocate the pages under a mutex.
  1363. *
  1364. * In order to not degrade performance with excessive locking, we try
  1365. * non-blocking allocations without a mutex first but on failure we fallback
  1366. * to blocking allocations with a mutex.
  1367. */
  1368. static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size)
  1369. {
  1370. struct crypt_config *cc = io->cc;
  1371. struct bio *clone;
  1372. unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1373. gfp_t gfp_mask = GFP_NOWAIT | __GFP_HIGHMEM;
  1374. unsigned i, len, remaining_size;
  1375. struct page *page;
  1376. retry:
  1377. if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))
  1378. mutex_lock(&cc->bio_alloc_lock);
  1379. clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, &cc->bs);
  1380. if (!clone)
  1381. goto out;
  1382. clone_init(io, clone);
  1383. remaining_size = size;
  1384. for (i = 0; i < nr_iovecs; i++) {
  1385. page = mempool_alloc(&cc->page_pool, gfp_mask);
  1386. if (!page) {
  1387. crypt_free_buffer_pages(cc, clone);
  1388. bio_put(clone);
  1389. gfp_mask |= __GFP_DIRECT_RECLAIM;
  1390. goto retry;
  1391. }
  1392. len = (remaining_size > PAGE_SIZE) ? PAGE_SIZE : remaining_size;
  1393. bio_add_page(clone, page, len, 0);
  1394. remaining_size -= len;
  1395. }
  1396. /* Allocate space for integrity tags */
  1397. if (dm_crypt_integrity_io_alloc(io, clone)) {
  1398. crypt_free_buffer_pages(cc, clone);
  1399. bio_put(clone);
  1400. clone = NULL;
  1401. }
  1402. out:
  1403. if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))
  1404. mutex_unlock(&cc->bio_alloc_lock);
  1405. return clone;
  1406. }
  1407. static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
  1408. {
  1409. struct bio_vec *bv;
  1410. struct bvec_iter_all iter_all;
  1411. bio_for_each_segment_all(bv, clone, iter_all) {
  1412. BUG_ON(!bv->bv_page);
  1413. mempool_free(bv->bv_page, &cc->page_pool);
  1414. }
  1415. }
  1416. static void crypt_io_init(struct dm_crypt_io *io, struct crypt_config *cc,
  1417. struct bio *bio, sector_t sector)
  1418. {
  1419. io->cc = cc;
  1420. io->base_bio = bio;
  1421. io->sector = sector;
  1422. io->error = 0;
  1423. io->ctx.r.req = NULL;
  1424. io->integrity_metadata = NULL;
  1425. io->integrity_metadata_from_pool = false;
  1426. atomic_set(&io->io_pending, 0);
  1427. }
  1428. static void crypt_inc_pending(struct dm_crypt_io *io)
  1429. {
  1430. atomic_inc(&io->io_pending);
  1431. }
  1432. static void kcryptd_io_bio_endio(struct work_struct *work)
  1433. {
  1434. struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
  1435. bio_endio(io->base_bio);
  1436. }
  1437. /*
  1438. * One of the bios was finished. Check for completion of
  1439. * the whole request and correctly clean up the buffer.
  1440. */
  1441. static void crypt_dec_pending(struct dm_crypt_io *io)
  1442. {
  1443. struct crypt_config *cc = io->cc;
  1444. struct bio *base_bio = io->base_bio;
  1445. blk_status_t error = io->error;
  1446. if (!atomic_dec_and_test(&io->io_pending))
  1447. return;
  1448. if (io->ctx.r.req)
  1449. crypt_free_req(cc, io->ctx.r.req, base_bio);
  1450. if (unlikely(io->integrity_metadata_from_pool))
  1451. mempool_free(io->integrity_metadata, &io->cc->tag_pool);
  1452. else
  1453. kfree(io->integrity_metadata);
  1454. base_bio->bi_status = error;
  1455. /*
  1456. * If we are running this function from our tasklet,
  1457. * we can't call bio_endio() here, because it will call
  1458. * clone_endio() from dm.c, which in turn will
  1459. * free the current struct dm_crypt_io structure with
  1460. * our tasklet. In this case we need to delay bio_endio()
  1461. * execution to after the tasklet is done and dequeued.
  1462. */
  1463. if (tasklet_trylock(&io->tasklet)) {
  1464. tasklet_unlock(&io->tasklet);
  1465. bio_endio(base_bio);
  1466. return;
  1467. }
  1468. INIT_WORK(&io->work, kcryptd_io_bio_endio);
  1469. queue_work(cc->io_queue, &io->work);
  1470. }
  1471. /*
  1472. * kcryptd/kcryptd_io:
  1473. *
  1474. * Needed because it would be very unwise to do decryption in an
  1475. * interrupt context.
  1476. *
  1477. * kcryptd performs the actual encryption or decryption.
  1478. *
  1479. * kcryptd_io performs the IO submission.
  1480. *
  1481. * They must be separated as otherwise the final stages could be
  1482. * starved by new requests which can block in the first stages due
  1483. * to memory allocation.
  1484. *
  1485. * The work is done per CPU global for all dm-crypt instances.
  1486. * They should not depend on each other and do not block.
  1487. */
  1488. static void crypt_endio(struct bio *clone)
  1489. {
  1490. struct dm_crypt_io *io = clone->bi_private;
  1491. struct crypt_config *cc = io->cc;
  1492. unsigned rw = bio_data_dir(clone);
  1493. blk_status_t error;
  1494. /*
  1495. * free the processed pages
  1496. */
  1497. if (rw == WRITE)
  1498. crypt_free_buffer_pages(cc, clone);
  1499. error = clone->bi_status;
  1500. bio_put(clone);
  1501. if (rw == READ && !error) {
  1502. kcryptd_queue_crypt(io);
  1503. return;
  1504. }
  1505. if (unlikely(error))
  1506. io->error = error;
  1507. crypt_dec_pending(io);
  1508. }
  1509. static void clone_init(struct dm_crypt_io *io, struct bio *clone)
  1510. {
  1511. struct crypt_config *cc = io->cc;
  1512. clone->bi_private = io;
  1513. clone->bi_end_io = crypt_endio;
  1514. bio_set_dev(clone, cc->dev->bdev);
  1515. clone->bi_opf = io->base_bio->bi_opf;
  1516. }
  1517. static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp)
  1518. {
  1519. struct crypt_config *cc = io->cc;
  1520. struct bio *clone;
  1521. /*
  1522. * We need the original biovec array in order to decrypt
  1523. * the whole bio data *afterwards* -- thanks to immutable
  1524. * biovecs we don't need to worry about the block layer
  1525. * modifying the biovec array; so leverage bio_clone_fast().
  1526. */
  1527. clone = bio_clone_fast(io->base_bio, gfp, &cc->bs);
  1528. if (!clone)
  1529. return 1;
  1530. crypt_inc_pending(io);
  1531. clone_init(io, clone);
  1532. clone->bi_iter.bi_sector = cc->start + io->sector;
  1533. if (dm_crypt_integrity_io_alloc(io, clone)) {
  1534. crypt_dec_pending(io);
  1535. bio_put(clone);
  1536. return 1;
  1537. }
  1538. submit_bio_noacct(clone);
  1539. return 0;
  1540. }
  1541. static void kcryptd_io_read_work(struct work_struct *work)
  1542. {
  1543. struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
  1544. crypt_inc_pending(io);
  1545. if (kcryptd_io_read(io, GFP_NOIO))
  1546. io->error = BLK_STS_RESOURCE;
  1547. crypt_dec_pending(io);
  1548. }
  1549. static void kcryptd_queue_read(struct dm_crypt_io *io)
  1550. {
  1551. struct crypt_config *cc = io->cc;
  1552. INIT_WORK(&io->work, kcryptd_io_read_work);
  1553. queue_work(cc->io_queue, &io->work);
  1554. }
  1555. static void kcryptd_io_write(struct dm_crypt_io *io)
  1556. {
  1557. struct bio *clone = io->ctx.bio_out;
  1558. submit_bio_noacct(clone);
  1559. }
  1560. #define crypt_io_from_node(node) rb_entry((node), struct dm_crypt_io, rb_node)
  1561. static int dmcrypt_write(void *data)
  1562. {
  1563. struct crypt_config *cc = data;
  1564. struct dm_crypt_io *io;
  1565. while (1) {
  1566. struct rb_root write_tree;
  1567. struct blk_plug plug;
  1568. spin_lock_irq(&cc->write_thread_lock);
  1569. continue_locked:
  1570. if (!RB_EMPTY_ROOT(&cc->write_tree))
  1571. goto pop_from_list;
  1572. set_current_state(TASK_INTERRUPTIBLE);
  1573. spin_unlock_irq(&cc->write_thread_lock);
  1574. if (unlikely(kthread_should_stop())) {
  1575. set_current_state(TASK_RUNNING);
  1576. break;
  1577. }
  1578. schedule();
  1579. set_current_state(TASK_RUNNING);
  1580. spin_lock_irq(&cc->write_thread_lock);
  1581. goto continue_locked;
  1582. pop_from_list:
  1583. write_tree = cc->write_tree;
  1584. cc->write_tree = RB_ROOT;
  1585. spin_unlock_irq(&cc->write_thread_lock);
  1586. BUG_ON(rb_parent(write_tree.rb_node));
  1587. /*
  1588. * Note: we cannot walk the tree here with rb_next because
  1589. * the structures may be freed when kcryptd_io_write is called.
  1590. */
  1591. blk_start_plug(&plug);
  1592. do {
  1593. io = crypt_io_from_node(rb_first(&write_tree));
  1594. rb_erase(&io->rb_node, &write_tree);
  1595. kcryptd_io_write(io);
  1596. } while (!RB_EMPTY_ROOT(&write_tree));
  1597. blk_finish_plug(&plug);
  1598. }
  1599. return 0;
  1600. }
  1601. static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async)
  1602. {
  1603. struct bio *clone = io->ctx.bio_out;
  1604. struct crypt_config *cc = io->cc;
  1605. unsigned long flags;
  1606. sector_t sector;
  1607. struct rb_node **rbp, *parent;
  1608. if (unlikely(io->error)) {
  1609. crypt_free_buffer_pages(cc, clone);
  1610. bio_put(clone);
  1611. crypt_dec_pending(io);
  1612. return;
  1613. }
  1614. /* crypt_convert should have filled the clone bio */
  1615. BUG_ON(io->ctx.iter_out.bi_size);
  1616. clone->bi_iter.bi_sector = cc->start + io->sector;
  1617. if ((likely(!async) && test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) ||
  1618. test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags)) {
  1619. submit_bio_noacct(clone);
  1620. return;
  1621. }
  1622. spin_lock_irqsave(&cc->write_thread_lock, flags);
  1623. if (RB_EMPTY_ROOT(&cc->write_tree))
  1624. wake_up_process(cc->write_thread);
  1625. rbp = &cc->write_tree.rb_node;
  1626. parent = NULL;
  1627. sector = io->sector;
  1628. while (*rbp) {
  1629. parent = *rbp;
  1630. if (sector < crypt_io_from_node(parent)->sector)
  1631. rbp = &(*rbp)->rb_left;
  1632. else
  1633. rbp = &(*rbp)->rb_right;
  1634. }
  1635. rb_link_node(&io->rb_node, parent, rbp);
  1636. rb_insert_color(&io->rb_node, &cc->write_tree);
  1637. spin_unlock_irqrestore(&cc->write_thread_lock, flags);
  1638. }
  1639. static bool kcryptd_crypt_write_inline(struct crypt_config *cc,
  1640. struct convert_context *ctx)
  1641. {
  1642. if (!test_bit(DM_CRYPT_WRITE_INLINE, &cc->flags))
  1643. return false;
  1644. /*
  1645. * Note: zone append writes (REQ_OP_ZONE_APPEND) do not have ordering
  1646. * constraints so they do not need to be issued inline by
  1647. * kcryptd_crypt_write_convert().
  1648. */
  1649. switch (bio_op(ctx->bio_in)) {
  1650. case REQ_OP_WRITE:
  1651. case REQ_OP_WRITE_SAME:
  1652. case REQ_OP_WRITE_ZEROES:
  1653. return true;
  1654. default:
  1655. return false;
  1656. }
  1657. }
  1658. static void kcryptd_crypt_write_continue(struct work_struct *work)
  1659. {
  1660. struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
  1661. struct crypt_config *cc = io->cc;
  1662. struct convert_context *ctx = &io->ctx;
  1663. int crypt_finished;
  1664. sector_t sector = io->sector;
  1665. blk_status_t r;
  1666. wait_for_completion(&ctx->restart);
  1667. reinit_completion(&ctx->restart);
  1668. r = crypt_convert(cc, &io->ctx, true, false);
  1669. if (r)
  1670. io->error = r;
  1671. crypt_finished = atomic_dec_and_test(&ctx->cc_pending);
  1672. if (!crypt_finished && kcryptd_crypt_write_inline(cc, ctx)) {
  1673. /* Wait for completion signaled by kcryptd_async_done() */
  1674. wait_for_completion(&ctx->restart);
  1675. crypt_finished = 1;
  1676. }
  1677. /* Encryption was already finished, submit io now */
  1678. if (crypt_finished) {
  1679. kcryptd_crypt_write_io_submit(io, 0);
  1680. io->sector = sector;
  1681. }
  1682. crypt_dec_pending(io);
  1683. }
  1684. static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
  1685. {
  1686. struct crypt_config *cc = io->cc;
  1687. struct convert_context *ctx = &io->ctx;
  1688. struct bio *clone;
  1689. int crypt_finished;
  1690. sector_t sector = io->sector;
  1691. blk_status_t r;
  1692. /*
  1693. * Prevent io from disappearing until this function completes.
  1694. */
  1695. crypt_inc_pending(io);
  1696. crypt_convert_init(cc, ctx, NULL, io->base_bio, sector);
  1697. clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size);
  1698. if (unlikely(!clone)) {
  1699. io->error = BLK_STS_IOERR;
  1700. goto dec;
  1701. }
  1702. io->ctx.bio_out = clone;
  1703. io->ctx.iter_out = clone->bi_iter;
  1704. sector += bio_sectors(clone);
  1705. crypt_inc_pending(io);
  1706. r = crypt_convert(cc, ctx,
  1707. test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags), true);
  1708. /*
  1709. * Crypto API backlogged the request, because its queue was full
  1710. * and we're in softirq context, so continue from a workqueue
  1711. * (TODO: is it actually possible to be in softirq in the write path?)
  1712. */
  1713. if (r == BLK_STS_DEV_RESOURCE) {
  1714. INIT_WORK(&io->work, kcryptd_crypt_write_continue);
  1715. queue_work(cc->crypt_queue, &io->work);
  1716. return;
  1717. }
  1718. if (r)
  1719. io->error = r;
  1720. crypt_finished = atomic_dec_and_test(&ctx->cc_pending);
  1721. if (!crypt_finished && kcryptd_crypt_write_inline(cc, ctx)) {
  1722. /* Wait for completion signaled by kcryptd_async_done() */
  1723. wait_for_completion(&ctx->restart);
  1724. crypt_finished = 1;
  1725. }
  1726. /* Encryption was already finished, submit io now */
  1727. if (crypt_finished) {
  1728. kcryptd_crypt_write_io_submit(io, 0);
  1729. io->sector = sector;
  1730. }
  1731. dec:
  1732. crypt_dec_pending(io);
  1733. }
  1734. static void kcryptd_crypt_read_done(struct dm_crypt_io *io)
  1735. {
  1736. crypt_dec_pending(io);
  1737. }
  1738. static void kcryptd_crypt_read_continue(struct work_struct *work)
  1739. {
  1740. struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
  1741. struct crypt_config *cc = io->cc;
  1742. blk_status_t r;
  1743. wait_for_completion(&io->ctx.restart);
  1744. reinit_completion(&io->ctx.restart);
  1745. r = crypt_convert(cc, &io->ctx, true, false);
  1746. if (r)
  1747. io->error = r;
  1748. if (atomic_dec_and_test(&io->ctx.cc_pending))
  1749. kcryptd_crypt_read_done(io);
  1750. crypt_dec_pending(io);
  1751. }
  1752. static void kcryptd_crypt_read_convert(struct dm_crypt_io *io)
  1753. {
  1754. struct crypt_config *cc = io->cc;
  1755. blk_status_t r;
  1756. crypt_inc_pending(io);
  1757. crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio,
  1758. io->sector);
  1759. r = crypt_convert(cc, &io->ctx,
  1760. test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags), true);
  1761. /*
  1762. * Crypto API backlogged the request, because its queue was full
  1763. * and we're in softirq context, so continue from a workqueue
  1764. */
  1765. if (r == BLK_STS_DEV_RESOURCE) {
  1766. INIT_WORK(&io->work, kcryptd_crypt_read_continue);
  1767. queue_work(cc->crypt_queue, &io->work);
  1768. return;
  1769. }
  1770. if (r)
  1771. io->error = r;
  1772. if (atomic_dec_and_test(&io->ctx.cc_pending))
  1773. kcryptd_crypt_read_done(io);
  1774. crypt_dec_pending(io);
  1775. }
  1776. static void kcryptd_async_done(struct crypto_async_request *async_req,
  1777. int error)
  1778. {
  1779. struct dm_crypt_request *dmreq = async_req->data;
  1780. struct convert_context *ctx = dmreq->ctx;
  1781. struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);
  1782. struct crypt_config *cc = io->cc;
  1783. /*
  1784. * A request from crypto driver backlog is going to be processed now,
  1785. * finish the completion and continue in crypt_convert().
  1786. * (Callback will be called for the second time for this request.)
  1787. */
  1788. if (error == -EINPROGRESS) {
  1789. complete(&ctx->restart);
  1790. return;
  1791. }
  1792. if (!error && cc->iv_gen_ops && cc->iv_gen_ops->post)
  1793. error = cc->iv_gen_ops->post(cc, org_iv_of_dmreq(cc, dmreq), dmreq);
  1794. if (error == -EBADMSG) {
  1795. char b[BDEVNAME_SIZE];
  1796. DMERR_LIMIT("%s: INTEGRITY AEAD ERROR, sector %llu", bio_devname(ctx->bio_in, b),
  1797. (unsigned long long)le64_to_cpu(*org_sector_of_dmreq(cc, dmreq)));
  1798. io->error = BLK_STS_PROTECTION;
  1799. } else if (error < 0)
  1800. io->error = BLK_STS_IOERR;
  1801. crypt_free_req(cc, req_of_dmreq(cc, dmreq), io->base_bio);
  1802. if (!atomic_dec_and_test(&ctx->cc_pending))
  1803. return;
  1804. /*
  1805. * The request is fully completed: for inline writes, let
  1806. * kcryptd_crypt_write_convert() do the IO submission.
  1807. */
  1808. if (bio_data_dir(io->base_bio) == READ) {
  1809. kcryptd_crypt_read_done(io);
  1810. return;
  1811. }
  1812. if (kcryptd_crypt_write_inline(cc, ctx)) {
  1813. complete(&ctx->restart);
  1814. return;
  1815. }
  1816. kcryptd_crypt_write_io_submit(io, 1);
  1817. }
  1818. static void kcryptd_crypt(struct work_struct *work)
  1819. {
  1820. struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
  1821. if (bio_data_dir(io->base_bio) == READ)
  1822. kcryptd_crypt_read_convert(io);
  1823. else
  1824. kcryptd_crypt_write_convert(io);
  1825. }
  1826. static void kcryptd_crypt_tasklet(unsigned long work)
  1827. {
  1828. kcryptd_crypt((struct work_struct *)work);
  1829. }
  1830. static void kcryptd_queue_crypt(struct dm_crypt_io *io)
  1831. {
  1832. struct crypt_config *cc = io->cc;
  1833. if ((bio_data_dir(io->base_bio) == READ && test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags)) ||
  1834. (bio_data_dir(io->base_bio) == WRITE && test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags))) {
  1835. /*
  1836. * in_irq(): Crypto API's skcipher_walk_first() refuses to work in hard IRQ context.
  1837. * irqs_disabled(): the kernel may run some IO completion from the idle thread, but
  1838. * it is being executed with irqs disabled.
  1839. */
  1840. if (in_irq() || irqs_disabled()) {
  1841. tasklet_init(&io->tasklet, kcryptd_crypt_tasklet, (unsigned long)&io->work);
  1842. tasklet_schedule(&io->tasklet);
  1843. return;
  1844. }
  1845. kcryptd_crypt(&io->work);
  1846. return;
  1847. }
  1848. INIT_WORK(&io->work, kcryptd_crypt);
  1849. queue_work(cc->crypt_queue, &io->work);
  1850. }
  1851. static void crypt_free_tfms_aead(struct crypt_config *cc)
  1852. {
  1853. if (!cc->cipher_tfm.tfms_aead)
  1854. return;
  1855. if (cc->cipher_tfm.tfms_aead[0] && !IS_ERR(cc->cipher_tfm.tfms_aead[0])) {
  1856. crypto_free_aead(cc->cipher_tfm.tfms_aead[0]);
  1857. cc->cipher_tfm.tfms_aead[0] = NULL;
  1858. }
  1859. kfree(cc->cipher_tfm.tfms_aead);
  1860. cc->cipher_tfm.tfms_aead = NULL;
  1861. }
  1862. static void crypt_free_tfms_skcipher(struct crypt_config *cc)
  1863. {
  1864. unsigned i;
  1865. if (!cc->cipher_tfm.tfms)
  1866. return;
  1867. for (i = 0; i < cc->tfms_count; i++)
  1868. if (cc->cipher_tfm.tfms[i] && !IS_ERR(cc->cipher_tfm.tfms[i])) {
  1869. crypto_free_skcipher(cc->cipher_tfm.tfms[i]);
  1870. cc->cipher_tfm.tfms[i] = NULL;
  1871. }
  1872. kfree(cc->cipher_tfm.tfms);
  1873. cc->cipher_tfm.tfms = NULL;
  1874. }
  1875. static void crypt_free_tfms(struct crypt_config *cc)
  1876. {
  1877. if (crypt_integrity_aead(cc))
  1878. crypt_free_tfms_aead(cc);
  1879. else
  1880. crypt_free_tfms_skcipher(cc);
  1881. }
  1882. static int crypt_alloc_tfms_skcipher(struct crypt_config *cc, char *ciphermode)
  1883. {
  1884. unsigned i;
  1885. int err;
  1886. cc->cipher_tfm.tfms = kcalloc(cc->tfms_count,
  1887. sizeof(struct crypto_skcipher *),
  1888. GFP_KERNEL);
  1889. if (!cc->cipher_tfm.tfms)
  1890. return -ENOMEM;
  1891. for (i = 0; i < cc->tfms_count; i++) {
  1892. cc->cipher_tfm.tfms[i] = crypto_alloc_skcipher(ciphermode, 0,
  1893. CRYPTO_ALG_ALLOCATES_MEMORY);
  1894. if (IS_ERR(cc->cipher_tfm.tfms[i])) {
  1895. err = PTR_ERR(cc->cipher_tfm.tfms[i]);
  1896. crypt_free_tfms(cc);
  1897. return err;
  1898. }
  1899. }
  1900. /*
  1901. * dm-crypt performance can vary greatly depending on which crypto
  1902. * algorithm implementation is used. Help people debug performance
  1903. * problems by logging the ->cra_driver_name.
  1904. */
  1905. DMDEBUG_LIMIT("%s using implementation \"%s\"", ciphermode,
  1906. crypto_skcipher_alg(any_tfm(cc))->base.cra_driver_name);
  1907. return 0;
  1908. }
  1909. static int crypt_alloc_tfms_aead(struct crypt_config *cc, char *ciphermode)
  1910. {
  1911. int err;
  1912. cc->cipher_tfm.tfms = kmalloc(sizeof(struct crypto_aead *), GFP_KERNEL);
  1913. if (!cc->cipher_tfm.tfms)
  1914. return -ENOMEM;
  1915. cc->cipher_tfm.tfms_aead[0] = crypto_alloc_aead(ciphermode, 0,
  1916. CRYPTO_ALG_ALLOCATES_MEMORY);
  1917. if (IS_ERR(cc->cipher_tfm.tfms_aead[0])) {
  1918. err = PTR_ERR(cc->cipher_tfm.tfms_aead[0]);
  1919. crypt_free_tfms(cc);
  1920. return err;
  1921. }
  1922. DMDEBUG_LIMIT("%s using implementation \"%s\"", ciphermode,
  1923. crypto_aead_alg(any_tfm_aead(cc))->base.cra_driver_name);
  1924. return 0;
  1925. }
  1926. static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode)
  1927. {
  1928. if (crypt_integrity_aead(cc))
  1929. return crypt_alloc_tfms_aead(cc, ciphermode);
  1930. else
  1931. return crypt_alloc_tfms_skcipher(cc, ciphermode);
  1932. }
  1933. static unsigned crypt_subkey_size(struct crypt_config *cc)
  1934. {
  1935. return (cc->key_size - cc->key_extra_size) >> ilog2(cc->tfms_count);
  1936. }
  1937. static unsigned crypt_authenckey_size(struct crypt_config *cc)
  1938. {
  1939. return crypt_subkey_size(cc) + RTA_SPACE(sizeof(struct crypto_authenc_key_param));
  1940. }
  1941. /*
  1942. * If AEAD is composed like authenc(hmac(sha256),xts(aes)),
  1943. * the key must be for some reason in special format.
  1944. * This funcion converts cc->key to this special format.
  1945. */
  1946. static void crypt_copy_authenckey(char *p, const void *key,
  1947. unsigned enckeylen, unsigned authkeylen)
  1948. {
  1949. struct crypto_authenc_key_param *param;
  1950. struct rtattr *rta;
  1951. rta = (struct rtattr *)p;
  1952. param = RTA_DATA(rta);
  1953. param->enckeylen = cpu_to_be32(enckeylen);
  1954. rta->rta_len = RTA_LENGTH(sizeof(*param));
  1955. rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM;
  1956. p += RTA_SPACE(sizeof(*param));
  1957. memcpy(p, key + enckeylen, authkeylen);
  1958. p += authkeylen;
  1959. memcpy(p, key, enckeylen);
  1960. }
  1961. static int crypt_setkey(struct crypt_config *cc)
  1962. {
  1963. unsigned subkey_size;
  1964. int err = 0, i, r;
  1965. /* Ignore extra keys (which are used for IV etc) */
  1966. subkey_size = crypt_subkey_size(cc);
  1967. if (crypt_integrity_hmac(cc)) {
  1968. if (subkey_size < cc->key_mac_size)
  1969. return -EINVAL;
  1970. crypt_copy_authenckey(cc->authenc_key, cc->key,
  1971. subkey_size - cc->key_mac_size,
  1972. cc->key_mac_size);
  1973. }
  1974. for (i = 0; i < cc->tfms_count; i++) {
  1975. if (crypt_integrity_hmac(cc))
  1976. r = crypto_aead_setkey(cc->cipher_tfm.tfms_aead[i],
  1977. cc->authenc_key, crypt_authenckey_size(cc));
  1978. else if (crypt_integrity_aead(cc))
  1979. r = crypto_aead_setkey(cc->cipher_tfm.tfms_aead[i],
  1980. cc->key + (i * subkey_size),
  1981. subkey_size);
  1982. else
  1983. r = crypto_skcipher_setkey(cc->cipher_tfm.tfms[i],
  1984. cc->key + (i * subkey_size),
  1985. subkey_size);
  1986. if (r)
  1987. err = r;
  1988. }
  1989. if (crypt_integrity_hmac(cc))
  1990. memzero_explicit(cc->authenc_key, crypt_authenckey_size(cc));
  1991. return err;
  1992. }
  1993. #ifdef CONFIG_KEYS
  1994. static bool contains_whitespace(const char *str)
  1995. {
  1996. while (*str)
  1997. if (isspace(*str++))
  1998. return true;
  1999. return false;
  2000. }
  2001. static int set_key_user(struct crypt_config *cc, struct key *key)
  2002. {
  2003. const struct user_key_payload *ukp;
  2004. ukp = user_key_payload_locked(key);
  2005. if (!ukp)
  2006. return -EKEYREVOKED;
  2007. if (cc->key_size != ukp->datalen)
  2008. return -EINVAL;
  2009. memcpy(cc->key, ukp->data, cc->key_size);
  2010. return 0;
  2011. }
  2012. #if defined(CONFIG_ENCRYPTED_KEYS) || defined(CONFIG_ENCRYPTED_KEYS_MODULE)
  2013. static int set_key_encrypted(struct crypt_config *cc, struct key *key)
  2014. {
  2015. const struct encrypted_key_payload *ekp;
  2016. ekp = key->payload.data[0];
  2017. if (!ekp)
  2018. return -EKEYREVOKED;
  2019. if (cc->key_size != ekp->decrypted_datalen)
  2020. return -EINVAL;
  2021. memcpy(cc->key, ekp->decrypted_data, cc->key_size);
  2022. return 0;
  2023. }
  2024. #endif /* CONFIG_ENCRYPTED_KEYS */
  2025. static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)
  2026. {
  2027. char *new_key_string, *key_desc;
  2028. int ret;
  2029. struct key_type *type;
  2030. struct key *key;
  2031. int (*set_key)(struct crypt_config *cc, struct key *key);
  2032. /*
  2033. * Reject key_string with whitespace. dm core currently lacks code for
  2034. * proper whitespace escaping in arguments on DM_TABLE_STATUS path.
  2035. */
  2036. if (contains_whitespace(key_string)) {
  2037. DMERR("whitespace chars not allowed in key string");
  2038. return -EINVAL;
  2039. }
  2040. /* look for next ':' separating key_type from key_description */
  2041. key_desc = strpbrk(key_string, ":");
  2042. if (!key_desc || key_desc == key_string || !strlen(key_desc + 1))
  2043. return -EINVAL;
  2044. if (!strncmp(key_string, "logon:", key_desc - key_string + 1)) {
  2045. type = &key_type_logon;
  2046. set_key = set_key_user;
  2047. } else if (!strncmp(key_string, "user:", key_desc - key_string + 1)) {
  2048. type = &key_type_user;
  2049. set_key = set_key_user;
  2050. #if defined(CONFIG_ENCRYPTED_KEYS) || defined(CONFIG_ENCRYPTED_KEYS_MODULE)
  2051. } else if (!strncmp(key_string, "encrypted:", key_desc - key_string + 1)) {
  2052. type = &key_type_encrypted;
  2053. set_key = set_key_encrypted;
  2054. #endif
  2055. } else {
  2056. return -EINVAL;
  2057. }
  2058. new_key_string = kstrdup(key_string, GFP_KERNEL);
  2059. if (!new_key_string)
  2060. return -ENOMEM;
  2061. key = request_key(type, key_desc + 1, NULL);
  2062. if (IS_ERR(key)) {
  2063. kfree_sensitive(new_key_string);
  2064. return PTR_ERR(key);
  2065. }
  2066. down_read(&key->sem);
  2067. ret = set_key(cc, key);
  2068. if (ret < 0) {
  2069. up_read(&key->sem);
  2070. key_put(key);
  2071. kfree_sensitive(new_key_string);
  2072. return ret;
  2073. }
  2074. up_read(&key->sem);
  2075. key_put(key);
  2076. /* clear the flag since following operations may invalidate previously valid key */
  2077. clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
  2078. ret = crypt_setkey(cc);
  2079. if (!ret) {
  2080. set_bit(DM_CRYPT_KEY_VALID, &cc->flags);
  2081. kfree_sensitive(cc->key_string);
  2082. cc->key_string = new_key_string;
  2083. } else
  2084. kfree_sensitive(new_key_string);
  2085. return ret;
  2086. }
  2087. static int get_key_size(char **key_string)
  2088. {
  2089. char *colon, dummy;
  2090. int ret;
  2091. if (*key_string[0] != ':')
  2092. return strlen(*key_string) >> 1;
  2093. /* look for next ':' in key string */
  2094. colon = strpbrk(*key_string + 1, ":");
  2095. if (!colon)
  2096. return -EINVAL;
  2097. if (sscanf(*key_string + 1, "%u%c", &ret, &dummy) != 2 || dummy != ':')
  2098. return -EINVAL;
  2099. *key_string = colon;
  2100. /* remaining key string should be :<logon|user>:<key_desc> */
  2101. return ret;
  2102. }
  2103. #else
  2104. static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)
  2105. {
  2106. return -EINVAL;
  2107. }
  2108. static int get_key_size(char **key_string)
  2109. {
  2110. return (*key_string[0] == ':') ? -EINVAL : (int)(strlen(*key_string) >> 1);
  2111. }
  2112. #endif /* CONFIG_KEYS */
  2113. static int crypt_set_key(struct crypt_config *cc, char *key)
  2114. {
  2115. int r = -EINVAL;
  2116. int key_string_len = strlen(key);
  2117. /* Hyphen (which gives a key_size of zero) means there is no key. */
  2118. if (!cc->key_size && strcmp(key, "-"))
  2119. goto out;
  2120. /* ':' means the key is in kernel keyring, short-circuit normal key processing */
  2121. if (key[0] == ':') {
  2122. r = crypt_set_keyring_key(cc, key + 1);
  2123. goto out;
  2124. }
  2125. /* clear the flag since following operations may invalidate previously valid key */
  2126. clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
  2127. /* wipe references to any kernel keyring key */
  2128. kfree_sensitive(cc->key_string);
  2129. cc->key_string = NULL;
  2130. /* Decode key from its hex representation. */
  2131. if (cc->key_size && hex2bin(cc->key, key, cc->key_size) < 0)
  2132. goto out;
  2133. r = crypt_setkey(cc);
  2134. if (!r)
  2135. set_bit(DM_CRYPT_KEY_VALID, &cc->flags);
  2136. out:
  2137. /* Hex key string not needed after here, so wipe it. */
  2138. memset(key, '0', key_string_len);
  2139. return r;
  2140. }
  2141. static int crypt_wipe_key(struct crypt_config *cc)
  2142. {
  2143. int r;
  2144. clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
  2145. get_random_bytes(&cc->key, cc->key_size);
  2146. /* Wipe IV private keys */
  2147. if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) {
  2148. r = cc->iv_gen_ops->wipe(cc);
  2149. if (r)
  2150. return r;
  2151. }
  2152. kfree_sensitive(cc->key_string);
  2153. cc->key_string = NULL;
  2154. r = crypt_setkey(cc);
  2155. memset(&cc->key, 0, cc->key_size * sizeof(u8));
  2156. return r;
  2157. }
  2158. static void crypt_calculate_pages_per_client(void)
  2159. {
  2160. unsigned long pages = (totalram_pages() - totalhigh_pages()) * DM_CRYPT_MEMORY_PERCENT / 100;
  2161. if (!dm_crypt_clients_n)
  2162. return;
  2163. pages /= dm_crypt_clients_n;
  2164. if (pages < DM_CRYPT_MIN_PAGES_PER_CLIENT)
  2165. pages = DM_CRYPT_MIN_PAGES_PER_CLIENT;
  2166. dm_crypt_pages_per_client = pages;
  2167. }
  2168. static void *crypt_page_alloc(gfp_t gfp_mask, void *pool_data)
  2169. {
  2170. struct crypt_config *cc = pool_data;
  2171. struct page *page;
  2172. /*
  2173. * Note, percpu_counter_read_positive() may over (and under) estimate
  2174. * the current usage by at most (batch - 1) * num_online_cpus() pages,
  2175. * but avoids potential spinlock contention of an exact result.
  2176. */
  2177. if (unlikely(percpu_counter_read_positive(&cc->n_allocated_pages) >= dm_crypt_pages_per_client) &&
  2178. likely(gfp_mask & __GFP_NORETRY))
  2179. return NULL;
  2180. page = alloc_page(gfp_mask);
  2181. if (likely(page != NULL))
  2182. percpu_counter_add(&cc->n_allocated_pages, 1);
  2183. return page;
  2184. }
  2185. static void crypt_page_free(void *page, void *pool_data)
  2186. {
  2187. struct crypt_config *cc = pool_data;
  2188. __free_page(page);
  2189. percpu_counter_sub(&cc->n_allocated_pages, 1);
  2190. }
  2191. static void crypt_dtr(struct dm_target *ti)
  2192. {
  2193. struct crypt_config *cc = ti->private;
  2194. ti->private = NULL;
  2195. if (!cc)
  2196. return;
  2197. if (cc->write_thread)
  2198. kthread_stop(cc->write_thread);
  2199. if (cc->io_queue)
  2200. destroy_workqueue(cc->io_queue);
  2201. if (cc->crypt_queue)
  2202. destroy_workqueue(cc->crypt_queue);
  2203. crypt_free_tfms(cc);
  2204. bioset_exit(&cc->bs);
  2205. mempool_exit(&cc->page_pool);
  2206. mempool_exit(&cc->req_pool);
  2207. mempool_exit(&cc->tag_pool);
  2208. WARN_ON(percpu_counter_sum(&cc->n_allocated_pages) != 0);
  2209. percpu_counter_destroy(&cc->n_allocated_pages);
  2210. if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
  2211. cc->iv_gen_ops->dtr(cc);
  2212. if (cc->dev)
  2213. dm_put_device(ti, cc->dev);
  2214. kfree_sensitive(cc->cipher_string);
  2215. kfree_sensitive(cc->key_string);
  2216. kfree_sensitive(cc->cipher_auth);
  2217. kfree_sensitive(cc->authenc_key);
  2218. mutex_destroy(&cc->bio_alloc_lock);
  2219. /* Must zero key material before freeing */
  2220. kfree_sensitive(cc);
  2221. spin_lock(&dm_crypt_clients_lock);
  2222. WARN_ON(!dm_crypt_clients_n);
  2223. dm_crypt_clients_n--;
  2224. crypt_calculate_pages_per_client();
  2225. spin_unlock(&dm_crypt_clients_lock);
  2226. }
  2227. static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode)
  2228. {
  2229. struct crypt_config *cc = ti->private;
  2230. if (crypt_integrity_aead(cc))
  2231. cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc));
  2232. else
  2233. cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc));
  2234. if (cc->iv_size)
  2235. /* at least a 64 bit sector number should fit in our buffer */
  2236. cc->iv_size = max(cc->iv_size,
  2237. (unsigned int)(sizeof(u64) / sizeof(u8)));
  2238. else if (ivmode) {
  2239. DMWARN("Selected cipher does not support IVs");
  2240. ivmode = NULL;
  2241. }
  2242. /* Choose ivmode, see comments at iv code. */
  2243. if (ivmode == NULL)
  2244. cc->iv_gen_ops = NULL;
  2245. else if (strcmp(ivmode, "plain") == 0)
  2246. cc->iv_gen_ops = &crypt_iv_plain_ops;
  2247. else if (strcmp(ivmode, "plain64") == 0)
  2248. cc->iv_gen_ops = &crypt_iv_plain64_ops;
  2249. else if (strcmp(ivmode, "plain64be") == 0)
  2250. cc->iv_gen_ops = &crypt_iv_plain64be_ops;
  2251. else if (strcmp(ivmode, "essiv") == 0)
  2252. cc->iv_gen_ops = &crypt_iv_essiv_ops;
  2253. else if (strcmp(ivmode, "benbi") == 0)
  2254. cc->iv_gen_ops = &crypt_iv_benbi_ops;
  2255. else if (strcmp(ivmode, "null") == 0)
  2256. cc->iv_gen_ops = &crypt_iv_null_ops;
  2257. else if (strcmp(ivmode, "eboiv") == 0)
  2258. cc->iv_gen_ops = &crypt_iv_eboiv_ops;
  2259. else if (strcmp(ivmode, "elephant") == 0) {
  2260. cc->iv_gen_ops = &crypt_iv_elephant_ops;
  2261. cc->key_parts = 2;
  2262. cc->key_extra_size = cc->key_size / 2;
  2263. if (cc->key_extra_size > ELEPHANT_MAX_KEY_SIZE)
  2264. return -EINVAL;
  2265. set_bit(CRYPT_ENCRYPT_PREPROCESS, &cc->cipher_flags);
  2266. } else if (strcmp(ivmode, "lmk") == 0) {
  2267. cc->iv_gen_ops = &crypt_iv_lmk_ops;
  2268. /*
  2269. * Version 2 and 3 is recognised according
  2270. * to length of provided multi-key string.
  2271. * If present (version 3), last key is used as IV seed.
  2272. * All keys (including IV seed) are always the same size.
  2273. */
  2274. if (cc->key_size % cc->key_parts) {
  2275. cc->key_parts++;
  2276. cc->key_extra_size = cc->key_size / cc->key_parts;
  2277. }
  2278. } else if (strcmp(ivmode, "tcw") == 0) {
  2279. cc->iv_gen_ops = &crypt_iv_tcw_ops;
  2280. cc->key_parts += 2; /* IV + whitening */
  2281. cc->key_extra_size = cc->iv_size + TCW_WHITENING_SIZE;
  2282. } else if (strcmp(ivmode, "random") == 0) {
  2283. cc->iv_gen_ops = &crypt_iv_random_ops;
  2284. /* Need storage space in integrity fields. */
  2285. cc->integrity_iv_size = cc->iv_size;
  2286. } else {
  2287. ti->error = "Invalid IV mode";
  2288. return -EINVAL;
  2289. }
  2290. return 0;
  2291. }
  2292. /*
  2293. * Workaround to parse HMAC algorithm from AEAD crypto API spec.
  2294. * The HMAC is needed to calculate tag size (HMAC digest size).
  2295. * This should be probably done by crypto-api calls (once available...)
  2296. */
  2297. static int crypt_ctr_auth_cipher(struct crypt_config *cc, char *cipher_api)
  2298. {
  2299. char *start, *end, *mac_alg = NULL;
  2300. struct crypto_ahash *mac;
  2301. if (!strstarts(cipher_api, "authenc("))
  2302. return 0;
  2303. start = strchr(cipher_api, '(');
  2304. end = strchr(cipher_api, ',');
  2305. if (!start || !end || ++start > end)
  2306. return -EINVAL;
  2307. mac_alg = kzalloc(end - start + 1, GFP_KERNEL);
  2308. if (!mac_alg)
  2309. return -ENOMEM;
  2310. strncpy(mac_alg, start, end - start);
  2311. mac = crypto_alloc_ahash(mac_alg, 0, CRYPTO_ALG_ALLOCATES_MEMORY);
  2312. kfree(mac_alg);
  2313. if (IS_ERR(mac))
  2314. return PTR_ERR(mac);
  2315. cc->key_mac_size = crypto_ahash_digestsize(mac);
  2316. crypto_free_ahash(mac);
  2317. cc->authenc_key = kmalloc(crypt_authenckey_size(cc), GFP_KERNEL);
  2318. if (!cc->authenc_key)
  2319. return -ENOMEM;
  2320. return 0;
  2321. }
  2322. static int crypt_ctr_cipher_new(struct dm_target *ti, char *cipher_in, char *key,
  2323. char **ivmode, char **ivopts)
  2324. {
  2325. struct crypt_config *cc = ti->private;
  2326. char *tmp, *cipher_api, buf[CRYPTO_MAX_ALG_NAME];
  2327. int ret = -EINVAL;
  2328. cc->tfms_count = 1;
  2329. /*
  2330. * New format (capi: prefix)
  2331. * capi:cipher_api_spec-iv:ivopts
  2332. */
  2333. tmp = &cipher_in[strlen("capi:")];
  2334. /* Separate IV options if present, it can contain another '-' in hash name */
  2335. *ivopts = strrchr(tmp, ':');
  2336. if (*ivopts) {
  2337. **ivopts = '\0';
  2338. (*ivopts)++;
  2339. }
  2340. /* Parse IV mode */
  2341. *ivmode = strrchr(tmp, '-');
  2342. if (*ivmode) {
  2343. **ivmode = '\0';
  2344. (*ivmode)++;
  2345. }
  2346. /* The rest is crypto API spec */
  2347. cipher_api = tmp;
  2348. /* Alloc AEAD, can be used only in new format. */
  2349. if (crypt_integrity_aead(cc)) {
  2350. ret = crypt_ctr_auth_cipher(cc, cipher_api);
  2351. if (ret < 0) {
  2352. ti->error = "Invalid AEAD cipher spec";
  2353. return -ENOMEM;
  2354. }
  2355. }
  2356. if (*ivmode && !strcmp(*ivmode, "lmk"))
  2357. cc->tfms_count = 64;
  2358. if (*ivmode && !strcmp(*ivmode, "essiv")) {
  2359. if (!*ivopts) {
  2360. ti->error = "Digest algorithm missing for ESSIV mode";
  2361. return -EINVAL;
  2362. }
  2363. ret = snprintf(buf, CRYPTO_MAX_ALG_NAME, "essiv(%s,%s)",
  2364. cipher_api, *ivopts);
  2365. if (ret < 0 || ret >= CRYPTO_MAX_ALG_NAME) {
  2366. ti->error = "Cannot allocate cipher string";
  2367. return -ENOMEM;
  2368. }
  2369. cipher_api = buf;
  2370. }
  2371. cc->key_parts = cc->tfms_count;
  2372. /* Allocate cipher */
  2373. ret = crypt_alloc_tfms(cc, cipher_api);
  2374. if (ret < 0) {
  2375. ti->error = "Error allocating crypto tfm";
  2376. return ret;
  2377. }
  2378. if (crypt_integrity_aead(cc))
  2379. cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc));
  2380. else
  2381. cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc));
  2382. return 0;
  2383. }
  2384. static int crypt_ctr_cipher_old(struct dm_target *ti, char *cipher_in, char *key,
  2385. char **ivmode, char **ivopts)
  2386. {
  2387. struct crypt_config *cc = ti->private;
  2388. char *tmp, *cipher, *chainmode, *keycount;
  2389. char *cipher_api = NULL;
  2390. int ret = -EINVAL;
  2391. char dummy;
  2392. if (strchr(cipher_in, '(') || crypt_integrity_aead(cc)) {
  2393. ti->error = "Bad cipher specification";
  2394. return -EINVAL;
  2395. }
  2396. /*
  2397. * Legacy dm-crypt cipher specification
  2398. * cipher[:keycount]-mode-iv:ivopts
  2399. */
  2400. tmp = cipher_in;
  2401. keycount = strsep(&tmp, "-");
  2402. cipher = strsep(&keycount, ":");
  2403. if (!keycount)
  2404. cc->tfms_count = 1;
  2405. else if (sscanf(keycount, "%u%c", &cc->tfms_count, &dummy) != 1 ||
  2406. !is_power_of_2(cc->tfms_count)) {
  2407. ti->error = "Bad cipher key count specification";
  2408. return -EINVAL;
  2409. }
  2410. cc->key_parts = cc->tfms_count;
  2411. chainmode = strsep(&tmp, "-");
  2412. *ivmode = strsep(&tmp, ":");
  2413. *ivopts = tmp;
  2414. /*
  2415. * For compatibility with the original dm-crypt mapping format, if
  2416. * only the cipher name is supplied, use cbc-plain.
  2417. */
  2418. if (!chainmode || (!strcmp(chainmode, "plain") && !*ivmode)) {
  2419. chainmode = "cbc";
  2420. *ivmode = "plain";
  2421. }
  2422. if (strcmp(chainmode, "ecb") && !*ivmode) {
  2423. ti->error = "IV mechanism required";
  2424. return -EINVAL;
  2425. }
  2426. cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL);
  2427. if (!cipher_api)
  2428. goto bad_mem;
  2429. if (*ivmode && !strcmp(*ivmode, "essiv")) {
  2430. if (!*ivopts) {
  2431. ti->error = "Digest algorithm missing for ESSIV mode";
  2432. kfree(cipher_api);
  2433. return -EINVAL;
  2434. }
  2435. ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME,
  2436. "essiv(%s(%s),%s)", chainmode, cipher, *ivopts);
  2437. } else {
  2438. ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME,
  2439. "%s(%s)", chainmode, cipher);
  2440. }
  2441. if (ret < 0 || ret >= CRYPTO_MAX_ALG_NAME) {
  2442. kfree(cipher_api);
  2443. goto bad_mem;
  2444. }
  2445. /* Allocate cipher */
  2446. ret = crypt_alloc_tfms(cc, cipher_api);
  2447. if (ret < 0) {
  2448. ti->error = "Error allocating crypto tfm";
  2449. kfree(cipher_api);
  2450. return ret;
  2451. }
  2452. kfree(cipher_api);
  2453. return 0;
  2454. bad_mem:
  2455. ti->error = "Cannot allocate cipher strings";
  2456. return -ENOMEM;
  2457. }
  2458. static int crypt_ctr_cipher(struct dm_target *ti, char *cipher_in, char *key)
  2459. {
  2460. struct crypt_config *cc = ti->private;
  2461. char *ivmode = NULL, *ivopts = NULL;
  2462. int ret;
  2463. cc->cipher_string = kstrdup(cipher_in, GFP_KERNEL);
  2464. if (!cc->cipher_string) {
  2465. ti->error = "Cannot allocate cipher strings";
  2466. return -ENOMEM;
  2467. }
  2468. if (strstarts(cipher_in, "capi:"))
  2469. ret = crypt_ctr_cipher_new(ti, cipher_in, key, &ivmode, &ivopts);
  2470. else
  2471. ret = crypt_ctr_cipher_old(ti, cipher_in, key, &ivmode, &ivopts);
  2472. if (ret)
  2473. return ret;
  2474. /* Initialize IV */
  2475. ret = crypt_ctr_ivmode(ti, ivmode);
  2476. if (ret < 0)
  2477. return ret;
  2478. /* Initialize and set key */
  2479. ret = crypt_set_key(cc, key);
  2480. if (ret < 0) {
  2481. ti->error = "Error decoding and setting key";
  2482. return ret;
  2483. }
  2484. /* Allocate IV */
  2485. if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) {
  2486. ret = cc->iv_gen_ops->ctr(cc, ti, ivopts);
  2487. if (ret < 0) {
  2488. ti->error = "Error creating IV";
  2489. return ret;
  2490. }
  2491. }
  2492. /* Initialize IV (set keys for ESSIV etc) */
  2493. if (cc->iv_gen_ops && cc->iv_gen_ops->init) {
  2494. ret = cc->iv_gen_ops->init(cc);
  2495. if (ret < 0) {
  2496. ti->error = "Error initialising IV";
  2497. return ret;
  2498. }
  2499. }
  2500. /* wipe the kernel key payload copy */
  2501. if (cc->key_string)
  2502. memset(cc->key, 0, cc->key_size * sizeof(u8));
  2503. return ret;
  2504. }
  2505. static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **argv)
  2506. {
  2507. struct crypt_config *cc = ti->private;
  2508. struct dm_arg_set as;
  2509. static const struct dm_arg _args[] = {
  2510. {0, 8, "Invalid number of feature args"},
  2511. };
  2512. unsigned int opt_params, val;
  2513. const char *opt_string, *sval;
  2514. char dummy;
  2515. int ret;
  2516. /* Optional parameters */
  2517. as.argc = argc;
  2518. as.argv = argv;
  2519. ret = dm_read_arg_group(_args, &as, &opt_params, &ti->error);
  2520. if (ret)
  2521. return ret;
  2522. while (opt_params--) {
  2523. opt_string = dm_shift_arg(&as);
  2524. if (!opt_string) {
  2525. ti->error = "Not enough feature arguments";
  2526. return -EINVAL;
  2527. }
  2528. if (!strcasecmp(opt_string, "allow_discards"))
  2529. ti->num_discard_bios = 1;
  2530. else if (!strcasecmp(opt_string, "same_cpu_crypt"))
  2531. set_bit(DM_CRYPT_SAME_CPU, &cc->flags);
  2532. else if (!strcasecmp(opt_string, "submit_from_crypt_cpus"))
  2533. set_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags);
  2534. else if (!strcasecmp(opt_string, "no_read_workqueue"))
  2535. set_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags);
  2536. else if (!strcasecmp(opt_string, "no_write_workqueue"))
  2537. set_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags);
  2538. else if (sscanf(opt_string, "integrity:%u:", &val) == 1) {
  2539. if (val == 0 || val > MAX_TAG_SIZE) {
  2540. ti->error = "Invalid integrity arguments";
  2541. return -EINVAL;
  2542. }
  2543. cc->on_disk_tag_size = val;
  2544. sval = strchr(opt_string + strlen("integrity:"), ':') + 1;
  2545. if (!strcasecmp(sval, "aead")) {
  2546. set_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags);
  2547. } else if (strcasecmp(sval, "none")) {
  2548. ti->error = "Unknown integrity profile";
  2549. return -EINVAL;
  2550. }
  2551. cc->cipher_auth = kstrdup(sval, GFP_KERNEL);
  2552. if (!cc->cipher_auth)
  2553. return -ENOMEM;
  2554. } else if (sscanf(opt_string, "sector_size:%hu%c", &cc->sector_size, &dummy) == 1) {
  2555. if (cc->sector_size < (1 << SECTOR_SHIFT) ||
  2556. cc->sector_size > 4096 ||
  2557. (cc->sector_size & (cc->sector_size - 1))) {
  2558. ti->error = "Invalid feature value for sector_size";
  2559. return -EINVAL;
  2560. }
  2561. if (ti->len & ((cc->sector_size >> SECTOR_SHIFT) - 1)) {
  2562. ti->error = "Device size is not multiple of sector_size feature";
  2563. return -EINVAL;
  2564. }
  2565. cc->sector_shift = __ffs(cc->sector_size) - SECTOR_SHIFT;
  2566. } else if (!strcasecmp(opt_string, "iv_large_sectors"))
  2567. set_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags);
  2568. else {
  2569. ti->error = "Invalid feature arguments";
  2570. return -EINVAL;
  2571. }
  2572. }
  2573. return 0;
  2574. }
  2575. #ifdef CONFIG_BLK_DEV_ZONED
  2576. static int crypt_report_zones(struct dm_target *ti,
  2577. struct dm_report_zones_args *args, unsigned int nr_zones)
  2578. {
  2579. struct crypt_config *cc = ti->private;
  2580. sector_t sector = cc->start + dm_target_offset(ti, args->next_sector);
  2581. args->start = cc->start;
  2582. return blkdev_report_zones(cc->dev->bdev, sector, nr_zones,
  2583. dm_report_zones_cb, args);
  2584. }
  2585. #endif
  2586. /*
  2587. * Construct an encryption mapping:
  2588. * <cipher> [<key>|:<key_size>:<user|logon>:<key_description>] <iv_offset> <dev_path> <start>
  2589. */
  2590. static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
  2591. {
  2592. struct crypt_config *cc;
  2593. const char *devname = dm_table_device_name(ti->table);
  2594. int key_size;
  2595. unsigned int align_mask;
  2596. unsigned long long tmpll;
  2597. int ret;
  2598. size_t iv_size_padding, additional_req_size;
  2599. char dummy;
  2600. if (argc < 5) {
  2601. ti->error = "Not enough arguments";
  2602. return -EINVAL;
  2603. }
  2604. key_size = get_key_size(&argv[1]);
  2605. if (key_size < 0) {
  2606. ti->error = "Cannot parse key size";
  2607. return -EINVAL;
  2608. }
  2609. cc = kzalloc(struct_size(cc, key, key_size), GFP_KERNEL);
  2610. if (!cc) {
  2611. ti->error = "Cannot allocate encryption context";
  2612. return -ENOMEM;
  2613. }
  2614. cc->key_size = key_size;
  2615. cc->sector_size = (1 << SECTOR_SHIFT);
  2616. cc->sector_shift = 0;
  2617. ti->private = cc;
  2618. spin_lock(&dm_crypt_clients_lock);
  2619. dm_crypt_clients_n++;
  2620. crypt_calculate_pages_per_client();
  2621. spin_unlock(&dm_crypt_clients_lock);
  2622. ret = percpu_counter_init(&cc->n_allocated_pages, 0, GFP_KERNEL);
  2623. if (ret < 0)
  2624. goto bad;
  2625. /* Optional parameters need to be read before cipher constructor */
  2626. if (argc > 5) {
  2627. ret = crypt_ctr_optional(ti, argc - 5, &argv[5]);
  2628. if (ret)
  2629. goto bad;
  2630. }
  2631. ret = crypt_ctr_cipher(ti, argv[0], argv[1]);
  2632. if (ret < 0)
  2633. goto bad;
  2634. if (crypt_integrity_aead(cc)) {
  2635. cc->dmreq_start = sizeof(struct aead_request);
  2636. cc->dmreq_start += crypto_aead_reqsize(any_tfm_aead(cc));
  2637. align_mask = crypto_aead_alignmask(any_tfm_aead(cc));
  2638. } else {
  2639. cc->dmreq_start = sizeof(struct skcipher_request);
  2640. cc->dmreq_start += crypto_skcipher_reqsize(any_tfm(cc));
  2641. align_mask = crypto_skcipher_alignmask(any_tfm(cc));
  2642. }
  2643. cc->dmreq_start = ALIGN(cc->dmreq_start, __alignof__(struct dm_crypt_request));
  2644. if (align_mask < CRYPTO_MINALIGN) {
  2645. /* Allocate the padding exactly */
  2646. iv_size_padding = -(cc->dmreq_start + sizeof(struct dm_crypt_request))
  2647. & align_mask;
  2648. } else {
  2649. /*
  2650. * If the cipher requires greater alignment than kmalloc
  2651. * alignment, we don't know the exact position of the
  2652. * initialization vector. We must assume worst case.
  2653. */
  2654. iv_size_padding = align_mask;
  2655. }
  2656. /* ...| IV + padding | original IV | original sec. number | bio tag offset | */
  2657. additional_req_size = sizeof(struct dm_crypt_request) +
  2658. iv_size_padding + cc->iv_size +
  2659. cc->iv_size +
  2660. sizeof(uint64_t) +
  2661. sizeof(unsigned int);
  2662. ret = mempool_init_kmalloc_pool(&cc->req_pool, MIN_IOS, cc->dmreq_start + additional_req_size);
  2663. if (ret) {
  2664. ti->error = "Cannot allocate crypt request mempool";
  2665. goto bad;
  2666. }
  2667. cc->per_bio_data_size = ti->per_io_data_size =
  2668. ALIGN(sizeof(struct dm_crypt_io) + cc->dmreq_start + additional_req_size,
  2669. ARCH_KMALLOC_MINALIGN);
  2670. ret = mempool_init(&cc->page_pool, BIO_MAX_PAGES, crypt_page_alloc, crypt_page_free, cc);
  2671. if (ret) {
  2672. ti->error = "Cannot allocate page mempool";
  2673. goto bad;
  2674. }
  2675. ret = bioset_init(&cc->bs, MIN_IOS, 0, BIOSET_NEED_BVECS);
  2676. if (ret) {
  2677. ti->error = "Cannot allocate crypt bioset";
  2678. goto bad;
  2679. }
  2680. mutex_init(&cc->bio_alloc_lock);
  2681. ret = -EINVAL;
  2682. if ((sscanf(argv[2], "%llu%c", &tmpll, &dummy) != 1) ||
  2683. (tmpll & ((cc->sector_size >> SECTOR_SHIFT) - 1))) {
  2684. ti->error = "Invalid iv_offset sector";
  2685. goto bad;
  2686. }
  2687. cc->iv_offset = tmpll;
  2688. ret = dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), &cc->dev);
  2689. if (ret) {
  2690. ti->error = "Device lookup failed";
  2691. goto bad;
  2692. }
  2693. ret = -EINVAL;
  2694. if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1 || tmpll != (sector_t)tmpll) {
  2695. ti->error = "Invalid device sector";
  2696. goto bad;
  2697. }
  2698. cc->start = tmpll;
  2699. /*
  2700. * For zoned block devices, we need to preserve the issuer write
  2701. * ordering. To do so, disable write workqueues and force inline
  2702. * encryption completion.
  2703. */
  2704. if (bdev_is_zoned(cc->dev->bdev)) {
  2705. set_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags);
  2706. set_bit(DM_CRYPT_WRITE_INLINE, &cc->flags);
  2707. }
  2708. if (crypt_integrity_aead(cc) || cc->integrity_iv_size) {
  2709. ret = crypt_integrity_ctr(cc, ti);
  2710. if (ret)
  2711. goto bad;
  2712. cc->tag_pool_max_sectors = POOL_ENTRY_SIZE / cc->on_disk_tag_size;
  2713. if (!cc->tag_pool_max_sectors)
  2714. cc->tag_pool_max_sectors = 1;
  2715. ret = mempool_init_kmalloc_pool(&cc->tag_pool, MIN_IOS,
  2716. cc->tag_pool_max_sectors * cc->on_disk_tag_size);
  2717. if (ret) {
  2718. ti->error = "Cannot allocate integrity tags mempool";
  2719. goto bad;
  2720. }
  2721. cc->tag_pool_max_sectors <<= cc->sector_shift;
  2722. }
  2723. ret = -ENOMEM;
  2724. cc->io_queue = alloc_workqueue("kcryptd_io/%s", WQ_MEM_RECLAIM, 1, devname);
  2725. if (!cc->io_queue) {
  2726. ti->error = "Couldn't create kcryptd io queue";
  2727. goto bad;
  2728. }
  2729. if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags))
  2730. cc->crypt_queue = alloc_workqueue("kcryptd/%s", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM,
  2731. 1, devname);
  2732. else
  2733. cc->crypt_queue = alloc_workqueue("kcryptd/%s",
  2734. WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND,
  2735. num_online_cpus(), devname);
  2736. if (!cc->crypt_queue) {
  2737. ti->error = "Couldn't create kcryptd queue";
  2738. goto bad;
  2739. }
  2740. spin_lock_init(&cc->write_thread_lock);
  2741. cc->write_tree = RB_ROOT;
  2742. cc->write_thread = kthread_create(dmcrypt_write, cc, "dmcrypt_write/%s", devname);
  2743. if (IS_ERR(cc->write_thread)) {
  2744. ret = PTR_ERR(cc->write_thread);
  2745. cc->write_thread = NULL;
  2746. ti->error = "Couldn't spawn write thread";
  2747. goto bad;
  2748. }
  2749. wake_up_process(cc->write_thread);
  2750. ti->num_flush_bios = 1;
  2751. ti->limit_swap_bios = true;
  2752. return 0;
  2753. bad:
  2754. crypt_dtr(ti);
  2755. return ret;
  2756. }
  2757. static int crypt_map(struct dm_target *ti, struct bio *bio)
  2758. {
  2759. struct dm_crypt_io *io;
  2760. struct crypt_config *cc = ti->private;
  2761. /*
  2762. * If bio is REQ_PREFLUSH or REQ_OP_DISCARD, just bypass crypt queues.
  2763. * - for REQ_PREFLUSH device-mapper core ensures that no IO is in-flight
  2764. * - for REQ_OP_DISCARD caller must use flush if IO ordering matters
  2765. */
  2766. if (unlikely(bio->bi_opf & REQ_PREFLUSH ||
  2767. bio_op(bio) == REQ_OP_DISCARD)) {
  2768. bio_set_dev(bio, cc->dev->bdev);
  2769. if (bio_sectors(bio))
  2770. bio->bi_iter.bi_sector = cc->start +
  2771. dm_target_offset(ti, bio->bi_iter.bi_sector);
  2772. return DM_MAPIO_REMAPPED;
  2773. }
  2774. /*
  2775. * Check if bio is too large, split as needed.
  2776. */
  2777. if (unlikely(bio->bi_iter.bi_size > (BIO_MAX_PAGES << PAGE_SHIFT)) &&
  2778. (bio_data_dir(bio) == WRITE || cc->on_disk_tag_size))
  2779. dm_accept_partial_bio(bio, ((BIO_MAX_PAGES << PAGE_SHIFT) >> SECTOR_SHIFT));
  2780. /*
  2781. * Ensure that bio is a multiple of internal sector encryption size
  2782. * and is aligned to this size as defined in IO hints.
  2783. */
  2784. if (unlikely((bio->bi_iter.bi_sector & ((cc->sector_size >> SECTOR_SHIFT) - 1)) != 0))
  2785. return DM_MAPIO_KILL;
  2786. if (unlikely(bio->bi_iter.bi_size & (cc->sector_size - 1)))
  2787. return DM_MAPIO_KILL;
  2788. io = dm_per_bio_data(bio, cc->per_bio_data_size);
  2789. crypt_io_init(io, cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector));
  2790. if (cc->on_disk_tag_size) {
  2791. unsigned tag_len = cc->on_disk_tag_size * (bio_sectors(bio) >> cc->sector_shift);
  2792. if (unlikely(tag_len > KMALLOC_MAX_SIZE) ||
  2793. unlikely(!(io->integrity_metadata = kmalloc(tag_len,
  2794. GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN)))) {
  2795. if (bio_sectors(bio) > cc->tag_pool_max_sectors)
  2796. dm_accept_partial_bio(bio, cc->tag_pool_max_sectors);
  2797. io->integrity_metadata = mempool_alloc(&cc->tag_pool, GFP_NOIO);
  2798. io->integrity_metadata_from_pool = true;
  2799. }
  2800. }
  2801. if (crypt_integrity_aead(cc))
  2802. io->ctx.r.req_aead = (struct aead_request *)(io + 1);
  2803. else
  2804. io->ctx.r.req = (struct skcipher_request *)(io + 1);
  2805. if (bio_data_dir(io->base_bio) == READ) {
  2806. if (kcryptd_io_read(io, GFP_NOWAIT))
  2807. kcryptd_queue_read(io);
  2808. } else
  2809. kcryptd_queue_crypt(io);
  2810. return DM_MAPIO_SUBMITTED;
  2811. }
  2812. static void crypt_status(struct dm_target *ti, status_type_t type,
  2813. unsigned status_flags, char *result, unsigned maxlen)
  2814. {
  2815. struct crypt_config *cc = ti->private;
  2816. unsigned i, sz = 0;
  2817. int num_feature_args = 0;
  2818. switch (type) {
  2819. case STATUSTYPE_INFO:
  2820. result[0] = '\0';
  2821. break;
  2822. case STATUSTYPE_TABLE:
  2823. DMEMIT("%s ", cc->cipher_string);
  2824. if (cc->key_size > 0) {
  2825. if (cc->key_string)
  2826. DMEMIT(":%u:%s", cc->key_size, cc->key_string);
  2827. else
  2828. for (i = 0; i < cc->key_size; i++)
  2829. DMEMIT("%02x", cc->key[i]);
  2830. } else
  2831. DMEMIT("-");
  2832. DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset,
  2833. cc->dev->name, (unsigned long long)cc->start);
  2834. num_feature_args += !!ti->num_discard_bios;
  2835. num_feature_args += test_bit(DM_CRYPT_SAME_CPU, &cc->flags);
  2836. num_feature_args += test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags);
  2837. num_feature_args += test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags);
  2838. num_feature_args += test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags);
  2839. num_feature_args += cc->sector_size != (1 << SECTOR_SHIFT);
  2840. num_feature_args += test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags);
  2841. if (cc->on_disk_tag_size)
  2842. num_feature_args++;
  2843. if (num_feature_args) {
  2844. DMEMIT(" %d", num_feature_args);
  2845. if (ti->num_discard_bios)
  2846. DMEMIT(" allow_discards");
  2847. if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags))
  2848. DMEMIT(" same_cpu_crypt");
  2849. if (test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags))
  2850. DMEMIT(" submit_from_crypt_cpus");
  2851. if (test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags))
  2852. DMEMIT(" no_read_workqueue");
  2853. if (test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags))
  2854. DMEMIT(" no_write_workqueue");
  2855. if (cc->on_disk_tag_size)
  2856. DMEMIT(" integrity:%u:%s", cc->on_disk_tag_size, cc->cipher_auth);
  2857. if (cc->sector_size != (1 << SECTOR_SHIFT))
  2858. DMEMIT(" sector_size:%d", cc->sector_size);
  2859. if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
  2860. DMEMIT(" iv_large_sectors");
  2861. }
  2862. break;
  2863. }
  2864. }
  2865. static void crypt_postsuspend(struct dm_target *ti)
  2866. {
  2867. struct crypt_config *cc = ti->private;
  2868. set_bit(DM_CRYPT_SUSPENDED, &cc->flags);
  2869. }
  2870. static int crypt_preresume(struct dm_target *ti)
  2871. {
  2872. struct crypt_config *cc = ti->private;
  2873. if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) {
  2874. DMERR("aborting resume - crypt key is not set.");
  2875. return -EAGAIN;
  2876. }
  2877. return 0;
  2878. }
  2879. static void crypt_resume(struct dm_target *ti)
  2880. {
  2881. struct crypt_config *cc = ti->private;
  2882. clear_bit(DM_CRYPT_SUSPENDED, &cc->flags);
  2883. }
  2884. /* Message interface
  2885. * key set <key>
  2886. * key wipe
  2887. */
  2888. static int crypt_message(struct dm_target *ti, unsigned argc, char **argv,
  2889. char *result, unsigned maxlen)
  2890. {
  2891. struct crypt_config *cc = ti->private;
  2892. int key_size, ret = -EINVAL;
  2893. if (argc < 2)
  2894. goto error;
  2895. if (!strcasecmp(argv[0], "key")) {
  2896. if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) {
  2897. DMWARN("not suspended during key manipulation.");
  2898. return -EINVAL;
  2899. }
  2900. if (argc == 3 && !strcasecmp(argv[1], "set")) {
  2901. /* The key size may not be changed. */
  2902. key_size = get_key_size(&argv[2]);
  2903. if (key_size < 0 || cc->key_size != key_size) {
  2904. memset(argv[2], '0', strlen(argv[2]));
  2905. return -EINVAL;
  2906. }
  2907. ret = crypt_set_key(cc, argv[2]);
  2908. if (ret)
  2909. return ret;
  2910. if (cc->iv_gen_ops && cc->iv_gen_ops->init)
  2911. ret = cc->iv_gen_ops->init(cc);
  2912. /* wipe the kernel key payload copy */
  2913. if (cc->key_string)
  2914. memset(cc->key, 0, cc->key_size * sizeof(u8));
  2915. return ret;
  2916. }
  2917. if (argc == 2 && !strcasecmp(argv[1], "wipe"))
  2918. return crypt_wipe_key(cc);
  2919. }
  2920. error:
  2921. DMWARN("unrecognised message received.");
  2922. return -EINVAL;
  2923. }
  2924. static int crypt_iterate_devices(struct dm_target *ti,
  2925. iterate_devices_callout_fn fn, void *data)
  2926. {
  2927. struct crypt_config *cc = ti->private;
  2928. return fn(ti, cc->dev, cc->start, ti->len, data);
  2929. }
  2930. static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits)
  2931. {
  2932. struct crypt_config *cc = ti->private;
  2933. /*
  2934. * Unfortunate constraint that is required to avoid the potential
  2935. * for exceeding underlying device's max_segments limits -- due to
  2936. * crypt_alloc_buffer() possibly allocating pages for the encryption
  2937. * bio that are not as physically contiguous as the original bio.
  2938. */
  2939. limits->max_segment_size = PAGE_SIZE;
  2940. limits->logical_block_size =
  2941. max_t(unsigned, limits->logical_block_size, cc->sector_size);
  2942. limits->physical_block_size =
  2943. max_t(unsigned, limits->physical_block_size, cc->sector_size);
  2944. limits->io_min = max_t(unsigned, limits->io_min, cc->sector_size);
  2945. }
  2946. static struct target_type crypt_target = {
  2947. .name = "crypt",
  2948. .version = {1, 22, 0},
  2949. .module = THIS_MODULE,
  2950. .ctr = crypt_ctr,
  2951. .dtr = crypt_dtr,
  2952. #ifdef CONFIG_BLK_DEV_ZONED
  2953. .features = DM_TARGET_ZONED_HM,
  2954. .report_zones = crypt_report_zones,
  2955. #endif
  2956. .map = crypt_map,
  2957. .status = crypt_status,
  2958. .postsuspend = crypt_postsuspend,
  2959. .preresume = crypt_preresume,
  2960. .resume = crypt_resume,
  2961. .message = crypt_message,
  2962. .iterate_devices = crypt_iterate_devices,
  2963. .io_hints = crypt_io_hints,
  2964. };
  2965. static int __init dm_crypt_init(void)
  2966. {
  2967. int r;
  2968. r = dm_register_target(&crypt_target);
  2969. if (r < 0)
  2970. DMERR("register failed %d", r);
  2971. return r;
  2972. }
  2973. static void __exit dm_crypt_exit(void)
  2974. {
  2975. dm_unregister_target(&crypt_target);
  2976. }
  2977. module_init(dm_crypt_init);
  2978. module_exit(dm_crypt_exit);
  2979. MODULE_AUTHOR("Jana Saout <jana@saout.de>");
  2980. MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption");
  2981. MODULE_LICENSE("GPL");