fips140-module.c 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright 2021 Google LLC
  4. * Author: Ard Biesheuvel <ardb@google.com>
  5. *
  6. * This file is the core of fips140.ko, which contains various crypto algorithms
  7. * that are also built into vmlinux. At load time, this module overrides the
  8. * built-in implementations of these algorithms with its implementations. It
  9. * also runs self-tests on these algorithms and verifies the integrity of its
  10. * code and data. If either of these steps fails, the kernel will panic.
  11. *
  12. * This module is intended to be loaded at early boot time in order to meet
  13. * FIPS 140 and NIAP FPT_TST_EXT.1 requirements. It shouldn't be used if you
  14. * don't need to meet these requirements.
  15. */
  16. #undef __DISABLE_EXPORTS
  17. #include <linux/ctype.h>
  18. #include <linux/module.h>
  19. #include <crypto/aead.h>
  20. #include <crypto/aes.h>
  21. #include <crypto/hash.h>
  22. #include <crypto/sha.h>
  23. #include <crypto/skcipher.h>
  24. #include <crypto/rng.h>
  25. #include <trace/hooks/fips140.h>
  26. #include "fips140-module.h"
  27. #include "internal.h"
  28. /*
  29. * FIPS 140-2 prefers the use of HMAC with a public key over a plain hash.
  30. */
  31. u8 __initdata fips140_integ_hmac_key[] = "The quick brown fox jumps over the lazy dog";
  32. /* this is populated by the build tool */
  33. u8 __initdata fips140_integ_hmac_digest[SHA256_DIGEST_SIZE];
  34. const u32 __initcall_start_marker __section(".initcalls._start");
  35. const u32 __initcall_end_marker __section(".initcalls._end");
  36. const u8 __fips140_text_start __section(".text.._start");
  37. const u8 __fips140_text_end __section(".text.._end");
  38. const u8 __fips140_rodata_start __section(".rodata.._start");
  39. const u8 __fips140_rodata_end __section(".rodata.._end");
  40. /*
  41. * We need this little detour to prevent Clang from detecting out of bounds
  42. * accesses to __fips140_text_start and __fips140_rodata_start, which only exist
  43. * to delineate the section, and so their sizes are not relevant to us.
  44. */
  45. const u32 *__initcall_start = &__initcall_start_marker;
  46. const u8 *__text_start = &__fips140_text_start;
  47. const u8 *__rodata_start = &__fips140_rodata_start;
  48. /*
  49. * The list of the crypto API algorithms (by cra_name) that will be unregistered
  50. * by this module, in preparation for the module registering its own
  51. * implementation(s) of them.
  52. *
  53. * All algorithms that will be declared as FIPS-approved in the module
  54. * certification must be listed here, to ensure that the non-FIPS-approved
  55. * implementations of these algorithms in the kernel image aren't used.
  56. *
  57. * For every algorithm in this list, the module should contain all the "same"
  58. * implementations that the kernel image does, including the C implementation as
  59. * well as any architecture-specific implementations. This is needed to avoid
  60. * performance regressions as well as the possibility of an algorithm being
  61. * unavailable on some CPUs. E.g., "xcbc(aes)" isn't in this list, as the
  62. * module doesn't have a C implementation of it (and it won't be FIPS-approved).
  63. *
  64. * Due to a quirk in the FIPS requirements, "gcm(aes)" isn't actually able to be
  65. * FIPS-approved. However, we otherwise treat it the same as the algorithms
  66. * that will be FIPS-approved, and therefore it's included in this list.
  67. *
  68. * When adding a new algorithm here, make sure to consider whether it needs a
  69. * self-test added to fips140_selftests[] as well.
  70. */
  71. static const struct {
  72. const char *name;
  73. bool approved;
  74. } fips140_algs_to_replace[] = {
  75. {"aes", true},
  76. {"cmac(aes)", true},
  77. {"ecb(aes)", true},
  78. {"cbc(aes)", true},
  79. {"cts(cbc(aes))", true},
  80. {"ctr(aes)", true},
  81. {"xts(aes)", true},
  82. {"gcm(aes)", false},
  83. {"hmac(sha1)", true},
  84. {"hmac(sha224)", true},
  85. {"hmac(sha256)", true},
  86. {"hmac(sha384)", true},
  87. {"hmac(sha512)", true},
  88. {"sha1", true},
  89. {"sha224", true},
  90. {"sha256", true},
  91. {"sha384", true},
  92. {"sha512", true},
  93. {"stdrng", true},
  94. {"jitterentropy_rng", false},
  95. };
  96. static bool __init fips140_should_unregister_alg(struct crypto_alg *alg)
  97. {
  98. int i;
  99. /*
  100. * All software algorithms are synchronous, hardware algorithms must
  101. * be covered by their own FIPS 140 certification.
  102. */
  103. if (alg->cra_flags & CRYPTO_ALG_ASYNC)
  104. return false;
  105. for (i = 0; i < ARRAY_SIZE(fips140_algs_to_replace); i++) {
  106. if (!strcmp(alg->cra_name, fips140_algs_to_replace[i].name))
  107. return true;
  108. }
  109. return false;
  110. }
  111. /*
  112. * FIPS 140-3 service indicators. FIPS 140-3 requires that all services
  113. * "provide an indicator when the service utilises an approved cryptographic
  114. * algorithm, security function or process in an approved manner". What this
  115. * means is very debatable, even with the help of the FIPS 140-3 Implementation
  116. * Guidance document. However, it was decided that a function that takes in an
  117. * algorithm name and returns whether that algorithm is approved or not will
  118. * meet this requirement. Note, this relies on some properties of the module:
  119. *
  120. * - The module doesn't distinguish between "services" and "algorithms"; its
  121. * services are simply its algorithms.
  122. *
  123. * - The status of an approved algorithm is never non-approved, since (a) the
  124. * module doesn't support operating in a non-approved mode, such as a mode
  125. * where the self-tests are skipped; (b) there are no cases where the module
  126. * supports non-approved settings for approved algorithms, e.g.
  127. * non-approved key sizes; and (c) this function isn't available to be
  128. * called until the module_init function has completed, so it's guaranteed
  129. * that the self-tests and integrity check have already passed.
  130. *
  131. * - The module does support some non-approved algorithms, so a single static
  132. * indicator ("return true;") would not be acceptable.
  133. */
  134. bool fips140_is_approved_service(const char *name)
  135. {
  136. size_t i;
  137. for (i = 0; i < ARRAY_SIZE(fips140_algs_to_replace); i++) {
  138. if (!strcmp(name, fips140_algs_to_replace[i].name))
  139. return fips140_algs_to_replace[i].approved;
  140. }
  141. return false;
  142. }
  143. EXPORT_SYMBOL_GPL(fips140_is_approved_service);
  144. /*
  145. * FIPS 140-3 requires that modules provide a "service" that outputs "the name
  146. * or module identifier and the versioning information that can be correlated
  147. * with a validation record". This function meets that requirement.
  148. *
  149. * Note: the module also prints this same information to the kernel log when it
  150. * is loaded. That might meet the requirement by itself. However, given the
  151. * vagueness of what counts as a "service", we provide this function too, just
  152. * in case the certification lab or CMVP is happier with an explicit function.
  153. *
  154. * Note: /sys/modules/fips140/scmversion also provides versioning information
  155. * about the module. However that file just shows the bare git commit ID, so it
  156. * probably isn't sufficient to meet the FIPS requirement, which seems to want
  157. * the "official" module name and version number used in the FIPS certificate.
  158. */
  159. const char *fips140_module_version(void)
  160. {
  161. return FIPS140_MODULE_NAME " " FIPS140_MODULE_VERSION;
  162. }
  163. EXPORT_SYMBOL_GPL(fips140_module_version);
  164. static LIST_HEAD(existing_live_algos);
  165. /*
  166. * Release a list of algorithms which have been removed from crypto_alg_list.
  167. *
  168. * Note that even though the list is a private list, we have to hold
  169. * crypto_alg_sem while iterating through it because crypto_unregister_alg() may
  170. * run concurrently (as we haven't taken a reference to the algorithms on the
  171. * list), and crypto_unregister_alg() will remove the algorithm from whichever
  172. * list it happens to be on, while holding crypto_alg_sem. That's okay, since
  173. * in that case crypto_unregister_alg() will handle the crypto_alg_put().
  174. */
  175. static void fips140_remove_final(struct list_head *list)
  176. {
  177. struct crypto_alg *alg;
  178. struct crypto_alg *n;
  179. /*
  180. * We need to take crypto_alg_sem to safely traverse the list (see
  181. * comment above), but we have to drop it when doing each
  182. * crypto_alg_put() as that may take crypto_alg_sem again.
  183. */
  184. down_write(&crypto_alg_sem);
  185. list_for_each_entry_safe(alg, n, list, cra_list) {
  186. list_del_init(&alg->cra_list);
  187. up_write(&crypto_alg_sem);
  188. crypto_alg_put(alg);
  189. down_write(&crypto_alg_sem);
  190. }
  191. up_write(&crypto_alg_sem);
  192. }
  193. static void __init unregister_existing_fips140_algos(void)
  194. {
  195. struct crypto_alg *alg, *tmp;
  196. LIST_HEAD(remove_list);
  197. LIST_HEAD(spawns);
  198. down_write(&crypto_alg_sem);
  199. /*
  200. * Find all registered algorithms that we care about, and move them to a
  201. * private list so that they are no longer exposed via the algo lookup
  202. * API. Subsequently, we will unregister them if they are not in active
  203. * use. If they are, we can't fully unregister them but we can ensure
  204. * that new users won't use them.
  205. */
  206. list_for_each_entry_safe(alg, tmp, &crypto_alg_list, cra_list) {
  207. if (!fips140_should_unregister_alg(alg))
  208. continue;
  209. if (refcount_read(&alg->cra_refcnt) == 1) {
  210. /*
  211. * This algorithm is not currently in use, but there may
  212. * be template instances holding references to it via
  213. * spawns. So let's tear it down like
  214. * crypto_unregister_alg() would, but without releasing
  215. * the lock, to prevent races with concurrent TFM
  216. * allocations.
  217. */
  218. alg->cra_flags |= CRYPTO_ALG_DEAD;
  219. list_move(&alg->cra_list, &remove_list);
  220. crypto_remove_spawns(alg, &spawns, NULL);
  221. } else {
  222. /*
  223. * This algorithm is live, i.e. it has TFMs allocated,
  224. * so we can't fully unregister it. It's not necessary
  225. * to dynamically redirect existing users to the FIPS
  226. * code, given that they can't be relying on FIPS
  227. * certified crypto in the first place. However, we do
  228. * need to ensure that new users will get the FIPS code.
  229. *
  230. * In most cases, setting alg->cra_priority to 0
  231. * achieves this. However, that isn't enough for
  232. * algorithms like "hmac(sha256)" that need to be
  233. * instantiated from a template, since existing
  234. * algorithms always take priority over a template being
  235. * instantiated. Therefore, we move the algorithm to
  236. * a private list so that algorithm lookups won't find
  237. * it anymore. To further distinguish it from the FIPS
  238. * algorithms, we also append "+orig" to its name.
  239. */
  240. pr_info("found already-live algorithm '%s' ('%s')\n",
  241. alg->cra_name, alg->cra_driver_name);
  242. alg->cra_priority = 0;
  243. strlcat(alg->cra_name, "+orig", CRYPTO_MAX_ALG_NAME);
  244. strlcat(alg->cra_driver_name, "+orig",
  245. CRYPTO_MAX_ALG_NAME);
  246. list_move(&alg->cra_list, &existing_live_algos);
  247. }
  248. }
  249. up_write(&crypto_alg_sem);
  250. fips140_remove_final(&remove_list);
  251. fips140_remove_final(&spawns);
  252. }
  253. static void __init unapply_text_relocations(void *section, int section_size,
  254. const Elf64_Rela *rela, int numrels)
  255. {
  256. while (numrels--) {
  257. u32 *place = (u32 *)(section + rela->r_offset);
  258. BUG_ON(rela->r_offset >= section_size);
  259. switch (ELF64_R_TYPE(rela->r_info)) {
  260. #ifdef CONFIG_ARM64
  261. case R_AARCH64_JUMP26:
  262. case R_AARCH64_CALL26:
  263. *place &= ~GENMASK(25, 0);
  264. break;
  265. case R_AARCH64_ADR_PREL_LO21:
  266. case R_AARCH64_ADR_PREL_PG_HI21:
  267. case R_AARCH64_ADR_PREL_PG_HI21_NC:
  268. *place &= ~(GENMASK(30, 29) | GENMASK(23, 5));
  269. break;
  270. case R_AARCH64_ADD_ABS_LO12_NC:
  271. case R_AARCH64_LDST8_ABS_LO12_NC:
  272. case R_AARCH64_LDST16_ABS_LO12_NC:
  273. case R_AARCH64_LDST32_ABS_LO12_NC:
  274. case R_AARCH64_LDST64_ABS_LO12_NC:
  275. case R_AARCH64_LDST128_ABS_LO12_NC:
  276. *place &= ~GENMASK(21, 10);
  277. break;
  278. default:
  279. pr_err("unhandled relocation type %llu\n",
  280. ELF64_R_TYPE(rela->r_info));
  281. BUG();
  282. #else
  283. #error
  284. #endif
  285. }
  286. rela++;
  287. }
  288. }
  289. static void __init unapply_rodata_relocations(void *section, int section_size,
  290. const Elf64_Rela *rela, int numrels)
  291. {
  292. while (numrels--) {
  293. void *place = section + rela->r_offset;
  294. BUG_ON(rela->r_offset >= section_size);
  295. switch (ELF64_R_TYPE(rela->r_info)) {
  296. #ifdef CONFIG_ARM64
  297. case R_AARCH64_ABS64:
  298. *(u64 *)place = 0;
  299. break;
  300. default:
  301. pr_err("unhandled relocation type %llu\n",
  302. ELF64_R_TYPE(rela->r_info));
  303. BUG();
  304. #else
  305. #error
  306. #endif
  307. }
  308. rela++;
  309. }
  310. }
  311. extern struct {
  312. u32 offset;
  313. u32 count;
  314. } fips140_rela_text, fips140_rela_rodata;
  315. static bool __init check_fips140_module_hmac(void)
  316. {
  317. struct crypto_shash *tfm = NULL;
  318. SHASH_DESC_ON_STACK(desc, dontcare);
  319. u8 digest[SHA256_DIGEST_SIZE];
  320. void *textcopy, *rodatacopy;
  321. int textsize, rodatasize;
  322. bool ok = false;
  323. int err;
  324. textsize = &__fips140_text_end - &__fips140_text_start;
  325. rodatasize = &__fips140_rodata_end - &__fips140_rodata_start;
  326. pr_info("text size : 0x%x\n", textsize);
  327. pr_info("rodata size: 0x%x\n", rodatasize);
  328. textcopy = kmalloc(textsize + rodatasize, GFP_KERNEL);
  329. if (!textcopy) {
  330. pr_err("Failed to allocate memory for copy of .text\n");
  331. goto out;
  332. }
  333. rodatacopy = textcopy + textsize;
  334. memcpy(textcopy, __text_start, textsize);
  335. memcpy(rodatacopy, __rodata_start, rodatasize);
  336. // apply the relocations in reverse on the copies of .text and .rodata
  337. unapply_text_relocations(textcopy, textsize,
  338. offset_to_ptr(&fips140_rela_text.offset),
  339. fips140_rela_text.count);
  340. unapply_rodata_relocations(rodatacopy, rodatasize,
  341. offset_to_ptr(&fips140_rela_rodata.offset),
  342. fips140_rela_rodata.count);
  343. fips140_inject_integrity_failure(textcopy);
  344. tfm = crypto_alloc_shash("hmac(sha256)", 0, 0);
  345. if (IS_ERR(tfm)) {
  346. pr_err("failed to allocate hmac tfm (%ld)\n", PTR_ERR(tfm));
  347. tfm = NULL;
  348. goto out;
  349. }
  350. desc->tfm = tfm;
  351. pr_info("using '%s' for integrity check\n",
  352. crypto_shash_driver_name(tfm));
  353. err = crypto_shash_setkey(tfm, fips140_integ_hmac_key,
  354. strlen(fips140_integ_hmac_key)) ?:
  355. crypto_shash_init(desc) ?:
  356. crypto_shash_update(desc, textcopy, textsize) ?:
  357. crypto_shash_finup(desc, rodatacopy, rodatasize, digest);
  358. /* Zeroizing this is important; see the comment below. */
  359. shash_desc_zero(desc);
  360. if (err) {
  361. pr_err("failed to calculate hmac shash (%d)\n", err);
  362. goto out;
  363. }
  364. if (memcmp(digest, fips140_integ_hmac_digest, sizeof(digest))) {
  365. pr_err("provided_digest : %*phN\n", (int)sizeof(digest),
  366. fips140_integ_hmac_digest);
  367. pr_err("calculated digest: %*phN\n", (int)sizeof(digest),
  368. digest);
  369. goto out;
  370. }
  371. ok = true;
  372. out:
  373. /*
  374. * FIPS 140-3 requires that all "temporary value(s) generated during the
  375. * integrity test" be zeroized (ref: FIPS 140-3 IG 9.7.B). There is no
  376. * technical reason to do this given that these values are public
  377. * information, but this is the requirement so we follow it.
  378. */
  379. crypto_free_shash(tfm);
  380. memzero_explicit(digest, sizeof(digest));
  381. kfree_sensitive(textcopy);
  382. return ok;
  383. }
  384. static void fips140_sha256(void *p, const u8 *data, unsigned int len, u8 *out,
  385. int *hook_inuse)
  386. {
  387. sha256(data, len, out);
  388. *hook_inuse = 1;
  389. }
  390. static void fips140_aes_expandkey(void *p, struct crypto_aes_ctx *ctx,
  391. const u8 *in_key, unsigned int key_len,
  392. int *err)
  393. {
  394. *err = aes_expandkey(ctx, in_key, key_len);
  395. }
  396. static void fips140_aes_encrypt(void *priv, const struct crypto_aes_ctx *ctx,
  397. u8 *out, const u8 *in, int *hook_inuse)
  398. {
  399. aes_encrypt(ctx, out, in);
  400. *hook_inuse = 1;
  401. }
  402. static void fips140_aes_decrypt(void *priv, const struct crypto_aes_ctx *ctx,
  403. u8 *out, const u8 *in, int *hook_inuse)
  404. {
  405. aes_decrypt(ctx, out, in);
  406. *hook_inuse = 1;
  407. }
  408. static bool update_fips140_library_routines(void)
  409. {
  410. int ret;
  411. ret = register_trace_android_vh_sha256(fips140_sha256, NULL) ?:
  412. register_trace_android_vh_aes_expandkey(fips140_aes_expandkey, NULL) ?:
  413. register_trace_android_vh_aes_encrypt(fips140_aes_encrypt, NULL) ?:
  414. register_trace_android_vh_aes_decrypt(fips140_aes_decrypt, NULL);
  415. return ret == 0;
  416. }
  417. /*
  418. * Initialize the FIPS 140 module.
  419. *
  420. * Note: this routine iterates over the contents of the initcall section, which
  421. * consists of an array of function pointers that was emitted by the linker
  422. * rather than the compiler. This means that these function pointers lack the
  423. * usual CFI stubs that the compiler emits when CFI codegen is enabled. So
  424. * let's disable CFI locally when handling the initcall array, to avoid
  425. * surpises.
  426. */
  427. static int __init __attribute__((__no_sanitize__("cfi")))
  428. fips140_init(void)
  429. {
  430. const u32 *initcall;
  431. pr_info("loading " FIPS140_MODULE_NAME " " FIPS140_MODULE_VERSION "\n");
  432. fips140_init_thread = current;
  433. unregister_existing_fips140_algos();
  434. /* iterate over all init routines present in this module and call them */
  435. for (initcall = __initcall_start + 1;
  436. initcall < &__initcall_end_marker;
  437. initcall++) {
  438. int (*init)(void) = offset_to_ptr(initcall);
  439. int err = init();
  440. /*
  441. * ENODEV is expected from initcalls that only register
  442. * algorithms that depend on non-present CPU features. Besides
  443. * that, errors aren't expected here.
  444. */
  445. if (err && err != -ENODEV) {
  446. pr_err("initcall %ps() failed: %d\n", init, err);
  447. goto panic;
  448. }
  449. }
  450. if (!fips140_run_selftests())
  451. goto panic;
  452. /*
  453. * It may seem backward to perform the integrity check last, but this
  454. * is intentional: the check itself uses hmac(sha256) which is one of
  455. * the algorithms that are replaced with versions from this module, and
  456. * the integrity check must use the replacement version. Also, to be
  457. * ready for FIPS 140-3, the integrity check algorithm must have already
  458. * been self-tested.
  459. */
  460. if (!check_fips140_module_hmac()) {
  461. pr_crit("integrity check failed -- giving up!\n");
  462. goto panic;
  463. }
  464. pr_info("integrity check passed\n");
  465. complete_all(&fips140_tests_done);
  466. if (!update_fips140_library_routines())
  467. goto panic;
  468. if (!fips140_eval_testing_init())
  469. goto panic;
  470. pr_info("module successfully loaded\n");
  471. return 0;
  472. panic:
  473. panic("FIPS 140 module load failure");
  474. }
  475. module_init(fips140_init);
  476. MODULE_IMPORT_NS(CRYPTO_INTERNAL);
  477. MODULE_LICENSE("GPL v2");
  478. /*
  479. * Crypto-related helper functions, reproduced here so that they will be
  480. * covered by the FIPS 140 integrity check.
  481. *
  482. * Non-cryptographic helper functions such as memcpy() can be excluded from the
  483. * FIPS module, but there is ambiguity about other helper functions like
  484. * __crypto_xor() and crypto_inc() which aren't cryptographic by themselves,
  485. * but are more closely associated with cryptography than e.g. memcpy(). To
  486. * err on the side of caution, we include copies of these in the FIPS module.
  487. */
  488. void __crypto_xor(u8 *dst, const u8 *src1, const u8 *src2, unsigned int len)
  489. {
  490. while (len >= 8) {
  491. *(u64 *)dst = *(u64 *)src1 ^ *(u64 *)src2;
  492. dst += 8;
  493. src1 += 8;
  494. src2 += 8;
  495. len -= 8;
  496. }
  497. while (len >= 4) {
  498. *(u32 *)dst = *(u32 *)src1 ^ *(u32 *)src2;
  499. dst += 4;
  500. src1 += 4;
  501. src2 += 4;
  502. len -= 4;
  503. }
  504. while (len >= 2) {
  505. *(u16 *)dst = *(u16 *)src1 ^ *(u16 *)src2;
  506. dst += 2;
  507. src1 += 2;
  508. src2 += 2;
  509. len -= 2;
  510. }
  511. while (len--)
  512. *dst++ = *src1++ ^ *src2++;
  513. }
  514. void crypto_inc(u8 *a, unsigned int size)
  515. {
  516. a += size;
  517. while (size--)
  518. if (++*--a)
  519. break;
  520. }