// SPDX-License-Identifier: GPL-2.0-only /* * Copyright 2021 Google LLC * Author: Ard Biesheuvel * * This file is the core of fips140.ko, which contains various crypto algorithms * that are also built into vmlinux. At load time, this module overrides the * built-in implementations of these algorithms with its implementations. It * also runs self-tests on these algorithms and verifies the integrity of its * code and data. If either of these steps fails, the kernel will panic. * * This module is intended to be loaded at early boot time in order to meet * FIPS 140 and NIAP FPT_TST_EXT.1 requirements. It shouldn't be used if you * don't need to meet these requirements. */ #undef __DISABLE_EXPORTS #include #include #include #include #include #include #include #include #include #include "fips140-module.h" #include "internal.h" /* * FIPS 140-2 prefers the use of HMAC with a public key over a plain hash. */ u8 __initdata fips140_integ_hmac_key[] = "The quick brown fox jumps over the lazy dog"; /* this is populated by the build tool */ u8 __initdata fips140_integ_hmac_digest[SHA256_DIGEST_SIZE]; const u32 __initcall_start_marker __section(".initcalls._start"); const u32 __initcall_end_marker __section(".initcalls._end"); const u8 __fips140_text_start __section(".text.._start"); const u8 __fips140_text_end __section(".text.._end"); const u8 __fips140_rodata_start __section(".rodata.._start"); const u8 __fips140_rodata_end __section(".rodata.._end"); /* * We need this little detour to prevent Clang from detecting out of bounds * accesses to __fips140_text_start and __fips140_rodata_start, which only exist * to delineate the section, and so their sizes are not relevant to us. */ const u32 *__initcall_start = &__initcall_start_marker; const u8 *__text_start = &__fips140_text_start; const u8 *__rodata_start = &__fips140_rodata_start; /* * The list of the crypto API algorithms (by cra_name) that will be unregistered * by this module, in preparation for the module registering its own * implementation(s) of them. * * All algorithms that will be declared as FIPS-approved in the module * certification must be listed here, to ensure that the non-FIPS-approved * implementations of these algorithms in the kernel image aren't used. * * For every algorithm in this list, the module should contain all the "same" * implementations that the kernel image does, including the C implementation as * well as any architecture-specific implementations. This is needed to avoid * performance regressions as well as the possibility of an algorithm being * unavailable on some CPUs. E.g., "xcbc(aes)" isn't in this list, as the * module doesn't have a C implementation of it (and it won't be FIPS-approved). * * Due to a quirk in the FIPS requirements, "gcm(aes)" isn't actually able to be * FIPS-approved. However, we otherwise treat it the same as the algorithms * that will be FIPS-approved, and therefore it's included in this list. * * When adding a new algorithm here, make sure to consider whether it needs a * self-test added to fips140_selftests[] as well. */ static const struct { const char *name; bool approved; } fips140_algs_to_replace[] = { {"aes", true}, {"cmac(aes)", true}, {"ecb(aes)", true}, {"cbc(aes)", true}, {"cts(cbc(aes))", true}, {"ctr(aes)", true}, {"xts(aes)", true}, {"gcm(aes)", false}, {"hmac(sha1)", true}, {"hmac(sha224)", true}, {"hmac(sha256)", true}, {"hmac(sha384)", true}, {"hmac(sha512)", true}, {"sha1", true}, {"sha224", true}, {"sha256", true}, {"sha384", true}, {"sha512", true}, {"stdrng", true}, {"jitterentropy_rng", false}, }; static bool __init fips140_should_unregister_alg(struct crypto_alg *alg) { int i; /* * All software algorithms are synchronous, hardware algorithms must * be covered by their own FIPS 140 certification. */ if (alg->cra_flags & CRYPTO_ALG_ASYNC) return false; for (i = 0; i < ARRAY_SIZE(fips140_algs_to_replace); i++) { if (!strcmp(alg->cra_name, fips140_algs_to_replace[i].name)) return true; } return false; } /* * FIPS 140-3 service indicators. FIPS 140-3 requires that all services * "provide an indicator when the service utilises an approved cryptographic * algorithm, security function or process in an approved manner". What this * means is very debatable, even with the help of the FIPS 140-3 Implementation * Guidance document. However, it was decided that a function that takes in an * algorithm name and returns whether that algorithm is approved or not will * meet this requirement. Note, this relies on some properties of the module: * * - The module doesn't distinguish between "services" and "algorithms"; its * services are simply its algorithms. * * - The status of an approved algorithm is never non-approved, since (a) the * module doesn't support operating in a non-approved mode, such as a mode * where the self-tests are skipped; (b) there are no cases where the module * supports non-approved settings for approved algorithms, e.g. * non-approved key sizes; and (c) this function isn't available to be * called until the module_init function has completed, so it's guaranteed * that the self-tests and integrity check have already passed. * * - The module does support some non-approved algorithms, so a single static * indicator ("return true;") would not be acceptable. */ bool fips140_is_approved_service(const char *name) { size_t i; for (i = 0; i < ARRAY_SIZE(fips140_algs_to_replace); i++) { if (!strcmp(name, fips140_algs_to_replace[i].name)) return fips140_algs_to_replace[i].approved; } return false; } EXPORT_SYMBOL_GPL(fips140_is_approved_service); /* * FIPS 140-3 requires that modules provide a "service" that outputs "the name * or module identifier and the versioning information that can be correlated * with a validation record". This function meets that requirement. * * Note: the module also prints this same information to the kernel log when it * is loaded. That might meet the requirement by itself. However, given the * vagueness of what counts as a "service", we provide this function too, just * in case the certification lab or CMVP is happier with an explicit function. * * Note: /sys/modules/fips140/scmversion also provides versioning information * about the module. However that file just shows the bare git commit ID, so it * probably isn't sufficient to meet the FIPS requirement, which seems to want * the "official" module name and version number used in the FIPS certificate. */ const char *fips140_module_version(void) { return FIPS140_MODULE_NAME " " FIPS140_MODULE_VERSION; } EXPORT_SYMBOL_GPL(fips140_module_version); static LIST_HEAD(existing_live_algos); /* * Release a list of algorithms which have been removed from crypto_alg_list. * * Note that even though the list is a private list, we have to hold * crypto_alg_sem while iterating through it because crypto_unregister_alg() may * run concurrently (as we haven't taken a reference to the algorithms on the * list), and crypto_unregister_alg() will remove the algorithm from whichever * list it happens to be on, while holding crypto_alg_sem. That's okay, since * in that case crypto_unregister_alg() will handle the crypto_alg_put(). */ static void fips140_remove_final(struct list_head *list) { struct crypto_alg *alg; struct crypto_alg *n; /* * We need to take crypto_alg_sem to safely traverse the list (see * comment above), but we have to drop it when doing each * crypto_alg_put() as that may take crypto_alg_sem again. */ down_write(&crypto_alg_sem); list_for_each_entry_safe(alg, n, list, cra_list) { list_del_init(&alg->cra_list); up_write(&crypto_alg_sem); crypto_alg_put(alg); down_write(&crypto_alg_sem); } up_write(&crypto_alg_sem); } static void __init unregister_existing_fips140_algos(void) { struct crypto_alg *alg, *tmp; LIST_HEAD(remove_list); LIST_HEAD(spawns); down_write(&crypto_alg_sem); /* * Find all registered algorithms that we care about, and move them to a * private list so that they are no longer exposed via the algo lookup * API. Subsequently, we will unregister them if they are not in active * use. If they are, we can't fully unregister them but we can ensure * that new users won't use them. */ list_for_each_entry_safe(alg, tmp, &crypto_alg_list, cra_list) { if (!fips140_should_unregister_alg(alg)) continue; if (refcount_read(&alg->cra_refcnt) == 1) { /* * This algorithm is not currently in use, but there may * be template instances holding references to it via * spawns. So let's tear it down like * crypto_unregister_alg() would, but without releasing * the lock, to prevent races with concurrent TFM * allocations. */ alg->cra_flags |= CRYPTO_ALG_DEAD; list_move(&alg->cra_list, &remove_list); crypto_remove_spawns(alg, &spawns, NULL); } else { /* * This algorithm is live, i.e. it has TFMs allocated, * so we can't fully unregister it. It's not necessary * to dynamically redirect existing users to the FIPS * code, given that they can't be relying on FIPS * certified crypto in the first place. However, we do * need to ensure that new users will get the FIPS code. * * In most cases, setting alg->cra_priority to 0 * achieves this. However, that isn't enough for * algorithms like "hmac(sha256)" that need to be * instantiated from a template, since existing * algorithms always take priority over a template being * instantiated. Therefore, we move the algorithm to * a private list so that algorithm lookups won't find * it anymore. To further distinguish it from the FIPS * algorithms, we also append "+orig" to its name. */ pr_info("found already-live algorithm '%s' ('%s')\n", alg->cra_name, alg->cra_driver_name); alg->cra_priority = 0; strlcat(alg->cra_name, "+orig", CRYPTO_MAX_ALG_NAME); strlcat(alg->cra_driver_name, "+orig", CRYPTO_MAX_ALG_NAME); list_move(&alg->cra_list, &existing_live_algos); } } up_write(&crypto_alg_sem); fips140_remove_final(&remove_list); fips140_remove_final(&spawns); } static void __init unapply_text_relocations(void *section, int section_size, const Elf64_Rela *rela, int numrels) { while (numrels--) { u32 *place = (u32 *)(section + rela->r_offset); BUG_ON(rela->r_offset >= section_size); switch (ELF64_R_TYPE(rela->r_info)) { #ifdef CONFIG_ARM64 case R_AARCH64_JUMP26: case R_AARCH64_CALL26: *place &= ~GENMASK(25, 0); break; case R_AARCH64_ADR_PREL_LO21: case R_AARCH64_ADR_PREL_PG_HI21: case R_AARCH64_ADR_PREL_PG_HI21_NC: *place &= ~(GENMASK(30, 29) | GENMASK(23, 5)); break; case R_AARCH64_ADD_ABS_LO12_NC: case R_AARCH64_LDST8_ABS_LO12_NC: case R_AARCH64_LDST16_ABS_LO12_NC: case R_AARCH64_LDST32_ABS_LO12_NC: case R_AARCH64_LDST64_ABS_LO12_NC: case R_AARCH64_LDST128_ABS_LO12_NC: *place &= ~GENMASK(21, 10); break; default: pr_err("unhandled relocation type %llu\n", ELF64_R_TYPE(rela->r_info)); BUG(); #else #error #endif } rela++; } } static void __init unapply_rodata_relocations(void *section, int section_size, const Elf64_Rela *rela, int numrels) { while (numrels--) { void *place = section + rela->r_offset; BUG_ON(rela->r_offset >= section_size); switch (ELF64_R_TYPE(rela->r_info)) { #ifdef CONFIG_ARM64 case R_AARCH64_ABS64: *(u64 *)place = 0; break; default: pr_err("unhandled relocation type %llu\n", ELF64_R_TYPE(rela->r_info)); BUG(); #else #error #endif } rela++; } } extern struct { u32 offset; u32 count; } fips140_rela_text, fips140_rela_rodata; static bool __init check_fips140_module_hmac(void) { struct crypto_shash *tfm = NULL; SHASH_DESC_ON_STACK(desc, dontcare); u8 digest[SHA256_DIGEST_SIZE]; void *textcopy, *rodatacopy; int textsize, rodatasize; bool ok = false; int err; textsize = &__fips140_text_end - &__fips140_text_start; rodatasize = &__fips140_rodata_end - &__fips140_rodata_start; pr_info("text size : 0x%x\n", textsize); pr_info("rodata size: 0x%x\n", rodatasize); textcopy = kmalloc(textsize + rodatasize, GFP_KERNEL); if (!textcopy) { pr_err("Failed to allocate memory for copy of .text\n"); goto out; } rodatacopy = textcopy + textsize; memcpy(textcopy, __text_start, textsize); memcpy(rodatacopy, __rodata_start, rodatasize); // apply the relocations in reverse on the copies of .text and .rodata unapply_text_relocations(textcopy, textsize, offset_to_ptr(&fips140_rela_text.offset), fips140_rela_text.count); unapply_rodata_relocations(rodatacopy, rodatasize, offset_to_ptr(&fips140_rela_rodata.offset), fips140_rela_rodata.count); fips140_inject_integrity_failure(textcopy); tfm = crypto_alloc_shash("hmac(sha256)", 0, 0); if (IS_ERR(tfm)) { pr_err("failed to allocate hmac tfm (%ld)\n", PTR_ERR(tfm)); tfm = NULL; goto out; } desc->tfm = tfm; pr_info("using '%s' for integrity check\n", crypto_shash_driver_name(tfm)); err = crypto_shash_setkey(tfm, fips140_integ_hmac_key, strlen(fips140_integ_hmac_key)) ?: crypto_shash_init(desc) ?: crypto_shash_update(desc, textcopy, textsize) ?: crypto_shash_finup(desc, rodatacopy, rodatasize, digest); /* Zeroizing this is important; see the comment below. */ shash_desc_zero(desc); if (err) { pr_err("failed to calculate hmac shash (%d)\n", err); goto out; } if (memcmp(digest, fips140_integ_hmac_digest, sizeof(digest))) { pr_err("provided_digest : %*phN\n", (int)sizeof(digest), fips140_integ_hmac_digest); pr_err("calculated digest: %*phN\n", (int)sizeof(digest), digest); goto out; } ok = true; out: /* * FIPS 140-3 requires that all "temporary value(s) generated during the * integrity test" be zeroized (ref: FIPS 140-3 IG 9.7.B). There is no * technical reason to do this given that these values are public * information, but this is the requirement so we follow it. */ crypto_free_shash(tfm); memzero_explicit(digest, sizeof(digest)); kfree_sensitive(textcopy); return ok; } static void fips140_sha256(void *p, const u8 *data, unsigned int len, u8 *out, int *hook_inuse) { sha256(data, len, out); *hook_inuse = 1; } static void fips140_aes_expandkey(void *p, struct crypto_aes_ctx *ctx, const u8 *in_key, unsigned int key_len, int *err) { *err = aes_expandkey(ctx, in_key, key_len); } static void fips140_aes_encrypt(void *priv, const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, int *hook_inuse) { aes_encrypt(ctx, out, in); *hook_inuse = 1; } static void fips140_aes_decrypt(void *priv, const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, int *hook_inuse) { aes_decrypt(ctx, out, in); *hook_inuse = 1; } static bool update_fips140_library_routines(void) { int ret; ret = register_trace_android_vh_sha256(fips140_sha256, NULL) ?: register_trace_android_vh_aes_expandkey(fips140_aes_expandkey, NULL) ?: register_trace_android_vh_aes_encrypt(fips140_aes_encrypt, NULL) ?: register_trace_android_vh_aes_decrypt(fips140_aes_decrypt, NULL); return ret == 0; } /* * Initialize the FIPS 140 module. * * Note: this routine iterates over the contents of the initcall section, which * consists of an array of function pointers that was emitted by the linker * rather than the compiler. This means that these function pointers lack the * usual CFI stubs that the compiler emits when CFI codegen is enabled. So * let's disable CFI locally when handling the initcall array, to avoid * surpises. */ static int __init __attribute__((__no_sanitize__("cfi"))) fips140_init(void) { const u32 *initcall; pr_info("loading " FIPS140_MODULE_NAME " " FIPS140_MODULE_VERSION "\n"); fips140_init_thread = current; unregister_existing_fips140_algos(); /* iterate over all init routines present in this module and call them */ for (initcall = __initcall_start + 1; initcall < &__initcall_end_marker; initcall++) { int (*init)(void) = offset_to_ptr(initcall); int err = init(); /* * ENODEV is expected from initcalls that only register * algorithms that depend on non-present CPU features. Besides * that, errors aren't expected here. */ if (err && err != -ENODEV) { pr_err("initcall %ps() failed: %d\n", init, err); goto panic; } } if (!fips140_run_selftests()) goto panic; /* * It may seem backward to perform the integrity check last, but this * is intentional: the check itself uses hmac(sha256) which is one of * the algorithms that are replaced with versions from this module, and * the integrity check must use the replacement version. Also, to be * ready for FIPS 140-3, the integrity check algorithm must have already * been self-tested. */ if (!check_fips140_module_hmac()) { pr_crit("integrity check failed -- giving up!\n"); goto panic; } pr_info("integrity check passed\n"); complete_all(&fips140_tests_done); if (!update_fips140_library_routines()) goto panic; if (!fips140_eval_testing_init()) goto panic; pr_info("module successfully loaded\n"); return 0; panic: panic("FIPS 140 module load failure"); } module_init(fips140_init); MODULE_IMPORT_NS(CRYPTO_INTERNAL); MODULE_LICENSE("GPL v2"); /* * Crypto-related helper functions, reproduced here so that they will be * covered by the FIPS 140 integrity check. * * Non-cryptographic helper functions such as memcpy() can be excluded from the * FIPS module, but there is ambiguity about other helper functions like * __crypto_xor() and crypto_inc() which aren't cryptographic by themselves, * but are more closely associated with cryptography than e.g. memcpy(). To * err on the side of caution, we include copies of these in the FIPS module. */ void __crypto_xor(u8 *dst, const u8 *src1, const u8 *src2, unsigned int len) { while (len >= 8) { *(u64 *)dst = *(u64 *)src1 ^ *(u64 *)src2; dst += 8; src1 += 8; src2 += 8; len -= 8; } while (len >= 4) { *(u32 *)dst = *(u32 *)src1 ^ *(u32 *)src2; dst += 4; src1 += 4; src2 += 4; len -= 4; } while (len >= 2) { *(u16 *)dst = *(u16 *)src1 ^ *(u16 *)src2; dst += 2; src1 += 2; src2 += 2; len -= 2; } while (len--) *dst++ = *src1++ ^ *src2++; } void crypto_inc(u8 *a, unsigned int size) { a += size; while (size--) if (++*--a) break; }