lib/rsa: Make fdt_add_bignum() available outside of RSA code

fdt_add_bignum() is useful for algorithms other than just RSA. To
allow its use for ECDSA, move it to a common file under lib/.

The new file is suffixed with '-libcrypto' because it has a direct
dependency on openssl. This is due to the use of the "BIGNUM *" type.

Signed-off-by: Alexandru Gagniuc <mr.nuke.me@gmail.com>
Reviewed-by: Simon Glass <sjg@chromium.org>

lib/fdt-libcrypto.c

@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (c) 2020, Alexandru Gagniuc <mr.nuke.me@gmail.com>
+ * Copyright (c) 2013, Google Inc.
+ */
+
+#include <libfdt.h>
+#include <u-boot/fdt-libcrypto.h>
+
+int fdt_add_bignum(void *blob, int noffset, const char *prop_name,
+		   BIGNUM *num, int num_bits)
+{
+	int nwords = num_bits / 32;
+	int size;
+	uint32_t *buf, *ptr;
+	BIGNUM *tmp, *big2, *big32, *big2_32;
+	BN_CTX *ctx;
+	int ret;
+
+	tmp = BN_new();
+	big2 = BN_new();
+	big32 = BN_new();
+	big2_32 = BN_new();
+
+	/*
+	 * Note: This code assumes that all of the above succeed, or all fail.
+	 * In practice memory allocations generally do not fail (unless the
+	 * process is killed), so it does not seem worth handling each of these
+	 * as a separate case. Technicaly this could leak memory on failure,
+	 * but a) it won't happen in practice, and b) it doesn't matter as we
+	 * will immediately exit with a failure code.
+	 */
+	if (!tmp || !big2 || !big32 || !big2_32) {
+		fprintf(stderr, "Out of memory (bignum)\n");
+		return -ENOMEM;
+	}
+	ctx = BN_CTX_new();
+	if (!ctx) {
+		fprintf(stderr, "Out of memory (bignum context)\n");
+		return -ENOMEM;
+	}
+	BN_set_word(big2, 2L);
+	BN_set_word(big32, 32L);
+	BN_exp(big2_32, big2, big32, ctx); /* B = 2^32 */
+
+	size = nwords * sizeof(uint32_t);
+	buf = malloc(size);
+	if (!buf) {
+		fprintf(stderr, "Out of memory (%d bytes)\n", size);
+		return -ENOMEM;
+	}
+
+	/* Write out modulus as big endian array of integers */
+	for (ptr = buf + nwords - 1; ptr >= buf; ptr--) {
+		BN_mod(tmp, num, big2_32, ctx); /* n = N mod B */
+		*ptr = cpu_to_fdt32(BN_get_word(tmp));
+		BN_rshift(num, num, 32); /*  N = N/B */
+	}
+
+	/*
+	 * We try signing with successively increasing size values, so this
+	 * might fail several times
+	 */
+	ret = fdt_setprop(blob, noffset, prop_name, buf, size);
+	free(buf);
+	BN_free(tmp);
+	BN_free(big2);
+	BN_free(big32);
+	BN_free(big2_32);
+
+	return ret ? -FDT_ERR_NOSPACE : 0;
+}