digsig.c

/*
 * Copyright (C) 2011 Nokia Corporation
 * Copyright (C) 2011 Intel Corporation
 *
 * Author:
 * Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
 *                 <dmitry.kasatkin@intel.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, version 2 of the License.
 *
 * File: sign.c
 *	implements signature (RSA) verification
 *	pkcs decoding is based on LibTomCrypt code
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/err.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/key.h>
#include <linux/crypto.h>
#include <crypto/hash.h>
#include <crypto/sha.h>
#include <keys/user-type.h>
#include <linux/mpi.h>
#include <linux/digsig.h>

static struct crypto_shash *shash;

static int pkcs_1_v1_5_decode_emsa(const unsigned char *msg,
			unsigned long  msglen,
			unsigned long  modulus_bitlen,
			unsigned char *out,
			unsigned long *outlen)
{
	unsigned long modulus_len, ps_len, i;

	modulus_len = (modulus_bitlen >> 3) + (modulus_bitlen & 7 ? 1 : 0);

	/* test message size */
	if ((msglen > modulus_len) || (modulus_len < 11))
		return -EINVAL;

	/* separate encoded message */
	if ((msg[0] != 0x00) || (msg[1] != (unsigned char)1))
		return -EINVAL;

	for (i = 2; i < modulus_len - 1; i++)
		if (msg[i] != 0xFF)
			break;

	/* separator check */
	if (msg[i] != 0)
		/* There was no octet with hexadecimal value 0x00
		to separate ps from m. */
		return -EINVAL;

	ps_len = i - 2;

	if (*outlen < (msglen - (2 + ps_len + 1))) {
		*outlen = msglen - (2 + ps_len + 1);
		return -EOVERFLOW;
	}

	*outlen = (msglen - (2 + ps_len + 1));
	memcpy(out, &msg[2 + ps_len + 1], *outlen);