Update build for OpenSSL-1.0.0a.
[dragonfly.git] / crypto / openssl / crypto / rsa / rsa_pmeth.c
CommitLineData
919b01cc
PA
1/* crypto/rsa/rsa_pmeth.c */
2/* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
3 * project 2006.
4 */
5/* ====================================================================
6 * Copyright (c) 2006 The OpenSSL Project. All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 *
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 *
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in
17 * the documentation and/or other materials provided with the
18 * distribution.
19 *
20 * 3. All advertising materials mentioning features or use of this
21 * software must display the following acknowledgment:
22 * "This product includes software developed by the OpenSSL Project
23 * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
24 *
25 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
26 * endorse or promote products derived from this software without
27 * prior written permission. For written permission, please contact
28 * licensing@OpenSSL.org.
29 *
30 * 5. Products derived from this software may not be called "OpenSSL"
31 * nor may "OpenSSL" appear in their names without prior written
32 * permission of the OpenSSL Project.
33 *
34 * 6. Redistributions of any form whatsoever must retain the following
35 * acknowledgment:
36 * "This product includes software developed by the OpenSSL Project
37 * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
38 *
39 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
40 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
41 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
42 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
43 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
44 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
45 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
46 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
47 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
48 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
49 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
50 * OF THE POSSIBILITY OF SUCH DAMAGE.
51 * ====================================================================
52 *
53 * This product includes cryptographic software written by Eric Young
54 * (eay@cryptsoft.com). This product includes software written by Tim
55 * Hudson (tjh@cryptsoft.com).
56 *
57 */
58
59#include <stdio.h>
60#include "cryptlib.h"
61#include <openssl/asn1t.h>
62#include <openssl/x509.h>
63#include <openssl/rsa.h>
64#include <openssl/bn.h>
65#include <openssl/evp.h>
01185282 66#include "../evp/evp_locl.h"
919b01cc
PA
67#include "rsa_locl.h"
68
69/* RSA pkey context structure */
70
71typedef struct
72 {
73 /* Key gen parameters */
74 int nbits;
75 BIGNUM *pub_exp;
76 /* Keygen callback info */
77 int gentmp[2];
78 /* RSA padding mode */
79 int pad_mode;
80 /* message digest */
81 const EVP_MD *md;
82 /* PSS/OAEP salt length */
83 int saltlen;
84 /* Temp buffer */
85 unsigned char *tbuf;
86 } RSA_PKEY_CTX;
87
88static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
89 {
90 RSA_PKEY_CTX *rctx;
91 rctx = OPENSSL_malloc(sizeof(RSA_PKEY_CTX));
92 if (!rctx)
93 return 0;
94 rctx->nbits = 1024;
95 rctx->pub_exp = NULL;
96 rctx->pad_mode = RSA_PKCS1_PADDING;
97 rctx->md = NULL;
98 rctx->tbuf = NULL;
99
100 rctx->saltlen = -2;
101
102 ctx->data = rctx;
103 ctx->keygen_info = rctx->gentmp;
104 ctx->keygen_info_count = 2;
105
106 return 1;
107 }
108
109static int pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
110 {
111 RSA_PKEY_CTX *dctx, *sctx;
112 if (!pkey_rsa_init(dst))
113 return 0;
114 sctx = src->data;
115 dctx = dst->data;
116 dctx->nbits = sctx->nbits;
117 if (sctx->pub_exp)
118 {
119 dctx->pub_exp = BN_dup(sctx->pub_exp);
120 if (!dctx->pub_exp)
121 return 0;
122 }
123 dctx->pad_mode = sctx->pad_mode;
124 dctx->md = sctx->md;
125 return 1;
126 }
127
128static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
129 {
130 if (ctx->tbuf)
131 return 1;
132 ctx->tbuf = OPENSSL_malloc(EVP_PKEY_size(pk->pkey));
133 if (!ctx->tbuf)
134 return 0;
135 return 1;
136 }
137
138static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
139 {
140 RSA_PKEY_CTX *rctx = ctx->data;
141 if (rctx)
142 {
143 if (rctx->pub_exp)
144 BN_free(rctx->pub_exp);
145 if (rctx->tbuf)
146 OPENSSL_free(rctx->tbuf);
147 OPENSSL_free(rctx);
148 }
149 }
150
151static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
152 const unsigned char *tbs, size_t tbslen)
153 {
154 int ret;
155 RSA_PKEY_CTX *rctx = ctx->data;
156 RSA *rsa = ctx->pkey->pkey.rsa;
157
158 if (rctx->md)
159 {
160 if (tbslen != (size_t)EVP_MD_size(rctx->md))
161 {
162 RSAerr(RSA_F_PKEY_RSA_SIGN,
163 RSA_R_INVALID_DIGEST_LENGTH);
164 return -1;
165 }
166 if (rctx->pad_mode == RSA_X931_PADDING)
167 {
168 if (!setup_tbuf(rctx, ctx))
169 return -1;
170 memcpy(rctx->tbuf, tbs, tbslen);
171 rctx->tbuf[tbslen] =
172 RSA_X931_hash_id(EVP_MD_type(rctx->md));
173 ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
174 sig, rsa, RSA_X931_PADDING);
175 }
176 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
177 {
178 unsigned int sltmp;
179 ret = RSA_sign(EVP_MD_type(rctx->md),
180 tbs, tbslen, sig, &sltmp, rsa);
181 if (ret <= 0)
182 return ret;
183 ret = sltmp;
184 }
185 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
186 {
187 if (!setup_tbuf(rctx, ctx))
188 return -1;
189 if (!RSA_padding_add_PKCS1_PSS(rsa, rctx->tbuf, tbs,
190 rctx->md, rctx->saltlen))
191 return -1;
192 ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
193 sig, rsa, RSA_NO_PADDING);
194 }
195 else
196 return -1;
197 }
198 else
199 ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
200 rctx->pad_mode);
201 if (ret < 0)
202 return ret;
203 *siglen = ret;
204 return 1;
205 }
206
207
208static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
209 unsigned char *rout, size_t *routlen,
210 const unsigned char *sig, size_t siglen)
211 {
212 int ret;
213 RSA_PKEY_CTX *rctx = ctx->data;
214
215 if (rctx->md)
216 {
217 if (rctx->pad_mode == RSA_X931_PADDING)
218 {
219 if (!setup_tbuf(rctx, ctx))
220 return -1;
221 ret = RSA_public_decrypt(siglen, sig,
222 rctx->tbuf, ctx->pkey->pkey.rsa,
223 RSA_X931_PADDING);
224 if (ret < 1)
225 return 0;
226 ret--;
227 if (rctx->tbuf[ret] !=
228 RSA_X931_hash_id(EVP_MD_type(rctx->md)))
229 {
230 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
231 RSA_R_ALGORITHM_MISMATCH);
232 return 0;
233 }
234 if (ret != EVP_MD_size(rctx->md))
235 {
236 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
237 RSA_R_INVALID_DIGEST_LENGTH);
238 return 0;
239 }
240 if (rout)
241 memcpy(rout, rctx->tbuf, ret);
242 }
243 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
244 {
245 size_t sltmp;
246 ret = int_rsa_verify(EVP_MD_type(rctx->md),
247 NULL, 0, rout, &sltmp,
248 sig, siglen, ctx->pkey->pkey.rsa);
249 if (ret <= 0)
250 return 0;
251 ret = sltmp;
252 }
253 else
254 return -1;
255 }
256 else
257 ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
258 rctx->pad_mode);
259 if (ret < 0)
260 return ret;
261 *routlen = ret;
262 return 1;
263 }
264
265static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
266 const unsigned char *sig, size_t siglen,
267 const unsigned char *tbs, size_t tbslen)
268 {
269 RSA_PKEY_CTX *rctx = ctx->data;
270 RSA *rsa = ctx->pkey->pkey.rsa;
271 size_t rslen;
272 if (rctx->md)
273 {
274 if (rctx->pad_mode == RSA_PKCS1_PADDING)
275 return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
276 sig, siglen, rsa);
277 if (rctx->pad_mode == RSA_X931_PADDING)
278 {
279 if (pkey_rsa_verifyrecover(ctx, NULL, &rslen,
280 sig, siglen) <= 0)
281 return 0;
282 }
283 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
284 {
285 int ret;
286 if (!setup_tbuf(rctx, ctx))
287 return -1;
288 ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
289 rsa, RSA_NO_PADDING);
290 if (ret <= 0)
291 return 0;
292 ret = RSA_verify_PKCS1_PSS(rsa, tbs, rctx->md,
293 rctx->tbuf, rctx->saltlen);
294 if (ret <= 0)
295 return 0;
296 return 1;
297 }
298 else
299 return -1;
300 }
301 else
302 {
303 if (!setup_tbuf(rctx, ctx))
304 return -1;
305 rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
306 rsa, rctx->pad_mode);
307 if (rslen == 0)
308 return 0;
309 }
310
311 if ((rslen != tbslen) || memcmp(tbs, rctx->tbuf, rslen))
312 return 0;
313
314 return 1;
315
316 }
317
318
319static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx,
320 unsigned char *out, size_t *outlen,
321 const unsigned char *in, size_t inlen)
322 {
323 int ret;
324 RSA_PKEY_CTX *rctx = ctx->data;
325 ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
326 rctx->pad_mode);
327 if (ret < 0)
328 return ret;
329 *outlen = ret;
330 return 1;
331 }
332
333static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx,
334 unsigned char *out, size_t *outlen,
335 const unsigned char *in, size_t inlen)
336 {
337 int ret;
338 RSA_PKEY_CTX *rctx = ctx->data;
339 ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
340 rctx->pad_mode);
341 if (ret < 0)
342 return ret;
343 *outlen = ret;
344 return 1;
345 }
346
347static int check_padding_md(const EVP_MD *md, int padding)
348 {
349 if (!md)
350 return 1;
351
352 if (padding == RSA_NO_PADDING)
353 {
354 RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_PADDING_MODE);
355 return 0;
356 }
357
358 if (padding == RSA_X931_PADDING)
359 {
360 if (RSA_X931_hash_id(EVP_MD_type(md)) == -1)
361 {
362 RSAerr(RSA_F_CHECK_PADDING_MD,
363 RSA_R_INVALID_X931_DIGEST);
364 return 0;
365 }
366 return 1;
367 }
368
369 return 1;
370 }
371
372
373static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
374 {
375 RSA_PKEY_CTX *rctx = ctx->data;
376 switch (type)
377 {
378 case EVP_PKEY_CTRL_RSA_PADDING:
379 if ((p1 >= RSA_PKCS1_PADDING) && (p1 <= RSA_PKCS1_PSS_PADDING))
380 {
381 if (!check_padding_md(rctx->md, p1))
382 return 0;
383 if (p1 == RSA_PKCS1_PSS_PADDING)
384 {
385 if (!(ctx->operation &
386 (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)))
387 goto bad_pad;
388 if (!rctx->md)
389 rctx->md = EVP_sha1();
390 }
391 if (p1 == RSA_PKCS1_OAEP_PADDING)
392 {
393 if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
394 goto bad_pad;
395 if (!rctx->md)
396 rctx->md = EVP_sha1();
397 }
398 rctx->pad_mode = p1;
399 return 1;
400 }
401 bad_pad:
402 RSAerr(RSA_F_PKEY_RSA_CTRL,
403 RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
404 return -2;
405
406 case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
407 if (p1 < -2)
408 return -2;
409 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING)
410 {
411 RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
412 return -2;
413 }
414 rctx->saltlen = p1;
415 return 1;
416
417 case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
418 if (p1 < 256)
419 {
420 RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_KEYBITS);
421 return -2;
422 }
423 rctx->nbits = p1;
424 return 1;
425
426 case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
427 if (!p2)
428 return -2;
429 rctx->pub_exp = p2;
430 return 1;
431
432 case EVP_PKEY_CTRL_MD:
433 if (!check_padding_md(p2, rctx->pad_mode))
434 return 0;
435 rctx->md = p2;
436 return 1;
437
438 case EVP_PKEY_CTRL_DIGESTINIT:
439 case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
440 case EVP_PKEY_CTRL_PKCS7_DECRYPT:
441 case EVP_PKEY_CTRL_PKCS7_SIGN:
442#ifndef OPENSSL_NO_CMS
443 case EVP_PKEY_CTRL_CMS_ENCRYPT:
444 case EVP_PKEY_CTRL_CMS_DECRYPT:
445 case EVP_PKEY_CTRL_CMS_SIGN:
446#endif
447 return 1;
448 case EVP_PKEY_CTRL_PEER_KEY:
449 RSAerr(RSA_F_PKEY_RSA_CTRL,
450 RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
451 return -2;
452
453 default:
454 return -2;
455
456 }
457 }
458
459static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
460 const char *type, const char *value)
461 {
462 if (!value)
463 {
464 RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_VALUE_MISSING);
465 return 0;
466 }
467 if (!strcmp(type, "rsa_padding_mode"))
468 {
469 int pm;
470 if (!strcmp(value, "pkcs1"))
471 pm = RSA_PKCS1_PADDING;
472 else if (!strcmp(value, "sslv23"))
473 pm = RSA_SSLV23_PADDING;
474 else if (!strcmp(value, "none"))
475 pm = RSA_NO_PADDING;
476 else if (!strcmp(value, "oeap"))
477 pm = RSA_PKCS1_OAEP_PADDING;
478 else if (!strcmp(value, "x931"))
479 pm = RSA_X931_PADDING;
480 else if (!strcmp(value, "pss"))
481 pm = RSA_PKCS1_PSS_PADDING;
482 else
483 {
484 RSAerr(RSA_F_PKEY_RSA_CTRL_STR,
485 RSA_R_UNKNOWN_PADDING_TYPE);
486 return -2;
487 }
488 return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
489 }
490
491 if (!strcmp(type, "rsa_pss_saltlen"))
492 {
493 int saltlen;
494 saltlen = atoi(value);
495 return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
496 }
497
498 if (!strcmp(type, "rsa_keygen_bits"))
499 {
500 int nbits;
501 nbits = atoi(value);
502 return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits);
503 }
504
505 if (!strcmp(type, "rsa_keygen_pubexp"))
506 {
507 int ret;
508 BIGNUM *pubexp = NULL;
509 if (!BN_asc2bn(&pubexp, value))
510 return 0;
511 ret = EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp);
512 if (ret <= 0)
513 BN_free(pubexp);
514 return ret;
515 }
516
517 return -2;
518 }
519
520static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
521 {
522 RSA *rsa = NULL;
523 RSA_PKEY_CTX *rctx = ctx->data;
524 BN_GENCB *pcb, cb;
525 int ret;
526 if (!rctx->pub_exp)
527 {
528 rctx->pub_exp = BN_new();
529 if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4))
530 return 0;
531 }
532 rsa = RSA_new();
533 if (!rsa)
534 return 0;
535 if (ctx->pkey_gencb)
536 {
537 pcb = &cb;
538 evp_pkey_set_cb_translate(pcb, ctx);
539 }
540 else
541 pcb = NULL;
542 ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
543 if (ret > 0)
544 EVP_PKEY_assign_RSA(pkey, rsa);
545 else
546 RSA_free(rsa);
547 return ret;
548 }
549
550const EVP_PKEY_METHOD rsa_pkey_meth =
551 {
552 EVP_PKEY_RSA,
553 EVP_PKEY_FLAG_AUTOARGLEN,
554 pkey_rsa_init,
555 pkey_rsa_copy,
556 pkey_rsa_cleanup,
557
558 0,0,
559
560 0,
561 pkey_rsa_keygen,
562
563 0,
564 pkey_rsa_sign,
565
566 0,
567 pkey_rsa_verify,
568
569 0,
570 pkey_rsa_verifyrecover,
571
572
573 0,0,0,0,
574
575 0,
576 pkey_rsa_encrypt,
577
578 0,
579 pkey_rsa_decrypt,
580
581 0,0,
582
583 pkey_rsa_ctrl,
584 pkey_rsa_ctrl_str
585
586
587 };