Attachment #194640 for bug #229029




		  uint8 * exponent)
{
	BN_CTX *ctx;
	BIGNUM *mod, *exp, *x, *y;
	uint8 inr[SEC_MAX_MODULUS_SIZE];
	int outlen;


	reverse(inr, len);

	ctx = BN_CTX_new();
	mod = BN_new();
	exp = BN_new();
	x = BN_new();
	y = BN_new();

	BN_bin2bn(modulus, modulus_size, mod);
	BN_bin2bn(exponent, SEC_EXPONENT_SIZE, exp);
	BN_bin2bn(inr, len, x);
	BN_mod_exp(y, x, exp, mod, ctx);
	outlen = BN_bn2bin(y, out);
	reverse(out, outlen);
	if (outlen < (int) modulus_size)
		memset(out + outlen, 0, modulus_size - outlen);

	BN_free(y);
	BN_clear_free(x);
	BN_free(exp);
	BN_free(mod);
	BN_CTX_free(ctx);
}



	   Kudos to Richard Levitte for the following (. intiutive .) 
	   lines of code that resets the OID and let's us extract the key. */

	X509_PUBKEY *key = NULL;
	X509_ALGOR *algor = NULL;

	key = X509_get_X509_PUBKEY(cert);
	algor = X509_PUBKEY_get0_param(NULL, NULL, 0, &algor, key);

	nid = OBJ_obj2nid(algor->algorithm);

	if ((nid == NID_md5WithRSAEncryption) || (nid == NID_shaWithRSAEncryption))
	{
		DEBUG_RDP5(("Re-setting algorithm type to RSA in server certificate\n"));
		X509_PUBKEY_set0_param(key, OBJ_nid2obj(NID_rsaEncryption),
				       0, NULL, NULL, 0);
	}
	epk = X509_get_pubkey(cert);
	if (NULL == epk)

{
	int len;

	BIGNUM *e = NULL;
	BIGNUM *n = NULL;

#if OPENSSL_VERSION_NUMBER < 0x10100000L
	e = rkey->e;
	n = rkey->n;
#else
	RSA_get0_key(rkey, &e, &n, NULL);
#endif

	if ((BN_num_bytes(e) > (int) max_exp_len) ||
	    (BN_num_bytes(n) > (int) max_mod_len))
	{
		return 1;
	}
	len = BN_bn2bin(e, exponent);
	reverse(exponent, len);
	len = BN_bn2bin(n, modulus);
	reverse(modulus, len);
	return 0;
}

rdssl_hmac_md5(const void *key, int key_len, const unsigned char *msg, int msg_len,
	       unsigned char *md)
{
	HMAC_CTX ctx;
	HMAC_CTX_init(&ctx);
	HMAC(EVP_md5(), key, key_len, msg, msg_len, md, NULL);
	HMAC_CTX_cleanup(&ctx);
}

Return to bug 229029

Lines 88-94 rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 * Link Here

(-)b/ssl.c (-25 / +40 lines)
88	uint8 * exponent)	88	uint8 * exponent)
89	{	89	{
90	BN_CTX *ctx;	90	BN_CTX *ctx;
91	BIGNUM mod, exp, x, y;	91	BIGNUM mod, exp, x, y;
92	uint8 inr[SEC_MAX_MODULUS_SIZE];	92	uint8 inr[SEC_MAX_MODULUS_SIZE];
93	int outlen;	93	int outlen;
94		94
Lines 98-121 rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 * Link Here
98	reverse(inr, len);	98	reverse(inr, len);
99		99
100	ctx = BN_CTX_new();	100	ctx = BN_CTX_new();
101	BN_init(&mod);	101	mod = BN_new();
102	BN_init(&exp);	102	exp = BN_new();
103	BN_init(&x);	103	x = BN_new();
104	BN_init(&y);	104	y = BN_new();
105		105
106	BN_bin2bn(modulus, modulus_size, &mod);	106	BN_bin2bn(modulus, modulus_size, mod);
107	BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);	107	BN_bin2bn(exponent, SEC_EXPONENT_SIZE, exp);
108	BN_bin2bn(inr, len, &x);	108	BN_bin2bn(inr, len, x);
109	BN_mod_exp(&y, &x, &exp, &mod, ctx);	109	BN_mod_exp(y, x, exp, mod, ctx);
110	outlen = BN_bn2bin(&y, out);	110	outlen = BN_bn2bin(y, out);
111	reverse(out, outlen);	111	reverse(out, outlen);
112	if (outlen < (int) modulus_size)	112	if (outlen < (int) modulus_size)
113	memset(out + outlen, 0, modulus_size - outlen);	113	memset(out + outlen, 0, modulus_size - outlen);
114		114
115	BN_free(&y);	115	BN_free(y);
116	BN_clear_free(&x);	116	BN_clear_free(x);
117	BN_free(&exp);	117	BN_free(exp);
118	BN_free(&mod);	118	BN_free(mod);
119	BN_CTX_free(ctx);	119	BN_CTX_free(ctx);
120	}	120	}
121		121
Lines 146-157 rdssl_cert_to_rkey(RDSSL_CERT * cert, uint32 * key_len) Link Here
146		146
147	Kudos to Richard Levitte for the following (. intiutive .)	147	Kudos to Richard Levitte for the following (. intiutive .)
148	lines of code that resets the OID and let's us extract the key. */	148	lines of code that resets the OID and let's us extract the key. */
149	nid = OBJ_obj2nid(cert->cert_info->key->algor->algorithm);	149
		150	X509_PUBKEY *key = NULL;
		151	X509_ALGOR *algor = NULL;
		152
		153	key = X509_get_X509_PUBKEY(cert);
		154	algor = X509_PUBKEY_get0_param(NULL, NULL, 0, &algor, key);
		155
		156	nid = OBJ_obj2nid(algor->algorithm);
		157
150	if ((nid == NID_md5WithRSAEncryption) \|\| (nid == NID_shaWithRSAEncryption))	158	if ((nid == NID_md5WithRSAEncryption) \|\| (nid == NID_shaWithRSAEncryption))
151	{	159	{
152	DEBUG_RDP5(("Re-setting algorithm type to RSA in server certificate\n"));	160	DEBUG_RDP5(("Re-setting algorithm type to RSA in server certificate\n"));
153	ASN1_OBJECT_free(cert->cert_info->key->algor->algorithm);	161	X509_PUBKEY_set0_param(key, OBJ_nid2obj(NID_rsaEncryption),
154	cert->cert_info->key->algor->algorithm = OBJ_nid2obj(NID_rsaEncryption);	162	0, NULL, NULL, 0);
155	}	163	}
156	epk = X509_get_pubkey(cert);	164	epk = X509_get_pubkey(cert);
157	if (NULL == epk)	165	if (NULL == epk)
Lines 201-214 rdssl_rkey_get_exp_mod(RDSSL_RKEY * rkey, uint8 * exponent, uint32 max_exp_len, Link Here
201	{	209	{
202	int len;	210	int len;
203		211
204	if ((BN_num_bytes(rkey->e) > (int) max_exp_len) \|\|	212	BIGNUM *e = NULL;
205	(BN_num_bytes(rkey->n) > (int) max_mod_len))	213	BIGNUM *n = NULL;
		214
		215	#if OPENSSL_VERSION_NUMBER < 0x10100000L
		216	e = rkey->e;
		217	n = rkey->n;
		218	#else
		219	RSA_get0_key(rkey, &e, &n, NULL);
		220	#endif
		221
		222	if ((BN_num_bytes(e) > (int) max_exp_len) \|\|
		223	(BN_num_bytes(n) > (int) max_mod_len))
206	{	224	{
207	return 1;	225	return 1;
208	}	226	}
209	len = BN_bn2bin(rkey->e, exponent);	227	len = BN_bn2bin(e, exponent);
210	reverse(exponent, len);	228	reverse(exponent, len);
211	len = BN_bn2bin(rkey->n, modulus);	229	len = BN_bn2bin(n, modulus);
212	reverse(modulus, len);	230	reverse(modulus, len);
213	return 0;	231	return 0;
214	}	232	}
Lines 229-236 void Link Here
229	rdssl_hmac_md5(const void key, int key_len, const unsigned char msg, int msg_len,	247	rdssl_hmac_md5(const void key, int key_len, const unsigned char msg, int msg_len,
230	unsigned char *md)	248	unsigned char *md)
231	{	249	{
232	HMAC_CTX ctx;
233	HMAC_CTX_init(&ctx);
234	HMAC(EVP_md5(), key, key_len, msg, msg_len, md, NULL);	250	HMAC(EVP_md5(), key, key_len, msg, msg_len, md, NULL);
235	HMAC_CTX_cleanup(&ctx);
236	}	251	}