0002-openssl11.patch 3.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130
  1. From bd6aa6acddf0ba640a49834807872f4cc0d0a773 Mon Sep 17 00:00:00 2001
  2. From: Jani Hakala <jjhakala@gmail.com>
  3. Date: Thu, 16 Jun 2016 14:28:15 +0300
  4. Subject: [PATCH] Fix OpenSSL 1.1 compability issues
  5. Some data types have been made opaque in OpenSSL version 1.1 so
  6. stack allocation and accessing struct fields directly does not work.
  7. Downloaded from upstream commit
  8. https://github.com/rdesktop/rdesktop/commit/bd6aa6acddf0ba640a49834807872f4cc0d0a773
  9. Signed-off-by: Bernd Kuhls <bernd.kuhls@t-online.de>
  10. ---
  11. ssl.c | 65 ++++++++++++++++++++++++++++++++++++-----------------------
  12. 1 file changed, 40 insertions(+), 25 deletions(-)
  13. diff --git a/ssl.c b/ssl.c
  14. index 48751255..032e9b9e 100644
  15. --- a/ssl.c
  16. +++ b/ssl.c
  17. @@ -88,7 +88,7 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 *
  18. uint8 * exponent)
  19. {
  20. BN_CTX *ctx;
  21. - BIGNUM mod, exp, x, y;
  22. + BIGNUM *mod, *exp, *x, *y;
  23. uint8 inr[SEC_MAX_MODULUS_SIZE];
  24. int outlen;
  25. @@ -98,24 +98,24 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 *
  26. reverse(inr, len);
  27. ctx = BN_CTX_new();
  28. - BN_init(&mod);
  29. - BN_init(&exp);
  30. - BN_init(&x);
  31. - BN_init(&y);
  32. -
  33. - BN_bin2bn(modulus, modulus_size, &mod);
  34. - BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);
  35. - BN_bin2bn(inr, len, &x);
  36. - BN_mod_exp(&y, &x, &exp, &mod, ctx);
  37. - outlen = BN_bn2bin(&y, out);
  38. + mod = BN_new();
  39. + exp = BN_new();
  40. + x = BN_new();
  41. + y = BN_new();
  42. +
  43. + BN_bin2bn(modulus, modulus_size, mod);
  44. + BN_bin2bn(exponent, SEC_EXPONENT_SIZE, exp);
  45. + BN_bin2bn(inr, len, x);
  46. + BN_mod_exp(y, x, exp, mod, ctx);
  47. + outlen = BN_bn2bin(y, out);
  48. reverse(out, outlen);
  49. if (outlen < (int) modulus_size)
  50. memset(out + outlen, 0, modulus_size - outlen);
  51. - BN_free(&y);
  52. - BN_clear_free(&x);
  53. - BN_free(&exp);
  54. - BN_free(&mod);
  55. + BN_free(y);
  56. + BN_clear_free(x);
  57. + BN_free(exp);
  58. + BN_free(mod);
  59. BN_CTX_free(ctx);
  60. }
  61. @@ -146,12 +146,20 @@ rdssl_cert_to_rkey(RDSSL_CERT * cert, uint32 * key_len)
  62. Kudos to Richard Levitte for the following (. intiutive .)
  63. lines of code that resets the OID and let's us extract the key. */
  64. - nid = OBJ_obj2nid(cert->cert_info->key->algor->algorithm);
  65. +
  66. + X509_PUBKEY *key = NULL;
  67. + X509_ALGOR *algor = NULL;
  68. +
  69. + key = X509_get_X509_PUBKEY(cert);
  70. + algor = X509_PUBKEY_get0_param(NULL, NULL, 0, &algor, key);
  71. +
  72. + nid = OBJ_obj2nid(algor->algorithm);
  73. +
  74. if ((nid == NID_md5WithRSAEncryption) || (nid == NID_shaWithRSAEncryption))
  75. {
  76. DEBUG_RDP5(("Re-setting algorithm type to RSA in server certificate\n"));
  77. - ASN1_OBJECT_free(cert->cert_info->key->algor->algorithm);
  78. - cert->cert_info->key->algor->algorithm = OBJ_nid2obj(NID_rsaEncryption);
  79. + X509_PUBKEY_set0_param(key, OBJ_nid2obj(NID_rsaEncryption),
  80. + 0, NULL, NULL, 0);
  81. }
  82. epk = X509_get_pubkey(cert);
  83. if (NULL == epk)
  84. @@ -201,14 +209,24 @@ rdssl_rkey_get_exp_mod(RDSSL_RKEY * rkey, uint8 * exponent, uint32 max_exp_len,
  85. {
  86. int len;
  87. - if ((BN_num_bytes(rkey->e) > (int) max_exp_len) ||
  88. - (BN_num_bytes(rkey->n) > (int) max_mod_len))
  89. + BIGNUM *e = NULL;
  90. + BIGNUM *n = NULL;
  91. +
  92. +#if OPENSSL_VERSION_NUMBER < 0x10100000L
  93. + e = rkey->e;
  94. + n = rkey->n;
  95. +#else
  96. + RSA_get0_key(rkey, &e, &n, NULL);
  97. +#endif
  98. +
  99. + if ((BN_num_bytes(e) > (int) max_exp_len) ||
  100. + (BN_num_bytes(n) > (int) max_mod_len))
  101. {
  102. return 1;
  103. }
  104. - len = BN_bn2bin(rkey->e, exponent);
  105. + len = BN_bn2bin(e, exponent);
  106. reverse(exponent, len);
  107. - len = BN_bn2bin(rkey->n, modulus);
  108. + len = BN_bn2bin(n, modulus);
  109. reverse(modulus, len);
  110. return 0;
  111. }
  112. @@ -229,8 +247,5 @@ void
  113. rdssl_hmac_md5(const void *key, int key_len, const unsigned char *msg, int msg_len,
  114. unsigned char *md)
  115. {
  116. - HMAC_CTX ctx;
  117. - HMAC_CTX_init(&ctx);
  118. HMAC(EVP_md5(), key, key_len, msg, msg_len, md, NULL);
  119. - HMAC_CTX_cleanup(&ctx);
  120. }