CryptoPkg/OpensslLib: Switch to upstream fix for OpenSSL RT#3955

A different fix for the excessive stack usage has been merged into
OpenSSL 1.1 as commit 8e704858f. Drop our own version and use a backport
of what was committed upstream.

Note: This requires the free() function to work correctly when passed
      a NULL argument (qv).

Contributed-under: TianoCore Contribution Agreement 1.0
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
Reviewed-by: Laszlo Ersek <lersek@redhat.com>
Reviewed-by: Qin Long <qin.long@intel.com>
Tested-by: Qin Long <qin.long@intel.com>

1 files changed, 137 insertions, 19 deletions

diff --git a/CryptoPkg/Library/OpensslLib/EDKII_openssl-1.0.2f.patch b/CryptoPkg/Library/OpensslLib/EDKII_openssl-1.0.2f.patch
index ec54de5725..8d5f4a8107 100644
--- a/CryptoPkg/Library/OpensslLib/EDKII_openssl-1.0.2f.patch
+++ b/CryptoPkg/Library/OpensslLib/EDKII_openssl-1.0.2f.patch
@@ -201,6 +201,63 @@ index abc6dc3..3a672e9 100644
  

  # define M_ASN1_New(arg,func) \

          if (((arg)=func()) == NULL) return(NULL)

+diff --git a/crypto/bn/bn_prime.c b/crypto/bn/bn_prime.c

+index 1d25687..e933ead 100644

+--- a/crypto/bn/bn_prime.c

++++ b/crypto/bn/bn_prime.c

+@@ -131,7 +131,7 @@

+ static int witness(BIGNUM *w, const BIGNUM *a, const BIGNUM *a1,

+                    const BIGNUM *a1_odd, int k, BN_CTX *ctx,

+                    BN_MONT_CTX *mont);

+-static int probable_prime(BIGNUM *rnd, int bits);

++static int probable_prime(BIGNUM *rnd, int bits, prime_t *mods);

+ static int probable_prime_dh(BIGNUM *rnd, int bits,

+                              const BIGNUM *add, const BIGNUM *rem,

+                              BN_CTX *ctx);

+@@ -166,9 +166,13 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,

+     BIGNUM *t;

+     int found = 0;

+     int i, j, c1 = 0;

+-    BN_CTX *ctx;

++    BN_CTX *ctx = NULL;

++    prime_t *mods = NULL;

+     int checks = BN_prime_checks_for_size(bits);

+ 

++    mods = OPENSSL_malloc(sizeof(*mods) * NUMPRIMES);

++    if (mods == NULL)

++	    goto err;

+     ctx = BN_CTX_new();

+     if (ctx == NULL)

+         goto err;

+@@ -179,7 +183,7 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,

+  loop:

+     /* make a random number and set the top and bottom bits */

+     if (add == NULL) {

+-        if (!probable_prime(ret, bits))

++        if (!probable_prime(ret, bits, mods))

+             goto err;

+     } else {

+         if (safe) {

+@@ -230,6 +234,7 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,

+     /* we have a prime :-) */

+     found = 1;

+  err:

++    OPENSSL_free(mods);

+     if (ctx != NULL) {

+         BN_CTX_end(ctx);

+         BN_CTX_free(ctx);

+@@ -375,10 +380,9 @@ static int witness(BIGNUM *w, const BIGNUM *a, const BIGNUM *a1,

+     return 1;

+ }

+ 

+-static int probable_prime(BIGNUM *rnd, int bits)

++static int probable_prime(BIGNUM *rnd, int bits, prime_t *mods)

+ {

+     int i;

+-    prime_t mods[NUMPRIMES];

+     BN_ULONG delta, maxdelta;

+ 

+  again:

 diff --git a/crypto/conf/conf.h b/crypto/conf/conf.h

 index 8d926d5..41cf38e 100644

 --- a/crypto/conf/conf.h

@@ -752,20 +809,29 @@ index 5747c73..fe465cc 100644
   * These functions write a private key in PKCS#8 format: it is a "drop in"

   * replacement for PEM_write_bio_PrivateKey() and friends. As usual if 'enc'

 diff --git a/crypto/pkcs7/pk7_smime.c b/crypto/pkcs7/pk7_smime.c

-index c4d3724..fd531c9 100644

+index c4d3724..0bc3d43 100644

 --- a/crypto/pkcs7/pk7_smime.c

 +++ b/crypto/pkcs7/pk7_smime.c

-@@ -254,7 +254,8 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store,

+@@ -64,6 +64,9 @@

+ #include <openssl/x509.h>

+ #include <openssl/x509v3.h>

+ 

++

++#define BUFFERSIZE 4096

++

+ static int pkcs7_copy_existing_digest(PKCS7 *p7, PKCS7_SIGNER_INFO *si);

+ 

+ PKCS7 *PKCS7_sign(X509 *signcert, EVP_PKEY *pkey, STACK_OF(X509) *certs,

+@@ -254,7 +257,7 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store,

      STACK_OF(PKCS7_SIGNER_INFO) *sinfos;

      PKCS7_SIGNER_INFO *si;

      X509_STORE_CTX cert_ctx;

 -    char buf[4096];

 +    char *buf = NULL;

-+    int bufsiz;

      int i, j = 0, k, ret = 0;

      BIO *p7bio = NULL;

      BIO *tmpin = NULL, *tmpout = NULL;

-@@ -274,12 +275,29 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store,

+@@ -274,12 +277,29 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store,

          PKCS7err(PKCS7_F_PKCS7_VERIFY, PKCS7_R_NO_CONTENT);

          return 0;

      }

@@ -795,32 +861,84 @@ index c4d3724..fd531c9 100644
  

      sinfos = PKCS7_get_signer_info(p7);

  

-@@ -355,9 +373,14 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store,

-     } else

+@@ -356,8 +376,12 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store,

          tmpout = out;

  

-+    bufsiz = 4096;

-+    buf = OPENSSL_malloc(bufsiz);

-+    if (buf == NULL) {

+     /* We now have to 'read' from p7bio to calculate digests etc. */

++    if ((buf = OPENSSL_malloc(BUFFERSIZE)) == NULL) {

++        PKCS7err(PKCS7_F_PKCS7_VERIFY, ERR_R_MALLOC_FAILURE);

 +        goto err;

 +    }

-     /* We now have to 'read' from p7bio to calculate digests etc. */

      for (;;) {

 -        i = BIO_read(p7bio, buf, sizeof(buf));

-+        i = BIO_read(p7bio, buf, bufsiz);

++        i = BIO_read(p7bio, buf, BUFFERSIZE);

          if (i <= 0)

              break;

          if (tmpout)

-@@ -394,6 +417,9 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store,

-     }

-     BIO_free_all(p7bio);

-     sk_X509_free(signers);

-+    if (buf != NULL) {

-+        OPENSSL_free(buf);

+@@ -388,6 +412,7 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store,

+     ret = 1;

+ 

+  err:

++    OPENSSL_free(buf);

+     if (tmpin == indata) {

+         if (indata)

+             BIO_pop(p7bio);

+@@ -506,7 +531,7 @@ int PKCS7_decrypt(PKCS7 *p7, EVP_PKEY *pkey, X509 *cert, BIO *data, int flags)

+ {

+     BIO *tmpmem;

+     int ret, i;

+-    char buf[4096];

++    char *buf = NULL;

+ 

+     if (!p7) {

+         PKCS7err(PKCS7_F_PKCS7_DECRYPT, PKCS7_R_INVALID_NULL_POINTER);

+@@ -550,24 +575,29 @@ int PKCS7_decrypt(PKCS7 *p7, EVP_PKEY *pkey, X509 *cert, BIO *data, int flags)

+         }

+         BIO_free_all(bread);

+         return ret;

+-    } else {

+-        for (;;) {

+-            i = BIO_read(tmpmem, buf, sizeof(buf));

+-            if (i <= 0) {

+-                ret = 1;

+-                if (BIO_method_type(tmpmem) == BIO_TYPE_CIPHER) {

+-                    if (!BIO_get_cipher_status(tmpmem))

+-                        ret = 0;

+-                }

+-

+-                break;

+-            }

+-            if (BIO_write(data, buf, i) != i) {

+-                ret = 0;

+-                break;

 +    }

-     return ret;

++    if ((buf = OPENSSL_malloc(BUFFERSIZE)) == NULL) {

++        PKCS7err(PKCS7_F_PKCS7_DECRYPT, ERR_R_MALLOC_FAILURE);

++        goto err;

++    }

++    for (;;) {

++        i = BIO_read(tmpmem, buf, BUFFERSIZE);

++        if (i <= 0) {

++            ret = 1;

++            if (BIO_method_type(tmpmem) == BIO_TYPE_CIPHER) {

++                if (!BIO_get_cipher_status(tmpmem))

++                    ret = 0;

+             }

++

++            break;

++        }

++        if (BIO_write(data, buf, i) != i) {

++            ret = 0;

++            break;

+         }

+-        BIO_free_all(tmpmem);

+-        return ret;

+     }

++err:

++    OPENSSL_free(buf);

++    BIO_free_all(tmpmem);

++    return ret;

  }

- 

 diff --git a/crypto/rand/rand_unix.c b/crypto/rand/rand_unix.c

 index 266111e..f60fac6 100644

 --- a/crypto/rand/rand_unix.c