Description:
This is similar to #29753, just YaSSL acts in different way, when 8k private keys are
read:

How to repeat:
Starting program: /usr/local/mysql-5.0/libexec/mysqld --skip-networking --socket=socket
--datadir=/Users/midom/Tests/certs/data --ssl-ca=ca-cert.pem --ssl-key=server-key.pem
--ssl-cert=server-cert.pem --datadir=/Users/midom/Tests/certs/data/
Reading symbols for shared libraries . done
070713 12:07:56 [Warning] Setting lower_case_table_names=2 because file system for
/Users/midom/Tests/certs/data/ is case insensitive
./../include/block.hpp:146: failed assertion `i < sz_'
#4  0x0039e51b in TaoCrypt::Base64Decoder::Decode (this=0xbffff4c4) at
./../include/block.hpp:146
        e1 = 28 '\034'
        e3 = 44 ','
        b2 = 113 'q'
        e2 = 88 'X'
        e4 = 110 'n'
        b1 = 249 '?'
        b3 = 167 '?'
        bytes = 5
        i = 4304924
        j = 1550
#5  0x0038ad76 in yaSSL::PemToDer (file=0xa000bda0, type=PrivateKey, info=0xbffff564) at
./../taocrypt/include/coding.hpp:80
        header = "-----BEGIN RSA PRIVATE KEY-----", '\0' <repeats 48 times>
        footer = "-----END RSA PRIVATE KEY-----", '\0' <repeats 50 times>
        begin = 32
        end = 6333
        foundEnd = false
        line = "-----END RSA PRIVATE
KEY-----\n\000YhrDRDQtw5p0/7IY3AcNKDUHv+XGn\n\000CH\n\000??$??? ????\005\000"
        tmp = {
  <Check> = {<No data fields>}, 
  members of input_buffer: 
  size_ = 0, 
  current_ = 0, 
  buffer_ = 0x4013200
"MIISKQIBAAKCBAEA1BZYf95sKL+WGiAhVznSV4B1f7g5E41wevaMZYqbIUGmD1/C\nw0+b4SN4D3IktWdbERNnU3A
uDJNiuCw1CI6d1pHk3xQB2T1dxGPtzh/37R+DekhC\nAUyhOBGOmodJybVPfDNCYcToecx43us0KdUpAZ4RDkGHsWE
aozrRpaGfUchdIhQF\n3Mrtg"..., 
  end_ = 0x4014a9d ""
}
        bytes = 0
        der = {
  buffer_ = {
    sz_ = 6301, 
    buffer_ = 0x4016c00
"MIISKQIBAAKCBAEA1BZYf95sKL+WGiAhVznSV4B1f7g5E41wevaMZYqbIUGmD1/C\nw0+b4SN4D3IktWdbERNnU3A
uDJNiuCw1CI6d1pHk3xQB2T1dxGPtzh/37R+DekhC\nAUyhOBGOmodJybVPfDNCYcToecx43us0KdUpAZ4RDkGHsWE
aozrRpaGfUchdIhQF\n3Mrtg"..., 
    allocator_ = {
      <AllocatorBase<TaoCrypt::byte>> = {<No data fields>}, <No data fields>}
  }, 
  current_ = 6300, 
  error_ = {
    what_ = NO_ERROR_E
  }
}
        b64Dec = {
  decoded_ = {
    sz_ = 4652, 
    buffer_ = 0x4018600 "0\202\022)\002\001", 
    allocator_ = {
      <AllocatorBase<TaoCrypt::byte>> = {<No data fields>}, <No data fields>}
  }, 
  coded_ = @0xbffff4b0
}
        sz = 6301
#6  0x00383c0c in yaSSL::read_file (ctx=0x2e00f10, file=0xbffffa59 "server-key.pem",
format=11, type=PrivateKey) at ssl.cpp:95
        info = {
  name =
"\001\000\000\000?\236????3\000\\???\000\017?\002\030????A8\000\020\017?\002C???\v\000\000
\000\002\000\000\000\000\017?\002?\207\216?O??\217\001\000\000\000??\203\217?C\037?\207\21
6?\020\017?\002\020\017?\002", 
  iv = "\000\000\001\000\001\000\000\000\" ?\002?{?\217?8\005?O??\217J??\217???\203", 
  ivSz = 0, 
  set = false
}
        x = (x509 *&) @0x2e00f18: 0x0
        format = 11
        input = (FILE *) 0xa000bda0
#7  0x0038430f in yaSSL_CTX_use_PrivateKey_file (ctx=0x2e00f10, file=0xbffffa59
"server-key.pem", format=11) at ssl.cpp:672
        ctx = (SSL_CTX *) 0x0
        file = 0x0
        format = 0
#8  0x0033eee8 in vio_set_cert_stuff (ctx=0x2e00f10, cert_file=0xbffffa73
"server-cert.pem", key_file=0xbffffa59 "server-key.pem") at viosslfactories.c:98
        _db_func_ = 0x381f08 "\211?\203?\024^]?U\211?\203?\030\213E\b\211\004$?_\206\001"
        _db_file_ = 0xbffff718 "x???+?3"
        _db_level_ = 48238336
        _db_framep_ = (char **) 0x2e00f10
        ctx = (class SSL_CTX *) 0x2e00f10
        key_file = 0xbffffa59 "server-key.pem"
#9  0x0033f62b in new_VioSSLFd (key_file=0xbffffa59 "server-key.pem", cert_file=0xbffffa73
"server-cert.pem", ca_file=0xbffffa43 "ca-cert.pem", ca_path=0x0, cipher=0x0,
method=0x2e00ef0) at viosslfactories.c:281
        dh = (DH *) 0xbffff75c
        ssl_fd = (struct st_VioSSLFd *) 0x2e00f00
        _db_func_ = 0x0
        _db_file_ = 0x103 <Address 0x103 out of bounds>
        _db_level_ = 0
        _db_framep_ = (char **) 0x2e00ef0
#10 0x0033f7d2 in new_VioSSLAcceptorFd (key_file=0xbffffa59 "server-key.pem",
cert_file=0xbffffa73 "server-cert.pem", ca_file=0xbffffa43 "ca-cert.pem", ca_path=0x0,
cipher=0x0) at viosslfactories.c:343
        ssl_fd = (struct st_VioSSLFd *) 0x5c8420
        key_file = 0x0
        cert_file = 0x0
        ca_file = 0x0
        ca_path = 0x0
        cipher = 0x0
#11 0x000898cf in main (argc=8, argv=0xbffff930) at mysqld.cc:3084
        argv = (char **) 0x5c8420
        stack_size = 196608

Suggested fix:
n/a

public key

Attachment: server-cert.pem (, text), 3.14 KiB.

private key

Attachment: server-key.pem (, text), 6.21 KiB.

A patch for this bug has been committed. After review, it may
be pushed to the relevant source trees for release in the next
version. You can access the patch from:

  http://lists.mysql.com/commits/31036

ChangeSet@1.2472, 2007-07-17 14:43:56-04:00, dkatz@damien-katzs-computer.local +5 -0
  Bug #29784  YaSSL assertion failure when reading 8k key.
  
  Fixed the yassl base64 decoding to correctly allocate a maximum decoded buffer size.

Pushed into 5.1.21-beta

Pushed into 5.0.48

Noted in 5.0.48, 5.1.21 changelogs.

An assertion failure occurred within yaSSL for very long keys.