Description:
I have an app that inserts 24 records every 3 seconds. Each record has 2 blob fields of exactly 1k. I'm using the setBytes method on the PreparedStatement class to set the array into the placeholders. After running for one day the app throws an OutOfMemoryError. I found the problem in the app using the NetBeans Profiler, and there was a leak in this method (PreparedStatement.setBytes), specifically by the use of the toByteArray method at ByteArrayOutputStream class.
The ByteArrayOutputStream class was recognized by Java team of Sun Microsystem as inefficient to release the memory and has been reported in many forums as the source of many memory leaks. Basically the leak comes from the creation of a double buffer (the intern and the returned)
How to repeat:
Develop an app that insert arrays of 1 or 2 Kbytes in a blob using a the method setBytes at PreparedStatement class, in a thread with a while true sleeping for a few milliseconds on each cycle.
Suggested fix:
I patch this method to avoid the using of the ByteArrayOutputStream class. In this patch the process can be a little bit slower because the double "for" over the data, but the memory leak is fixed.
there is my patch:
protected void setBytes(int parameterIndex, byte[] x,
boolean checkForIntroducer, boolean escapeForMBChars)
throws SQLException {
if (x == null) {
setNull(parameterIndex, java.sql.Types.BINARY);
} else {
String connectionEncoding = this.connection.getEncoding();
if (this.connection.isNoBackslashEscapesSet()
|| (escapeForMBChars
&& this.connection.getUseUnicode()
&& connectionEncoding != null
&& CharsetMapping.isMultibyteCharset(connectionEncoding))) {
// Send as hex
byte buffer[]=new byte[(x.length * 2) + 3];
//ByteArrayOutputStream bOut = new ByteArrayOutputStream((x.length * 2) + 3);
int index = 0;
buffer[index ++]='x';
buffer[index ++]='\'';
//bOut.write('x');
//bOut.write('\'');
for (int i = 0; i < x.length; i++) {
int lowBits = (x[i] & 0xff) / 16;
int highBits = (x[i] & 0xff) % 16;
buffer[index++]= (byte)HEX_DIGITS[lowBits];
buffer[index++]= (byte)HEX_DIGITS[highBits];
//bOut.write(HEX_DIGITS[lowBits]);
//bOut.write(HEX_DIGITS[highBits]);
}
buffer[index++]= '\'';
//bOut.write('\'');
setInternal(parameterIndex,/* bOut.toByteArray()*/buffer);
buffer = null;
return;
}
// escape them
int numBytes = x.length;
int pad = 2;
boolean needsIntroducer = checkForIntroducer
&& this.connection.versionMeetsMinimum(4, 1, 0);
if (needsIntroducer) {
pad += 7;
}
//contar la cantidad de caracteres \n,\r,\0,\',\",\032, \\
for(int i=0;i<x.length;i++){
if(x[i]=='\n' || x[i]=='\r' ||
x[i]=='\032' || x[i]=='\'' ||
x[i]=='"' || x[i]=='\\' ||
x[i]==0)
pad++;
}
int index = 0;
byte[] buffer = new byte[numBytes+pad];
//ByteArrayOutputStream bOut = new ByteArrayOutputStream(numBytes + pad);
if (needsIntroducer) {
buffer[index++]=('_');
buffer[index++]=('b');
buffer[index++]=('i');
buffer[index++]=('n');
buffer[index++]=('a');
buffer[index++]=('r');
buffer[index++]=('y');
//bOut.write('_');
//bOut.write('b');
//bOut.write('i');
//bOut.write('n');
//bOut.write('a');
//bOut.write('r');
//bOut.write('y');
}
buffer[index++]='\'';
//bOut.write('\'');
for (int i = 0; i < numBytes; ++i) {
byte b = x[i];
switch (b) {
case 0: /* Must be escaped for 'mysql' */
buffer[index++]='\\';
buffer[index++]='0';
//bOut.write('\\');
//bOut.write('0');
break;
case '\n': /* Must be escaped for logs */
buffer[index++]='\\';
buffer[index++]='n';
//bOut.write('\\');
//bOut.write('n');
break;
case '\r':
buffer[index++]='\\';
buffer[index++]='r';
//bOut.write('\\');
//bOut.write('r');
break;
case '\\':
buffer[index++]='\\';
buffer[index++]='\\';
//bOut.write('\\');
//bOut.write('\\');
break;
case '\'':
buffer[index++]='\\';
buffer[index++]='\'';
//bOut.write('\\');
//bOut.write('\'');
break;
case '"': /* Better safe than sorry */
buffer[index++]='\\';
buffer[index++]='"';
//bOut.write('\\');
//bOut.write('"');
break;
case '\032': /* This gives problems on Win32 */
buffer[index++]='\\';
buffer[index++]='Z';
//bOut.write('\\');
//bOut.write('Z');
break;
default:
buffer[index++]=b;
//bOut.write(b);
}
}
buffer[index++]='\'';
//bOut.write('\'');
setInternal(parameterIndex, /*bOut.toByteArray()*/buffer);
}
}
Description: I have an app that inserts 24 records every 3 seconds. Each record has 2 blob fields of exactly 1k. I'm using the setBytes method on the PreparedStatement class to set the array into the placeholders. After running for one day the app throws an OutOfMemoryError. I found the problem in the app using the NetBeans Profiler, and there was a leak in this method (PreparedStatement.setBytes), specifically by the use of the toByteArray method at ByteArrayOutputStream class. The ByteArrayOutputStream class was recognized by Java team of Sun Microsystem as inefficient to release the memory and has been reported in many forums as the source of many memory leaks. Basically the leak comes from the creation of a double buffer (the intern and the returned) How to repeat: Develop an app that insert arrays of 1 or 2 Kbytes in a blob using a the method setBytes at PreparedStatement class, in a thread with a while true sleeping for a few milliseconds on each cycle. Suggested fix: I patch this method to avoid the using of the ByteArrayOutputStream class. In this patch the process can be a little bit slower because the double "for" over the data, but the memory leak is fixed. there is my patch: protected void setBytes(int parameterIndex, byte[] x, boolean checkForIntroducer, boolean escapeForMBChars) throws SQLException { if (x == null) { setNull(parameterIndex, java.sql.Types.BINARY); } else { String connectionEncoding = this.connection.getEncoding(); if (this.connection.isNoBackslashEscapesSet() || (escapeForMBChars && this.connection.getUseUnicode() && connectionEncoding != null && CharsetMapping.isMultibyteCharset(connectionEncoding))) { // Send as hex byte buffer[]=new byte[(x.length * 2) + 3]; //ByteArrayOutputStream bOut = new ByteArrayOutputStream((x.length * 2) + 3); int index = 0; buffer[index ++]='x'; buffer[index ++]='\''; //bOut.write('x'); //bOut.write('\''); for (int i = 0; i < x.length; i++) { int lowBits = (x[i] & 0xff) / 16; int highBits = (x[i] & 0xff) % 16; buffer[index++]= (byte)HEX_DIGITS[lowBits]; buffer[index++]= (byte)HEX_DIGITS[highBits]; //bOut.write(HEX_DIGITS[lowBits]); //bOut.write(HEX_DIGITS[highBits]); } buffer[index++]= '\''; //bOut.write('\''); setInternal(parameterIndex,/* bOut.toByteArray()*/buffer); buffer = null; return; } // escape them int numBytes = x.length; int pad = 2; boolean needsIntroducer = checkForIntroducer && this.connection.versionMeetsMinimum(4, 1, 0); if (needsIntroducer) { pad += 7; } //contar la cantidad de caracteres \n,\r,\0,\',\",\032, \\ for(int i=0;i<x.length;i++){ if(x[i]=='\n' || x[i]=='\r' || x[i]=='\032' || x[i]=='\'' || x[i]=='"' || x[i]=='\\' || x[i]==0) pad++; } int index = 0; byte[] buffer = new byte[numBytes+pad]; //ByteArrayOutputStream bOut = new ByteArrayOutputStream(numBytes + pad); if (needsIntroducer) { buffer[index++]=('_'); buffer[index++]=('b'); buffer[index++]=('i'); buffer[index++]=('n'); buffer[index++]=('a'); buffer[index++]=('r'); buffer[index++]=('y'); //bOut.write('_'); //bOut.write('b'); //bOut.write('i'); //bOut.write('n'); //bOut.write('a'); //bOut.write('r'); //bOut.write('y'); } buffer[index++]='\''; //bOut.write('\''); for (int i = 0; i < numBytes; ++i) { byte b = x[i]; switch (b) { case 0: /* Must be escaped for 'mysql' */ buffer[index++]='\\'; buffer[index++]='0'; //bOut.write('\\'); //bOut.write('0'); break; case '\n': /* Must be escaped for logs */ buffer[index++]='\\'; buffer[index++]='n'; //bOut.write('\\'); //bOut.write('n'); break; case '\r': buffer[index++]='\\'; buffer[index++]='r'; //bOut.write('\\'); //bOut.write('r'); break; case '\\': buffer[index++]='\\'; buffer[index++]='\\'; //bOut.write('\\'); //bOut.write('\\'); break; case '\'': buffer[index++]='\\'; buffer[index++]='\''; //bOut.write('\\'); //bOut.write('\''); break; case '"': /* Better safe than sorry */ buffer[index++]='\\'; buffer[index++]='"'; //bOut.write('\\'); //bOut.write('"'); break; case '\032': /* This gives problems on Win32 */ buffer[index++]='\\'; buffer[index++]='Z'; //bOut.write('\\'); //bOut.write('Z'); break; default: buffer[index++]=b; //bOut.write(b); } } buffer[index++]='\''; //bOut.write('\''); setInternal(parameterIndex, /*bOut.toByteArray()*/buffer); } }