From 3a82388f883a057697c1fbf02a478f201c727a7e Mon Sep 17 00:00:00 2001 From: Marcin Szymborski Date: Thu, 9 Jul 2020 11:58:27 +0200 Subject: [PATCH] Add maxResultBuffer property. --- src/com/mysql/jdbc/ConnectionProperties.java | 4 + .../mysql/jdbc/ConnectionPropertiesImpl.java | 11 + .../jdbc/LocalizedErrorMessages.properties | 3 + .../mysql/jdbc/MultiHostMySQLConnection.java | 8 + src/com/mysql/jdbc/MysqlIO.java | 80 +++++- src/com/mysql/jdbc/counters/Counter.java | 44 +++ .../counters/ResultByteBufferCounter.java | 64 +++++ .../jdbc2/optional/ConnectionWrapper.java | 8 + ...nnectionPropertyMaxResultBufferParser.java | 251 ++++++++++++++++++ .../regression/ResultSetRegressionTest.java | 64 +++++ ...tionPropertyMaxResultBufferParserTest.java | 87 ++++++ 11 files changed, 616 insertions(+), 8 deletions(-) create mode 100644 src/com/mysql/jdbc/counters/Counter.java create mode 100644 src/com/mysql/jdbc/counters/ResultByteBufferCounter.java create mode 100644 src/com/mysql/jdbc/util/ConnectionPropertyMaxResultBufferParser.java create mode 100644 src/testsuite/util/ConnectionPropertyMaxResultBufferParserTest.java diff --git a/src/com/mysql/jdbc/ConnectionProperties.java b/src/com/mysql/jdbc/ConnectionProperties.java index 79ae5388..283a5035 100644 --- a/src/com/mysql/jdbc/ConnectionProperties.java +++ b/src/com/mysql/jdbc/ConnectionProperties.java @@ -1433,4 +1433,8 @@ public boolean getEnableEscapeProcessing(); public void setEnableEscapeProcessing(boolean flag); + + public String getMaxResultBuffer(); + + public void setMaxResultBuffer(String maxResultBuffer); } diff --git a/src/com/mysql/jdbc/ConnectionPropertiesImpl.java b/src/com/mysql/jdbc/ConnectionPropertiesImpl.java index 8dd12950..babb706d 100644 --- a/src/com/mysql/jdbc/ConnectionPropertiesImpl.java +++ b/src/com/mysql/jdbc/ConnectionPropertiesImpl.java @@ -1340,6 +1340,9 @@ public ExceptionInterceptor getExceptionInterceptor() { private BooleanConnectionProperty enableEscapeProcessing = new BooleanConnectionProperty("enableEscapeProcessing", true, Messages.getString("ConnectionProperties.enableEscapeProcessing"), "5.1.37", PERFORMANCE_CATEGORY, Integer.MIN_VALUE); + private StringConnectionProperty maxResultBuffer = new StringConnectionProperty("maxResultBuffer", null, + Messages.getString("ConnectionProperties.maxResultBuffer"), "5.1.50", PERFORMANCE_CATEGORY, Integer.MIN_VALUE); + protected DriverPropertyInfo[] exposeAsDriverPropertyInfoInternal(Properties info, int slotsToReserve) throws SQLException { initializeProperties(info); @@ -4992,4 +4995,12 @@ public boolean getEnableEscapeProcessing() { public void setEnableEscapeProcessing(boolean flag) { this.enableEscapeProcessing.setValue(flag); } + + public String getMaxResultBuffer() { + return this.maxResultBuffer.getValueAsString(); + } + + public void setMaxResultBuffer(String maxResultBuffer) { + this.maxResultBuffer.setValue(maxResultBuffer); + } } diff --git a/src/com/mysql/jdbc/LocalizedErrorMessages.properties b/src/com/mysql/jdbc/LocalizedErrorMessages.properties index aa6cea01..6a7ea83c 100644 --- a/src/com/mysql/jdbc/LocalizedErrorMessages.properties +++ b/src/com/mysql/jdbc/LocalizedErrorMessages.properties @@ -663,6 +663,7 @@ ConnectionProperties.readOnlyPropagatesToServer=Should the driver issue appropri ConnectionProperties.enabledSSLCipherSuites=If "useSSL" is set to "true", overrides the cipher suites enabled for use on the underlying SSL sockets. This may be required when using external JSSE providers or to specify cipher suites compatible with both MySQL server and used JVM. ConnectionProperties.enabledTLSProtocols=If "useSSL" is set to "true", overrides the TLS protocols enabled for use on the underlying SSL sockets. This may be used to restrict connections to specific TLS versions. ConnectionProperties.enableEscapeProcessing=Sets the default escape processing behavior for Statement objects. The method Statement.setEscapeProcessing() can be used to specify the escape processing behavior for an individual Statement object. Default escape processing behavior in prepared statements must be defined with the property 'processEscapeCodesForPrepStmts'. +ConnectionProperties.maxResultBuffer=Specifies size of buffer during fetching result set. Can be specified as specified size or percent of heap memory. # # Error Messages for Connection Properties @@ -708,3 +709,5 @@ ReplicationConnectionProxy.badValueForReadFromMasterWhenNoSlaves=Bad value ''{0} ReplicationConnectionProxy.badValueForReplicationEnableJMX=Bad value ''{0}'' for property "replicationEnableJMX". ReplicationConnectionProxy.initializationWithEmptyHostsLists=A replication connection cannot be initialized without master hosts and slave hosts, simultaneously. ReplicationConnectionProxy.noHostsInconsistentState=The replication connection is an inconsistent state due to non existing hosts in both its internal hosts lists. + +PropertyDefinition.1=The connection property ''{0}'' acceptable patterns are: {1}. The value ''{2}'' is not acceptable. \ No newline at end of file diff --git a/src/com/mysql/jdbc/MultiHostMySQLConnection.java b/src/com/mysql/jdbc/MultiHostMySQLConnection.java index 8e3e2717..5527a11f 100644 --- a/src/com/mysql/jdbc/MultiHostMySQLConnection.java +++ b/src/com/mysql/jdbc/MultiHostMySQLConnection.java @@ -2495,6 +2495,14 @@ public void setEnableEscapeProcessing(boolean flag) { getActiveMySQLConnection().setEnableEscapeProcessing(flag); } + public String getMaxResultBuffer() { + return getActiveMySQLConnection().getMaxResultBuffer(); + } + + public void setMaxResultBuffer(String maxResultBuffer) { + getActiveMySQLConnection().setMaxResultBuffer(maxResultBuffer); + } + public boolean isUseSSLExplicit() { return getActiveMySQLConnection().isUseSSLExplicit(); } diff --git a/src/com/mysql/jdbc/MysqlIO.java b/src/com/mysql/jdbc/MysqlIO.java index 8f85cff3..e70376d9 100644 --- a/src/com/mysql/jdbc/MysqlIO.java +++ b/src/com/mysql/jdbc/MysqlIO.java @@ -59,12 +59,17 @@ import com.mysql.jdbc.authentication.MysqlNativePasswordPlugin; import com.mysql.jdbc.authentication.MysqlOldPasswordPlugin; import com.mysql.jdbc.authentication.Sha256PasswordPlugin; +import com.mysql.jdbc.counters.Counter; +import com.mysql.jdbc.counters.ResultByteBufferCounter; import com.mysql.jdbc.exceptions.MySQLStatementCancelledException; import com.mysql.jdbc.exceptions.MySQLTimeoutException; import com.mysql.jdbc.profiler.ProfilerEvent; +import com.mysql.jdbc.util.ConnectionPropertyMaxResultBufferParser; import com.mysql.jdbc.util.ReadAheadInputStream; import com.mysql.jdbc.util.ResultSetUtil; +import javax.xml.parsers.ParserConfigurationException; + /** * This class is used by Connection for communicating with the MySQL server. */ @@ -247,6 +252,8 @@ private ExceptionInterceptor exceptionInterceptor; private int authPluginDataLength = 0; + private final Counter resultByteCounter; + /** * Constructor: Connect to the MySQL server and setup a stream connection. * @@ -331,8 +338,17 @@ public MysqlIO(String host, int port, Properties props, String socketFactoryClas if (this.connection.getLogSlowQueries()) { calculateSlowQueryThreshold(); } + + //set up threshold for query result buffer + long maxResultBuffer = ConnectionPropertyMaxResultBufferParser + .parseProperty(props.getProperty("maxResultBuffer")); + + this.resultByteCounter = new ResultByteBufferCounter(maxResultBuffer); + } catch (IOException ioEx) { throw SQLError.createCommunicationsException(this.connection, 0, 0, ioEx, getExceptionInterceptor()); + } catch (ParserConfigurationException e) { + throw SQLError.createSQLException(e.getMessage(), SQLError.SQL_STATE_INVALID_CONNECTION_ATTRIBUTE, getExceptionInterceptor()); } } @@ -864,6 +880,11 @@ protected Buffer checkErrorPacket() throws SQLException { return checkErrorPacket(-1); } + + private Buffer checkErrorPacket(boolean isDataLoading) throws SQLException { + return checkErrorPacket(-1, isDataLoading); + } + /** * Determines if the database charset is the same as the platform charset */ @@ -1989,7 +2010,7 @@ final ResultSetRow nextRow(Field[] fields, int columnCount, boolean isBinaryEnco Buffer rowPacket = null; if (existingRowPacket == null) { - rowPacket = checkErrorPacket(); + rowPacket = checkErrorPacket(true); if (!useBufferRowExplicit && useBufferRowIfPossible) { if (rowPacket.getBufLength() > this.useBufferRowSizeThreshold) { @@ -2030,6 +2051,8 @@ final ResultSetRow nextRow(Field[] fields, int columnCount, boolean isBinaryEnco readServerStatusForResultSets(rowPacket); + resultByteCounter.resetCounter(); + return null; } @@ -2051,6 +2074,8 @@ final ResultSetRow nextRow(Field[] fields, int columnCount, boolean isBinaryEnco rowPacket.setPosition(rowPacket.getPosition() - 1); readServerStatusForResultSets(rowPacket); + resultByteCounter.resetCounter(); + return null; } @@ -2068,7 +2093,7 @@ final ResultSetRow nextRowFast(Field[] fields, int columnCount, boolean isBinary // Have we stumbled upon a multi-packet? if (packetLength == this.maxThreeBytes) { - reuseAndReadPacket(this.reusablePacket, packetLength); + reuseAndReadPacket(this.reusablePacket, packetLength, true); // Go back to "old" way which uses packets return nextRow(fields, columnCount, isBinaryEncoded, resultSetConcurrency, useBufferRowIfPossible, useBufferRowExplicit, canReuseRowPacket, @@ -2078,7 +2103,7 @@ final ResultSetRow nextRowFast(Field[] fields, int columnCount, boolean isBinary // Does this go over the threshold where we should use a BufferRow? if (packetLength > this.useBufferRowSizeThreshold) { - reuseAndReadPacket(this.reusablePacket, packetLength); + reuseAndReadPacket(this.reusablePacket, packetLength, true); // Go back to "old" way which uses packets return nextRow(fields, columnCount, isBinaryEncoded, resultSetConcurrency, true, true, false, this.reusablePacket); @@ -2160,6 +2185,8 @@ final ResultSetRow nextRowFast(Field[] fields, int columnCount, boolean isBinary } } + this.resultByteCounter.resetCounter(); + return null; // last data packet } @@ -2203,6 +2230,7 @@ final ResultSetRow nextRowFast(Field[] fields, int columnCount, boolean isBinary } else if (len == 0) { rowData[i] = Constants.EMPTY_BYTE_ARRAY; } else { + this.resultByteCounter.increaseCounter(len); rowData[i] = new byte[len]; int bytesRead = readFully(this.mysqlInput, rowData[i], 0, len); @@ -3414,10 +3442,14 @@ private void reclaimLargeReusablePacket() { * @throws SQLException */ private final Buffer reuseAndReadPacket(Buffer reuse) throws SQLException { - return reuseAndReadPacket(reuse, -1); + return reuseAndReadPacket(reuse, -1, false); + } + + private final Buffer reuseAndReadPacket(Buffer reuse, boolean isDataLoading) throws SQLException { + return reuseAndReadPacket(reuse, -1, isDataLoading); } - private final Buffer reuseAndReadPacket(Buffer reuse, int existingPacketLength) throws SQLException { + private final Buffer reuseAndReadPacket(Buffer reuse, int existingPacketLength, boolean isDataLoading) throws SQLException { try { reuse.setWasMultiPacket(false); @@ -3436,6 +3468,8 @@ private final Buffer reuseAndReadPacket(Buffer reuse, int existingPacketLength) packetLength = existingPacketLength; } + increaseCounterIfReadingData(isDataLoading, packetLength); + if (this.traceProtocol) { StringBuilder traceMessageBuf = new StringBuilder(); @@ -3501,7 +3535,7 @@ private final Buffer reuseAndReadPacket(Buffer reuse, int existingPacketLength) // it's multi-packet isMultiPacket = true; - packetLength = readRemainingMultiPackets(reuse, multiPacketSeq); + packetLength = readRemainingMultiPackets(reuse, multiPacketSeq, isDataLoading); } if (!isMultiPacket) { @@ -3531,7 +3565,7 @@ private final Buffer reuseAndReadPacket(Buffer reuse, int existingPacketLength) } - private int readRemainingMultiPackets(Buffer reuse, byte multiPacketSeq) throws IOException, SQLException { + private int readRemainingMultiPackets(Buffer reuse, byte multiPacketSeq, boolean isDataLoading) throws IOException, SQLException { int packetLength = -1; Buffer multiPacket = null; @@ -3543,6 +3577,9 @@ private int readRemainingMultiPackets(Buffer reuse, byte multiPacketSeq) throws } packetLength = (this.packetHeaderBuf[0] & 0xff) + ((this.packetHeaderBuf[1] & 0xff) << 8) + ((this.packetHeaderBuf[2] & 0xff) << 16); + + increaseCounterIfReadingData(isDataLoading, packetLength); + if (multiPacket == null) { multiPacket = new Buffer(packetLength); } @@ -3584,6 +3621,12 @@ private int readRemainingMultiPackets(Buffer reuse, byte multiPacketSeq) throws return packetLength; } + private void increaseCounterIfReadingData(boolean isDataLoading, int packetLength) throws IOException { + if (isDataLoading) { + resultByteCounter.increaseCounter(packetLength); + } + } + /** * @param multiPacketSeq * @throws CommunicationsException @@ -3850,6 +3893,23 @@ private final ResultSetImpl sendFileToServer(StatementImpl callingStatement, Str * @throws CommunicationsException */ private Buffer checkErrorPacket(int command) throws SQLException { + return checkErrorPacket(command, false); + } + + /** + * Checks for errors in the reply packet, and if none, returns the reply + * packet, ready for reading + * + * @param command + * the command being issued (if used) + * @param isDataLoading + * flag if result data is reading + * + * @throws SQLException + * if an error packet was received + * @throws CommunicationsException + */ + private Buffer checkErrorPacket(int command, boolean isDataLoading) throws SQLException { //int statusCode = 0; Buffer resultPacket = null; this.serverStatus = 0; @@ -3857,7 +3917,7 @@ private Buffer checkErrorPacket(int command) throws SQLException { try { // Check return value, if we get a java.io.EOFException, the server has gone away. We'll pass it on up the exception chain and let someone higher up // decide what to do (barf, reconnect, etc). - resultPacket = reuseAndReadPacket(this.reusablePacket); + resultPacket = reuseAndReadPacket(this.reusablePacket, isDataLoading); } catch (SQLException sqlEx) { // Don't wrap SQL Exceptions throw sqlEx; @@ -5013,4 +5073,8 @@ private void appendCharsetByteForHandshake(Buffer packet, String enc) throws SQL public boolean isEOFDeprecated() { return (this.clientParam & CLIENT_DEPRECATE_EOF) != 0; } + + public Counter getResultByteCounter() { + return this.resultByteCounter; + } } diff --git a/src/com/mysql/jdbc/counters/Counter.java b/src/com/mysql/jdbc/counters/Counter.java new file mode 100644 index 00000000..239e58e9 --- /dev/null +++ b/src/com/mysql/jdbc/counters/Counter.java @@ -0,0 +1,44 @@ +/* + Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + + The MySQL Connector/J is licensed under the terms of the GPLv2 + , like most MySQL Connectors. + There are special exceptions to the terms and conditions of the GPLv2 as it is applied to + this software, see the FOSS License Exception + . + + This program is free software; you can redistribute it and/or modify it under the terms + of the GNU General Public License as published by the Free Software Foundation; version 2 + of the License. + + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with this + program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth + Floor, Boston, MA 02110-1301 USA + + */ + +package com.mysql.jdbc.counters; + +import java.io.IOException; + +public interface Counter { + + /** + * Increases counter of reading query result bytes + * + * @param count + * count of query result bytes + * @throws IOException + * throw, when query result is larger then threshold + */ + void increaseCounter(long count) throws IOException; + + /** + * Reset counter to 0 + */ + void resetCounter(); +} diff --git a/src/com/mysql/jdbc/counters/ResultByteBufferCounter.java b/src/com/mysql/jdbc/counters/ResultByteBufferCounter.java new file mode 100644 index 00000000..0e182388 --- /dev/null +++ b/src/com/mysql/jdbc/counters/ResultByteBufferCounter.java @@ -0,0 +1,64 @@ +/* + Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + + The MySQL Connector/J is licensed under the terms of the GPLv2 + , like most MySQL Connectors. + There are special exceptions to the terms and conditions of the GPLv2 as it is applied to + this software, see the FOSS License Exception + . + + This program is free software; you can redistribute it and/or modify it under the terms + of the GNU General Public License as published by the Free Software Foundation; version 2 + of the License. + + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with this + program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth + Floor, Boston, MA 02110-1301 USA + + */ + +package com.mysql.jdbc.counters; + + +import java.io.IOException; + +/** + * Check if query result is not larger then max result buffer size threshold + */ +public class ResultByteBufferCounter implements Counter { + + private long resultByteBufferCounter; + private long maxResultBuffer; + + public ResultByteBufferCounter(long maxResultBuffer) { + this.resultByteBufferCounter = 0; + this.maxResultBuffer = maxResultBuffer; + } + + public void increaseCounter(long count) throws IOException { + if (this.maxResultBuffer != -1) { + this.resultByteBufferCounter += count; + if (this.resultByteBufferCounter > this.maxResultBuffer) { + long counterValue = this.resultByteBufferCounter; + resetCounter(); + throw new IOException(new StringBuilder("Result set exceeded maxResultBuffer limit. Received: ") + .append(counterValue) + .append("; Current limit: ") + .append(this.maxResultBuffer) + .toString()); + } + } + } + + public void resetCounter() { + this.resultByteBufferCounter = 0; + } + + public long getResultByteBufferCounter() { + return this.resultByteBufferCounter; + } +} diff --git a/src/com/mysql/jdbc/jdbc2/optional/ConnectionWrapper.java b/src/com/mysql/jdbc/jdbc2/optional/ConnectionWrapper.java index 9e5440b7..2507a4df 100644 --- a/src/com/mysql/jdbc/jdbc2/optional/ConnectionWrapper.java +++ b/src/com/mysql/jdbc/jdbc2/optional/ConnectionWrapper.java @@ -2898,6 +2898,14 @@ public void setEnableEscapeProcessing(boolean flag) { this.mc.setEnableEscapeProcessing(flag); } + public String getMaxResultBuffer() { + return this.mc.getMaxResultBuffer(); + } + + public void setMaxResultBuffer(String maxResultBuffer) { + this.mc.setMaxResultBuffer(maxResultBuffer); + } + public boolean isUseSSLExplicit() { return this.mc.isUseSSLExplicit(); } diff --git a/src/com/mysql/jdbc/util/ConnectionPropertyMaxResultBufferParser.java b/src/com/mysql/jdbc/util/ConnectionPropertyMaxResultBufferParser.java new file mode 100644 index 00000000..3ae375f2 --- /dev/null +++ b/src/com/mysql/jdbc/util/ConnectionPropertyMaxResultBufferParser.java @@ -0,0 +1,251 @@ +/* + Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + + The MySQL Connector/J is licensed under the terms of the GPLv2 + , like most MySQL Connectors. + There are special exceptions to the terms and conditions of the GPLv2 as it is applied to + this software, see the FOSS License Exception + . + + This program is free software; you can redistribute it and/or modify it under the terms + of the GNU General Public License as published by the Free Software Foundation; version 2 + of the License. + + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with this + program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth + Floor, Boston, MA 02110-1301 USA + + */ + +package com.mysql.jdbc.util; + +import java.lang.management.ManagementFactory; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.xml.parsers.ParserConfigurationException; + +import src.com.mysql.jdbc.Messages; + +public class ConnectionPropertyMaxResultBufferParser { + + private static final Logger LOGGER = Logger.getLogger(ConnectionPropertyMaxResultBufferParser.class.getName()); + + private static final String[] PERCENT_PHRASES = new String[] { "p", "pct", "percent" }; + public static final String MAX_RESULT_BUFFER_ACCETABLE_PATTERNS = "10 - bytes; 10K - kilobytes; 10M - megabytes; 10G - gigabytes; 10T - terabytes; 10p, 10pct, 10percent - percentage of heap memory"; + + /** + * Method to parse value of max result buffer size. + * + * @param value + * string containing size of bytes with optional multiplier (T, G, M or K) or percent + * value to declare max percent of heap memory to use. + * @return + * value of max result buffer size. + * @throws ParserConfigurationException + * when given value can't be parsed. + */ + public static long parseProperty(String value) throws ParserConfigurationException { + long result = -1; + if (checkIfValueContainsPercent(value)) { + result = parseBytePercentValue(value); + } else if (checkIfValueExistsToBeParsed(value)) { + result = parseByteValue(value); + } + result = adjustResultSize(result); + return result; + } + + /** + * Method to check if given value can contain percent declaration of size of max result buffer. + * + * @param value + * Value to check. + * @return + * Result if value contains percent. + */ + private static boolean checkIfValueContainsPercent(String value) { + return (value != null) && (getPercentPhraseLengthIfContains(value) != -1); + } + + /** + * Method to get percent value of max result buffer size dependable on actual free memory. This + * method doesn't check other possibilities of value declaration. + * + * @param value + * string containing percent used to define max result buffer. + * @return + * percent value of max result buffer size. + * @throws ParserConfigurationException + * Exception when given value can't be parsed. + */ + private static long parseBytePercentValue(String value) throws ParserConfigurationException { + long result = -1; + int length; + + if (checkIfValueExistsToBeParsed(value)) { + length = getPercentPhraseLengthIfContains(value); + + if (length == -1) { + throw new ParserConfigurationException( + Messages.getString("PropertyDefinition.1", new Object[] { "maxResultBuffer", MAX_RESULT_BUFFER_ACCETABLE_PATTERNS, value })); + } + + result = calculatePercentOfMemory(value, length); + } + return result; + } + + /** + * Method to get length of percent phrase existing in given string, only if one of phrases exist + * on the length of string. + * + * @param valueToCheck + * String which is gonna be checked if contains percent phrase. + * @return + * Length of phrase inside string, returns -1 when no phrase found. + */ + private static int getPercentPhraseLengthIfContains(String valueToCheck) { + int result = -1; + for (String phrase : PERCENT_PHRASES) { + int indx = getPhraseLengthIfContains(valueToCheck, phrase); + if (indx != -1) { + result = indx; + } + } + return result; + } + + /** + * Method to get length of given phrase in given string to check, method checks if phrase exist on + * the end of given string. + * + * @param valueToCheck + * String which gonna be checked if contains phrase. + * @param phrase + * Phrase to be looked for on the end of given string. + * @return + * Length of phrase inside string, returns -1 when phrase wasn't found. + */ + private static int getPhraseLengthIfContains(String valueToCheck, String phrase) { + int searchValueLength = phrase.length(); + + if (valueToCheck.length() > searchValueLength) { + String subValue = valueToCheck.substring(valueToCheck.length() - searchValueLength); + if (subValue.equals(phrase)) { + return searchValueLength; + } + } + return -1; + } + + /** + * Method to calculate percent of given max heap memory. + * + * @param value + * String which contains percent + percent phrase which gonna be use during calculations. + * @param percentPhraseLength + * Length of percent phrase inside given value. + * @return + * Size of byte buffer based on percent of max heap memory. + */ + private static long calculatePercentOfMemory(String value, int percentPhraseLength) { + String realValue = value.substring(0, value.length() - percentPhraseLength); + double percent = Double.parseDouble(realValue) / 100; + long result = (long) (percent * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()); + return result; + } + + /** + * Method to check if given value has any chars to be parsed. + * + * @param value + * Value to be checked. + * @return + * Result if value can be parsed. + */ + private static boolean checkIfValueExistsToBeParsed(String value) { + return value != null && value.length() != 0; + } + + /** + * Method to get size based on given string value. String can contains just a number or number + + * multiplier sign (like T, G, M or K). + * + * @param value + * Given string to be parsed. + * @return + * Size based on given string. + * @throws ParserConfigurationException + * Exception when given value can't be parsed. + */ + private static long parseByteValue(String value) throws ParserConfigurationException { + long result = -1; + long multiplier = 1; + long mul = 1000; + String realValue; + char sign = value.charAt(value.length() - 1); + + switch (sign) { + + case 'T': + case 't': + multiplier *= mul; + + case 'G': + case 'g': + multiplier *= mul; + + case 'M': + case 'm': + multiplier *= mul; + + case 'K': + case 'k': + multiplier *= mul; + realValue = value.substring(0, value.length() - 1); + result = Integer.parseInt(realValue) * multiplier; + break; + + case '%': + return result; + + default: + if (sign >= '0' && sign <= '9') { + result = Long.parseLong(value); + } else { + + throw new ParserConfigurationException( + Messages.getString("PropertyDefinition.1", new Object[] { "maxResultBuffer", MAX_RESULT_BUFFER_ACCETABLE_PATTERNS, value })); + } + break; + } + return result; + } + + /** + * Method to adjust result memory limit size. If given memory is larger than 90% of max heap + * memory then it gonna be reduced to 90% of max heap memory. + * + * @param value + * Size to be adjusted. + * @return + * Adjusted size (original size or 90% of max heap memory) + */ + private static long adjustResultSize(long value) { + if (value > 0.9 * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()) { + long newResult = (long) (0.9 * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()); + + LOGGER.log(Level.WARNING, new StringBuilder("WARNING! Required to allocate ").append(value) + .append(" bytes, which exceeded possible heap memory size. Assigned ").append(newResult).append(" bytes as limit.").toString()); + + value = newResult; + } + return value; + } + +} diff --git a/src/testsuite/regression/ResultSetRegressionTest.java b/src/testsuite/regression/ResultSetRegressionTest.java index 9d3885fa..ba4e2bd9 100644 --- a/src/testsuite/regression/ResultSetRegressionTest.java +++ b/src/testsuite/regression/ResultSetRegressionTest.java @@ -73,6 +73,7 @@ import com.mysql.jdbc.TimeUtil; import com.mysql.jdbc.Util; +import com.mysql.jdbc.counters.ResultByteBufferCounter; import testsuite.BaseTestCase; import testsuite.BufferingLogger; @@ -6020,4 +6021,67 @@ public void testBug20913289() throws Exception { } } } + + /** + * Verify if ResultByteBufferCounter works properly. + * @throws Exception + */ + public void testMaxResultBuffer() throws Exception { + createTable("testMaxResultBuffer", "(value VARCHAR(10))"); + for (int i=0; i < 200; i++) { + this.stmt.execute("INSERT INTO testMaxResultBuffer(value) VALUES ('123456789X');"); + } + Connection con = null; + Properties props = new Properties(); + try { + props.setProperty("maxResultBuffer", "3000"); + con = getConnectionWithProps(props); + Statement stm = con.createStatement(); + stm.execute("SELECT * FROM testMaxResultBuffer"); + ResultByteBufferCounter counter = (ResultByteBufferCounter)((MySQLConnection)con).getIO().getResultByteCounter(); + assertEquals("The result byte counter should be reset after the data has been successfully retrieved",0,counter.getResultByteBufferCounter()); + } finally { + this.stmt.execute("TRUNCATE TABLE testMaxResultBuffer"); + if (con != null) { + con.close(); + } + } + } + + /** + * Verify if ResultByteBufferCounter throws proper exception + * when reading result larger than maxResultBuffer property value. + * @throws Exception + */ + public void testMaxResultBufferException() throws Exception { + createTable("testMaxResultBuffer", "(value VARCHAR(10))"); + for (int i=0; i < 200; i++) { + this.stmt.execute("INSERT INTO testMaxResultBuffer(value) VALUES ('123456789X');"); + } + + try { + assertThrows("CommunicationsException should be thrown", CommunicationsException.class, + new Callable() { + public Void call() throws Exception { + Connection con = null; + Properties props = new Properties(); + try { + props.setProperty("maxResultBuffer", "1000"); + con = getConnectionWithProps(props); + Statement stm = con.createStatement(); + stm.execute("SELECT * FROM testMaxResultBuffer"); + } finally { + if (con != null) { + con.close(); + } + } + return null; + } + }); + } finally { + this.stmt.execute("TRUNCATE TABLE testMaxResultBuffer"); + } + + } + } diff --git a/src/testsuite/util/ConnectionPropertyMaxResultBufferParserTest.java b/src/testsuite/util/ConnectionPropertyMaxResultBufferParserTest.java new file mode 100644 index 00000000..c9d01000 --- /dev/null +++ b/src/testsuite/util/ConnectionPropertyMaxResultBufferParserTest.java @@ -0,0 +1,87 @@ +/* + Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + + The MySQL Connector/J is licensed under the terms of the GPLv2 + , like most MySQL Connectors. + There are special exceptions to the terms and conditions of the GPLv2 as it is applied to + this software, see the FOSS License Exception + . + + This program is free software; you can redistribute it and/or modify it under the terms + of the GNU General Public License as published by the Free Software Foundation; version 2 + of the License. + + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with this + program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth + Floor, Boston, MA 02110-1301 USA + + */ + +package testsuite.util; + +import java.lang.management.ManagementFactory; +import java.util.Arrays; +import java.util.Collection; +import java.util.concurrent.Callable; + +import javax.xml.parsers.ParserConfigurationException; + +import src.com.mysql.jdbc.util.ConnectionPropertyMaxResultBufferParser; +import src.testsuite.BaseTestCase; + +public class ConnectionPropertyMaxResultBufferParserTest extends BaseTestCase { + + public ConnectionPropertyMaxResultBufferParserTest(String name) { + super(name); + } + + /** + * Runs all test cases in this test suite + * + * @param args + */ + public static void main(String[] args) { + junit.textui.TestRunner.run(ConnectionPropertyMaxResultBufferParserTest.class); + } + + public void testGetMaxResultBufferValue() { + try { + Collection data = data(); + for (Object[] item : data) { + long result = ConnectionPropertyMaxResultBufferParser.parseProperty((String) item[0]); + assertEquals("Expected :" + (Long) item[1] + " get: " + result, ((Long) item[1]).longValue(), result); + } + } catch (ParserConfigurationException e) { + //shouldn't occur + fail(); + } + } + + public void testGetParserConfigurationException() { + assertThrows(ParserConfigurationException.class, new Callable() { + public Void call() throws Exception { + ConnectionPropertyMaxResultBufferParser.parseProperty("abc"); + return null; + } + }); + } + + private Collection data() { + Object[][] data = new Object[][] { { "100", 100L }, { "10K", 10L * 1000 }, { "25M", 25L * 1000 * 1000 }, + //next two should be too big + { "35G", (long) (0.90 * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()) }, + { "1T", (long) (0.90 * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()) }, + //percent test + { "5p", (long) (0.05 * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()) }, + { "10pct", (long) (0.10 * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()) }, + { "15percent", (long) (0.15 * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()) }, + //for testing empty property + { "", -1 }, { null, -1 } }; + return Arrays.asList(data); + } + +}