From 67006de245b53a2c5c20ba8d33b9f061dab2c10f Mon Sep 17 00:00:00 2001 From: Marcin Szymborski Date: Tue, 7 Jul 2020 17:16:08 +0200 Subject: [PATCH] Add maxResultBuffer property. --- ...nnectionPropertyMaxResultBufferParser.java | 255 ++++++++++++++++++ .../mysql/cj/conf/PropertyDefinitions.java | 2 + .../java/com/mysql/cj/conf/PropertyKey.java | 1 + .../exceptions/MaxResultBufferException.java | 58 ++++ .../com/mysql/cj/protocol/MessageReader.java | 29 ++ .../cj/protocol/ProtocolEntityReader.java | 10 + .../a/DebugBufferingPacketReader.java | 11 +- .../cj/protocol/a/MultiPacketReader.java | 13 +- .../mysql/cj/protocol/a/NativeProtocol.java | 42 +++ .../protocol/a/ResultByteBufferCounter.java | 72 +++++ .../cj/protocol/a/ResultsetRowReader.java | 14 +- .../cj/protocol/a/SimplePacketReader.java | 23 +- .../protocol/a/TimeTrackingPacketReader.java | 11 +- .../cj/protocol/a/TracingPacketReader.java | 12 +- .../cj/LocalizedErrorMessages.properties | 4 +- ...tionPropertyMaxResultBufferParserTest.java | 94 +++++++ .../cj/protocol/a/SimplePacketReaderTest.java | 44 ++- .../regression/ResultSetRegressionTest.java | 50 ++++ 18 files changed, 731 insertions(+), 14 deletions(-) create mode 100644 src/main/core-api/java/com/mysql/cj/conf/ConnectionPropertyMaxResultBufferParser.java create mode 100644 src/main/core-api/java/com/mysql/cj/exceptions/MaxResultBufferException.java create mode 100644 src/main/protocol-impl/java/com/mysql/cj/protocol/a/ResultByteBufferCounter.java create mode 100644 src/test/java/com/mysql/cj/conf/ConnectionPropertyMaxResultBufferParserTest.java diff --git a/src/main/core-api/java/com/mysql/cj/conf/ConnectionPropertyMaxResultBufferParser.java b/src/main/core-api/java/com/mysql/cj/conf/ConnectionPropertyMaxResultBufferParser.java new file mode 100644 index 00000000..9a8ab226 --- /dev/null +++ b/src/main/core-api/java/com/mysql/cj/conf/ConnectionPropertyMaxResultBufferParser.java @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License, version 2.0, as published by the + * Free Software Foundation. + * + * This program is also distributed with certain software (including but not + * limited to OpenSSL) that is licensed under separate terms, as designated in a + * particular file or component or in included license documentation. The + * authors of MySQL hereby grant you an additional permission to link the + * program and your derivative works with the separately licensed software that + * they have included with MySQL. + * + * Without limiting anything contained in the foregoing, this file, which is + * part of MySQL Connector/J, is also subject to the Universal FOSS Exception, + * version 1.0, a copy of which can be found at + * http://oss.oracle.com/licenses/universal-foss-exception. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, + * for more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package com.mysql.cj.conf; + +import java.lang.management.ManagementFactory; + +import javax.xml.parsers.ParserConfigurationException; + +import com.mysql.cj.log.Log; + +public class ConnectionPropertyMaxResultBufferParser { + + private static Log logger; + + private static final String[] PERCENT_PHRASES = new String[] { "p", "pct", "percent" }; + + /** + * Method to parse value of max result buffer size. + * + * @param value + * string containing size of bytes with optional multiplier (T, G, M or K) or percent + * value to declare max percent of heap memory to use. + * @param log + * reference to log object + * @return + * value of max result buffer size. + * @throws ParserConfigurationException + * when given value can't be parsed. + */ + public static long parseProperty(String value, Log log) throws ParserConfigurationException { + logger = log; + long result = -1; + if (checkIfValueContainsPercent(value)) { + result = parseBytePercentValue(value); + } else if (checkIfValueExistsToBeParsed(value)) { + result = parseByteValue(value); + } + result = adjustResultSize(result); + return result; + } + + /** + * Method to check if given value can contain percent declaration of size of max result buffer. + * + * @param value + * Value to check. + * @return + * Result if value contains percent. + */ + private static boolean checkIfValueContainsPercent(String value) { + return (value != null) && (getPercentPhraseLengthIfContains(value) != -1); + } + + /** + * Method to get percent value of max result buffer size dependable on actual free memory. This + * method doesn't check other possibilities of value declaration. + * + * @param value + * string containing percent used to define max result buffer. + * @return + * percent value of max result buffer size. + * @throws ParserConfigurationException + * Exception when given value can't be parsed. + */ + private static long parseBytePercentValue(String value) throws ParserConfigurationException { + long result = -1; + int length; + + if (checkIfValueExistsToBeParsed(value)) { + length = getPercentPhraseLengthIfContains(value); + + if (length == -1) { + throw new ParserConfigurationException("10 - bytes; 10K - kilobytes; 10M - megabytes; 10G - gigabytes; 10T - terabytes; " + + "10p, 10pct, 10percent - percentage of heap memory"); + } + + result = calculatePercentOfMemory(value, length); + } + return result; + } + + /** + * Method to get length of percent phrase existing in given string, only if one of phrases exist + * on the length of string. + * + * @param valueToCheck + * String which is gonna be checked if contains percent phrase. + * @return + * Length of phrase inside string, returns -1 when no phrase found. + */ + private static int getPercentPhraseLengthIfContains(String valueToCheck) { + int result = -1; + for (String phrase : PERCENT_PHRASES) { + int indx = getPhraseLengthIfContains(valueToCheck, phrase); + if (indx != -1) { + result = indx; + } + } + return result; + } + + /** + * Method to get length of given phrase in given string to check, method checks if phrase exist on + * the end of given string. + * + * @param valueToCheck + * String which gonna be checked if contains phrase. + * @param phrase + * Phrase to be looked for on the end of given string. + * @return + * Length of phrase inside string, returns -1 when phrase wasn't found. + */ + private static int getPhraseLengthIfContains(String valueToCheck, String phrase) { + int searchValueLength = phrase.length(); + + if (valueToCheck.length() > searchValueLength) { + String subValue = valueToCheck.substring(valueToCheck.length() - searchValueLength); + if (subValue.equals(phrase)) { + return searchValueLength; + } + } + return -1; + } + + /** + * Method to calculate percent of given max heap memory. + * + * @param value + * String which contains percent + percent phrase which gonna be use during calculations. + * @param percentPhraseLength + * Length of percent phrase inside given value. + * @return + * Size of byte buffer based on percent of max heap memory. + */ + private static long calculatePercentOfMemory(String value, int percentPhraseLength) { + String realValue = value.substring(0, value.length() - percentPhraseLength); + double percent = Double.parseDouble(realValue) / 100; + long result = (long) (percent * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()); + return result; + } + + /** + * Method to check if given value has any chars to be parsed. + * + * @param value + * Value to be checked. + * @return + * Result if value can be parsed. + */ + private static boolean checkIfValueExistsToBeParsed(String value) { + return value != null && value.length() != 0; + } + + /** + * Method to get size based on given string value. String can contains just a number or number + + * multiplier sign (like T, G, M or K). + * + * @param value + * Given string to be parsed. + * @return + * Size based on given string. + * @throws ParserConfigurationException + * Exception when given value can't be parsed. + */ + private static long parseByteValue(String value) throws ParserConfigurationException { + long result = -1; + long multiplier = 1; + long mul = 1000; + String realValue; + char sign = value.charAt(value.length() - 1); + + switch (sign) { + + case 'T': + case 't': + multiplier *= mul; + + case 'G': + case 'g': + multiplier *= mul; + + case 'M': + case 'm': + multiplier *= mul; + + case 'K': + case 'k': + multiplier *= mul; + realValue = value.substring(0, value.length() - 1); + result = Integer.parseInt(realValue) * multiplier; + break; + + case '%': + return result; + + default: + if (sign >= '0' && sign <= '9') { + result = Long.parseLong(value); + } else { + throw new ParserConfigurationException("10 - bytes; 10K - kilobytes; 10M - megabytes; 10G - gigabytes; 10T - terabytes; " + + "10p, 10pct, 10percent - percentage of heap memory"); + } + break; + } + return result; + } + + /** + * Method to adjust result memory limit size. If given memory is larger than 90% of max heap + * memory then it gonna be reduced to 90% of max heap memory. + * + * @param value + * Size to be adjusted. + * @return + * Adjusted size (original size or 90% of max heap memory) + */ + private static long adjustResultSize(long value) { + if (value > 0.9 * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()) { + long newResult = (long) (0.9 * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()); + + logger.logWarn(new StringBuilder("WARNING! Required to allocate ").append(value) + .append(" bytes, which exceeded possible heap memory size. Assigned ").append(newResult).append(" bytes as limit.").toString()); + + value = newResult; + } + return value; + } +} diff --git a/src/main/core-api/java/com/mysql/cj/conf/PropertyDefinitions.java b/src/main/core-api/java/com/mysql/cj/conf/PropertyDefinitions.java index ed7eb963..2d0f7e1b 100644 --- a/src/main/core-api/java/com/mysql/cj/conf/PropertyDefinitions.java +++ b/src/main/core-api/java/com/mysql/cj/conf/PropertyDefinitions.java @@ -674,6 +674,8 @@ new BooleanPropertyDefinition(PropertyKey.enableEscapeProcessing, DEFAULT_VALUE_TRUE, RUNTIME_MODIFIABLE, Messages.getString("ConnectionProperties.enableEscapeProcessing"), "6.0.1", CATEGORY_PERFORMANCE, Integer.MIN_VALUE), + new StringPropertyDefinition(PropertyKey.maxResultBuffer, DEFAULT_VALUE_NULL_STRING, RUNTIME_MODIFIABLE, + Messages.getString("ConnectionProperties.maxResultBuffer"), "8.0.21", CATEGORY_PERFORMANCE, Integer.MIN_VALUE), // // CATEGORY_DEBUGING_PROFILING diff --git a/src/main/core-api/java/com/mysql/cj/conf/PropertyKey.java b/src/main/core-api/java/com/mysql/cj/conf/PropertyKey.java index be5d4c4d..293a0370 100644 --- a/src/main/core-api/java/com/mysql/cj/conf/PropertyKey.java +++ b/src/main/core-api/java/com/mysql/cj/conf/PropertyKey.java @@ -154,6 +154,7 @@ maxAllowedPacket("maxAllowedPacket", true), // maxQuerySizeToLog("maxQuerySizeToLog", true), // maxReconnects("maxReconnects", true), // + maxResultBuffer("maxResultBuffer", true), maxRows("maxRows", true), // metadataCacheSize("metadataCacheSize", true), // netTimeoutForStreamingResults("netTimeoutForStreamingResults", true), // diff --git a/src/main/core-api/java/com/mysql/cj/exceptions/MaxResultBufferException.java b/src/main/core-api/java/com/mysql/cj/exceptions/MaxResultBufferException.java new file mode 100644 index 00000000..1c9f085d --- /dev/null +++ b/src/main/core-api/java/com/mysql/cj/exceptions/MaxResultBufferException.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License, version 2.0, as published by the + * Free Software Foundation. + * + * This program is also distributed with certain software (including but not + * limited to OpenSSL) that is licensed under separate terms, as designated in a + * particular file or component or in included license documentation. The + * authors of MySQL hereby grant you an additional permission to link the + * program and your derivative works with the separately licensed software that + * they have included with MySQL. + * + * Without limiting anything contained in the foregoing, this file, which is + * part of MySQL Connector/J, is also subject to the Universal FOSS Exception, + * version 1.0, a copy of which can be found at + * http://oss.oracle.com/licenses/universal-foss-exception. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, + * for more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package com.mysql.cj.exceptions; + +/** + * Exception throws when trying to read more data then maxResultBuffer property + */ +public class MaxResultBufferException extends RuntimeException { + + private static final long serialVersionUID = -3541531839780888929L; + + public MaxResultBufferException() { + super(); + } + + public MaxResultBufferException(String message) { + super(message); + } + + public MaxResultBufferException(String message, Throwable cause) { + super(message, cause); + } + + public MaxResultBufferException(Throwable cause) { + super(cause); + } + + protected MaxResultBufferException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/src/main/core-api/java/com/mysql/cj/protocol/MessageReader.java b/src/main/core-api/java/com/mysql/cj/protocol/MessageReader.java index 408a01b2..69974187 100644 --- a/src/main/core-api/java/com/mysql/cj/protocol/MessageReader.java +++ b/src/main/core-api/java/com/mysql/cj/protocol/MessageReader.java @@ -34,6 +34,7 @@ import com.mysql.cj.exceptions.CJOperationNotSupportedException; import com.mysql.cj.exceptions.ExceptionFactory; +import com.mysql.cj.protocol.a.ResultByteBufferCounter; public interface MessageReader { @@ -61,6 +62,25 @@ */ M readMessage(Optional reuse, H header) throws IOException; + /** + * Read message from server into to the given {@link Message} instance or into the new one if not present. + * For asynchronous channel it synchronously reads the next message in the stream, blocking until the message is read fully. + * Could throw CJCommunicationsException wrapping an {@link IOException} during read or parse + * + * @param reuse + * {@link Message} object to reuse. May be ignored by implementation. + * @param header + * {@link MessageHeader} instance + * @param isRowReading + * row reading flag + * @return {@link Message} instance + * @throws IOException + * if an error occurs + */ + default M readMessage(Optional reuse, H header, boolean isRowReading) throws IOException { + throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); + } + /** * Read message from server into to the given {@link Message} instance or into the new one if not present. * For asynchronous channel it synchronously reads the next message in the stream, blocking until the message is read fully. @@ -137,4 +157,13 @@ default void stopAfterNextMessage() { // no-op } + /** + * Set result byte buffer counter if it is null + * + * @param counter + * ResultByteBufferCounter object instance + */ + default void setResultByteBufferCounterIfNoExist(ResultByteBufferCounter counter) { + // no-op + } } diff --git a/src/main/core-api/java/com/mysql/cj/protocol/ProtocolEntityReader.java b/src/main/core-api/java/com/mysql/cj/protocol/ProtocolEntityReader.java index adc5219a..52901163 100644 --- a/src/main/core-api/java/com/mysql/cj/protocol/ProtocolEntityReader.java +++ b/src/main/core-api/java/com/mysql/cj/protocol/ProtocolEntityReader.java @@ -33,6 +33,7 @@ import com.mysql.cj.exceptions.CJOperationNotSupportedException; import com.mysql.cj.exceptions.ExceptionFactory; +import com.mysql.cj.protocol.a.ResultByteBufferCounter; public interface ProtocolEntityReader { @@ -74,4 +75,13 @@ default T read(int maxRows, boolean streamResults, M resultPacket, ColumnDefinit throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not allowed"); } + /** + * Set result byte buffer counter if it is null + * + * @param counter + * ResultByteBufferCounter object instance + */ + default void setResultByteBufferCounterIfNoExist(ResultByteBufferCounter counter) { + throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not allowed"); + } } diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/DebugBufferingPacketReader.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/DebugBufferingPacketReader.java index 6b553a69..83ab4e1a 100644 --- a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/DebugBufferingPacketReader.java +++ b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/DebugBufferingPacketReader.java @@ -96,8 +96,13 @@ public NativePacketHeader readHeader() throws IOException { @Override public NativePacketPayload readMessage(Optional reuse, NativePacketHeader header) throws IOException { + return this.readMessage(reuse, header, false); + } + + @Override + public NativePacketPayload readMessage(Optional reuse, NativePacketHeader header, boolean isRowReading) throws IOException { int packetLength = header.getMessageSize(); - NativePacketPayload buf = this.packetReader.readMessage(reuse, header); + NativePacketPayload buf = this.packetReader.readMessage(reuse, header, isRowReading); int bytesToDump = Math.min(MAX_PACKET_DUMP_LENGTH, packetLength); String PacketPayloadImpl = StringUtils.dumpAsHex(buf.getByteBuffer(), bytesToDump); @@ -145,4 +150,8 @@ public void resetMessageSequence() { return this.packetReader; } + @Override + public void setResultByteBufferCounterIfNoExist(ResultByteBufferCounter counter) { + this.packetReader.setResultByteBufferCounterIfNoExist(counter); + } } diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/MultiPacketReader.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/MultiPacketReader.java index 63fb4b6f..f392e110 100644 --- a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/MultiPacketReader.java +++ b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/MultiPacketReader.java @@ -56,9 +56,14 @@ public NativePacketHeader readHeader() throws IOException { @Override public NativePacketPayload readMessage(Optional reuse, NativePacketHeader header) throws IOException { + return this.readMessage(reuse, header, false); + } + + @Override + public NativePacketPayload readMessage(Optional reuse, NativePacketHeader header, boolean isRowReading) throws IOException { int packetLength = header.getMessageSize(); - NativePacketPayload buf = this.packetReader.readMessage(reuse, header); + NativePacketPayload buf = this.packetReader.readMessage(reuse, header, isRowReading); if (packetLength == NativeConstants.MAX_PACKET_SIZE) { // it's a multi-packet @@ -81,7 +86,7 @@ public NativePacketPayload readMessage(Optional reuse, Nati throw new IOException(Messages.getString("PacketReader.10")); } - this.packetReader.readMessage(Optional.of(multiPacket), hdr); + this.packetReader.readMessage(Optional.of(multiPacket), hdr, isRowReading); buf.writeBytes(StringLengthDataType.STRING_FIXED, multiPacket.getByteBuffer(), 0, multiPacketLength); @@ -113,4 +118,8 @@ public void resetMessageSequence() { return this.packetReader; } + @Override + public void setResultByteBufferCounterIfNoExist(ResultByteBufferCounter counter) { + this.packetReader.setResultByteBufferCounterIfNoExist(counter); + } } diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/NativeProtocol.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/NativeProtocol.java index bf81e801..c063bc9c 100644 --- a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/NativeProtocol.java +++ b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/NativeProtocol.java @@ -55,6 +55,8 @@ import java.util.TimeZone; import java.util.function.Supplier; +import javax.xml.parsers.ParserConfigurationException; + import com.mysql.cj.CharsetMapping; import com.mysql.cj.Constants; import com.mysql.cj.MessageBuilder; @@ -66,6 +68,7 @@ import com.mysql.cj.ServerVersion; import com.mysql.cj.Session; import com.mysql.cj.TransactionEventHandler; +import com.mysql.cj.conf.ConnectionPropertyMaxResultBufferParser; import com.mysql.cj.conf.PropertyKey; import com.mysql.cj.conf.PropertySet; import com.mysql.cj.conf.RuntimeProperty; @@ -79,6 +82,7 @@ import com.mysql.cj.exceptions.DataTruncationException; import com.mysql.cj.exceptions.ExceptionFactory; import com.mysql.cj.exceptions.FeatureNotAvailableException; +import com.mysql.cj.exceptions.MaxResultBufferException; import com.mysql.cj.exceptions.MysqlErrorNumbers; import com.mysql.cj.exceptions.PasswordExpiredException; import com.mysql.cj.exceptions.WrongArgumentException; @@ -195,6 +199,8 @@ private BaseMetricsHolder metricsHolder; + private ResultByteBufferCounter counter; + /** * The comment (if any) that we'll prepend to all queries * sent to the server (to show up in "SHOW PROCESSLIST") @@ -285,6 +291,20 @@ public void init(Session sess, SocketConnection phConnection, PropertySet propSe protocolEntityClassToBinaryReader.put(Resultset.class, new BinaryResultsetReader(this)); this.PROTOCOL_ENTITY_CLASS_TO_BINARY_READER = Collections.unmodifiableMap(protocolEntityClassToBinaryReader); + this.counter = getResultByteBufferCounter(); + + } + + private ResultByteBufferCounter getResultByteBufferCounter() { + try { + long maxResultBuffer = ConnectionPropertyMaxResultBufferParser + .parseProperty(this.propertySet.getStringProperty(PropertyKey.maxResultBuffer).getValue(), this.log); + return new ResultByteBufferCounter(maxResultBuffer); + } catch (ParserConfigurationException e) { + throw ExceptionFactory.createException(Messages.getString("PropertyDefinition.2", + new Object[] { PropertyKey.maxResultBuffer, e.getMessage(), this.propertySet.getStringProperty(PropertyKey.maxResultBuffer).getValue() }), + this.exceptionInterceptor); + } } @Override @@ -1012,6 +1032,9 @@ public void reclaimLargeReusablePacket() { return rs; + } catch (MaxResultBufferException mrbEx) { + throw ExceptionFactory.createCommunicationsException(this.propertySet, this.serverSession, this.getPacketSentTimeHolder(), + this.getPacketReceivedTimeHolder(), mrbEx, getExceptionInterceptor()); } catch (CJException sqlEx) { if (this.queryInterceptors != null) { // TODO why doing this? @@ -1574,6 +1597,7 @@ public static MysqlType findMysqlType(PropertySet propertySet, int mysqlTypeId, if (sr == null) { throw ExceptionFactory.createException(FeatureNotAvailableException.class, "ProtocolEntityReader isn't available for class " + requiredClass); } + setResultByteBufferCounterForResultRowsSet(requiredClass, sr); return sr.read(protocolEntityFactory); } @@ -2167,4 +2191,22 @@ public void initServerSession() { } } } + + public ResultByteBufferCounter getCounter() { + return this.counter; + } + + /** + * Set ResultByteBufferCounter if requiredClass is ResultsetRow + * @param requiredClass + * class + * @param sr + * instance of ProtocolEntityReader + */ + private void setResultByteBufferCounterForResultRowsSet(Class requiredClass, + ProtocolEntityReader sr) { + if (ResultsetRow.class.equals(requiredClass)) { + sr.setResultByteBufferCounterIfNoExist(this.counter); + } + } } diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/ResultByteBufferCounter.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/ResultByteBufferCounter.java new file mode 100644 index 00000000..eafbef04 --- /dev/null +++ b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/ResultByteBufferCounter.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License, version 2.0, as published by the + * Free Software Foundation. + * + * This program is also distributed with certain software (including but not + * limited to OpenSSL) that is licensed under separate terms, as designated in a + * particular file or component or in included license documentation. The + * authors of MySQL hereby grant you an additional permission to link the + * program and your derivative works with the separately licensed software that + * they have included with MySQL. + * + * Without limiting anything contained in the foregoing, this file, which is + * part of MySQL Connector/J, is also subject to the Universal FOSS Exception, + * version 1.0, a copy of which can be found at + * http://oss.oracle.com/licenses/universal-foss-exception. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, + * for more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package com.mysql.cj.protocol.a; + +import com.mysql.cj.Messages; +import com.mysql.cj.exceptions.MaxResultBufferException; + +public class ResultByteBufferCounter { + + private long resultByteBufferCounter; + private final long maxResultBuffer; + + public ResultByteBufferCounter(long maxResultBuffer) { + this.resultByteBufferCounter = 0; + this.maxResultBuffer = maxResultBuffer; + } + + /** + * Increases counter of reading query result bytes + * @param count + * count of query result bytes + * @throws MaxResultBufferException + * throw, when query result is larger then threshold + */ + public void increaseCounter(long count) { + if (this.maxResultBuffer != -1) { + this.resultByteBufferCounter += count; + if (this.resultByteBufferCounter > this.maxResultBuffer) { + throw new MaxResultBufferException(Messages.getString("ConnectionString.27", + new Object[]{this.resultByteBufferCounter, this.maxResultBuffer})); + } + } + } + + /** + * Reset counter to 0 + */ + public void resetCounter() { + this.resultByteBufferCounter = 0; + } + + public long getResultByteBufferCounter() { + return this.resultByteBufferCounter; + } +} diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/ResultsetRowReader.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/ResultsetRowReader.java index d5fb8043..63495cfc 100644 --- a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/ResultsetRowReader.java +++ b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/ResultsetRowReader.java @@ -47,6 +47,8 @@ protected RuntimeProperty useBufferRowSizeThreshold; + private ResultByteBufferCounter counter; + public ResultsetRowReader(NativeProtocol prot) { this.protocol = prot; @@ -70,9 +72,11 @@ public ResultsetRow read(ProtocolEntityFactory maxAllowedPacket; private byte readPacketSequence = -1; + private ResultByteBufferCounter counter; public SimplePacketReader(SocketConnection socketConnection, RuntimeProperty maxAllowedPacket) { this.socketConnection = socketConnection; @@ -84,6 +86,11 @@ public NativePacketHeader readHeader() throws IOException { @Override public NativePacketPayload readMessage(Optional reuse, NativePacketHeader header) throws IOException { + return this.readMessage(reuse, header, false); + } + + @Override + public NativePacketPayload readMessage(Optional reuse, NativePacketHeader header, boolean isRowReading) throws IOException { try { int packetLength = header.getMessageSize(); NativePacketPayload buf; @@ -103,6 +110,7 @@ public NativePacketPayload readMessage(Optional reuse, Nati } else { buf = new NativePacketPayload(new byte[packetLength]); } + increaseResultBufferCounter(packetLength, isRowReading); // Read the data from the server int numBytesRead = this.socketConnection.getMysqlInput().readFully(buf.getByteBuffer(), 0, packetLength); @@ -111,7 +119,7 @@ public NativePacketPayload readMessage(Optional reuse, Nati } return buf; - } catch (IOException e) { + } catch (IOException | MaxResultBufferException e) { try { this.socketConnection.forceClose(); } catch (Exception ex) { @@ -131,4 +139,17 @@ public void resetMessageSequence() { this.readPacketSequence = 0; } + @Override + public void setResultByteBufferCounterIfNoExist(ResultByteBufferCounter counter) { + if (this.counter == null) { + this.counter = counter; + } + } + + private void increaseResultBufferCounter(int packerLength, boolean isRowReading) throws IOException { + if (isRowReading) { + Optional.ofNullable(this.counter) + .ifPresent(c -> c.increaseCounter(packerLength)); + } + } } diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/TimeTrackingPacketReader.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/TimeTrackingPacketReader.java index 470b02b0..47cfe3a9 100644 --- a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/TimeTrackingPacketReader.java +++ b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/TimeTrackingPacketReader.java @@ -54,7 +54,12 @@ public NativePacketHeader readHeader() throws IOException { @Override public NativePacketPayload readMessage(Optional reuse, NativePacketHeader header) throws IOException { - NativePacketPayload buf = this.packetReader.readMessage(reuse, header); + return this.readMessage(reuse, header, false); + } + + @Override + public NativePacketPayload readMessage(Optional reuse, NativePacketHeader header, boolean isRowReading) throws IOException { + NativePacketPayload buf = this.packetReader.readMessage(reuse, header, isRowReading); this.lastPacketReceivedTimeMs = System.currentTimeMillis(); return buf; } @@ -84,4 +89,8 @@ public void resetMessageSequence() { return this.packetReader; } + @Override + public void setResultByteBufferCounterIfNoExist(ResultByteBufferCounter counter) { + this.packetReader.setResultByteBufferCounterIfNoExist(counter); + } } diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/TracingPacketReader.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/TracingPacketReader.java index 4f8d1b38..7e1498d6 100644 --- a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/TracingPacketReader.java +++ b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/TracingPacketReader.java @@ -71,8 +71,13 @@ public NativePacketHeader readHeader() throws IOException { @Override public NativePacketPayload readMessage(Optional reuse, NativePacketHeader header) throws IOException { + return this.readMessage(reuse, header, false); + } + + @Override + public NativePacketPayload readMessage(Optional reuse, NativePacketHeader header, boolean isRowReading) throws IOException { int packetLength = header.getMessageSize(); - NativePacketPayload buf = this.packetReader.readMessage(reuse, header); + NativePacketPayload buf = this.packetReader.readMessage(reuse, header, isRowReading); StringBuilder traceMessageBuf = new StringBuilder(); @@ -110,4 +115,9 @@ public void resetMessageSequence() { return this.packetReader; } + @Override + public void setResultByteBufferCounterIfNoExist(ResultByteBufferCounter counter) { + this.packetReader.setResultByteBufferCounterIfNoExist(counter); + } + } diff --git a/src/main/resources/com/mysql/cj/LocalizedErrorMessages.properties b/src/main/resources/com/mysql/cj/LocalizedErrorMessages.properties index 4be581de..62af7af6 100644 --- a/src/main/resources/com/mysql/cj/LocalizedErrorMessages.properties +++ b/src/main/resources/com/mysql/cj/LocalizedErrorMessages.properties @@ -141,7 +141,7 @@ ConnectionString.23=''{0}'' cannot be set to false with DNS SRV lookup enabled. ConnectionString.24=Using named pipes with DNS SRV lookup is not allowed. ConnectionString.25=The option ''{0}'' cannot be set. Live management of connections is not supported with DNS SRV lookup. ConnectionString.26=Unable to locate any hosts for {0}. - +ConnectionString.27=The result set exceeds maxResultBuffer limit. Received: {0} ; Current limit: {1}. ConnectionWrapper.0=Can''t set autocommit to ''true'' on an XAConnection ConnectionWrapper.1=Can''t call commit() on an XAConnection associated with a global transaction @@ -914,6 +914,7 @@ ConnectionProperties.enabledTLSProtocols=If "useSSL" is set to "true", overrides ConnectionProperties.enableEscapeProcessing=Sets the default escape processing behavior for Statement objects. The method Statement.setEscapeProcessing() can be used to specify the escape processing behavior for an individual Statement object. Default escape processing behavior in prepared statements must be defined with the property ''processEscapeCodesForPrepStmts''. ConnectionProperties.replicationConnectionGroup=Logical group of replication connections within a classloader, used to manage different groups independently. If not specified, live management of replication connections is disabled. ConnectionProperties.dnsSrv=Should the driver use the given host name to lookup for DNS SRV records and use the resulting list of hosts in a multi-host failover connection? Note that a single host name and no port must be provided when this option is enabled. +ConnectionProperties.maxResultBuffer=Specifies size of buffer during fetching result set. Can be specified as specified size or percent of heap memory. ConnectionProperties.sslMode=By default, network connections are SSL encrypted; this property permits secure connections to be turned off, or a different levels of security to be chosen. The following values are allowed: "DISABLED" - Establish unencrypted connections; "PREFERRED" - (default) Establish encrypted connections if the server enabled them, otherwise fall back to unencrypted connections; "REQUIRED" - Establish secure connections if the server enabled them, fail otherwise; "VERIFY_CA" - Like "REQUIRED" but additionally verify the server TLS certificate against the configured Certificate Authority (CA) certificates; "VERIFY_IDENTITY" - Like "VERIFY_CA", but additionally verify that the server certificate matches the host to which the connection is attempted.[CR] This property replaced the deprecated legacy properties "useSSL", "requireSSL", and "verifyServerCertificate", which are still accepted but translated into a value for "sslMode" if "sslMode" is not explicitly set: "useSSL=false" is translated to "sslMode=DISABLED"; '{'"useSSL=true", "requireSSL=false", "verifyServerCertificate=false"'}' is translated to "sslMode=PREFERRED"; '{'"useSSL=true", "requireSSL=true", "verifyServerCertificate=false"'}' is translated to "sslMode=REQUIRED"; '{'"useSSL=true" AND "verifyServerCertificate=true"'}' is translated to "sslMode=VERIFY_CA". There is no equivalent legacy settings for "sslMode=VERIFY_IDENTITY". Note that, for ALL server versions, the default setting of "sslMode" is "PREFERRED", and it is equivalent to the legacy settings of "useSSL=true", "requireSSL=false", and "verifyServerCertificate=false", which are different from their default settings for Connector/J 8.0.12 and earlier in some situations. Applications that continue to use the legacy properties and rely on their old default settings should be reviewed.[CR] The legacy properties are ignored if "sslMode" is set explicitly. If none of "sslMode" or "useSSL" is set explicitly, the default setting of "sslMode=PREFERRED" applies. ConnectionProperties.useAsyncProtocol=Use asynchronous variant of X Protocol @@ -934,3 +935,4 @@ ConnectionProperties.xdevapiCompressionAlgorithm=A comma-delimited list of tripl ConnectionProperties.unknown=Property is not defined in Connector/J but used in connection URL. PropertyDefinition.1=The connection property ''{0}'' acceptable values are: {1}. The value ''{2}'' is not acceptable. +PropertyDefinition.2=The connection property ''{0}'' acceptable patterns are: {1}. The value ''{2}'' is not acceptable. diff --git a/src/test/java/com/mysql/cj/conf/ConnectionPropertyMaxResultBufferParserTest.java b/src/test/java/com/mysql/cj/conf/ConnectionPropertyMaxResultBufferParserTest.java new file mode 100644 index 00000000..270055d0 --- /dev/null +++ b/src/test/java/com/mysql/cj/conf/ConnectionPropertyMaxResultBufferParserTest.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License, version 2.0, as published by the + * Free Software Foundation. + * + * This program is also distributed with certain software (including but not + * limited to OpenSSL) that is licensed under separate terms, as designated in a + * particular file or component or in included license documentation. The + * authors of MySQL hereby grant you an additional permission to link the + * program and your derivative works with the separately licensed software that + * they have included with MySQL. + * + * Without limiting anything contained in the foregoing, this file, which is + * part of MySQL Connector/J, is also subject to the Universal FOSS Exception, + * version 1.0, a copy of which can be found at + * http://oss.oracle.com/licenses/universal-foss-exception. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, + * for more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package com.mysql.cj.conf; + +import com.mysql.cj.log.Log; +import com.mysql.cj.log.StandardLogger; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import javax.xml.parsers.ParserConfigurationException; +import java.lang.management.ManagementFactory; +import java.util.Arrays; +import java.util.Collection; + +import static org.junit.Assert.*; + +@RunWith(Parameterized.class) +public class ConnectionPropertyMaxResultBufferParserTest { + + private final Log log = new StandardLogger(ConnectionPropertyMaxResultBufferParser.class.getName()); + + @Parameterized.Parameter(0) + public String valueToParse; + + @Parameterized.Parameter(1) + public long expectedResult; + + @Parameterized.Parameters(name = "{index}: Test with valueToParse={0}, expectedResult={1}") + public static Collection data() { + Object[][] data = new Object[][]{ + {"100", 100L}, + {"10K", 10L * 1000}, + {"25M", 25L * 1000 * 1000}, + //next two should be too big + {"35G", (long) (0.90 * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax())}, + {"1T", (long) (0.90 * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax())}, + //percent test + {"5p", (long) (0.05 * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax())}, + {"10pct", (long) (0.10 * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax())}, + {"15percent", + (long) (0.15 * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax())}, + //for testing empty property + {"", -1}, + {null, -1} + }; + return Arrays.asList(data); + } + + @Test + public void testGetMaxResultBufferValue() { + try { + long result = ConnectionPropertyMaxResultBufferParser.parseProperty(valueToParse, log); + assertEquals(expectedResult, result); + } catch (ParserConfigurationException e) { + //shouldn't occur + fail(); + } + } + + @Test(expected = ParserConfigurationException.class) + public void testGetParserConfigurationException() throws ParserConfigurationException { + long result = ConnectionPropertyMaxResultBufferParser.parseProperty("abc", log); + fail(); + } + +} \ No newline at end of file diff --git a/src/test/java/com/mysql/cj/protocol/a/SimplePacketReaderTest.java b/src/test/java/com/mysql/cj/protocol/a/SimplePacketReaderTest.java index 22f6bbf6..dc5f1663 100644 --- a/src/test/java/com/mysql/cj/protocol/a/SimplePacketReaderTest.java +++ b/src/test/java/com/mysql/cj/protocol/a/SimplePacketReaderTest.java @@ -29,9 +29,7 @@ package com.mysql.cj.protocol.a; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; @@ -45,12 +43,14 @@ import org.junit.Test; +import com.mysql.cj.Messages; import com.mysql.cj.conf.PropertyKey; import com.mysql.cj.conf.PropertySet; import com.mysql.cj.conf.RuntimeProperty; import com.mysql.cj.exceptions.CJPacketTooBigException; import com.mysql.cj.exceptions.ExceptionInterceptor; import com.mysql.cj.exceptions.FeatureNotAvailableException; +import com.mysql.cj.exceptions.MaxResultBufferException; import com.mysql.cj.exceptions.SSLParamsException; import com.mysql.cj.jdbc.JdbcPropertySetImpl; import com.mysql.cj.log.Log; @@ -121,9 +121,7 @@ public void truncatedPacketHeaderRead() throws IOException { // trivial payload test @Test public void readBasicPayload() throws IOException { - RuntimeProperty maxAllowedPacket = new JdbcPropertySetImpl().getProperty(PropertyKey.maxAllowedPacket); - SocketConnection connection = new FixedBufferSocketConnection(new byte[] { 3, 2, 1, 6, 5, 4 }); - MessageReader reader = new SimplePacketReader(connection, maxAllowedPacket); + MessageReader reader = prepareSimpleSimplePacketReader(); NativePacketPayload b = reader.readMessage(Optional.empty(), new NativePacketHeader(new byte[] { 3, 0, 0, 0 })); assertEquals(3, b.getByteBuffer()[0]); assertEquals(2, b.getByteBuffer()[1]); @@ -220,6 +218,40 @@ public void heuristicTestWithRandomPackets() throws IOException { } } + //Test if ResultByteBufferCounter works properly + @Test + public void readLessOrEqualsDataThenMaxResultBufferTest() throws IOException { + MessageReader reader = prepareSimpleSimplePacketReader(); + ResultByteBufferCounter counter = new ResultByteBufferCounter(6); + reader.setResultByteBufferCounterIfNoExist(counter); + NativePacketPayload b1 = reader.readMessage(Optional.empty(), new NativePacketHeader(new byte[] { 3, 0, 0, 0 }), true); + assertEquals("Counter should be equals to count of read data", 3, counter.getResultByteBufferCounter()); + NativePacketPayload b2 = reader.readMessage(Optional.empty(), new NativePacketHeader(new byte[] { 3, 0, 0, 0 }), true); + assertEquals("Counter should be equals to count of read data", 6, counter.getResultByteBufferCounter()); + } + + //Test if ResultByteBufferCounter throws correct exception, when trying to read more data than maxResultBuffer + @Test + public void maxResultBufferExceptionTest() { + MaxResultBufferException mrbExc = assertThrows(MaxResultBufferException.class, () -> { + MessageReader reader = prepareSimpleSimplePacketReader(); + ResultByteBufferCounter counter = new ResultByteBufferCounter(5); + reader.setResultByteBufferCounterIfNoExist(counter); + NativePacketPayload b = reader.readMessage(Optional.empty(), new NativePacketHeader(new byte[] { 6, 0, 0, 0 }), true); + }); + + String expectedMessage = Messages.getString("ConnectionString.27", new Object[] { 6, 5 }); + String actualMessage = mrbExc.getMessage(); + + assertEquals("Exception message should looks like: " + expectedMessage, expectedMessage, actualMessage); + } + + private MessageReader prepareSimpleSimplePacketReader() { + RuntimeProperty maxAllowedPacket = new JdbcPropertySetImpl().getProperty(PropertyKey.maxAllowedPacket); + SocketConnection connection = new FixedBufferSocketConnection(new byte[] { 3, 2, 1, 6, 5, 4 }); + return new SimplePacketReader(connection, maxAllowedPacket); + } + // TODO any boundary conditions or large packet issues? public static class FixedBufferSocketConnection extends MockSocketConnection { diff --git a/src/test/java/testsuite/regression/ResultSetRegressionTest.java b/src/test/java/testsuite/regression/ResultSetRegressionTest.java index 7c506d17..5aefbb07 100644 --- a/src/test/java/testsuite/regression/ResultSetRegressionTest.java +++ b/src/test/java/testsuite/regression/ResultSetRegressionTest.java @@ -84,6 +84,7 @@ import com.mysql.cj.Messages; import com.mysql.cj.MysqlType; +import com.mysql.cj.NativeSession; import com.mysql.cj.conf.DefaultPropertySet; import com.mysql.cj.conf.PropertyDefinitions.DatabaseTerm; import com.mysql.cj.conf.PropertyKey; @@ -91,6 +92,7 @@ import com.mysql.cj.exceptions.ExceptionInterceptor; import com.mysql.cj.exceptions.ExceptionInterceptorChain; import com.mysql.cj.exceptions.MysqlErrorNumbers; +import com.mysql.cj.exceptions.MaxResultBufferException; import com.mysql.cj.jdbc.JdbcConnection; import com.mysql.cj.jdbc.MysqlSQLXML; import com.mysql.cj.jdbc.ServerPreparedStatement; @@ -102,6 +104,7 @@ import com.mysql.cj.jdbc.result.UpdatableResultSet; import com.mysql.cj.log.Log; import com.mysql.cj.protocol.InternalDate; +import com.mysql.cj.protocol.a.ResultByteBufferCounter; import com.mysql.cj.protocol.a.result.NativeResultset; import com.mysql.cj.protocol.a.result.ResultsetRowsCursor; import com.mysql.cj.protocol.a.result.ResultsetRowsStreaming; @@ -7345,4 +7348,51 @@ public void testBug97724() throws Exception { assertEquals(3.3d, this.rs.getDouble(1)); assertFalse(this.rs.next()); } + + public void testMaxResultBuffer() throws Exception { + createTable("testMaxResultBuffer", "(value VARCHAR(10))"); + for (int i=0; i < 200; i++) { + this.stmt.execute("INSERT INTO testMaxResultBuffer(value) VALUES ('123456789X');"); + } + Connection con = null; + Properties props = new Properties(); + try { + props.setProperty(PropertyKey.maxResultBuffer.getKeyName(), "3000"); + con = getConnectionWithProps(props); + Statement stm = con.createStatement(); + stm.execute("SELECT * FROM testMaxResultBuffer"); + ResultByteBufferCounter counter = ((NativeSession) ((JdbcConnection) con).getSession()).getProtocol().getCounter(); + assertEquals("The result byte counter should be reset after the data has been successfully retrieved",0,counter.getResultByteBufferCounter()); + } finally { + if (con != null) { + con.close(); + } + } + } + + public void testMaxResultBufferException() throws Exception { + createTable("testMaxResultBuffer", "(value VARCHAR(10))"); + for (int i=0; i < 200; i++) { + this.stmt.execute("INSERT INTO testMaxResultBuffer(value) VALUES ('123456789X');"); + } + + CommunicationsException cExec = assertThrows("CommunicationsException should be thrown", CommunicationsException.class, () -> { + Connection con = null; + Properties props = new Properties(); + try { + props.setProperty(PropertyKey.maxResultBuffer.getKeyName(), "1000"); + con = getConnectionWithProps(props); + Statement stm = con.createStatement(); + stm.execute("SELECT * FROM testMaxResultBuffer"); + } finally { + if (con != null) { + con.close(); + } + } + return null; + }); + + assertTrue("Expected MaxResultBufferException",cExec.getCause().getCause() instanceof MaxResultBufferException); + + } }