Description:
I got the error: unpack requires a string argument of length 8 when calling cursor.fetchone() or cursor.fetchall() or cursor.fetchmany().
In [3]: cursor = connector.cursor(prepared=True)
In [4]: stmt = "SELECT * FROM my_table WHERE my_key=%s"
In [5]: cursor.execute(stmt, ('my_value',))
In [6]: cursor.fetchone()
---------------------------------------------------------------------------
error Traceback (most recent call last)
<ipython-input-6-5501e92f1036> in <module>()
----> 1 cursor.fetchone()
/usr/lib/python2.7/dist-packages/mysql/connector/cursor.pyc in fetchone(self)
1041 Returns a tuple or None.
1042 """
-> 1043 return self._fetch_row() or None
1044
1045 def fetchmany(self, size=None):
/usr/lib/python2.7/dist-packages/mysql/connector/cursor.pyc in _fetch_row(self)
709 if self._nextrow == (None, None):
710 (row, eof) = self._connection.get_row(
--> 711 binary=self._binary, columns=self.description)
712 else:
713 (row, eof) = self._nextrow
/usr/lib/python2.7/dist-packages/mysql/connector/connection.pyc in get_row(self, binary, columns)
599 Returns a tuple.
600 """
--> 601 (rows, eof) = self.get_rows(count=1, binary=binary, columns=columns)
602 if len(rows):
603 return (rows[0], eof)
/usr/lib/python2.7/dist-packages/mysql/connector/connection.pyc in get_rows(self, count, binary, columns)
580 if binary:
581 rows = self._protocol.read_binary_result(
--> 582 self._socket, columns, count)
583 else:
584 rows = self._protocol.read_text_result(self._socket, count)
/usr/lib/python2.7/dist-packages/mysql/connector/protocol.pyc in read_binary_result(self, sock, columns, count)
398 elif packet[4] == '\x00':
399 eof = None
--> 400 values = self._parse_binary_values(columns, packet[5:])
401 if eof is None and values is not None:
402 rows.append(values)
/usr/lib/python2.7/dist-packages/mysql/connector/protocol.pyc in _parse_binary_values(self, fields, packet)
349 """Parse values from a binary result packet"""
350 null_bitmap_length = (len(fields) + 7 + 2) // 8
--> 351 null_bitmap = utils.intread(packet[0:null_bitmap_length])
352 packet = packet[null_bitmap_length:]
353
/usr/lib/python2.7/dist-packages/mysql/connector/utils.pyc in intread(buf)
43 else:
44 tmp = buf + '\x00'*(8-length)
---> 45 return struct.unpack('<Q', tmp)[0]
46 except:
47 raise
my_table has 81 columns.
This issue doesn't reproduce if I only select a few columns, i.e. changing the stmt to SELECT my_col1,my_col2,my_col3 from my_table WHERE my_key=%s.
I suspect that this is a real bug in mysql connector/python. The bug is either in protocol.py: MySQLProtocol._parse_binary_values() or in utils.py: intread().
The code in utils.py: intread() expects a string of not more than 8 bytes, but protocol.py: MySQLProtocol._parse_binary_values() passes in a longer string when the number of columns exceeds 62 (8 * 8 - 2).
Below is the relevant code snippet:
In protocol.py:
348: def _parse_binary_values(self, fields, packet):
349: """Parse values from a binary result packet"""
350: null_bitmap_length = (len(fields) + 7 + 2) // 8
351: null_bitmap = utils.intread(packet[0:null_bitmap_length])
352:
353: values = []
354: for pos, field in enumerate(fields):
355: if null_bitmap & 1 << (pos + 2):
In utils.py:
32: def intread(buf):
33: """Unpacks the given buffer to an integer"""
34: try:
35: if isinstance(buf, int):
36: return buf
37: length = len(buf)
38: if length == 1:
39: return int(ord(buf))
40: if length <= 4:
41: tmp = buf + '\x00'*(4-length)
42: return struct.unpack('<I', tmp)[0]
43: else:
44: tmp = buf + '\x00'*(8-length)
45: return struct.unpack('<Q', tmp)[0]
46: except:
47: raise
How to repeat:
Below is the sequence of calls which reproduces the issue.
1) create a table with more than 62 columns
2) connect to the table
3) cursor = connector.cursor(prepared=True)
4) stmt = "SELECT * FROM my_table WHERE my_key=%s"
5) cursor.execute(stmt, ('my_value',))
6) cursor.fetchone()
Suggested fix:
I am able to resolve my issue by making the following changes:
In protocol.py:
348: def _parse_binary_values(self, fields, packet):
349: """Parse values from a binary result packet"""
350: null_bitmap_length = (len(fields) + 7 + 2) // 8
351: null_bitmap = utils.read_bitmap(packet[0:null_bitmap_length])
352:
353: values = []
354: for pos, field in enumerate(fields):
355: if null_bitmap.test_bit(pos + 2):
In utils.py (adding the following lines):
class MySqlIntBitmap(object):
def __init__(self, value):
self.int_bitmap = value
def test_bit(self, bit):
return (self.int_bitmap & (1 << bit))
class MySqlByteArrayBitmap(object):
def __init__(self, value):
self.byte_array_bitmap = value
def test_bit(self, bit):
index = bit // 8
entry_bit = bit % 8
return (self.byte_array_bitmap[index] & (1 << entry_bit))
def read_bitmap(buf):
"""Unpacks the given buffer to a bitmap"""
try:
if isinstance(buf, int):
return MySqlIntBitmap(buf)
length = len(buf)
if length == 1:
return MySqlIntBitmap(int(ord(buf)))
if length <= 4:
tmp = buf + '\x00'*(4-length)
return MySqlIntBitmap(struct.unpack('<I', tmp)[0])
elif length <= 8:
tmp = buf + '\x00'*(8-length)
return MySqlIntBitmap(struct.unpack('<Q', tmp)[0])
else:
return MySqlByteArrayBitmap([int(ord(i)) for i in buf])
except:
raise
Description: I got the error: unpack requires a string argument of length 8 when calling cursor.fetchone() or cursor.fetchall() or cursor.fetchmany(). In [3]: cursor = connector.cursor(prepared=True) In [4]: stmt = "SELECT * FROM my_table WHERE my_key=%s" In [5]: cursor.execute(stmt, ('my_value',)) In [6]: cursor.fetchone() --------------------------------------------------------------------------- error Traceback (most recent call last) <ipython-input-6-5501e92f1036> in <module>() ----> 1 cursor.fetchone() /usr/lib/python2.7/dist-packages/mysql/connector/cursor.pyc in fetchone(self) 1041 Returns a tuple or None. 1042 """ -> 1043 return self._fetch_row() or None 1044 1045 def fetchmany(self, size=None): /usr/lib/python2.7/dist-packages/mysql/connector/cursor.pyc in _fetch_row(self) 709 if self._nextrow == (None, None): 710 (row, eof) = self._connection.get_row( --> 711 binary=self._binary, columns=self.description) 712 else: 713 (row, eof) = self._nextrow /usr/lib/python2.7/dist-packages/mysql/connector/connection.pyc in get_row(self, binary, columns) 599 Returns a tuple. 600 """ --> 601 (rows, eof) = self.get_rows(count=1, binary=binary, columns=columns) 602 if len(rows): 603 return (rows[0], eof) /usr/lib/python2.7/dist-packages/mysql/connector/connection.pyc in get_rows(self, count, binary, columns) 580 if binary: 581 rows = self._protocol.read_binary_result( --> 582 self._socket, columns, count) 583 else: 584 rows = self._protocol.read_text_result(self._socket, count) /usr/lib/python2.7/dist-packages/mysql/connector/protocol.pyc in read_binary_result(self, sock, columns, count) 398 elif packet[4] == '\x00': 399 eof = None --> 400 values = self._parse_binary_values(columns, packet[5:]) 401 if eof is None and values is not None: 402 rows.append(values) /usr/lib/python2.7/dist-packages/mysql/connector/protocol.pyc in _parse_binary_values(self, fields, packet) 349 """Parse values from a binary result packet""" 350 null_bitmap_length = (len(fields) + 7 + 2) // 8 --> 351 null_bitmap = utils.intread(packet[0:null_bitmap_length]) 352 packet = packet[null_bitmap_length:] 353 /usr/lib/python2.7/dist-packages/mysql/connector/utils.pyc in intread(buf) 43 else: 44 tmp = buf + '\x00'*(8-length) ---> 45 return struct.unpack('<Q', tmp)[0] 46 except: 47 raise my_table has 81 columns. This issue doesn't reproduce if I only select a few columns, i.e. changing the stmt to SELECT my_col1,my_col2,my_col3 from my_table WHERE my_key=%s. I suspect that this is a real bug in mysql connector/python. The bug is either in protocol.py: MySQLProtocol._parse_binary_values() or in utils.py: intread(). The code in utils.py: intread() expects a string of not more than 8 bytes, but protocol.py: MySQLProtocol._parse_binary_values() passes in a longer string when the number of columns exceeds 62 (8 * 8 - 2). Below is the relevant code snippet: In protocol.py: 348: def _parse_binary_values(self, fields, packet): 349: """Parse values from a binary result packet""" 350: null_bitmap_length = (len(fields) + 7 + 2) // 8 351: null_bitmap = utils.intread(packet[0:null_bitmap_length]) 352: 353: values = [] 354: for pos, field in enumerate(fields): 355: if null_bitmap & 1 << (pos + 2): In utils.py: 32: def intread(buf): 33: """Unpacks the given buffer to an integer""" 34: try: 35: if isinstance(buf, int): 36: return buf 37: length = len(buf) 38: if length == 1: 39: return int(ord(buf)) 40: if length <= 4: 41: tmp = buf + '\x00'*(4-length) 42: return struct.unpack('<I', tmp)[0] 43: else: 44: tmp = buf + '\x00'*(8-length) 45: return struct.unpack('<Q', tmp)[0] 46: except: 47: raise How to repeat: Below is the sequence of calls which reproduces the issue. 1) create a table with more than 62 columns 2) connect to the table 3) cursor = connector.cursor(prepared=True) 4) stmt = "SELECT * FROM my_table WHERE my_key=%s" 5) cursor.execute(stmt, ('my_value',)) 6) cursor.fetchone() Suggested fix: I am able to resolve my issue by making the following changes: In protocol.py: 348: def _parse_binary_values(self, fields, packet): 349: """Parse values from a binary result packet""" 350: null_bitmap_length = (len(fields) + 7 + 2) // 8 351: null_bitmap = utils.read_bitmap(packet[0:null_bitmap_length]) 352: 353: values = [] 354: for pos, field in enumerate(fields): 355: if null_bitmap.test_bit(pos + 2): In utils.py (adding the following lines): class MySqlIntBitmap(object): def __init__(self, value): self.int_bitmap = value def test_bit(self, bit): return (self.int_bitmap & (1 << bit)) class MySqlByteArrayBitmap(object): def __init__(self, value): self.byte_array_bitmap = value def test_bit(self, bit): index = bit // 8 entry_bit = bit % 8 return (self.byte_array_bitmap[index] & (1 << entry_bit)) def read_bitmap(buf): """Unpacks the given buffer to a bitmap""" try: if isinstance(buf, int): return MySqlIntBitmap(buf) length = len(buf) if length == 1: return MySqlIntBitmap(int(ord(buf))) if length <= 4: tmp = buf + '\x00'*(4-length) return MySqlIntBitmap(struct.unpack('<I', tmp)[0]) elif length <= 8: tmp = buf + '\x00'*(8-length) return MySqlIntBitmap(struct.unpack('<Q', tmp)[0]) else: return MySqlByteArrayBitmap([int(ord(i)) for i in buf]) except: raise