Bug #75175 Using a BinaryField in Django will raise an exception on UPDATE/INSERT

• Submitted: 11 Dec 2014 10:25
• Modified: 19 Jun 2015 17:03
• Reporter: Lee Packham
• Status: Closed
• Category: Connector / Python
• Severity: S2 (Serious)
• Version: 2.0.2
• OS: Any
• CPU Architecture: Any

[11 Dec 2014 10:25] Lee Packham

Description:
When writing to a table with a BinaryField from Django you get a UnicodeDecodeError due to the binary being in the statement.

How to repeat:
1. Create a table with a binary field
2. Create small Django project with the binary field
3. Try to save binary into the field.

----> u.save()

/Users/leepa/Code/py3/venvs/generic34/lib/python3.4/site-packages/django/db/models/base.py in save(self, force_insert, force_update, using, update_fields)
    589
    590         self.save_base(using=using, force_insert=force_insert,
--> 591                        force_update=force_update, update_fields=update_fields)
    592     save.alters_data = True
    593

/Users/leepa/Code/py3/venvs/generic34/lib/python3.4/site-packages/django/db/models/base.py in save_base(self, raw, force_insert, force_update, using, update_fields)
    617             if not raw:
    618                 self._save_parents(cls, using, update_fields)
--> 619             updated = self._save_table(raw, cls, force_insert, force_update, using, update_fields)
    620         # Store the database on which the object was saved
    621         self._state.db = using

/Users/leepa/Code/py3/venvs/generic34/lib/python3.4/site-packages/django/db/models/base.py in _save_table(self, raw, cls, force_insert, force_update, using, update_fields)
    679             forced_update = update_fields or force_update
    680             updated = self._do_update(base_qs, using, pk_val, values, update_fields,
--> 681                                       forced_update)
    682             if force_update and not updated:
    683                 raise DatabaseError("Forced update did not affect any rows.")

/Users/leepa/Code/py3/venvs/generic34/lib/python3.4/site-packages/django/db/models/base.py in _do_update(self, base_qs, using, pk_val, values, update_fields, forced_update)
    723             else:
    724                 return False
--> 725         return filtered._update(values) > 0
    726
    727     def _do_insert(self, manager, using, fields, update_pk, raw):

/Users/leepa/Code/py3/venvs/generic34/lib/python3.4/site-packages/django/db/models/query.py in _update(self, values)
    598         query.add_update_fields(values)
    599         self._result_cache = None
--> 600         return query.get_compiler(self.db).execute_sql(CURSOR)
    601     _update.alters_data = True
    602     _update.queryset_only = False

/Users/leepa/Code/py3/venvs/generic34/lib/python3.4/site-packages/django/db/models/sql/compiler.py in execute_sql(self, result_type)
   1002         related queries are not available.
   1003         """
-> 1004         cursor = super(SQLUpdateCompiler, self).execute_sql(result_type)
   1005         try:
   1006             rows = cursor.rowcount if cursor else 0

/Users/leepa/Code/py3/venvs/generic34/lib/python3.4/site-packages/django/db/models/sql/compiler.py in execute_sql(self, result_type)
    784         cursor = self.connection.cursor()
    785         try:
--> 786             cursor.execute(sql, params)
    787         except Exception:
    788             cursor.close()

/Users/leepa/Code/py3/venvs/generic34/lib/python3.4/site-packages/django/db/backends/utils.py in execute(self, sql, params)
     83             stop = time()
     84             duration = stop - start
---> 85             sql = self.db.ops.last_executed_query(self.cursor, sql, params)
     86             self.db.queries.append({
     87                 'sql': sql,

/Users/leepa/Code/py3/venvs/generic34/lib/python3.4/site-packages/mysql/connector/django/base.py in last_executed_query(self, cursor, sql, params)
    369
    370     def last_executed_query(self, cursor, sql, params):
--> 371         return cursor.statement
    372
    373     def no_limit_value(self):

/Users/leepa/Code/py3/venvs/generic34/lib/python3.4/site-packages/mysql/connector/django/base.py in __getattr__(self, attr)
    145     def __getattr__(self, attr):
    146         """Return attribute of wrapped cursor"""
--> 147         return getattr(self.cursor, attr)
    148
    149     def __iter__(self):

/Users/leepa/Code/py3/venvs/generic34/lib/python3.4/site-packages/mysql/connector/cursor.py in statement(self)
    855         print(self._executed.strip())
    856         try:
--> 857             return self._executed.strip().decode('utf8')
    858         except AttributeError:
    859             return self._executed.strip()

UnicodeDecodeError: 'utf-8' codec can't decode byte 0x89 in position 80: invalid start byte

This is because the Binary Field can't be decoded. That's fine and should be treated the same as the AttributeError.

There's no workaround for this without editing the code in the library. Hence the Severity.

Suggested fix:
Change the exception in CusorBase.statement to:

    except (AttributeError, UnicodeDecodeError):

[19 Jun 2015 17:03] Paul DuBois

Noted in 2.1.3 changelog.

Writing to a table with a BinaryField from Django resulted in a
UnicodeDecodeError exception.