| Bug #16763 | Hangs or Connection unexpectedly terminated on read operation (not thread-safe) | ||
|---|---|---|---|
| Submitted: | 25 Jan 2006 1:22 | Modified: | 9 Oct 2006 20:08 |
| Reporter: | Yvan Rodrigues | Email Updates: | |
| Status: | Can't repeat | Impact on me: | |
| Category: | Connector / NET | Severity: | S2 (Serious) |
| Version: | 1.0.7 and 5.0.1 | OS: | Windows (Windows XP Home) |
| Assigned to: | CPU Architecture: | Any | |
[25 Jan 2006 1:32]
Yvan Rodrigues
Often the exact same condition is experienced without the EXCEPTION. That is after being issue a Fill() command the library will hang in an endless loop. Maybe it will time out but I've given up after 5 minutes.
Here is an example of this condition:
Call stack:
system.dll!System.Net.Sockets.Socket.Receive(byte[] buffer = {Length=4096}, int offset = 0, int size = 4096, System.Net.Sockets.SocketFlags socketFlags = None) + 0x145 bytes
> mysql.data.dll!MySql.Data.Common.SocketStream.Read(byte[] buffer = {Length=4096}, int offset = 0, int count = 4096) Line 99 + 0x1b bytes C#
mscorlib.dll!System.IO.BufferedStream.Read(byte[] array = {Length=1024}, int offset = 28, int count = 996) + 0x1d5 bytes
mysql.data.dll!MySql.Data.MySqlClient.PacketReader.Read(byte[] buffer = {Length=1024}, long pos = 28, long len = 996) Line 223 + 0x1d bytes C#
mysql.data.dll!MySql.Data.MySqlClient.PacketReader.Skip(long count = 5522246) Line 158 + 0x80 bytes C#
mysql.data.dll!MySql.Data.MySqlClient.PacketReader.OpenPacket() Line 124 C#
mysql.data.dll!MySql.Data.MySqlClient.NativeDriver.GetFieldMetaData() Line 522 C#
mysql.data.dll!MySql.Data.MySqlClient.NativeDriver.ReadFieldMetadata(int count = 72, MySql.Data.MySqlClient.MySqlField[] fields = {Length=72}) Line 507 + 0x16 bytes C#
mysql.data.dll!MySql.Data.MySqlClient.CommandResult.Load() Line 157 C#
mysql.data.dll!MySql.Data.MySqlClient.MySqlDataReader.NextResult() Line 673 + 0x14 bytes C#
mysql.data.dll!MySql.Data.MySqlClient.MySqlCommand.ExecuteReader(System.Data.CommandBehavior behavior = SequentialAccess) Line 406 + 0xb bytes C#
mysql.data.dll!MySql.Data.MySqlClient.MySqlCommand.System.Data.IDbCommand.ExecuteReader(System.Data.CommandBehavior behavior = SequentialAccess) Line 370 + 0xb bytes C#
system.data.dll!System.Data.Common.DbDataAdapter.FillFromCommand(System.Object data = {MisMabel1.MisMabel_Schema}, int startRecord = 0, int maxRecords = 0, string srcTable = "Table", System.Data.IDbCommand command = {MySql.Data.MySqlClient.MySqlCommand}, System.Data.CommandBehavior behavior = Default) + 0x93 bytes
system.data.dll!System.Data.Common.DbDataAdapter.Fill(System.Data.DataSet dataSet = {MisMabel1.MisMabel_Schema}, int startRecord = 0, int maxRecords = 0, string srcTable = "Table", System.Data.IDbCommand command = {MySql.Data.MySqlClient.MySqlCommand}, System.Data.CommandBehavior behavior = Default) + 0xba bytes
system.data.dll!System.Data.Common.DbDataAdapter.Fill(System.Data.DataSet dataSet = {MisMabel1.MisMabel_Schema}) + 0x50 bytes
MisMabel1.exe!MisMabel1.ToolsStandardsItems.ToolsStandardsItems(MySql.Data.MySqlClient.MySqlConnection mySqlConnection = {MySql.Data.MySqlClient.MySqlConnection}) Line 70 + 0x14 bytes C#
MisMabel1.exe!MisMabel1.Form1.ToolsStandards_Items_Click(System.Object sender = {System.Windows.Forms.MenuItem}, System.EventArgs e = {System.EventArgs}) Line 923 + 0x1a bytes C#
system.windows.forms.dll!System.Windows.Forms.MenuItem.OnClick(System.EventArgs e = {System.EventArgs}) + 0x8f bytes
system.windows.forms.dll!MenuItemData.Execute() + 0x1e bytes
system.windows.forms.dll!System.Windows.Forms.Command.Invoke() + 0x4c bytes
system.windows.forms.dll!System.Windows.Forms.Command.DispatchID(int id = 294) + 0x2d bytes
system.windows.forms.dll!System.Windows.Forms.Control.WmCommand(System.Windows.Forms.Message m = {System.Windows.Forms.Message}) + 0x49 bytes
system.windows.forms.dll!System.Windows.Forms.Control.WndProc(System.Windows.Forms.Message m = {System.Windows.Forms.Message}) + 0x238 bytes
system.windows.forms.dll!System.Windows.Forms.ScrollableControl.WndProc(System.Windows.Forms.Message m = {System.Windows.Forms.Message}) + 0x7d bytes
system.windows.forms.dll!System.Windows.Forms.ContainerControl.WndProc(System.Windows.Forms.Message m = {System.Windows.Forms.Message}) + 0x42 bytes
system.windows.forms.dll!System.Windows.Forms.Form.WndProc(System.Windows.Forms.Message m = {System.Windows.Forms.Message}) + 0x2c1 bytes
system.windows.forms.dll!ControlNativeWindow.OnMessage(System.Windows.Forms.Message m = {System.Windows.Forms.Message}) + 0x19 bytes
system.windows.forms.dll!ControlNativeWindow.WndProc(System.Windows.Forms.Message m = {System.Windows.Forms.Message}) + 0xda bytes
system.windows.forms.dll!System.Windows.Forms.NativeWindow.DebuggableCallback(int hWnd = 26478380, int msg = 273, int wparam = 294, int lparam = 0) + 0x39 bytes
system.windows.forms.dll!System.Windows.Forms.Application.ComponentManager.System.Windows.Forms.UnsafeNativeMethods+IMsoComponentManager.FPushMessageLoop(int dwComponentID = 1, int reason = -1, int pvLoopData = 0) + 0x2fa bytes
system.windows.forms.dll!ThreadContext.RunMessageLoopInner(int reason = -1, System.Windows.Forms.ApplicationContext context = {System.Windows.Forms.ApplicationContext}) + 0x1db bytes
system.windows.forms.dll!ThreadContext.RunMessageLoop(int reason = -1, System.Windows.Forms.ApplicationContext context = {System.Windows.Forms.ApplicationContext}) + 0x4f bytes
system.windows.forms.dll!System.Windows.Forms.Application.Run(System.Windows.Forms.Form mainForm = {MisMabel1.Form1}) + 0x37 bytes
MisMabel1.exe!MisMabel1.Form1.Main() Line 709 C#
Output:
'DefaultDomain': Loaded 'c:\windows\microsoft.net\framework\v1.1.4322\mscorlib.dll', No symbols loaded.
'MisMabel1': Loaded 'C:\Documents and Settings\Yvan\My Documents\Mabel MIS\dev\MisMabel1\bin\Debug\MisMabel1.exe', Symbols loaded.
'MisMabel1.exe': Loaded 'c:\windows\assembly\gac\system.windows.forms\1.0.5000.0__b77a5c561934e089\system.windows.forms.dll', No symbols loaded.
'MisMabel1.exe': Loaded 'c:\windows\assembly\gac\system\1.0.5000.0__b77a5c561934e089\system.dll', No symbols loaded.
'MisMabel1.exe': Loaded 'c:\windows\assembly\gac\system.drawing\1.0.5000.0__b03f5f7f11d50a3a\system.drawing.dll', No symbols loaded.
'MisMabel1.exe': Loaded 'c:\documents and settings\yvan\my documents\mabel mis\dev\mismabel1\bin\debug\mysql.data.dll', Symbols loaded.
'MisMabel1.exe': Loaded 'c:\windows\assembly\gac\system.data\1.0.5000.0__b77a5c561934e089\system.data.dll', No symbols loaded.
'MisMabel1.exe': Loaded 'c:\windows\assembly\gac\system.xml\1.0.5000.0__b77a5c561934e089\system.xml.dll', No symbols loaded.
'MisMabel1.exe': Loaded 'c:\windows\assembly\gac\accessibility\1.0.5000.0__b03f5f7f11d50a3a\accessibility.dll', No symbols loaded.
Again this hangs on line 99 of SocketStream.cs which looks like
return socket.Receive(buffer, offset, count, SocketFlags.None);
And to reiterate, everything works as designed under 1.0.6
Thanks for your help
[25 Jan 2006 3:05]
Yvan Rodrigues
Sorry: C# in Visual Studio 2003 using .NET 1.1
[3 Mar 2006 12:26]
Valeriy Kravchuk
Thank you for a detailed problem report. Please, specify the exact version of MySQL server you use, for completeness. Have you upgraded only Connector (1.0.6 --> 1.0.7) or server version also changed?
[4 Mar 2006 21:02]
Yvan Rodrigues
The server is running 4.0.23-standard
[4 Mar 2006 21:04]
Yvan Rodrigues
As far as I know, the only variable is the Connector version. That is, if I complile with 1.0.7 is breaks (either using the pre-compiled dll or compiling my own from source) and when I use 1.0.6 it is fine.
[14 Apr 2006 13:12]
Tonci Grgin
Thanks for your problem report. I was unable to verify it on XP pro. If you wish you can reopen this report but please post data dump and complete code which reproduces this error.
[9 Oct 2006 19:54]
Yvan Rodrigues
I have had to work on this for many hours last night, and I think I have determined the problem.
I am now on:
Windows XP
Visual Studio.NET 2005
C#
.NET Runtime 2.0
Connector/.NET 5.0.1 beta (Assembly 5.0.1.0)
Server is 4.1.21-standard
Problem is as follows:
- Application was designed to have a single MySql.Data.MySqlClient.MySqlConnection object, and keep it open while the application is running
- MySqlDataAdapters and MySqlCommands use this open connection (without problems so long as there were not network problems)
- another thread, a Systems.Timers.Timer that triggers a method to MySql.Data.MySqlClient.MySqlConnection.Ping the current connection (an attempt to keep-alive and determine connection state) would upset the current read operation on the main thread, causing it to either
(a) get "stuck" indefinitely in the current Read operation
(b) throw an exception in a seemingly random place (during a socket read operation, often:
IndexOutOfRangeException in line 324 of MySqlStream.cs "Probable I/O race condition detected while copying memory. The I/O package is not thread safe by default. In multithreaded applications, a stream must be accessed in a thread-safe way, such as a thread-safe wrapper returned by TextReader's or TextWriter's Synchronized methods. This also applies to classes like StreamWriter and StreamReader."
Code to reproduce, add something like this to project (simplified):
// create the mysql timer
this.MySqlTimer = new System.Timers.Timer(5000);
this.MySqlTimer.Elapsed += new System.Timers.ElapsedEventHandler(MySqlTimer_Elapsed);
private void MySqlTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
{
this.mySqlConnection.Ping();
return;
}
SUMMARY:
- read operations are not thread-safe
WORKAROUND:
- do not share MySqlConnections between threads
IMPORTANT:
- This works fine with Connector/NET versions 1.0.6 and previous -- in fact my application has been use in a production environment for almost two years, using 1.0.6
Since this is not documented in the changelog, it was either an unintentional change, possibly using different calls to the base socket classes.
- IF an exception is thrown
[9 Oct 2006 20:08]
Yvan Rodrigues
Could this be re-opened and either fixed (did <=1.0.6 use different read operations?) or perhaps better documented. The current docs do say "Public static (Shared in Visual Basic) members of this type are safe for multithreaded operations. Instance members are not guaranteed to be thread-safe." so I guess this is "as designed" but I do find it interesting that pre-1.0.7 is different.
Description: This is a 1.0.7 problem. My code works fine under 1.0.6 and previous on the same project. For no particular reason (frequently) the application will "hang" due to "EXCEPTION: Connection unexpectedly terminated" at line 99 in SocketStream.cs. The last line in my code is MySqlDataAdapter.Fill This is not consistent -- that is, any Fill() in my code could trigger the problem. The network connection is stable (and works fine under 1.0.6) Here is a call stack: system.dll!System.Net.Sockets.Socket.Receive(byte[] buffer = {Length=1024}, int offset = 7, int size = 1017, System.Net.Sockets.SocketFlags socketFlags = None) + 0x145 bytes mysql.data.dll!MySql.Data.Common.SocketStream.Read(byte[] buffer = {Length=1024}, int offset = 7, int count = 1017) Line 99 + 0x1b bytes C# mscorlib.dll!System.IO.BufferedStream.Read(byte[] array = {Length=1024}, int offset = 0, int count = 1024) + 0x23f bytes mysql.data.dll!MySql.Data.MySqlClient.PacketReader.Read(byte[] buffer = {Length=1024}, long pos = 0, long len = 1024) Line 223 + 0x1d bytes C# mysql.data.dll!MySql.Data.MySqlClient.PacketReader.Skip(long count = 7546397) Line 158 + 0x80 bytes C# mysql.data.dll!MySql.Data.MySqlClient.PacketReader.OpenPacket() Line 124 C# mysql.data.dll!MySql.Data.MySqlClient.NativeDriver.OpenDataRow(int fieldCount = 0, bool isBinary = false) Line 445 C# mysql.data.dll!MySql.Data.MySqlClient.CommandResult.Consume() Line 226 + 0xe bytes C# mysql.data.dll!MySql.Data.MySqlClient.MySqlDataReader.Close() Line 132 C# mysql.data.dll!MySql.Data.MySqlClient.MySqlDataReader.System.IDisposable.Dispose() Line 98 C# system.data.dll!System.Data.Common.DbDataAdapter.FillFromCommand(System.Object data = {MisMabel1.MisMabel_Schema}, int startRecord = 0, int maxRecords = 0, string srcTable = "Standards_Users", System.Data.IDbCommand command = {MySql.Data.MySqlClient.MySqlCommand}, System.Data.CommandBehavior behavior = Default) + 0x126 bytes system.data.dll!System.Data.Common.DbDataAdapter.Fill(System.Data.DataSet dataSet = {MisMabel1.MisMabel_Schema}, int startRecord = 0, int maxRecords = 0, string srcTable = "Standards_Users", System.Data.IDbCommand command = {MySql.Data.MySqlClient.MySqlCommand}, System.Data.CommandBehavior behavior = Default) + 0xba bytes system.data.dll!System.Data.Common.DbDataAdapter.Fill(System.Data.DataSet dataSet = {MisMabel1.MisMabel_Schema}, string srcTable = "Standards_Users") + 0x4b bytes > MisMabel1.exe!MisMabel1.EditOrders.EditOrders(MySql.Data.MySqlClient.MySqlConnection mySqlConnection = {MySql.Data.MySqlClient.MySqlConnection}) Line 270 + 0x32 bytes C# MisMabel1.exe!MisMabel1.Form1.EditOrders_Click(System.Object sender = {System.Windows.Forms.MenuItem}, System.EventArgs e = {System.EventArgs}) Line 929 + 0x1a bytes C# system.windows.forms.dll!System.Windows.Forms.MenuItem.OnClick(System.EventArgs e = {System.EventArgs}) + 0x8f bytes system.windows.forms.dll!MenuItemData.Execute() + 0x1e bytes system.windows.forms.dll!System.Windows.Forms.Command.Invoke() + 0x4c bytes system.windows.forms.dll!System.Windows.Forms.Command.DispatchID(int id = 265) + 0x2d bytes system.windows.forms.dll!System.Windows.Forms.Control.WmCommand(System.Windows.Forms.Message m = {System.Windows.Forms.Message}) + 0x49 bytes system.windows.forms.dll!System.Windows.Forms.Control.WndProc(System.Windows.Forms.Message m = {System.Windows.Forms.Message}) + 0x238 bytes system.windows.forms.dll!System.Windows.Forms.ScrollableControl.WndProc(System.Windows.Forms.Message m = {System.Windows.Forms.Message}) + 0x7d bytes system.windows.forms.dll!System.Windows.Forms.ContainerControl.WndProc(System.Windows.Forms.Message m = {System.Windows.Forms.Message}) + 0x42 bytes system.windows.forms.dll!System.Windows.Forms.Form.WndProc(System.Windows.Forms.Message m = {System.Windows.Forms.Message}) + 0x2c1 bytes system.windows.forms.dll!ControlNativeWindow.OnMessage(System.Windows.Forms.Message m = {System.Windows.Forms.Message}) + 0x19 bytes system.windows.forms.dll!ControlNativeWindow.WndProc(System.Windows.Forms.Message m = {System.Windows.Forms.Message}) + 0xda bytes system.windows.forms.dll!System.Windows.Forms.NativeWindow.DebuggableCallback(int hWnd = 27265780, int msg = 273, int wparam = 265, int lparam = 0) + 0x39 bytes system.windows.forms.dll!System.Windows.Forms.Application.ComponentManager.System.Windows.Forms.UnsafeNativeMethods+IMsoComponentManager.FPushMessageLoop(int dwComponentID = 1, int reason = -1, int pvLoopData = 0) + 0x2fa bytes system.windows.forms.dll!ThreadContext.RunMessageLoopInner(int reason = -1, System.Windows.Forms.ApplicationContext context = {System.Windows.Forms.ApplicationContext}) + 0x1db bytes system.windows.forms.dll!ThreadContext.RunMessageLoop(int reason = -1, System.Windows.Forms.ApplicationContext context = {System.Windows.Forms.ApplicationContext}) + 0x4f bytes system.windows.forms.dll!System.Windows.Forms.Application.Run(System.Windows.Forms.Form mainForm = {MisMabel1.Form1}) + 0x37 bytes MisMabel1.exe!MisMabel1.Form1.Main() Line 709 C# Here is the output: 'DefaultDomain': Loaded 'c:\windows\microsoft.net\framework\v1.1.4322\mscorlib.dll', No symbols loaded. 'MisMabel1': Loaded 'C:\Documents and Settings\Yvan\My Documents\Mabel MIS\dev\MisMabel1\bin\Debug\MisMabel1.exe', Symbols loaded. 'MisMabel1.exe': Loaded 'c:\windows\assembly\gac\system.windows.forms\1.0.5000.0__b77a5c561934e089\system.windows.forms.dll', No symbols loaded. 'MisMabel1.exe': Loaded 'c:\windows\assembly\gac\system\1.0.5000.0__b77a5c561934e089\system.dll', No symbols loaded. 'MisMabel1.exe': Loaded 'c:\windows\assembly\gac\system.drawing\1.0.5000.0__b03f5f7f11d50a3a\system.drawing.dll', No symbols loaded. 'MisMabel1.exe': Loaded 'c:\documents and settings\yvan\my documents\mabel mis\dev\mismabel1\bin\debug\mysql.data.dll', Symbols loaded. 'MisMabel1.exe': Loaded 'c:\windows\assembly\gac\system.data\1.0.5000.0__b77a5c561934e089\system.data.dll', No symbols loaded. 'MisMabel1.exe': Loaded 'c:\windows\assembly\gac\system.xml\1.0.5000.0__b77a5c561934e089\system.xml.dll', No symbols loaded. 'MisMabel1.exe': Loaded 'c:\windows\assembly\gac\accessibility\1.0.5000.0__b03f5f7f11d50a3a\accessibility.dll', No symbols loaded. EXCEPTION: Connection unexpectedly terminated If I can think of anything else, I'll add it How to repeat: A Windows Forms application with many MySqlDataAdapter.Fill calls. Here is the code that produced this exception BUT it can happen anywhere. // // mySqlConnection // this.mySqlConnection = mySqlConnection; // populate a dataset containing the countries for the drop-down this.mySqlDataAdapter1 = new MySql.Data.MySqlClient.MySqlDataAdapter(); IT BROKE HERE -> this.mySqlDataAdapter1.SelectCommand = new MySql.Data.MySqlClient.MySqlCommand("SELECT * FROM Standards_Countries ORDER BY Name ASC", this.mySqlConnection); this.mySqlDataAdapter1.Fill(this.Collections, "Standards_Countries"); this.mySqlDataAdapter1.SelectCommand = new MySql.Data.MySqlClient.MySqlCommand("SELECT * FROM Standards_Provinces ORDER BY Name ASC", this.mySqlConnection); this.mySqlDataAdapter1.Fill(this.Collections, "Standards_Provinces"); this.mySqlDataAdapter1.SelectCommand = new MySql.Data.MySqlClient.MySqlCommand("SELECT * FROM Standards_Currencies ORDER BY Code ASC", this.mySqlConnection); this.mySqlDataAdapter1.Fill(this.Collections, "Standards_Currencies"); this.mySqlDataAdapter1.SelectCommand = new MySql.Data.MySqlClient.MySqlCommand("SELECT * FROM Standards_Coupons WHERE IsActive = 1 ORDER BY Code ASC", this.mySqlConnection); this.mySqlDataAdapter1.Fill(this.Collections, "Standards_Coupons"); this.mySqlDataAdapter1.SelectCommand = new MySql.Data.MySqlClient.MySqlCommand("SELECT * FROM Standards_Users WHERE IsAgent = 1 OR IsFundraiser = 1 ORDER BY WebName ASC", this.mySqlConnection); this.mySqlDataAdapter1.Fill(this.Collections, "Standards_Users"); // populate the dropdowns this.mySqlDataAdapter1.SelectCommand = new MySql.Data.MySqlClient.MySqlCommand("SELECT * FROM Standards_OrderMethods ORDER BY Code ASC", this.mySqlConnection); this.mySqlDataAdapter1.Fill(this.Collections, "Standards_OrderMethods"); this.mySqlDataAdapter1.SelectCommand = new MySql.Data.MySqlClient.MySqlCommand("SELECT * FROM Standards_PaymentTypes ORDER BY Name ASC", this.mySqlConnection); this.mySqlDataAdapter1.Fill(this.Collections, "Standards_PaymentTypes"); this.mySqlDataAdapter1.SelectCommand = new MySql.Data.MySqlClient.MySqlCommand("SELECT * FROM Standards_Colours WHERE IsActive = 1 ORDER BY Name ASC", this.mySqlConnection); this.mySqlDataAdapter1.Fill(this.Collections, "Standards_Colours"); this.mySqlDataAdapter1.SelectCommand = new MySql.Data.MySqlClient.MySqlCommand("SELECT * FROM Standards_Icons WHERE IsActive = 1 ORDER BY Name ASC", this.mySqlConnection); this.mySqlDataAdapter1.Fill(this.Collections, "Standards_Icons"); this.mySqlDataAdapter1.SelectCommand = new MySql.Data.MySqlClient.MySqlCommand("SELECT * FROM Standards_Items WHERE IsActive = 1 ORDER BY Name ASC", this.mySqlConnection); this.mySqlDataAdapter1.Fill(this.Collections, "Standards_Items"); this.mySqlDataAdapter1.SelectCommand = new MySql.Data.MySqlClient.MySqlCommand("SELECT * FROM Standards_Taxes ORDER BY ShortName ASC", this.mySqlConnection); this.mySqlDataAdapter1.Fill(this.Collections, "Standards_Taxes");