I am using MySQL version Server version: 5.5.12-eb-5.5.12-log on on linux 2.6.32-279.el6.x86_64, CentOS release 6.3 (Final).
Many sessions on MySQL hang, no high load/high IO usage
Using Pstack and
http://poormansprofiler.org/ to trace mysql, the result is as be below,
48 __lll_lock_wait,_L_lock_995,pthread_mutex_lock,inline_mysql_mutex_lock,optimized,ha_commit_trans,trans_commit,mysql_execute_command,mysql_parse,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,pfs_spawn_thread,start_thread,clone
41 __lll_lock_wait,_L_lock_995,pthread_mutex_lock,inline_mysql_mutex_lock,optimized,ha_commit_trans,trans_commit_stmt,mysql_execute_command,mysql_parse,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,pfs_spawn_thread,start_thread,clone
2 read,my_real_read,my_net_read,do_command,do_handle_one_connection,handle_one_connection,pfs_spawn_thread,start_thread,clone
1 sigwait,signal_hand,pfs_spawn_thread,start_thread,clone
1 read,vio_read_buff,my_real_read,my_net_read,cli_safe_read,handle_slave_io,pfs_spawn_thread,start_thread,clone
1 pthread_cond_wait@@GLIBC_2.3.2,os_cond_wait,reset_sig_count=2),srv_purge_thread,start_thread,clone
1 pthread_cond_wait@@GLIBC_2.3.2,os_cond_wait,reset_sig_count=25140),srv_master_thread,start_thread,clone
1 pthread_cond_wait@@GLIBC_2.3.2,inline_mysql_cond_wait,thd=0x4674ed70,,Event_queue::get_top_for_execution_if_time,Event_scheduler::run,event_scheduler_thread,pfs_spawn_thread,start_thread,clone
1 pthread_cond_wait@@GLIBC_2.3.2,inline_mysql_cond_wait,need_lock=false),MYSQL_BIN_LOG::rotate_and_purge,MYSQL_BIN_LOG::unlog,ha_commit_trans,trans_commit,mysql_execute_command,mysql_parse,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,pfs_spawn_thread,start_thread,clone
1 pthread_cond_wait@@GLIBC_2.3.2,cache_thread,optimized,do_handle_one_connection,handle_one_connection,pfs_spawn_thread,start_thread,clone
1 pthread_cond_timedwait@@GLIBC_2.3.2,os_cond_wait_timed,time_in_usec=<value,srv_monitor_thread,start_thread,clone
1 pthread_cond_timedwait@@GLIBC_2.3.2,os_cond_wait_timed,time_in_usec=<value,srv_lock_timeout_thread,start_thread,clone
1 pthread_cond_timedwait@@GLIBC_2.3.2,os_cond_wait_timed,time_in_usec=<value,srv_error_monitor_thread,start_thread,clone
1 pthread_cond_timedwait@@GLIBC_2.3.2,inline_mysql_cond_timedwait,term_lock=0x4670eae0,,terminate_slave_threads,stop_slave,mysql_execute_command,mysql_parse,dispatch_command,do_command,do_handle_one_connection,handle_one_connection,pfs_spawn_thread,start_thread,clone
1 poll,handle_connections_sockets,mysqld_main,__libc_start_main,_start
1 ??,os_aio_linux_collect,message1=0x7f71c2ee7e78,,fil_aio_wait,io_handler_thread,start_thread,clone
1 ??,os_aio_linux_collect,message1=0x7f6734f02e78,,fil_aio_wait,io_handler_thread,start_thread,clone
1 ??,os_aio_linux_collect,message1=0x7f6734501e78,,fil_aio_wait,io_handler_thread,start_thread,clone
Thread 759 (Thread 0x7f6722089700 (LWP 122062)):
#0 0x00000030cd20b43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00000000007a189b in Event_queue::cond_wait(THD*, timespec*, char const*, char const*, unsigned int) ()
#2 0x00000000007a1961 in Event_queue::get_top_for_execution_if_time(THD*, Event_queue_element_for_exec**) ()
#3 0x00000000007a0ace in Event_scheduler::run(THD*) ()
#4 0x00000000007a0f1e in event_scheduler_thread ()
#5 0x00000000007e85db in pfs_spawn_thread ()
#6 0x00000030cd207851 in start_thread () from /lib64/libpthread.so.0
#7 0x00000030ccee767d in clone () from /lib64/libc.so.6
Thread 758 (Thread 0x7f6722048700 (LWP 122084)):
#0 0x00000030cd20e054 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00000030cd2093be in _L_lock_995 () from /lib64/libpthread.so.0
#2 0x00000030cd209326 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00000000007f2669 in innobase_xa_prepare(handlerton*, THD*, bool) ()
#4 0x0000000000685327 in ha_commit_trans(THD*, bool) ()
#5 0x000000000061e404 in trans_commit(THD*) ()
#6 0x0000000000575568 in mysql_execute_command(THD*) ()
#7 0x000000000057937b in mysql_parse(THD*, char*, unsigned int, Parser_state*) ()
#8 0x000000000057a879 in dispatch_command(enum_server_command, THD*, char*, unsigned int) ()
#9 0x000000000057abe4 in do_command(THD*) ()
#10 0x000000000061025b in do_handle_one_connection(THD*) ()
#11 0x00000000006104a4 in handle_one_connection ()
#12 0x00000000007e85db in pfs_spawn_thread ()
#13 0x00000030cd207851 in start_thread () from /lib64/libpthread.so.0
#14 0x00000030ccee767d in clone () from /lib64/libc.so.6
Thread 757 (Thread 0x7f6722007700 (LWP 122103)):
#0 0x00000030cd20e054 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00000030cd209388 in _L_lock_854 () from /lib64/libpthread.so.0
#2 0x00000030cd209257 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x000000000073d40c in Log_event::read_log_event(st_io_cache*, String*, st_mysql_mutex*) ()
#4 0x000000000058d9e8 in mysql_binlog_send(THD*, char*, unsigned long long, unsigned short) ()
#5 0x0000000000579eca in dispatch_command(enum_server_command, THD*, char*, unsigned int) ()
#6 0x000000000057abe4 in do_command(THD*) ()
#7 0x000000000061025b in do_handle_one_connection(THD*) ()
#8 0x00000000006104a4 in handle_one_connection ()
#9 0x00000000007e85db in pfs_spawn_thread ()
#10 0x00000030cd207851 in start_thread () from /lib64/libpthread.so.0
#11 0x00000030ccee767d in clone () from /lib64/libc.so.6
Thread 169 (Thread 0x7f66f247e700 (LWP 73094)):
#0 0x00000030cd20e054 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00000030cd209388 in _L_lock_854 () from /lib64/libpthread.so.0
#2 0x00000030cd209257 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000000000727c0f in MYSQL_BIN_LOG::find_log_pos(st_log_info*, char const*, bool) ()
#4 0x000000000058d218 in mysql_binlog_send(THD*, char*, unsigned long long, unsigned short) ()
#5 0x0000000000579eca in dispatch_command(enum_server_command, THD*, char*, unsigned int) ()
#6 0x000000000057abe4 in do_command(THD*) ()
#7 0x000000000061025b in do_handle_one_connection(THD*) ()
#8 0x00000000006104a4 in handle_one_connection ()
#9 0x00000000007e85db in pfs_spawn_thread ()
#10 0x00000030cd207851 in start_thread () from /lib64/libpthread.so.0
#11 0x00000030ccee767d in clone () from /lib64/libc.so.6
Thread 168 (Thread 0x7f66f243d700 (LWP 73095)):
#0 0x00000030cd20e054 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00000030cd209388 in _L_lock_854 () from /lib64/libpthread.so.0
#2 0x00000030cd209257 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000000000727c0f in MYSQL_BIN_LOG::find_log_pos(st_log_info*, char const*, bool) ()
#4 0x000000000058d218 in mysql_binlog_send(THD*, char*, unsigned long long, unsigned short) ()
#5 0x0000000000579eca in dispatch_command(enum_server_command, THD*, char*, unsigned int) ()
#6 0x000000000057abe4 in do_command(THD*) ()
#7 0x000000000061025b in do_handle_one_connection(THD*) ()
#8 0x00000000006104a4 in handle_one_connection ()
#9 0x00000000007e85db in pfs_spawn_thread ()