Table hits 700k rows mysqld starts reading 200k per second

[Tim Nelson]

I have a strange problem that when the main table in my application reaches 700k'ish rows all of a sudden mysqld 5.0.95 starts running out of control. IO Wait goes to 50% and sar -b yields:

Linux 2.6.18-308.4.1.el5 (AUSVDDSGPS1) 06/14/2013

11:39:03 AM tps rtps wtps bread/s bwrtn/s
11:39:04 AM 1230.69 1230.69 0.00 273362.38 0.00
11:39:05 AM 1495.00 1396.00 99.00 288096.00 2088.00
11:39:06 AM 1538.38 1538.38 0.00 333139.39 0.00
11:39:07 AM 1523.00 1523.00 0.00 304240.00 0.00
11:39:08 AM 2057.84 2057.84 0.00 288745.10 0.00

200k reads per second !!! (this normally < 100)

"show processlist" normally shows 1 or 2 entries, but in this case all kinds of selects and inserts show as locked.

At first I thought it was a hardware issue or something with the VMware instance it is running on, but the admins tell me there are no errors.

Current thought this is a tuning issue with MySQL. This is really a single table database with a fixed workload (no ad-hoc queries etc).

Can anyone spot anything?


------------- TABLE DEFINITION ---------------------

CREATE TABLE `track` (
`customer` varchar(10) default NULL,
`truck` varchar(20) default NULL,
`latitude` varchar(20) default NULL,
`longitude` varchar(20) default NULL,
`beenread` char(1) default NULL,
`pdate` date default NULL,
`ptime` time default NULL,
`speed` varchar(20) default NULL,
`heading` varchar(20) default NULL,
`signature` longtext,
`ordernum` varchar(40) default NULL,
`invoice` varchar(40) default '',
`status` varchar(40) default '',
`pict1` longblob,
`pict2` longblob,
`pict3` longblob,
`ddsbeenread` char(1) default NULL,
`split` varchar(64) default NULL,
`transid` varchar(64) default NULL,
`runid` varchar(64) default NULL,
`scheduleid` varchar(64) default NULL,
`assetid` varchar(64) default NULL,
`pict4` longblob,
`pict5` longblob,
`pict6` longblob,
KEY `track_1` (`customer`,`truck`,`pdate`,`ptime`),
KEY `track_2` (`customer`,`pdate`,`ptime`),
KEY `track_3` (`customer`,`truck`,`ddsbeenread`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;


--------- SHOW VARIABLES VALUES -----------

bdb_log_buffer_size 262144
bulk_insert_buffer_size 8388608
innodb_buffer_pool_awe_mem_mb 0
innodb_buffer_pool_size 8388608
innodb_log_buffer_size 1048576
join_buffer_size 131072
key_buffer_size 8384512
myisam_sort_buffer_size 8388608
net_buffer_length 16384
preload_buffer_size 32768
read_buffer_size 131072
read_rnd_buffer_size 262144
sort_buffer_size 2097144
bdb_cache_size 8384512
binlog_cache_size 32768
delayed_queue_size 1000
innodb_additional_mem_pool_size 1048576
innodb_log_file_size 5242880
key_cache_block_size 1024
large_page_size 0
max_binlog_cache_size 18446744073709547520
max_binlog_size 1073741824
max_heap_table_size 16777216
max_join_size 18446744073709551615
max_relay_log_size 0
myisam_data_pointer_size 6
myisam_max_sort_file_size 9223372036853727232
myisam_mmap_size 18446744073709551615
query_alloc_block_size 8192
query_cache_size 0
query_prealloc_size 8192
range_alloc_block_size 4096
thread_cache_size 0
tmp_table_size 33554432
transaction_alloc_block_size 8192
transaction_prealloc_size 4096


---------------- SHOW STATUS ------------------------

+-----------------------------------+-----------+
| Variable_name | Value |
+-----------------------------------+-----------+
| Aborted_clients | 33 |
| Aborted_connects | 0 |
| Binlog_cache_disk_use | 0 |
| Binlog_cache_use | 0 |
| Bytes_received | 101 |
| Bytes_sent | 157 |
| Com_admin_commands | 0 |
| Com_alter_db | 0 |
| Com_alter_table | 0 |
| Com_analyze | 0 |
| Com_backup_table | 0 |
| Com_begin | 0 |
| Com_call_procedure | 0 |
| Com_change_db | 0 |
| Com_change_master | 0 |
| Com_check | 0 |
| Com_checksum | 0 |
| Com_commit | 0 |
| Com_create_db | 0 |
| Com_create_function | 0 |
| Com_create_index | 0 |
| Com_create_table | 0 |
| Com_create_user | 0 |
| Com_dealloc_sql | 0 |
| Com_delete | 0 |
| Com_delete_multi | 0 |
| Com_do | 0 |
| Com_drop_db | 0 |
| Com_drop_function | 0 |
| Com_drop_index | 0 |
| Com_drop_table | 0 |
| Com_drop_user | 0 |
| Com_execute_sql | 0 |
| Com_flush | 0 |
| Com_grant | 0 |
| Com_ha_close | 0 |
| Com_ha_open | 0 |
| Com_ha_read | 0 |
| Com_help | 0 |
| Com_insert | 0 |
| Com_insert_select | 0 |
| Com_kill | 0 |
| Com_load | 0 |
| Com_load_master_data | 0 |
| Com_load_master_table | 0 |
| Com_lock_tables | 0 |
| Com_optimize | 0 |
| Com_preload_keys | 0 |
| Com_prepare_sql | 0 |
| Com_purge | 0 |
| Com_purge_before_date | 0 |
| Com_rename_table | 0 |
| Com_repair | 0 |
| Com_replace | 0 |
| Com_replace_select | 0 |
| Com_reset | 0 |
| Com_restore_table | 0 |
| Com_revoke | 0 |
| Com_revoke_all | 0 |
| Com_rollback | 0 |
| Com_savepoint | 0 |
| Com_select | 1 |
| Com_set_option | 0 |
| Com_show_binlog_events | 0 |
| Com_show_binlogs | 0 |
| Com_show_charsets | 0 |
| Com_show_collations | 0 |
| Com_show_column_types | 0 |
| Com_show_create_db | 0 |
| Com_show_create_table | 0 |
| Com_show_databases | 0 |
| Com_show_errors | 0 |
| Com_show_fields | 0 |
| Com_show_grants | 0 |
| Com_show_innodb_status | 0 |
| Com_show_keys | 0 |
| Com_show_logs | 0 |
| Com_show_master_status | 0 |
| Com_show_ndb_status | 0 |
| Com_show_new_master | 0 |
| Com_show_open_tables | 0 |
| Com_show_privileges | 0 |
| Com_show_processlist | 0 |
| Com_show_slave_hosts | 0 |
| Com_show_slave_status | 0 |
| Com_show_status | 1 |
| Com_show_storage_engines | 0 |
| Com_show_tables | 0 |
| Com_show_triggers | 0 |
| Com_show_variables | 0 |
| Com_show_warnings | 0 |
| Com_slave_start | 0 |
| Com_slave_stop | 0 |
| Com_stmt_close | 0 |
| Com_stmt_execute | 0 |
| Com_stmt_fetch | 0 |
| Com_stmt_prepare | 0 |
| Com_stmt_reset | 0 |
| Com_stmt_send_long_data | 0 |
| Com_truncate | 0 |
| Com_unlock_tables | 0 |
| Com_update | 0 |
| Com_update_multi | 0 |
| Com_xa_commit | 0 |
| Com_xa_end | 0 |
| Com_xa_prepare | 0 |
| Com_xa_recover | 0 |
| Com_xa_rollback | 0 |
| Com_xa_start | 0 |
| Compression | OFF |
| Connections | 1359760 |
| Created_tmp_disk_tables | 0 |
| Created_tmp_files | 28 |
| Created_tmp_tables | 1 |
| Delayed_errors | 0 |
| Delayed_insert_threads | 0 |
| Delayed_writes | 0 |
| Flush_commands | 1 |
| Handler_commit | 0 |
| Handler_delete | 0 |
| Handler_discover | 0 |
| Handler_prepare | 0 |
| Handler_read_first | 0 |
| Handler_read_key | 0 |
| Handler_read_next | 0 |
| Handler_read_prev | 0 |
| Handler_read_rnd | 0 |
| Handler_read_rnd_next | 0 |
| Handler_rollback | 0 |
| Handler_savepoint | 0 |
| Handler_savepoint_rollback | 0 |
| Handler_update | 0 |
| Handler_write | 132 |
| Innodb_buffer_pool_pages_data | 24 |
| Innodb_buffer_pool_pages_dirty | 0 |
| Innodb_buffer_pool_pages_flushed | 0 |
| Innodb_buffer_pool_pages_free | 488 |
| Innodb_buffer_pool_pages_misc | 0 |
| Innodb_buffer_pool_pages_total | 512 |
| Innodb_buffer_pool_read_ahead_rnd | 1 |
| Innodb_buffer_pool_read_ahead_seq | 0 |
| Innodb_buffer_pool_read_requests | 108 |
| Innodb_buffer_pool_reads | 17 |
| Innodb_buffer_pool_wait_free | 0 |
| Innodb_buffer_pool_write_requests | 0 |
| Innodb_data_fsyncs | 3 |
| Innodb_data_pending_fsyncs | 0 |
| Innodb_data_pending_reads | 0 |
| Innodb_data_pending_writes | 0 |
| Innodb_data_read | 2576384 |
| Innodb_data_reads | 30 |
| Innodb_data_writes | 3 |
| Innodb_data_written | 1536 |
| Innodb_dblwr_pages_written | 0 |
| Innodb_dblwr_writes | 0 |
| Innodb_log_waits | 0 |
| Innodb_log_write_requests | 0 |
| Innodb_log_writes | 1 |
| Innodb_os_log_fsyncs | 3 |
| Innodb_os_log_pending_fsyncs | 0 |
| Innodb_os_log_pending_writes | 0 |
| Innodb_os_log_written | 512 |
| Innodb_page_size | 16384 |
| Innodb_pages_created | 0 |
| Innodb_pages_read | 24 |
| Innodb_pages_written | 0 |
| Innodb_row_lock_current_waits | 0 |
| Innodb_row_lock_time | 0 |
| Innodb_row_lock_time_avg | 0 |
| Innodb_row_lock_time_max | 0 |
| Innodb_row_lock_waits | 0 |
| Innodb_rows_deleted | 0 |
| Innodb_rows_inserted | 0 |
| Innodb_rows_read | 0 |
| Innodb_rows_updated | 0 |
| Key_blocks_not_flushed | 0 |
| Key_blocks_unused | 0 |
| Key_blocks_used | 6694 |
| Key_read_requests | 374220236 |
| Key_reads | 12853772 |
| Key_write_requests | 9612883 |
| Key_writes | 5882547 |
| Last_query_cost | 0.000000 |
| Max_used_connections | 130 |
| Not_flushed_delayed_rows | 0 |
| Open_files | 66 |
| Open_streams | 0 |
| Open_tables | 64 |
| Opened_tables | 0 |
| Prepared_stmt_count | 0 |
| Qcache_free_blocks | 0 |
| Qcache_free_memory | 0 |
| Qcache_hits | 0 |
| Qcache_inserts | 0 |
| Qcache_lowmem_prunes | 0 |
| Qcache_not_cached | 0 |
| Qcache_queries_in_cache | 0 |
| Qcache_total_blocks | 0 |
| Queries | 5752670 |
| Questions | 2 |
| Rpl_status | NULL |
| Select_full_join | 0 |
| Select_full_range_join | 0 |
| Select_range | 0 |
| Select_range_check | 0 |
| Select_scan | 1 |
| Slave_open_temp_tables | 0 |
| Slave_retried_transactions | 0 |
| Slave_running | OFF |
| Slow_launch_threads | 0 |
| Slow_queries | 0 |
| Sort_merge_passes | 0 |
| Sort_range | 0 |
| Sort_rows | 0 |
| Sort_scan | 0 |
| Ssl_accept_renegotiates | 0 |
| Ssl_accepts | 0 |
| Ssl_callback_cache_hits | 0 |
| Ssl_cipher | |
| Ssl_cipher_list | |
| Ssl_client_connects | 0 |
| Ssl_connect_renegotiates | 0 |
| Ssl_ctx_verify_depth | 0 |
| Ssl_ctx_verify_mode | 0 |
| Ssl_default_timeout | 0 |
| Ssl_finished_accepts | 0 |
| Ssl_finished_connects | 0 |
| Ssl_session_cache_hits | 0 |
| Ssl_session_cache_misses | 0 |
| Ssl_session_cache_mode | NONE |
| Ssl_session_cache_overflows | 0 |
| Ssl_session_cache_size | 0 |
| Ssl_session_cache_timeouts | 0 |
| Ssl_sessions_reused | 0 |
| Ssl_used_session_cache_entries | 0 |
| Ssl_verify_depth | 0 |
| Ssl_verify_mode | 0 |
| Ssl_version | |
| Table_locks_immediate | 2963588 |
| Table_locks_waited | 67418 |
| Tc_log_max_pages_used | 0 |
| Tc_log_page_size | 0 |
| Tc_log_page_waits | 0 |
| Threads_cached | 0 |
| Threads_connected | 1 |
| Threads_created | 1359759 |
| Threads_running | 1 |
| Uptime | 575111 |
+-----------------------------------+-----------+



Edited 2 time(s). Last edit at 06/14/2013 11:06AM by Tim Nelson.