Mysql crashing with sig 6

[Vikash Jain]

Hi

I am facing one issue here, To avoid table locking at the time of data extraction/loading we are closing all query connections of MySQL by killing all queries, while performing this task MySQL getting crashed with signal-6.

Below I have pasted mysqld error log entries for this, and suggest if you have any thought about this.

One thing which I observed is, both the time when query having mysql-udf in it caused this problem.. But wanted to understand is that the problem.

FYI: I wrote finyr.so and it works perfectly for all select queries.

*** glibc detected *** /usr/sbin/mysqld: double free or corruption (!prev): 0x0000000015406fc0 ***
======= Backtrace: =========
/lib64/libc.so.6[0x3e67871ce2]
/lib64/libc.so.6(cfree+0x8c)[0x3e6787590c]
/lib64/libc.so.6(fclose+0x14b)[0x3e67860d0b]
/usr/lib64/mysql/plugin/finyr.so(deinit_my_mysql_Log+0x22)[0x2aaac596d9c5]
/usr/lib64/mysql/plugin/finyr.so(getrange_deinit+0x16)[0x2aaac596d946]
/usr/sbin/mysqld(_ZN11udf_handler7cleanupEv+0x94)[0x543c84]
/usr/sbin/mysqld(_ZN13Item_udf_func7cleanupEv+0x10)[0x543cb0]
/usr/sbin/mysqld(_ZN11Query_arena10free_itemsEv+0x22)[0x5aef42]
/usr/sbin/mysqld(_ZN3THD19cleanup_after_queryEv+0x90)[0x5b2680]
/usr/sbin/mysqld(_Z11mysql_parseP3THDPKcjPS2_+0x190)[0x5d51b0]
/usr/sbin/mysqld(_Z16dispatch_command19enum_server_commandP3THDPcj+0x1030)[0x5d6260]
/usr/sbin/mysqld(_Z10do_commandP3THD+0xe4)[0x5d6834]
/usr/sbin/mysqld(handle_one_connection+0x6f0)[0x5c9850]
/lib64/libpthread.so.0[0x3e68406367]
/lib64/libc.so.6(clone+0x6d)[0x3e678d30ad]
======= Memory map: ========
00400000-00b1e000 r-xp 00000000 68:07 928792 /usr/sbin/mysqld
00d1d000-00d7f000 rw-p 0071d000 68:07 928792 /usr/sbin/mysqld
00d7f000-00d96000 rw-p 00d7f000 00:00 0
00f7e000-01005000 rw-p 0077e000 68:07 928792 /usr/sbin/mysqld
14bdd000-17d03000 rw-p 14bdd000 00:00 0 [heap]
404ee000-404ef000 ---p 404ee000 00:00 0
404ef000-40eef000 rw-p 404ef000 00:00 0
40eef000-40ef0000 ---p 40eef000 00:00 0
40ef0000-418f0000 rw-p 40ef0000 00:00 0
418f0000-418f1000 ---p 418f0000 00:00 0
418f1000-41931000 rw-p 418f1000 00:00 0
41931000-41932000 ---p 41931000 00:00 0
41932000-41972000 rw-p 41932000 00:00 0
419aa000-419ab000 ---p 419aa000 00:00 0
419ab000-419eb000 rw-p 419ab000 00:00 0
419eb000-419ec000 ---p 419eb000 00:00 0
419ec000-41a2c000 rw-p 419ec000 00:00 0
41a2c000-41a2d000 ---p 41a2c000 00:00 0
41a2d000-41a6d000 rw-p 41a2d000 00:00 0
41a6d000-41a6e000 ---p 41a6d000 00:00 0
41a6e000-41aae000 rw-p 41a6e000 00:00 0
41aae000-41aaf000 ---p 41aae000 00:00 0
41aaf000-41aef000 rw-p 41aaf000 00:00 0
41aef000-41af0000 ---p 41aef000 00:00 0
41af0000-41b30000 rw-p 41af0000 00:00 0
41b59000-41b5a000 ---p 41b59000 00:00 0
41b5a000-4255a000 rw-p 41b5a000 00:00 0
4255a000-4255b000 ---p 4255a000 00:00 0
4255b000-42f5b000 rw-p 4255b000 00:00 0
42f5b000-42f5c000 ---p 42f5b000 00:00 0
42f5c000-4395c000 rw-p 42f5c000 00:00 0
4395c000-4395d000 ---p 4395c000 00:00 0
4395d000-4435d000 rw-p 4395d000 00:00 0
4435d000-4435e000 ---p 4435d000 00:00 0
4435e000-44d5e000 rw-p 4435e000 00:00 0
44d5e000-44d5f000 ---p 44d5e000 00:00 0
44d5f000-4575f000 rw-p 44d5f000 00:00 0
4575f000-45760000 ---p 4575f000 00:00 0
45760000-46160000 rw-p 45760000 00:00 0
46160000-46161000 ---p 46160000 00:00 0
46161000-461a1000 rw-p 46161000 00:00 0
461a1000-461a2000 ---p 461a1000 00:00 0
461a2000-461e2000 rw-p 461a2000 00:00 0
461e2000-461e3000 ---p 461e2000 00:00 0
461e3000-46223000 rw-p 461e3000 00:00 0
46223000-46224000 ---p 46223000 00:00 0
46224000-46264000 rw-p 46224000 00:00 0
46264000-46265000 ---p 46264000 00:00 0
46265000-462a5000 rw-p 46265000 00:00 0
462a5000-462a6000 ---p 462a5000 00:00 0
462a6000-462e6000 rw-p 462a6000 00:00 0
462e6000-462e7000 ---p 462e6000 00:00 0
462e7000-46327000 rw-p 462e7000 00:00 0
46327000-46328000 ---p 46327000 00:00 0
46328000-46368000 rw-p 46328000 00:00 0
46368000-46369000 ---p 46368000 00:00 0
46369000-463a9000 rw-p 46369000 00:00 0
463a9000-463aa000 ---p 463a9000 00:00 0
463aa000-463ea000 rw-p 463aa000 00:00 0
463ea000-463eb000 ---p 463ea000 00:00 0
463eb000-4642b000 rw-p 463eb000 00:00 0
4642b000-4642c000 ---p 4642b000 00:00 0
4642c000-4646c000 rw-p 4642c000 00:00 0
4646c000-4646d000 ---p 4646c000 00:00 0
4646d000-464ad000 rw-p 4646d000 00:00 0
464ad000-464ae000 ---p 464ad000 00:00 0
464ae000-464ee000 rw-p 464ae000 00:00 0
464ee000-464ef000 ---p 464ee000 00:00 0
464ef000-4652f000 rw-p 464ef000 00:00 0
4652f000-46530000 ---p 4652f000 00:00 0
46530000-46570000 rw-p 46530000 00:00 0
46570000-46571000 ---p 46570000 00:00 0
46571000-465b1000 rw-p 46571000 00:00 0
465b1000-465b2000 ---p 465b1000 00:00 0
465b2000-465f2000 rw-p 465b2000 00:00 0
465f2000-465f3000 ---p 465f2000 00:00 0
465f3000-46633000 rw-p 465f3000 00:00 0
46633000-46634000 ---p 46633000 00:00 0
46634000-46674000 rw-p 46634000 00:00 0
46674000-46675000 ---p 46674000 00:00 0
46675000-466b5000 rw-p 46675000 00:00 0
466b5000-466b6000 ---p 466b5000 00:00 0
466b6000-466f6000 rw-p 466b6000 00:00 0
466f6000-466f7000 ---p 466f6000 00:00 0
466f7000-46737000 rw-p 466f7000 00:00 0
46737000-46738000 ---p 46737000 00:00 0
46738000-46778000 rw-p 46738000 00:00 0
46778000-46779000 ---p 46778000 00:00 0
46779000-467b9000 rw-p 46779000 00:00 0
467b9000-467ba000 ---p 467b9000 00:00 0
467ba000-467fa000 rw-p 467ba000 00:00 0
467fa000-467fb000 ---p 467fa000 00:00 0
467fb000-4683b000 rw-p 467fb000 00:00 0
3e67400000-3e6741c000 r-xp 00000000 68:07 327682 /lib64/ld-2.5.so
3e6761b000-3e6761c000 r--p 0001b000 68:07 327682 /lib64/ld-2.5.so
3e6761c000-3e6761d000 rw-p 0001c000 68:07 327682 /lib64/ld-2.5.so
3e67800000-3e6794c000 r-xp 00000000 68:07 327690 /lib64/libc-2.5.so
3e6794c000-3e67b4c000 ---p 0014c000 68:07 327690 /lib64/libc-2.5.so
3e67b4c000-3e67b50000 r--p 0014c000 68:07 327690 /lib64/libc-2.5.so
3e67b50000-3e67b51000 rw-p 00150000 68:07 327690 /lib64/libc-2.5.so
3e67b51000-3e67b56000 rw-p 3e67b51000 00:00 0
3e67c00000-3e67c82000 r-xp 00000000 68:07 327722 /lib64/libm-2.5.so
3e67c82000-3e67e81000 ---p 00082000 68:07 327722 /lib64/libm-2.5.so
3e67e81000-3e67e82000 r--p 00081000 68:07 327722 /lib64/libm-2.5.so
3e67e82000-3e67e83000 rw-p 00082000 68:07 327722 /lib64/libm-2.5.so
3e68000000-3e68002000 r-xp 00000000 68:07 327716 /lib64/libdl-2.5.so
3e68002000-3e68202000 ---p 00002000 68:07 327716 /lib64/libdl-2.5.so
3e68202000-3e68203000 r--p 00002000 68:07 327716 /lib64/libdl-2.5.so
3e68203000-3e68204000 rw-p 00003000 68:07 327716 /lib64/libdl-2.5.so
3e68400000-3e68416000 r-xp 00000000 68:07 327698 /lib64/libpthread-2.5.so
3e68416000-3e68615000 ---p 00016000 68:07 327698 /lib64/libpthread-2.5.so
3e68615000-3e68616000 r--p 00015000 68:07 327698 /lib64/libpthread-2.5.so
3e68616000-3e68617000 rw-p 00016000 68:07 327698 /lib64/libpthread-2.5.so
3e68617000-3e6861b000 rw-p 3e68617000 00:00 0
3e69800000-3e6980d000 r-xp 00000000 68:07 327879 /lib64/libgcc_s-4.1.2-20080825.so.1
3e6980d000-3e69a0d000 ---p 0000d000 68:07 327879 /lib64/libgcc_s-4.1.2-20080825.so.1
3e69a0d000-3e69a0e000 rw-p 0000d000 68:07 327879 /lib64/libgcc_s-4.1.2-20080825.so.1
3e71c00000-3e71c15000 r-xp 00000000 68:07 327714 /lib64/libnsl-2.5.so
3e71c15000-3e71e14000 ---p 00015000 68:07 327714 /lib64/libnsl-2.5.so
3e71e14000-3e71e15000 r--p 00014000 68:07 327714 /lib64/libnsl-2.5.so
3e71e15000-3e71e16000 rw-p 00015000 68:07 327714 /lib64/libnsl-2.5.so
3e71e16000-3e71e18000 rw-p 3e71e16000 00:00 0
3e72000000-3e72009000 r-xp 00000000 68:07 327718 /lib64/libcrypt-2.5.so
3e72009000-3e72208000 ---p 00009000 68:07 327718 /lib64/libcrypt-2.5.so
3e72208000-3e72209000 r--p 00008000 68:07 327718 /lib64/libcrypt-2.5.so
3e72209000-3e7220a000 rw-p 00009000 68:07 327718 /lib64/libcrypt-2.5.so
3e7220a000-3e72238000 rw-p 3e7220a000 00:00 0
2aaaaaae2000-2aaaaaaec000 r-xp 00000000 68:07 327715 /lib64/libnss_files-2.5.so
2aaaaaaec000-2aaaaaceb000 ---p 0000a000 68:07 327715 /lib64/libnss_files-2.5.so
2aaaaaceb000-2aaaaacec000 r--p 00009000 68:07 327715 /lib64/libnss_files-2.5.so
2aaaaacec000-2aaaaaced000 rw-p 0000a000 68:07 327715 /lib64/libnss_files-2.5.so
2aaaaaced000-2aaac596a000 rw-p 2aaaaaced000 00:00 0
2aaac596a000-2aaac596f000 r-xp 00000000 68:07 1411452 /usr/lib64/mysql/plugin/finyr.so
2aaac596f000-2aaac5a6e000 ---p 00005000 68:07 1411452 /usr/lib64/mysql/plugin/finyr.so
2aaac5a6e000-2aaac5a6f000 rw-p 00004000 68:07 1411452 /usr/lib64/mysql/plugin/finyr.so
2aaac5b8d000-2aaac5d93000 rw-p 2aaac5b8d000 00:00 0
2aaac5d93000-2aaac5d94000 r-xp 00000000 68:07 1410730 /usr/lib64/mysql/plugin/gep_days.so
2aaac5d94000-2aaac5f93000 ---p 00001000 68:07 1410730 /usr/lib64/mysql/plugin/gep_days.so
2aaac5f93000-2aaac5f94000 rw-p 00000000 68:07 1410730 /usr/lib64/mysql/plugin/gep_days.so
2aaac8000000-2aaac9934000 rw-p 2aaac8000000 00:00 0
2aaac9934000-2aaacc000000 ---p 2aaac9934000 00:00 0
2b9669330000-2b9669331000 rw-p 2b9669330000 00:00 0
2b966934b000-2b966934f000 rw-p 2b966934b000 00:00 0
7fff41765000-7fff4177a000 rw-p 7ffffffea000 00:00 0 [stack]
ffffffffff600000-ffffffffffe00000 ---p 00000000 00:00 0 [vdso]
101201 21:00:01 - mysqld got signal 6 ;
This could be because you hit a bug. It is also possible that this binary
or one of the libraries it was linked against is corrupt, improperly built,
or misconfigured. This error can also be caused by malfunctioning hardware.
We will try our best to scrape up some info that will hopefully help diagnose
the problem, but since we have already crashed, something is definitely wrong
and this may fail.

key_buffer_size=402653184
read_buffer_size=2097152
max_used_connections=34
max_threads=151
threads_connected=27
It is possible that mysqld could use up to
key_buffer_size + (read_buffer_size + sort_buffer_size)*max_threads = 1013266 K
bytes of memory
Hope that's ok; if not, decrease some variables in the equation.

thd: 0x14c06b10
Attempting backtrace. You can use the following information to find out
where mysqld died. If you see no messages after this, something went
terribly wrong...
stack_bottom = 0x463a7f10 thread_stack 0x40000
/usr/sbin/mysqld(my_print_stacktrace+0x2e)[0x850ffe]
/usr/sbin/mysqld(handle_segfault+0x322)[0x5c2652]
/lib64/libpthread.so.0[0x3e6840e4c0]
/lib64/libc.so.6(gsignal+0x35)[0x3e67830215]
/lib64/libc.so.6(abort+0x110)[0x3e67831cc0]
/lib64/libc.so.6[0x3e6786a7fb]
/lib64/libc.so.6[0x3e67871ce2]
/lib64/libc.so.6(cfree+0x8c)[0x3e6787590c]
/lib64/libc.so.6(fclose+0x14b)[0x3e67860d0b]
/usr/lib64/mysql/plugin/finyr.so(deinit_my_mysql_Log+0x22)[0x2aaac596d9c5]
/usr/lib64/mysql/plugin/finyr.so(getrange_deinit+0x16)[0x2aaac596d946]
/usr/sbin/mysqld(_ZN11udf_handler7cleanupEv+0x94)[0x543c84]
/usr/sbin/mysqld(_ZN13Item_udf_func7cleanupEv+0x10)[0x543cb0]
/usr/sbin/mysqld(_ZN11Query_arena10free_itemsEv+0x22)[0x5aef42]
/usr/sbin/mysqld(_ZN3THD19cleanup_after_queryEv+0x90)[0x5b2680]
/usr/sbin/mysqld(_Z11mysql_parseP3THDPKcjPS2_+0x190)[0x5d51b0]
/usr/sbin/mysqld(_Z16dispatch_command19enum_server_commandP3THDPcj+0x1030)[0x5d6260]
/usr/sbin/mysqld(_Z10do_commandP3THD+0xe4)[0x5d6834]
/usr/sbin/mysqld(handle_one_connection+0x6f0)[0x5c9850]
/lib64/libpthread.so.0[0x3e68406367]
/lib64/libc.so.6(clone+0x6d)[0x3e678d30ad]
Trying to get some variables.
Some pointers may be invalid and cause the dump to abort...
thd->query at 0x1547ba30 = select 1179671, getrange(datediff( RBT_SUBSCRIBER_PROFILE.PROVISIONING_DATE, RBT_SUBSCRIBER_PROFILE.DE_PROVISIONING_DATE), '0,10,20,30','New-Subs'),month(trn_crbt.sdate), count(trn_crbt.subscriber_id) from trn_crbt, mst_circle, free_paid, RBT_SUBSCRIBER_PROFILE where trn_crbt.sdate < '30' and year(trn_crbt.sdate) = '2010' and mst_circle.circle not like 'T%' and mst_circle.circle = 'BIH' and free_paid.fp_name = 'Paid' and flag in(5, 7) and error_code !=555 and trn_crbt.circle=mst_circle.circle_id and trn_crbt.free_paid=free_paid.fp_id and trn_crbt.subscriber_id=RBT_SUBSCRIBER_PROFILE.SUBSCRIBER_ID group by getrange(datediff( RBT_SUBSCRIBER_PROFILE.PROVISIONING_DATE, RBT_SUBSCRIBER_PROFILE.DE_PROVISIONING_DATE), '0,10,20,30','New-Subs'),month(trn_crbt.sdate)
thd->thread_id=56
thd->killed=KILL_CONNECTION
The manual page at http://dev.mysql.com/doc/mysql/en/crashing.html contains
information that should help you find out what is causing the crash.