Assertion `r == 0' failed in ft/cachetable /cachetable.cc:579 | abort (sig=6) in toku_do_backtrace_abort()

Description

GDB

+bt
#0 0x00007fc59b8ce771 in pthread_kill () from /lib64/libpthread.so.0
#1 0x000000000067634d in handle_fatal_signal (sig=6) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.
0/sql/signal_handler.cc:236
#2 <signal handler called>
#3 0x00007fc59a4d25c9 in raise () from /lib64/libc.so.6
#4 0x00007fc59a4d3cd8 in abort () from /lib64/libc.so.6
#5 0x00007fc57c92cf48 in toku_do_backtrace_abort() [clone .80651] () at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-s
erver-5.6.21-69.0/storage/tokudb/ft-index/portability/toku_assert.cc:199
#6 0x00007fc57c92cfd2 in toku_do_assert_zero_fail (expr=expr@entry=18446744073709551615, expr_as_string=expr_as_string@entry=0x7fc57c99f4c0 "r", function=f
unction@entry=0x7fc57c9b9810 <ZZ20toku_cachefile_closePP9cachefileb10toku_lsnE12FUNCTION_.28427.11176> "toku_cachefile_close", file=file@entry=0x7fc57
c9a5800 "/mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/storage/tokudb/ft-index/ft/cachetable/cachetab
le.cc", line=line@entry=579, caller_errno=<optimized out>) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.2
1-69.0/storage/tokudb/ft-index/portability/toku_assert.cc:229
#7 0x00007fc57c94e344 in toku_cachefile_close (cfp=<optimized out>, oplsn_valid=<optimized out>, oplsn=...) at /mnt/workspace/percona-server-5.6-binaries-o
pt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/storage/tokudb/ft-index/ft/cachetable/cachetable.cc:579
#8 0x00007fc57c94e45b in toku_ft_remove_reference (ft=0x7fc4923f8e00, oplsn_valid=oplsn_valid@entry=false, oplsn=oplsn@entry=..., remove_ref=<optimized out
>, extra=extra@entry=0x0) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/storage/tokudb/ft-index/ft
/ft.cc:959
#9 0x00007fc57c94e69d in toku_ft_remove_txn_ref (ft=<optimized out>) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-s
erver-5.6.21-69.0/storage/tokudb/ft-index/ft/ft.cc:828
#10 0x00007fc57c94e6ac in remove_txn (h=@0x7fc565f2f740: 0x7fc4923f8e00, idx=idx@entry=12, txn=txn@entry=0x7fc573ba2800) at /mnt/workspace/percona-server-5.
6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/storage/tokudb/ft-index/ft/txn/txn.cc:615
#11 0x00007fc57c94e7bb in toku::omt::iterate_internal (this=this@entry=0x7fc573ba2980, right=right@entry=27, subtree=..., idx=<optimized out>, iterate_extra
=iterate_extra@entry=0x7fc573ba2800, left=0) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/storage
/tokudb/ft-index/util/omt.cc:850
#12 0x00007fc57c94e879 in iterate_internal (left=0, iterate_extra=0x7fc573ba2800, idx=<optimized out>, subtree=..., right=27, this=0x7fc573ba2980) at /mnt/w
orkspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/storage/tokudb/ft-index/util/omt.cc:846
#13 toku::omt::iterate_internal (this=this@entry=0x7fc573ba2980, right=right@entry=27, subtree=..., idx=<optimized out>, iterate_extra=iterate_extra@entry=0x7fc573ba2800, left=0) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/storage/tokudb/ft-index/util/omt.cc:846
#14 0x00007fc57c94e845 in iterate_internal (left=0, iterate_extra=0x7fc573ba2800, idx=<optimized out>, subtree=..., right=27, this=0x7fc573ba2980) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/storage/tokudb/ft-index/util/omt.cc:846
#15 toku::omt::iterate_internal (this=0x7fc573ba2980, right=27, subtree=..., idx=<optimized out>, iterate_extra=<optimized out>, left=0) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/storage/tokudb/ft-index/util/omt.cc:854
#16 0x00007fc57c940856 in toku_txn_commit (txn=0x7fc4943f23a0, flags=<optimized out>, poll=<optimized out>, poll_extra=<optimized out>, release_mo_lock=<optimized out>, low_priority=<optimized out>) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/storage/tokudb/ft-index/src/ydb_txn.cc:187
#17 0x00007fc57c8a52ee in commit_txn_with_progress (thd=0x7fc4afd8a000, flags=0, txn=0x7fc4943f23a0) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/storage/tokudb/hatoku_hton.cc:742
#18 tokudb_commit (hton=<optimized out>, thd=0x7fc4afd8a000, all=<optimized out>) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/storage/tokudb/hatoku_hton.cc:785
#19 0x00000000005ba0e6 in ha_commit_low (thd=thd@entry=0x7fc4afd8a000, all=all@entry=false, run_after_commit=run_after_commit@entry=true) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/sql/handler.cc:1557
#20 0x00000000006654d8 in TC_LOG_MMAP::commit (this=0x1372ea0 <tc_log_mmap>, thd=0x7fc4afd8a000, all=<optimized out>) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/sql/log.cc:2949
#21 0x00000000005ba90f in ha_commit_trans (thd=thd@entry=0x7fc4afd8a000, all=all@entry=false, ignore_global_read_lock=ignore_global_read_lock@entry=false) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/sql/handler.cc:1500
#22 0x000000000077f459 in trans_commit_stmt (thd=thd@entry=0x7fc4afd8a000) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/sql/transaction.cc:434
#23 0x00000000006f5204 in mysql_execute_command (thd=thd@entry=0x7fc4afd8a000) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/sql/sql_parse.cc:5500
#24 0x00000000006fbc68 in mysql_parse (thd=thd@entry=0x7fc4afd8a000, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x7fc565f31160) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/sql/sql_parse.cc:6773
#25 0x00000000006fd3d9 in dispatch_command (command=<optimized out>, thd=thd@entry=0x7fc4afd8a000, packet=0x7fc4afd8e001 " LOAD DATA INFILE '/ssd/qa56opt/55/tmp/gentest20477.tmp' INTO TABLE `table500_tokudb_default_int`", packet_length=<optimized out>) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/sql/sql_parse.cc:1432
#26 0x00000000006ff139 in do_command (thd=thd@entry=0x7fc4afd8a000) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/sql/sql_parse.cc:1049
#27 0x0000000000787418 in threadpool_process_request (thd=0x7fc4afd8a000) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/sql/threadpool_common.cc:311
#28 0x00000000007883ed in handle_event (connection=0x7fc56579fc10) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/sql/threadpool_unix.cc:1553
#29 worker_main (param=param@entry=0x1394e00 <all_groups+1024>) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/sql/threadpool_unix.cc:1606
#30 0x0000000000af79a3 in pfs_spawn_thread (arg=0x7fc5637fdec0) at /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/storage/perfschema/pfs.cc:1860
#31 0x00007fc59b8c9df3 in start_thread () from /lib64/libpthread.so.0
#32 0x00007fc59a59301d in clone () from /lib64/libc.so.6
(gdb) +set logging off

======Error info
Engine status:
BUILD_ID = 0
Env panic code: 2
Env panic string: /mnt/workspace/percona-server-5.6-binaries-opt-yassl/label_exp/centos6-64/percona-server-5.6.21-69.0/storage/tokudb/ft-index/ft/cachetable
/cachetable.cc:579 toku_cachefile_close: Assertion `r == 0' failed (errno=2) (r=18446744073709551615)


terribly wrong...
stack_bottom = 7fc565f31d48 thread_stack 0x40000
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/bin/mysqld(my_print_stacktrace+0x2c)[0x8e90ac]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/bin/mysqld(handle_fatal_signal+0x461)[0x6763e1]
/lib64/libpthread.so.0(+0xf130)[0x7fc59b8d1130]
/lib64/libc.so.6(gsignal+0x39)[0x7fc59a4d25c9]
/lib64/libc.so.6(abort+0x148)[0x7fc59a4d3cd8]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/lib/mysql/plugin/ha_tokudb.so(+0xd8f48)[0x7fc57c92cf48]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/lib/mysql/plugin/ha_tokudb.so(+0xd8fd2)[0x7fc57c92cfd2]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/lib/mysql/plugin/ha_tokudb.so(Z20toku_cachefile_closePP9cachefileb10_toku_lsn+0x234)[0x7fc57c94e344]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/lib/mysql/plugin/ha_tokudb.so(Z24toku_ft_remove_referenceP2ftb10toku_lsnPFvS0_PvES2+0x7b)[0x7fc57c94e45b]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/lib/mysql/plugin/ha_tokudb.so(_Z10remove_txnRKP2ftjP7tokutxn+0xc)[0x7fc57c94e6ac]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/lib/mysql/plugin/ha_tokudb.so(+0xfa7bb)[0x7fc57c94e7bb]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/lib/mysql/plugin/ha_tokudb.so(+0xfa879)[0x7fc57c94e879]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/lib/mysql/plugin/ha_tokudb.so(+0xfa845)[0x7fc57c94e845]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/lib/mysql/plugin/ha_tokudb.so(+0xec856)[0x7fc57c940856]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/lib/mysql/plugin/ha_tokudb.so(+0x512ee)[0x7fc57c8a52ee]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/bin/mysqld(_Z13ha_commit_lowP3THDbb+0x86)[0x5ba0e6]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/bin/mysqld(_ZN11TC_LOG_MMAP6commitEP3THDb+0x38)[0x6654d8]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/bin/mysqld(_Z15ha_commit_transP3THDbb+0x29f)[0x5ba90f]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/bin/mysqld(_Z17trans_commit_stmtP3THD+0x29)[0x77f459]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/bin/mysqld(_Z21mysql_execute_commandP3THD+0x5a4)[0x6f5204]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/bin/mysqld(_Z11mysql_parseP3THDPcjP12Parser_state+0x5c8)[0x6fbc68]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/bin/mysqld(_Z16dispatch_command19enum_server_commandP3THDPcj+0xff9)[0x6fd3d9]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/bin/mysqld(_Z26threadpool_process_requestP3THD+0xf8)[0x787418]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/bin/mysqld[0x7883ed]
/ssd/qa56opt/Percona-Server-5.6.21-rel69.0-670.Linux.x86_64/bin/mysqld(pfs_spawn_thread+0x143)[0xaf79a3]
/lib64/libpthread.so.0(+0x7df3)[0x7fc59b8c9df3]
/lib64/libc.so.6(clone+0x6d)[0x7fc59a59301d]

Trying to get some variables.
Some pointers may be invalid and cause the dump to abort.
Query (7fc4a1ffd010): LOAD DATA INFILE '/ssd/qa56opt/55/tmp/gentest20477.tmp' INTO TABLE `table500_tokudb_default_int`
Connection ID (thread ID): 10
Status: NOT_KILLED

Environment

None

Status

Assignee

Rich Prohaska

Reporter

Ramesh Sivaraman

Labels

None

External issue ID

None

Freshdesk Tickets

None

Fix versions

Priority

Major
Configure