Database locked during delete operation

Description

During massive deleteion processes whole database stop responding. New queries hangs in "Opening tables" state.

TokuDB_locks

1 25444 rows in set (0.50 sec)

pt-pmp

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 1044 pthread_cond_wait,ha_tokudb::??(ha_tokudb.so),start_thread(libpthread.so.0),clone(libc.so.6),?? 867 poll(libc.so.6),vio_io_wait,vio_socket_io_wait,vio_read,::??,my_net_read_packet,do_command,do_handle_one_connection,handle_one_connection,::??,start_thread(libpthread.so.0),clone(libc.so.6),?? 487 pthread_cond_wait,one_thread_per_connection_end,do_handle_one_connection,handle_one_connection,::??,start_thread(libpthread.so.0),clone(libc.so.6),?? 359 __lll_lock_wait(libpthread.so.0),_L_lock_1145(libpthread.so.0),pthread_mutex_lock(libpthread.so.0),TOKUDB_SHARE::get_share(ha_tokudb.so),ha_tokudb::open(ha_tokudb.so),handler::ha_open,open_table_from_share,open_table,open_tables,open_and_lock_tables,::??,mysql_execute_command,::??,dispatch_command,do_handle_one_connection,handle_one_connection,::??,start_thread(libpthread.so.0),clone(libc.so.6),?? 100 libaio::??(libaio.so.1),::??,::??,::??,start_thread(libpthread.so.0),clone(libc.so.6),?? 48 __lll_lock_wait(libpthread.so.0),_L_lock_926(libpthread.so.0),pthread_mutex_lock(libpthread.so.0),toku_db_open_iname(ha_tokudb.so),ha_tokudb::??(ha_tokudb.so),ha_tokudb::??(ha_tokudb.so),tokudb::metadata::open(ha_tokudb.so),ha_tokudb::??(ha_tokudb.so),ha_tokudb::get_status(ha_tokudb.so),ha_tokudb::initialize_share(ha_tokudb.so),ha_tokudb::open(ha_tokudb.so),handler::ha_open,open_table_from_share,open_table,open_tables,open_and_lock_tables,::??,mysql_execute_command,::??,dispatch_command,do_handle_one_connection,handle_one_connection,::??,start_thread(libpthread.so.0),clone(libc.so.6),?? 43 __lll_lock_wait(libpthread.so.0),_L_lock_926(libpthread.so.0),pthread_mutex_lock(libpthread.so.0),ha_tokudb::??(ha_tokudb.so),toku_db_close(ha_tokudb.so),ha_tokudb::??(ha_tokudb.so),TOKUDB_SHARE::release(ha_tokudb.so),ha_tokudb::__close(ha_tokudb.so),ha_tokudb::close(ha_tokudb.so),closefrm,intern_close_table,tc_release_table,close_thread_table,close_thread_tables,mysql_execute_command,::??,dispatch_command,do_handle_one_connection,handle_one_connection,::??,start_thread(libpthread.so.0),clone(libc.so.6),?? 40 pthread_cond_timedwait,::??,::??,start_thread(libpthread.so.0),clone(libc.so.6),?? 30 select(libc.so.6),::??,::??,start_thread(libpthread.so.0),clone(libc.so.6),?? 30 libz::??(libz.so.1),libz::??(libz.so.1),deflate(libz.so.1),toku_compress(ha_tokudb.so),compress_nocrc_sub_block(ha_tokudb.so),toku_serialize_ftnode_to_memory(ha_tokudb.so),toku_serialize_ftnode_to(ha_tokudb.so),toku_ftnode_flush_callback(ha_tokudb.so),ha_tokudb::??(ha_tokudb.so),ha_tokudb::??(ha_tokudb.so),start_thread(libpthread.so.0),clone(libc.so.6),?? 24 __lll_lock_wait(libpthread.so.0),_L_lock_926(libpthread.so.0),pthread_mutex_lock(libpthread.so.0),toku_db_open_iname(ha_tokudb.so),ha_tokudb::??(ha_tokudb.so),ha_tokudb::??(ha_tokudb.so),ha_tokudb::open_secondary_dictionary(ha_tokudb.so),ha_tokudb::initialize_share(ha_tokudb.so),ha_tokudb::open(ha_tokudb.so),handler::ha_open,open_table_from_share,open_table,open_tables,open_and_lock_tables,::??,mysql_execute_command,::??,dispatch_command,do_handle_one_connection,handle_one_connection,::??,start_thread(libpthread.so.0),clone(libc.so.6),?? 22 __lll_lock_wait(libpthread.so.0),_L_lock_926(libpthread.so.0),pthread_mutex_lock(libpthread.so.0),ha_tokudb::??(ha_tokudb.so),toku_db_close(ha_tokudb.so),ha_tokudb::??(ha_tokudb.so),TOKUDB_SHARE::release(ha_tokudb.so),ha_tokudb::__close(ha_tokudb.so),ha_tokudb::close(ha_tokudb.so),closefrm,intern_close_table,tc_add_table,open_table,open_tables,open_and_lock_tables,::??,mysql_execute_command,::??,dispatch_command,do_handle_one_connection,handle_one_connection,::??,start_thread(libpthread.so.0),clone(libc.so.6),?? 20 pthread_cond_wait,::??,::??,start_thread(libpthread.so.0),clone(libc.so.6),?? 20 pthread_cond_wait,handle_rpl_parallel_thread,::??,start_thread(libpthread.so.0),clone(libc.so.6),?? 20 pthread_cond_timedwait,::??,::??,::??,start_thread(libpthread.so.0),clone(libc.so.6),?? 20 libz::??(libz.so.1),deflate(libz.so.1),toku_compress(ha_tokudb.so),compress_nocrc_sub_block(ha_tokudb.so),toku_serialize_ftnode_to_memory(ha_tokudb.so),toku_serialize_ftnode_to(ha_tokudb.so),toku_ftnode_flush_callback(ha_tokudb.so),ha_tokudb::??(ha_tokudb.so),ha_tokudb::??(ha_tokudb.so),start_thread(libpthread.so.0),clone(libc.so.6),?? 18 nanosleep(libc.so.6),usleep(libc.so.6),ha_tokudb::??(ha_tokudb.so),start_thread(libpthread.so.0),clone(libc.so.6),?? 18 __lll_lock_wait(libpthread.so.0),_L_lock_1145(libpthread.so.0),pthread_mutex_lock(libpthread.so.0),TOKUDB_SHARE::get_share(ha_tokudb.so),ha_tokudb::open(ha_tokudb.so),handler::ha_open,open_table_from_share,open_table,open_tables,open_and_lock_tables,mysql_delete,mysql_execute_command,::??,dispatch_command,do_handle_one_connection,handle_one_connection,::??,start_thread(libpthread.so.0),clone(libc.so.6),?? 16 __lll_lock_wait(libpthread.so.0),_L_lock_1145(libpthread.so.0),pthread_mutex_lock(libpthread.so.0),TOKUDB_SHARE::get_share(ha_tokudb.so),ha_tokudb::open(ha_tokudb.so),handler::ha_open,open_table_from_share,open_table,open_tables,open_and_lock_tables,mysql_insert,mysql_execute_command,::??,dispatch_command,do_handle_one_connection,handle_one_connection,::??,start_thread(libpthread.so.0),clone(libc.so.6),?? 14 __lll_lock_wait(libpthread.so.0),_L_lock_926(libpthread.so.0),pthread_mutex_lock(libpthread.so.0),toku_ft_open_close_lock(ha_tokudb.so),ha_tokudb::??(ha_tokudb.so),toku_ft_handle_open(ha_tokudb.so),toku_db_open_iname(ha_tokudb.so),ha_tokudb::??(ha_tokudb.so),ha_tokudb::??(ha_tokudb.so),tokudb::metadata::open(ha_tokudb.so),ha_tokudb::??(ha_tokudb.so),ha_tokudb::get_status(ha_tokudb.so),ha_tokudb::initialize_share(ha_tokudb.so),ha_tokudb::open(ha_tokudb.so),handler::ha_open,open_table_from_share,open_table,open_tables,open_and_lock_tables,::??,mysql_execute_command,::??,dispatch_command,do_handle_one_connection,handle_one_connection,::??,start_thread(libpthread.so.0),clone(libc.so.6),?? 14 __lll_lock_wait(libpthread.so.0),_L_lock_926(libpthread.so.0),pthread_mutex_lock(libpthread.so.0),toku_ft_open_close_lock(ha_tokudb.so),ha_tokudb::??(ha_tokudb.so),toku_ft_handle_open(ha_tokudb.so),toku_db_open_iname(ha_tokudb.so),ha_tokudb::??(ha_tokudb.so),ha_tokudb::??(ha_tokudb.so),ha_tokudb::open_main_dictionary(ha_tokudb.so),ha_tokudb::initialize_share(ha_tokudb.so),ha_tokudb::open(ha_tokudb.so),handler::ha_open,open_table_from_share,open_table,open_tables,open_and_lock_tables,::??,mysql_execute_command,::??,dispatch_command,do_handle_one_connection,handle_one_connection,::??,start_thread(libpthread.so.0),clone(libc.so.6),??

table with many deletions

1 2 3 4 5 6 7 8 9 10 11 12 `idarchive` int(10) unsigned NOT NULL, `name` varchar(255) NOT NULL, `idsite` int(10) unsigned DEFAULT NULL, `date1` date DEFAULT NULL, `date2` date DEFAULT NULL, `period` tinyint(3) unsigned DEFAULT NULL, `ts_archived` datetime DEFAULT NULL, `value` double DEFAULT NULL, PRIMARY KEY (`idarchive`,`name`), KEY `index_idsite_dates_period` (`idsite`,`date1`,`date2`,`period`,`ts_archived`), KEY `index_period_archived` (`period`,`ts_archived`) ) ENGINE=TokuDB

versions

1 2 | tokudb_version | 5.6.33-79.0 | | version | 10.1.19-MariaDB-1~wheezy

We face this problem from 2 months almost every day so it looks confirmed.
I found also this

Environment

None

Status

Assignee

Unassigned

Reporter

Jacek Lubzinski

Labels

None

External issue ID

None

External issue ID

None

External issue ID

None

External issue ID

None

External issue ID

None

External issue ID

None

External issue ID

None

External issue ID

None

External issue ID

None

Freshdesk Tickets

None

Priority

Major
Configure