diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index 67cc2860f8661..0eb4cf56c0427 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -1779,7 +1779,7 @@ copy_back() if it exists. */ ds_tmp = ds_create(dst_dir, DS_TYPE_LOCAL); - if (!(ret = copy_or_move_file(ds_tmp, LOG_FILE_NAME, LOG_FILE_NAME, + if (!(ret = copy_or_move_file(ds_tmp, "ib_logfile0", "ib_logfile0", dst_dir, 1))) { goto cleanup; } @@ -1890,7 +1890,7 @@ copy_back() } /* skip the redo log (it was already copied) */ - if (!strcmp(filename, LOG_FILE_NAME)) { + if (!strcmp(filename, "ib_logfile0")) { continue; } diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 26ec7a2fa264e..a16c6d3a4b43d 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -2745,7 +2745,7 @@ static bool innodb_init() srv_log_group_home_dir= xtrabackup_target_dir; bool ret; - const std::string ib_logfile0{get_log_file_path()}; + const std::string ib_logfile0{log_sys.get_circular_path()}; os_file_delete_if_exists_func(ib_logfile0.c_str(), nullptr); os_file_t file= os_file_create_func(ib_logfile0.c_str(), OS_FILE_CREATE, @@ -4925,6 +4925,8 @@ static bool backup_wait_for_commit_lsn() lsn_t last_lsn= recv_sys.lsn; /* read the latest checkpoint lsn */ + log_sys.last_checkpoint_lsn= 0; + recv_sys.file_checkpoint= 0; if (recv_sys.find_checkpoint() == DB_SUCCESS && log_sys.is_latest()) { metadata_to_lsn= log_sys.last_checkpoint_lsn; @@ -5550,6 +5552,7 @@ static bool xtrabackup_backup_func() srv_n_purge_threads = 1; srv_read_only_mode = TRUE; + recv_sys.rpo = LSN_MAX; srv_operation = SRV_OPERATION_BACKUP; log_file_op = backup_file_op; @@ -5642,10 +5645,11 @@ static bool xtrabackup_backup_func() /* open the log file */ memset(&stat_info, 0, sizeof(MY_STAT)); - dst_log_file = ds_open(backup_datasinks.m_redo, LOG_FILE_NAME, &stat_info); + dst_log_file = + ds_open(backup_datasinks.m_redo, "ib_logfile0", &stat_info); if (dst_log_file == NULL) { - msg("Error: failed to open the target stream for '%s'.", - LOG_FILE_NAME); + msg("Error: failed to open the target stream" + " for 'ib_logfile0'."); goto fail; } diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index eb5e38427f7f6..99eb5ea0cdfcc 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -166,6 +166,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc ../sql/opt_hints.cc ../sql/opt_hints.h ../sql/opt_trace_ddl_info.cc ../sql/opt_trace_ddl_info.h ../sql/sql_path.cc + ../sql/sql_backup.cc ${GEN_SOURCES} ${MYSYS_LIBWRAP_SOURCE} ) diff --git a/mysql-test/collections/buildbot_suites.bat b/mysql-test/collections/buildbot_suites.bat index 053057872bf40..129a2f67d79de 100644 --- a/mysql-test/collections/buildbot_suites.bat +++ b/mysql-test/collections/buildbot_suites.bat @@ -6,6 +6,7 @@ innodb,^ versioning,^ plugins,^ mariabackup,^ +backup,^ roles,^ auth_gssapi,^ mysql_sha2,^ diff --git a/mysql-test/include/innodb_encrypt_log.inc b/mysql-test/include/innodb_encrypt_log.inc index 5beebeae81f07..fd3e0ceea5758 100644 --- a/mysql-test/include/innodb_encrypt_log.inc +++ b/mysql-test/include/innodb_encrypt_log.inc @@ -2,3 +2,7 @@ # (see include/innodb_encrypt_log.combinations) --source include/have_innodb.inc +if ($MTR_COMBINATION_CRYPT) +{ +--source ../suite/encryption/include/skip_innodb_log_archive.inc +} diff --git a/mysql-test/main/analyze_engine_stats2.result b/mysql-test/main/analyze_engine_stats2.result index 881a2a0a633a0..f6cf1fa25f518 100644 --- a/mysql-test/main/analyze_engine_stats2.result +++ b/mysql-test/main/analyze_engine_stats2.result @@ -18,7 +18,7 @@ repeat(uuid(), 7) from seq_1_to_16384; commit; SET GLOBAL innodb_fast_shutdown=0; -# restart +# restart: --innodb-log-recovery-start=0 set log_slow_verbosity='engine'; set long_query_time=0.0; set @js='$analyze_output'; diff --git a/mysql-test/main/analyze_engine_stats2.test b/mysql-test/main/analyze_engine_stats2.test index 22ade3a746f12..e17666c44b595 100644 --- a/mysql-test/main/analyze_engine_stats2.test +++ b/mysql-test/main/analyze_engine_stats2.test @@ -32,6 +32,8 @@ from seq_1_to_16384; commit; SET GLOBAL innodb_fast_shutdown=0; +# Avoid any crash recovery that would load pages. +--let $restart_parameters=--innodb-log-recovery-start=0 source include/restart_mysqld.inc; set log_slow_verbosity='engine'; set long_query_time=0.0; diff --git a/mysql-test/main/analyze_stmt_prefetch_count.result b/mysql-test/main/analyze_stmt_prefetch_count.result index d55b416c32a85..25fc8aae96853 100644 --- a/mysql-test/main/analyze_stmt_prefetch_count.result +++ b/mysql-test/main/analyze_stmt_prefetch_count.result @@ -19,7 +19,7 @@ commit; # (in the test's .opt file we've disabled buffer pool saving/loading # and also tried to disable any background activity) SET GLOBAL innodb_fast_shutdown=0; -# restart +# restart: --innodb-log-recovery-start=0 set @innodb_pages_read0= (select variable_value from information_schema.session_status diff --git a/mysql-test/main/analyze_stmt_prefetch_count.test b/mysql-test/main/analyze_stmt_prefetch_count.test index 3e52830a0d06f..a9dad5cc18baa 100644 --- a/mysql-test/main/analyze_stmt_prefetch_count.test +++ b/mysql-test/main/analyze_stmt_prefetch_count.test @@ -28,6 +28,8 @@ commit; --echo # (in the test's .opt file we've disabled buffer pool saving/loading --echo # and also tried to disable any background activity) SET GLOBAL innodb_fast_shutdown=0; +# Avoid any crash recovery that would load pages. +--let $restart_parameters=--innodb-log-recovery-start=0 --source include/restart_mysqld.inc set @innodb_pages_read0= diff --git a/mysql-test/main/backup_server.result b/mysql-test/main/backup_server.result new file mode 100644 index 0000000000000..7a064d7e5fe11 --- /dev/null +++ b/mysql-test/main/backup_server.result @@ -0,0 +1,6 @@ +BACKUP SERVER TO '$datadir/some_directory'; +ERROR HY000: Incorrect arguments to BACKUP SERVER TO +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; diff --git a/mysql-test/main/backup_server.test b/mysql-test/main/backup_server.test new file mode 100644 index 0000000000000..50cf326d39838 --- /dev/null +++ b/mysql-test/main/backup_server.test @@ -0,0 +1,10 @@ +--let $datadir=`select @@datadir` +--error ER_WRONG_ARGUMENTS +evalp BACKUP SERVER TO '$datadir/some_directory'; +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--rmdir $MYSQLTEST_VARDIR/some_directory +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--rmdir $MYSQLTEST_VARDIR/some_directory diff --git a/mysql-test/main/grant_backup_server.result b/mysql-test/main/grant_backup_server.result new file mode 100644 index 0000000000000..a4d85292dee45 --- /dev/null +++ b/mysql-test/main/grant_backup_server.result @@ -0,0 +1,27 @@ +CREATE USER user1@localhost IDENTIFIED BY ''; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the RELOAD privilege(s) for this operation +disconnect con1; +connection default; +GRANT SELECT ON test.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the RELOAD privilege(s) for this operation +disconnect con1; +connection default; +GRANT RELOAD ON test.* TO user1@localhost; +ERROR HY000: Incorrect usage of DB GRANT and GLOBAL PRIVILEGES +GRANT RELOAD ON *.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the SELECT privilege(s) for this operation +disconnect con1; +connection default; +GRANT SELECT ON *.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO '$datadir/some_directory'; +ERROR HY000: Incorrect arguments to BACKUP SERVER TO +disconnect con1; +connection default; +DROP USER user1@localhost; diff --git a/mysql-test/main/grant_backup_server.test b/mysql-test/main/grant_backup_server.test new file mode 100644 index 0000000000000..726abb19908bb --- /dev/null +++ b/mysql-test/main/grant_backup_server.test @@ -0,0 +1,29 @@ +--source include/not_embedded.inc +CREATE USER user1@localhost IDENTIFIED BY ''; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +GRANT SELECT ON test.* TO user1@localhost; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +--error ER_WRONG_USAGE +GRANT RELOAD ON test.* TO user1@localhost; +GRANT RELOAD ON *.* TO user1@localhost; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +GRANT SELECT ON *.* TO user1@localhost; +--connect con1,localhost,user1 +--let $datadir=`select @@datadir` +--error ER_WRONG_ARGUMENTS +evalp BACKUP SERVER TO '$datadir/some_directory'; +--disconnect con1 +--connection default +DROP USER user1@localhost; diff --git a/mysql-test/main/mysqld--help.result b/mysql-test/main/mysqld--help.result index 3e6804a150dc8..00575695fa90f 100644 --- a/mysql-test/main/mysqld--help.result +++ b/mysql-test/main/mysqld--help.result @@ -2056,7 +2056,7 @@ performance-schema-max-socket-classes 10 performance-schema-max-socket-instances -1 performance-schema-max-sql-text-length 1024 performance-schema-max-stage-classes 170 -performance-schema-max-statement-classes 227 +performance-schema-max-statement-classes 228 performance-schema-max-statement-stack 10 performance-schema-max-table-handles -1 performance-schema-max-table-instances -1 diff --git a/mysql-test/mariadb-test-run.pl b/mysql-test/mariadb-test-run.pl index c3816654d08fe..8b2e671588adf 100755 --- a/mysql-test/mariadb-test-run.pl +++ b/mysql-test/mariadb-test-run.pl @@ -180,6 +180,7 @@ END main- archive- atomic- + backup- binlog- binlog_encryption- binlog_in_engine- @@ -327,7 +328,7 @@ END my $opt_debug_sync_timeout= 300; # Default timeout for WAIT_FOR actions. my $warn_seconds = 60; -my $rebootstrap_re= '--innodb[-_](?:page[-_]size|checksum[-_]algorithm|undo[-_]tablespaces|log[-_]group[-_]home[-_]dir|data[-_]home[-_]dir)|data[-_]file[-_]path|force_rebootstrap'; +my $rebootstrap_re= '--innodb[-_](?:page[-_]size|checksum[-_]algorithm|undo[-_]tablespaces|log[-_](group[-_]home[-_]dir|archive)|data[-_]home[-_]dir)|data[-_]file[-_]path|force_rebootstrap'; sub testcase_timeout ($) { return $opt_testcase_timeout * 60; } sub check_timeout ($) { return testcase_timeout($_[0]); } @@ -3146,7 +3147,7 @@ sub mysql_install_db { # need to be given to the bootstrap process as well as the # server process. foreach my $extra_opt ( @opt_extra_mysqld_opt ) { - if ($extra_opt =~ /--innodb/) { + if ($extra_opt =~ /--((loose|skip)[-_])*innodb/) { mtr_add_arg($args, $extra_opt); } } diff --git a/mysql-test/suite/backup/backup_innodb.combinations b/mysql-test/suite/backup/backup_innodb.combinations new file mode 100644 index 0000000000000..7fd419bba7692 --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.combinations @@ -0,0 +1,4 @@ +[archived] +innodb_log_archive=ON +[circular] +innodb_log_archive=OFF diff --git a/mysql-test/suite/backup/backup_innodb.result b/mysql-test/suite/backup/backup_innodb.result new file mode 100644 index 0000000000000..ee2d34ae10bc9 --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.result @@ -0,0 +1,21 @@ +CREATE TABLE t(i INT PRIMARY KEY) ENGINE=INNODB; +INSERT INTO t VALUES(1); +BEGIN; +DELETE FROM t; +connect backup,localhost,root; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +disconnect backup; +connection default; +COMMIT; +SELECT * FROM t; +i +ibdata1 +mysql +test +undo001 +undo002 +undo003 +# restart +SELECT * FROM t; +i +DROP TABLE t; diff --git a/mysql-test/suite/backup/backup_innodb.test b/mysql-test/suite/backup/backup_innodb.test new file mode 100644 index 0000000000000..cbad0c803b1e0 --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.test @@ -0,0 +1,25 @@ +--source include/have_innodb.inc + +CREATE TABLE t(i INT PRIMARY KEY) ENGINE=INNODB; +INSERT INTO t VALUES(1); +BEGIN; +DELETE FROM t; + +--connect backup,localhost,root +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--disconnect backup +--connection default + +COMMIT; +SELECT * FROM t; + +let $datadir=`SELECT @@datadir`; +--source include/shutdown_mysqld.inc +--list_files $MYSQLTEST_VARDIR/some_directory +#--rmdir $datadir +#--move_file $MYSQLTEST_VARDIR/some_directory $datadir +--rmdir $MYSQLTEST_VARDIR/some_directory +--source include/start_mysqld.inc + +SELECT * FROM t; +DROP TABLE t; diff --git a/mysql-test/suite/backup/suite.pm b/mysql-test/suite/backup/suite.pm new file mode 100644 index 0000000000000..e24cb5b5185b1 --- /dev/null +++ b/mysql-test/suite/backup/suite.pm @@ -0,0 +1,10 @@ +package My::Suite::Backup; + +@ISA = qw(My::Suite); +use My::Find; +use File::Basename; +use strict; + +return "Not run for embedded server" if $::opt_embedded_server; + +bless { }; diff --git a/mysql-test/suite/binlog_in_engine/recovery.result b/mysql-test/suite/binlog_in_engine/recovery.result index fb149d6910538..17a44dbe8457a 100644 --- a/mysql-test/suite/binlog_in_engine/recovery.result +++ b/mysql-test/suite/binlog_in_engine/recovery.result @@ -38,6 +38,21 @@ binlog-000000.ibb # Gtid # # BEGIN GTID #-#-# binlog-000000.ibb # Query # # use `test`; INSERT INTO t1 VALUES (1) binlog-000000.ibb # Xid # # COMMIT /* XID */ binlog-000000.ibb # Gtid # # BEGIN GTID #-#-# +binlog-000000.ibb # Query # # use `mtr`; INSERT INTO test_suppressions (pattern) VALUES ( NAME_CONST('pattern',_utf8mb4'InnoDB: innodb_read_only prevents crash recovery' COLLATE 'utf8mb4_uca1400_ai_ci')) +binlog-000000.ibb # Query # # COMMIT +binlog-000000.ibb # Gtid # # BEGIN GTID #-#-# +binlog-000000.ibb # Query # # use `mtr`; INSERT INTO test_suppressions (pattern) VALUES ( NAME_CONST('pattern',_utf8mb4'InnoDB: Plugin initialization aborted' COLLATE 'utf8mb4_uca1400_ai_ci')) +binlog-000000.ibb # Query # # COMMIT +binlog-000000.ibb # Gtid # # BEGIN GTID #-#-# +binlog-000000.ibb # Query # # use `mtr`; INSERT INTO test_suppressions (pattern) VALUES ( NAME_CONST('pattern',_utf8mb4'Plugin \'InnoDB\' registration as a STORAGE ENGINE failed' COLLATE 'utf8mb4_uca1400_ai_ci')) +binlog-000000.ibb # Query # # COMMIT +binlog-000000.ibb # Gtid # # BEGIN GTID #-#-# +binlog-000000.ibb # Query # # use `mtr`; INSERT INTO test_suppressions (pattern) VALUES ( NAME_CONST('pattern',_utf8mb4'unknown option \'--innodb-invalid-option\'' COLLATE 'utf8mb4_uca1400_ai_ci')) +binlog-000000.ibb # Query # # COMMIT +binlog-000000.ibb # Gtid # # BEGIN GTID #-#-# +binlog-000000.ibb # Query # # use `mtr`; INSERT INTO test_suppressions (pattern) VALUES ( NAME_CONST('pattern',_utf8mb4'\\[ERROR\\] Aborting' COLLATE 'utf8mb4_uca1400_ai_ci')) +binlog-000000.ibb # Query # # COMMIT +binlog-000000.ibb # Gtid # # BEGIN GTID #-#-# binlog-000000.ibb # Query # # use `test`; INSERT INTO t1 VALUES (2) binlog-000000.ibb # Query # # use `test`; INSERT INTO t1 VALUES (3) binlog-000000.ibb # Xid # # COMMIT /* XID */ diff --git a/mysql-test/suite/binlog_in_engine/recovery.test b/mysql-test/suite/binlog_in_engine/recovery.test index 5d9a286501359..716b93f020039 100644 --- a/mysql-test/suite/binlog_in_engine/recovery.test +++ b/mysql-test/suite/binlog_in_engine/recovery.test @@ -12,6 +12,7 @@ INSERT INTO t1 VALUES (1); --let $no_checkpoint_flush= 1 --let no_checkpoint_kill= 1 +--source ../../suite/innodb/include/no_checkpoint_prepare.inc --source ../../suite/innodb/include/no_checkpoint_start.inc --let $file= query_get_value(SHOW MASTER STATUS, File, 1) --let $pos= query_get_value(SHOW MASTER STATUS, Position, 1) diff --git a/mysql-test/suite/binlog_in_engine/recovery_large.test b/mysql-test/suite/binlog_in_engine/recovery_large.test index 33ba8f65af8f0..f801e84201dac 100644 --- a/mysql-test/suite/binlog_in_engine/recovery_large.test +++ b/mysql-test/suite/binlog_in_engine/recovery_large.test @@ -32,6 +32,7 @@ while ($i < $pre_count) { --let $no_checkpoint_flush= 1 --let $no_checkpoint_kill= 1 +--source ../../suite/innodb/include/no_checkpoint_prepare.inc --source ../../suite/innodb/include/no_checkpoint_start.inc --let $file= query_get_value(SHOW MASTER STATUS, File, 1) --let $pos= query_get_value(SHOW MASTER STATUS, Position, 1) diff --git a/mysql-test/suite/binlog_in_engine/recovery_no_prealloc.result b/mysql-test/suite/binlog_in_engine/recovery_no_prealloc.result index 1c9180d5c4cc5..048eea27d8db3 100644 --- a/mysql-test/suite/binlog_in_engine/recovery_no_prealloc.result +++ b/mysql-test/suite/binlog_in_engine/recovery_no_prealloc.result @@ -30,6 +30,31 @@ binlog-000000.ibb # Annotate_rows # # INSERT INTO t1 VALUES (2, REPEAT('2', 2000 binlog-000000.ibb # Table_map # # table_id: # (test.t1) binlog-000000.ibb # Write_rows_v1 # # table_id: # flags: STMT_END_F binlog-000000.ibb # Xid # # COMMIT /* XID */ +binlog-000000.ibb # Gtid # # BEGIN GTID #-#-# +binlog-000000.ibb # Annotate_rows # # INSERT INTO test_suppressions (pattern) VALUES ( NAME_CONST('pattern',_utf8mb4'InnoDB: innodb_read_only prevents crash recovery' COLLATE 'utf8mb4_uca1400_ai_ci')) +binlog-000000.ibb # Table_map # # table_id: # (mtr.test_suppressions) +binlog-000000.ibb # Write_rows_v1 # # table_id: # flags: STMT_END_F +binlog-000000.ibb # Query # # COMMIT +binlog-000000.ibb # Gtid # # BEGIN GTID #-#-# +binlog-000000.ibb # Annotate_rows # # INSERT INTO test_suppressions (pattern) VALUES ( NAME_CONST('pattern',_utf8mb4'InnoDB: Plugin initialization aborted' COLLATE 'utf8mb4_uca1400_ai_ci')) +binlog-000000.ibb # Table_map # # table_id: # (mtr.test_suppressions) +binlog-000000.ibb # Write_rows_v1 # # table_id: # flags: STMT_END_F +binlog-000000.ibb # Query # # COMMIT +binlog-000000.ibb # Gtid # # BEGIN GTID #-#-# +binlog-000000.ibb # Annotate_rows # # INSERT INTO test_suppressions (pattern) VALUES ( NAME_CONST('pattern',_utf8mb4'Plugin \'InnoDB\' registration as a STORAGE ENGINE failed' COLLATE 'utf8mb4_uca1400_ai_ci')) +binlog-000000.ibb # Table_map # # table_id: # (mtr.test_suppressions) +binlog-000000.ibb # Write_rows_v1 # # table_id: # flags: STMT_END_F +binlog-000000.ibb # Query # # COMMIT +binlog-000000.ibb # Gtid # # BEGIN GTID #-#-# +binlog-000000.ibb # Annotate_rows # # INSERT INTO test_suppressions (pattern) VALUES ( NAME_CONST('pattern',_utf8mb4'unknown option \'--innodb-invalid-option\'' COLLATE 'utf8mb4_uca1400_ai_ci')) +binlog-000000.ibb # Table_map # # table_id: # (mtr.test_suppressions) +binlog-000000.ibb # Write_rows_v1 # # table_id: # flags: STMT_END_F +binlog-000000.ibb # Query # # COMMIT +binlog-000000.ibb # Gtid # # BEGIN GTID #-#-# +binlog-000000.ibb # Annotate_rows # # INSERT INTO test_suppressions (pattern) VALUES ( NAME_CONST('pattern',_utf8mb4'\\[ERROR\\] Aborting' COLLATE 'utf8mb4_uca1400_ai_ci')) +binlog-000000.ibb # Table_map # # table_id: # (mtr.test_suppressions) +binlog-000000.ibb # Write_rows_v1 # # table_id: # flags: STMT_END_F +binlog-000000.ibb # Query # # COMMIT include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info binlog-000001.ibb # Gtid # # BEGIN GTID #-#-# diff --git a/mysql-test/suite/binlog_in_engine/recovery_no_prealloc.test b/mysql-test/suite/binlog_in_engine/recovery_no_prealloc.test index 4f7e90f8b1f86..10936a23dee11 100644 --- a/mysql-test/suite/binlog_in_engine/recovery_no_prealloc.test +++ b/mysql-test/suite/binlog_in_engine/recovery_no_prealloc.test @@ -27,6 +27,7 @@ INSERT INTO t1 VALUES (2, REPEAT('2', 200000)); --let $no_checkpoint_flush= 1 --let no_checkpoint_kill= 1 +--source ../../suite/innodb/include/no_checkpoint_prepare.inc --source ../../suite/innodb/include/no_checkpoint_start.inc # Then fill up rest of binlog-000000.ibb, and some but not all of diff --git a/mysql-test/suite/encryption/include/skip_innodb_log_archive.inc b/mysql-test/suite/encryption/include/skip_innodb_log_archive.inc new file mode 100644 index 0000000000000..3fa44408a68a3 --- /dev/null +++ b/mysql-test/suite/encryption/include/skip_innodb_log_archive.inc @@ -0,0 +1,12 @@ +--disable_query_log +SET STATEMENT sql_log_bin=0 FOR +call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed\\."); +SET STATEMENT sql_log_bin=0 FOR +call mtr.add_suppression("InnoDB: Plugin initialization aborted"); +SET STATEMENT sql_log_bin=0 FOR +call mtr.add_suppression("InnoDB: ib_0.*\\.log does not match innodb_encrypt_log"); +--enable_query_log +if (`SELECT COUNT(*)=0 FROM information_schema.global_variables where variable_name='innodb_log_archive' and variable_value='OFF'`) +{ + --skip Test requires innodb_log_archive=OFF +} diff --git a/mysql-test/suite/encryption/r/doublewrite_debug.result b/mysql-test/suite/encryption/r/doublewrite_debug.result index a4d4995b3327e..b5b6db9f29386 100644 --- a/mysql-test/suite/encryption/r/doublewrite_debug.result +++ b/mysql-test/suite/encryption/r/doublewrite_debug.result @@ -82,7 +82,7 @@ set global innodb_buf_flush_list_now = 1; # restart FOUND 1 /InnoDB: Encrypted page \[page id: space=[1-9]*, page number=3\] in file .*test.t1.ibd looks corrupted/ in mysqld.1.err select * from t1; -ERROR 42000: Unknown storage engine 'InnoDB' +Got one of the listed errors # shutdown server # remove datadir # xtrabackup move back diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change.result b/mysql-test/suite/encryption/r/innodb-bad-key-change.result index 5fe5eb27fc5d2..a47eb634b26a7 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change.result @@ -32,10 +32,10 @@ foobar 1 foobar 2 # Restart server with keysbad3.txt -# restart: --file-key-management-filename=MYSQL_TEST_DIR/std_data/keysbad3.txt +# restart: --file-key-management-filename=MYSQL_TEST_DIR/std_data/keysbad3.txt --innodb-log-recovery-start=0 SELECT * FROM t1; ERROR HY000: Table `test`.`t1` is corrupted. Please drop the table and recreate. -# restart: --file-key-management-filename=MYSQL_TEST_DIR/std_data/keysbad3.txt +# restart: --file-key-management-filename=MYSQL_TEST_DIR/std_data/keysbad3.txt --innodb-log-recovery-start=0 DROP TABLE t1; # Start server with keys3.txt # restart: --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys3.txt @@ -44,7 +44,7 @@ CREATE TABLE t2 (c VARCHAR(8), id int not null primary key, b int, key(b)) ENGIN INSERT INTO t2 VALUES ('foobar',1,2); # Restart server with keys2.txt -# restart: --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt +# restart: --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt --innodb-log-recovery-start=0 SELECT * FROM t2; ERROR HY000: Table `test`.`t2` is corrupted. Please drop the table and recreate. SELECT * FROM t2 where id = 1; diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result index 95ed9a2c04245..7b46b65cf0707 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result @@ -13,7 +13,7 @@ call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot decrypt \\[page id: space=" CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; INSERT INTO t1 VALUES (1,'foo'),(2,'bar'); -# restart: --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys3.txt +# restart: --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys3.txt --innodb-log-recovery-start=0 SELECT * FROM t1; ERROR HY000: Table `test`.`t1` is corrupted. Please drop the table and recreate. SHOW WARNINGS; @@ -40,7 +40,7 @@ Level Code Message FLUSH TABLES t1 FOR EXPORT; backup: t1 UNLOCK TABLES; -# restart: --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys3.txt +# restart: --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys3.txt --innodb-log-recovery-start=0 ALTER TABLE t1 DISCARD TABLESPACE; ERROR HY000: Table `test`.`t1` is corrupted. Please drop the table and recreate. DROP TABLE t1; diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change3.result b/mysql-test/suite/encryption/r/innodb-bad-key-change3.result index 2c257aa6ef7a0..1696f006e0802 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change3.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change3.result @@ -22,7 +22,7 @@ backup: t1 UNLOCK TABLES; ALTER TABLE t1 DISCARD TABLESPACE; restore: t1 .ibd and .cfg files -# restart +# restart: --innodb-log-recovery-start=0 ALTER TABLE t1 IMPORT TABLESPACE; ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SHOW CREATE TABLE t1; diff --git a/mysql-test/suite/encryption/r/innodb-compressed-blob.result b/mysql-test/suite/encryption/r/innodb-compressed-blob.result index c8294763f7e54..8bff9a783cbc9 100644 --- a/mysql-test/suite/encryption/r/innodb-compressed-blob.result +++ b/mysql-test/suite/encryption/r/innodb-compressed-blob.result @@ -16,7 +16,7 @@ insert into t1 values (1, repeat('secret',6000)); insert into t2 values (1, repeat('secret',6000)); insert into t3 values (1, repeat('secret',6000)); # Restart mysqld --file-key-management-filename=keys3.txt -# restart: --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys3.txt +# restart: --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys3.txt --innodb-log-recovery-start=0 select count(*) from t1 FORCE INDEX (b) where b like 'secret%'; ERROR HY000: Table `test`.`t1` is corrupted. Please drop the table and recreate. select count(*) from t2 FORCE INDEX (b) where b like 'secret%'; diff --git a/mysql-test/suite/encryption/r/innodb-encryption-disable.result b/mysql-test/suite/encryption/r/innodb-encryption-disable.result index 13bd8ac4a34fa..7903ae4d5ce6e 100644 --- a/mysql-test/suite/encryption/r/innodb-encryption-disable.result +++ b/mysql-test/suite/encryption/r/innodb-encryption-disable.result @@ -23,7 +23,7 @@ CREATE TABLE `t1` ( ) ENGINE=InnoDB; insert into t1 values (1,2,'maria','db','encryption'); alter table t1 encrypted='yes' `encryption_key_id`=1; -# restart: --innodb-encrypt-tables=OFF +# restart: --innodb-encrypt-tables=OFF --innodb-log-recovery-start=0 select * from t1; ERROR HY000: Table `test`.`t1` is corrupted. Please drop the table and recreate. select * from t5; diff --git a/mysql-test/suite/encryption/r/innodb-force-corrupt.result b/mysql-test/suite/encryption/r/innodb-force-corrupt.result index e559ddc066f03..077a6ab5f3368 100644 --- a/mysql-test/suite/encryption/r/innodb-force-corrupt.result +++ b/mysql-test/suite/encryption/r/innodb-force-corrupt.result @@ -18,7 +18,7 @@ INSERT INTO t3 select * from t1; COMMIT; # Backup tables before corrupting # Corrupt tables -# restart +# restart: --innodb-log-recovery-start=0 SELECT * FROM t1; ERROR HY000: Table `test`.`t1` is corrupted. Please drop the table and recreate. SELECT * FROM t2; diff --git a/mysql-test/suite/encryption/r/innodb-missing-key.result b/mysql-test/suite/encryption/r/innodb-missing-key.result index 14fb0275f1c0e..e93fbafdf84c5 100644 --- a/mysql-test/suite/encryption/r/innodb-missing-key.result +++ b/mysql-test/suite/encryption/r/innodb-missing-key.result @@ -26,7 +26,7 @@ INSERT INTO t2 SELECT * FROM t1; INSERT INTO t3 SELECT * FROM t1; # Restart server with keys3.txt -# restart: --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys3.txt +# restart: --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys3.txt --innodb-log-recovery-start=0 set global innodb_encryption_rotate_key_age = 1; use test; CREATE TABLE t4(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=YES ENCRYPTION_KEY_ID=1; diff --git a/mysql-test/suite/encryption/r/innodb-page_encryption_compression.result b/mysql-test/suite/encryption/r/innodb-page_encryption_compression.result index 762cc12e7e9f3..5ad6daf1ff86a 100644 --- a/mysql-test/suite/encryption/r/innodb-page_encryption_compression.result +++ b/mysql-test/suite/encryption/r/innodb-page_encryption_compression.result @@ -49,7 +49,7 @@ select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; variable_value > 0 1 -# restart: --innodb-encrypt-tables=OFF +# restart: --innodb-encrypt-tables=OFF --innodb-log-recovery-start=0 set global innodb_compression_algorithm = 1; alter table innodb_normal engine=innodb page_compressed=DEFAULT; show create table innodb_normal; diff --git a/mysql-test/suite/encryption/r/innodb-read-only.result b/mysql-test/suite/encryption/r/innodb-read-only.result index c40f5c4f9ba34..ed51c41822dce 100644 --- a/mysql-test/suite/encryption/r/innodb-read-only.result +++ b/mysql-test/suite/encryption/r/innodb-read-only.result @@ -8,7 +8,7 @@ WHERE NAME LIKE '%encrypt%'; SELECT @encryption_threads_running; @encryption_threads_running 4 -# restart: --innodb-read-only=1 --innodb-encrypt-tables=1 +# restart: --innodb-read-only=1 --innodb-encrypt-tables=1 --innodb-log-recovery-start=0 SET GLOBAL innodb_encryption_threads=4; SET @encryption_threads_running=0; SELECT COUNT(*) INTO @encryption_threads_running diff --git a/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result b/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result index 3c3e4831d8a0f..b7bdee10daa90 100644 --- a/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result +++ b/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result @@ -20,6 +20,13 @@ AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS FOUND 1 /InnoDB: Upgrade after a crash is not supported. This redo log was created before MariaDB 10\.2\.2, and we did not find a valid checkpoint/ in mysqld.1.err # empty redo log from before MariaDB 10.2.2 +# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-recovery-target=12345 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +COUNT(*) +0 +FOUND 1 /InnoDB: cannot fulfill innodb_log_recovery_target=12345!=/ in mysqld.1.err # restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_corruption --innodb-force-recovery=5 --innodb-log-file-size=4m SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' diff --git a/mysql-test/suite/encryption/t/bulk_insert.test b/mysql-test/suite/encryption/t/bulk_insert.test index ce7804cb4e067..94b40f0152232 100644 --- a/mysql-test/suite/encryption/t/bulk_insert.test +++ b/mysql-test/suite/encryption/t/bulk_insert.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/skip_innodb_log_archive.inc # innodb_encrypt_log --source include/have_sequence.inc --source include/have_file_key_management_plugin.inc diff --git a/mysql-test/suite/encryption/t/corrupted_during_recovery.test b/mysql-test/suite/encryption/t/corrupted_during_recovery.test index 27b7d31dfffe6..a4fb6d43d63dd 100644 --- a/mysql-test/suite/encryption/t/corrupted_during_recovery.test +++ b/mysql-test/suite/encryption/t/corrupted_during_recovery.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source ../../suite/innodb/include/no_checkpoint_prepare.inc --source include/have_file_key_management_plugin.inc --disable_query_log @@ -18,6 +19,7 @@ CREATE TABLE t1(a BIGINT PRIMARY KEY) ENGINE=InnoDB, ENCRYPTED=YES; INSERT INTO t1 VALUES(1); # Force a redo log checkpoint. let $restart_noprint=2; +let $restart_parameters=--innodb-log-recovery-start=0; --source include/restart_mysqld.inc --source ../../suite/innodb/include/no_checkpoint_start.inc CREATE TABLE t2(a BIGINT PRIMARY KEY) ENGINE=InnoDB, ENCRYPTED=YES; @@ -58,7 +60,7 @@ SELECT * FROM t1; call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t[1].ibd looks corrupted; key_version="); call mtr.add_suppression("Table .*t1.* is corrupted. Please drop the table and recreate."); --enable_query_log -let $restart_parameters=--innodb_force_recovery=1 --skip-innodb-buffer-pool-load-at-startup; +let $restart_parameters=--innodb-log-recovery-start=0 --innodb_force_recovery=1 --skip-innodb-buffer-pool-load-at-startup; --source include/restart_mysqld.inc --error ER_TABLE_CORRUPT diff --git a/mysql-test/suite/encryption/t/debug_key_management.test b/mysql-test/suite/encryption/t/debug_key_management.test index 9638391e69058..3211687c0c294 100644 --- a/mysql-test/suite/encryption/t/debug_key_management.test +++ b/mysql-test/suite/encryption/t/debug_key_management.test @@ -1,4 +1,5 @@ -- source include/have_innodb.inc +-- source include/skip_innodb_log_archive.inc # innodb_encrypt_log -- source include/have_debug.inc -- source include/innodb_undo_tablespaces.inc -- source include/not_embedded.inc diff --git a/mysql-test/suite/encryption/t/doublewrite_debug.test b/mysql-test/suite/encryption/t/doublewrite_debug.test index 4f2215240441f..55d61357fda8f 100644 --- a/mysql-test/suite/encryption/t/doublewrite_debug.test +++ b/mysql-test/suite/encryption/t/doublewrite_debug.test @@ -1,7 +1,9 @@ --source include/have_innodb.inc +--source ../../suite/innodb/include/skip_innodb_log_archive.inc #mariadb-backup --source include/have_debug.inc --source include/not_embedded.inc --source include/have_file_key_management_plugin.inc +--source ../../suite/innodb/include/no_checkpoint_prepare.inc call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t[12]\\.ibd looks corrupted"); call mtr.add_suppression("InnoDB: Unable to apply log to corrupted page "); call mtr.add_suppression("InnoDB: Plugin initialization aborted"); @@ -233,7 +235,7 @@ let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; let SEARCH_PATTERN=InnoDB: Encrypted page \\[page id: space=[1-9]*, page number=3\\] in file .*test.t1.ibd looks corrupted; --source include/search_pattern_in_file.inc ---error ER_UNKNOWN_STORAGE_ENGINE +--error ER_UNKNOWN_STORAGE_ENGINE,ER_TABLE_CORRUPT select * from t1; --source ../../mariabackup/include/restart_and_restore.inc diff --git a/mysql-test/suite/encryption/t/encrypt_and_grep.test b/mysql-test/suite/encryption/t/encrypt_and_grep.test index 648ad80780c93..988604db58ecd 100644 --- a/mysql-test/suite/encryption/t/encrypt_and_grep.test +++ b/mysql-test/suite/encryption/t/encrypt_and_grep.test @@ -1,4 +1,5 @@ -- source include/have_innodb.inc +-- source include/skip_innodb_log_archive.inc # innodb_encrypt_log -- source include/innodb_undo_tablespaces.inc -- source include/have_file_key_management_plugin.inc diff --git a/mysql-test/suite/encryption/t/encryption_force.test b/mysql-test/suite/encryption/t/encryption_force.test index 3c6f039184b96..28b49c866b489 100644 --- a/mysql-test/suite/encryption/t/encryption_force.test +++ b/mysql-test/suite/encryption/t/encryption_force.test @@ -1,4 +1,5 @@ -- source include/have_innodb.inc +-- source include/skip_innodb_log_archive.inc # innodb_encrypt_log -- source include/have_partition.inc -- source include/have_example_key_management_plugin.inc diff --git a/mysql-test/suite/encryption/t/file_creation.test b/mysql-test/suite/encryption/t/file_creation.test index 6b0126831a4ca..939da080133ce 100644 --- a/mysql-test/suite/encryption/t/file_creation.test +++ b/mysql-test/suite/encryption/t/file_creation.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source ../../suite/innodb/include/no_checkpoint_prepare.inc --source include/have_example_key_management_plugin.inc let $restart_noprint=2; # embedded does not support restart diff --git a/mysql-test/suite/encryption/t/innochecksum.test b/mysql-test/suite/encryption/t/innochecksum.test index e440e3d3344f8..bb2eb0e1a9359 100644 --- a/mysql-test/suite/encryption/t/innochecksum.test +++ b/mysql-test/suite/encryption/t/innochecksum.test @@ -7,6 +7,7 @@ # Require InnoDB -- source include/have_innodb.inc -- source include/have_file_key_management_plugin.inc +-- source include/skip_innodb_log_archive.inc # depends on innodb_log_archive=OFF checkpoint logic -- source include/innodb_page_size_small.inc -- source include/innodb_checksum_algorithm.inc -- source include/maybe_debug.inc diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change.test b/mysql-test/suite/encryption/t/innodb-bad-key-change.test index e867261ff514d..18d6678fbea4e 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change.test @@ -42,7 +42,7 @@ SELECT * FROM t1; --echo --echo # Restart server with keysbad3.txt --- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keysbad3.txt +-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keysbad3.txt --innodb-log-recovery-start=0 -- source include/restart_mysqld.inc --disable_warnings @@ -50,7 +50,7 @@ SELECT * FROM t1; SELECT * FROM t1; --enable_warnings --- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keysbad3.txt +-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keysbad3.txt --innodb-log-recovery-start=0 -- source include/restart_mysqld.inc --replace_regex /key_id [1-9][0-9]*/\1 / @@ -71,7 +71,7 @@ INSERT INTO t2 VALUES ('foobar',1,2); --echo --echo # Restart server with keys2.txt --- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt +-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt --innodb-log-recovery-start=0 -- source include/restart_mysqld.inc --disable_warnings diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test index eec4872c0a0e1..cbf04bc0c8f4e 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test @@ -29,7 +29,7 @@ CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; INSERT INTO t1 VALUES (1,'foo'),(2,'bar'); ---let $restart_parameters=--plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt +--let $restart_parameters=--plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt --innodb-log-recovery-start=0 --source include/restart_mysqld.inc --error ER_TABLE_CORRUPT @@ -60,7 +60,7 @@ ib_backup_tablespaces("test", "t1"); EOF UNLOCK TABLES; ---let $restart_parameters=--plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt +--let $restart_parameters=--plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt --innodb-log-recovery-start=0 --source include/restart_mysqld.inc --error ER_TABLE_CORRUPT diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change3.test b/mysql-test/suite/encryption/t/innodb-bad-key-change3.test index f4065290938f2..532b3ecc116cc 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change3.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change3.test @@ -65,6 +65,7 @@ EOF --write_line "restart:--innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys2.txt" $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --enable_reconnect --source include/wait_until_connected_again.inc +--let $restart_parameters=--innodb-log-recovery-start=0 --source include/restart_mysqld.inc --error ER_GET_ERRMSG diff --git a/mysql-test/suite/encryption/t/innodb-compressed-blob.test b/mysql-test/suite/encryption/t/innodb-compressed-blob.test index 091ed4003a8ef..0fd0efeded43c 100644 --- a/mysql-test/suite/encryption/t/innodb-compressed-blob.test +++ b/mysql-test/suite/encryption/t/innodb-compressed-blob.test @@ -25,7 +25,7 @@ insert into t2 values (1, repeat('secret',6000)); insert into t3 values (1, repeat('secret',6000)); --echo # Restart mysqld --file-key-management-filename=keys3.txt --- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt +-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt --innodb-log-recovery-start=0 -- source include/restart_mysqld.inc --error ER_TABLE_CORRUPT diff --git a/mysql-test/suite/encryption/t/innodb-encryption-disable.test b/mysql-test/suite/encryption/t/innodb-encryption-disable.test index 676edd54f986b..dab821a440e63 100644 --- a/mysql-test/suite/encryption/t/innodb-encryption-disable.test +++ b/mysql-test/suite/encryption/t/innodb-encryption-disable.test @@ -41,7 +41,7 @@ CREATE TABLE `t1` ( insert into t1 values (1,2,'maria','db','encryption'); alter table t1 encrypted='yes' `encryption_key_id`=1; ---let $restart_parameters=--innodb-encrypt-tables=OFF +--let $restart_parameters=--innodb-encrypt-tables=OFF --innodb-log-recovery-start=0 --source include/restart_mysqld.inc --error ER_TABLE_CORRUPT diff --git a/mysql-test/suite/encryption/t/innodb-force-corrupt.test b/mysql-test/suite/encryption/t/innodb-force-corrupt.test index 3cab108199080..a14f5e4526f8f 100644 --- a/mysql-test/suite/encryption/t/innodb-force-corrupt.test +++ b/mysql-test/suite/encryption/t/innodb-force-corrupt.test @@ -68,7 +68,9 @@ print FILE pack("H*", "c001cafedeadb017"); close FILE or die "close"; EOF +--let $restart_parameters=--innodb-log-recovery-start=0 --source include/start_mysqld.inc +--let $restart_parameters= --error ER_TABLE_CORRUPT SELECT * FROM t1; diff --git a/mysql-test/suite/encryption/t/innodb-key-rotation-disable.test b/mysql-test/suite/encryption/t/innodb-key-rotation-disable.test index 1bd69365f6892..33936e1704768 100644 --- a/mysql-test/suite/encryption/t/innodb-key-rotation-disable.test +++ b/mysql-test/suite/encryption/t/innodb-key-rotation-disable.test @@ -1,4 +1,5 @@ -- source include/have_innodb.inc +-- source include/skip_innodb_log_archive.inc # innodb_encrypt_log -- source include/have_file_key_management_plugin.inc # not embedded because of restarts -- source include/not_embedded.inc diff --git a/mysql-test/suite/encryption/t/innodb-missing-key.test b/mysql-test/suite/encryption/t/innodb-missing-key.test index 07e2248411319..f2fc0e815bdad 100644 --- a/mysql-test/suite/encryption/t/innodb-missing-key.test +++ b/mysql-test/suite/encryption/t/innodb-missing-key.test @@ -39,7 +39,7 @@ INSERT INTO t3 SELECT * FROM t1; --echo --echo # Restart server with keys3.txt --- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt +-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt --innodb-log-recovery-start=0 -- source include/restart_mysqld.inc set global innodb_encryption_rotate_key_age = 1; diff --git a/mysql-test/suite/encryption/t/innodb-page_encryption_compression.test b/mysql-test/suite/encryption/t/innodb-page_encryption_compression.test index 5fe6f68657661..dfd845400fba6 100644 --- a/mysql-test/suite/encryption/t/innodb-page_encryption_compression.test +++ b/mysql-test/suite/encryption/t/innodb-page_encryption_compression.test @@ -42,7 +42,7 @@ FLUSH TABLES innodb_dynamic FOR EXPORT; UNLOCK TABLES; select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; ---let $restart_parameters=--innodb-encrypt-tables=OFF +--let $restart_parameters=--innodb-encrypt-tables=OFF --innodb-log-recovery-start=0 --source include/restart_mysqld.inc # zlib diff --git a/mysql-test/suite/encryption/t/innodb-page_encryption_log_encryption.test b/mysql-test/suite/encryption/t/innodb-page_encryption_log_encryption.test index a736c7292ad35..3c64c039f3a45 100644 --- a/mysql-test/suite/encryption/t/innodb-page_encryption_log_encryption.test +++ b/mysql-test/suite/encryption/t/innodb-page_encryption_log_encryption.test @@ -1,4 +1,5 @@ -- source include/have_innodb.inc +-- source include/skip_innodb_log_archive.inc # innodb_encrypt_log -- source include/not_embedded.inc -- source include/have_file_key_management_plugin.inc diff --git a/mysql-test/suite/encryption/t/innodb-read-only.test b/mysql-test/suite/encryption/t/innodb-read-only.test index d85257f371185..a8da36136af0f 100644 --- a/mysql-test/suite/encryption/t/innodb-read-only.test +++ b/mysql-test/suite/encryption/t/innodb-read-only.test @@ -38,7 +38,7 @@ SELECT @encryption_threads_running; # MDEV-11835: InnoDB: Failing assertion: free_slot != NULL on # restarting server with encryption and read-only # ---let $restart_parameters= --innodb-read-only=1 --innodb-encrypt-tables=1 +--let $restart_parameters= --innodb-read-only=1 --innodb-encrypt-tables=1 --innodb-log-recovery-start=0 --source include/restart_mysqld.inc # Server read-only mode diff --git a/mysql-test/suite/encryption/t/innodb-redo-badkey.test b/mysql-test/suite/encryption/t/innodb-redo-badkey.test index bacc71dd2c86e..85dc8e57559eb 100644 --- a/mysql-test/suite/encryption/t/innodb-redo-badkey.test +++ b/mysql-test/suite/encryption/t/innodb-redo-badkey.test @@ -1,4 +1,5 @@ -- source include/have_innodb.inc +--source ../../suite/innodb/include/no_checkpoint_prepare.inc -- source include/have_file_key_management_plugin.inc # embedded does not support restart -- source include/not_embedded.inc diff --git a/mysql-test/suite/encryption/t/innodb-redo-nokeys.test b/mysql-test/suite/encryption/t/innodb-redo-nokeys.test index 05505011da784..58ae8dc69fd8c 100644 --- a/mysql-test/suite/encryption/t/innodb-redo-nokeys.test +++ b/mysql-test/suite/encryption/t/innodb-redo-nokeys.test @@ -1,4 +1,5 @@ -- source include/have_innodb.inc +--source ../../suite/innodb/include/no_checkpoint_prepare.inc -- source include/have_file_key_management_plugin.inc # embedded does not support restart -- source include/not_embedded.inc diff --git a/mysql-test/suite/encryption/t/innodb_encrypt_freed.test b/mysql-test/suite/encryption/t/innodb_encrypt_freed.test index 408e874a3b242..b37ff06dac1f7 100644 --- a/mysql-test/suite/encryption/t/innodb_encrypt_freed.test +++ b/mysql-test/suite/encryption/t/innodb_encrypt_freed.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/skip_innodb_log_archive.inc # innodb_encrypt_log --source include/have_example_key_management_plugin.inc --source include/have_debug.inc --source include/not_embedded.inc diff --git a/mysql-test/suite/encryption/t/innodb_encrypt_log.test b/mysql-test/suite/encryption/t/innodb_encrypt_log.test index 5448a606ba807..cdb807c157780 100644 --- a/mysql-test/suite/encryption/t/innodb_encrypt_log.test +++ b/mysql-test/suite/encryption/t/innodb_encrypt_log.test @@ -1,4 +1,5 @@ -- source include/have_innodb.inc +-- source include/skip_innodb_log_archive.inc # innodb_encrypt_log -- source include/not_embedded.inc -- source filekeys_plugin.inc diff --git a/mysql-test/suite/encryption/t/innodb_encrypt_log_corruption.test b/mysql-test/suite/encryption/t/innodb_encrypt_log_corruption.test index f1642e83e32e0..19858188cf1ff 100644 --- a/mysql-test/suite/encryption/t/innodb_encrypt_log_corruption.test +++ b/mysql-test/suite/encryption/t/innodb_encrypt_log_corruption.test @@ -1,3 +1,4 @@ +--source include/skip_innodb_log_archive.inc # innodb_encrypt_log --let $no_cleanup=1 --source ../../innodb/t/log_corruption.test diff --git a/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test b/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test index 6cfc7e3452d61..3d21300c7e0dc 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test +++ b/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test @@ -1,4 +1,5 @@ -- source include/have_innodb.inc +-- source include/skip_innodb_log_archive.inc # innodb_encrypt_log -- source include/have_example_key_management_plugin.inc -- source include/not_embedded.inc # This test is too slow for valgrind and causes innodb semaphores to time out diff --git a/mysql-test/suite/encryption/t/innodb_encryption.test b/mysql-test/suite/encryption/t/innodb_encryption.test index 2b0b2b8d7fb5c..113bbf152605d 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption.test +++ b/mysql-test/suite/encryption/t/innodb_encryption.test @@ -2,6 +2,7 @@ # # -- source include/have_innodb.inc +-- source include/skip_innodb_log_archive.inc # innodb_encrypt_log -- source include/have_example_key_management_plugin.inc -- source include/innodb_undo_tablespaces.inc diff --git a/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test b/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test index e33aaec3e21c0..c1f51be89883f 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test +++ b/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test @@ -1,4 +1,5 @@ -- source include/have_innodb.inc +-- source include/skip_innodb_log_archive.inc # innodb_encrypt_log -- source include/have_example_key_management_plugin.inc -- source include/not_valgrind.inc -- source include/not_embedded.inc diff --git a/mysql-test/suite/encryption/t/innodb_encryption_tables.test b/mysql-test/suite/encryption/t/innodb_encryption_tables.test index d03bc890ba4ed..3a8d244a1e3f4 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption_tables.test +++ b/mysql-test/suite/encryption/t/innodb_encryption_tables.test @@ -1,4 +1,5 @@ -- source include/have_innodb.inc +-- source include/skip_innodb_log_archive.inc # innodb_encrypt_log -- source include/have_example_key_management_plugin.inc -- source include/not_embedded.inc # We can't run this test under valgrind as it 'takes forever' diff --git a/mysql-test/suite/encryption/t/innodb_first_page.test b/mysql-test/suite/encryption/t/innodb_first_page.test index db4d8eb3b16f5..838fc8396a9a8 100644 --- a/mysql-test/suite/encryption/t/innodb_first_page.test +++ b/mysql-test/suite/encryption/t/innodb_first_page.test @@ -3,6 +3,7 @@ # --source include/have_innodb.inc +--source include/skip_innodb_log_archive.inc # innodb_encrypt_log --source include/have_file_key_management_plugin.inc --source include/innodb_undo_tablespaces.inc diff --git a/mysql-test/suite/encryption/t/innodb_onlinealter_encryption.test b/mysql-test/suite/encryption/t/innodb_onlinealter_encryption.test index dc6d1e6f93c86..0b5c70339f704 100644 --- a/mysql-test/suite/encryption/t/innodb_onlinealter_encryption.test +++ b/mysql-test/suite/encryption/t/innodb_onlinealter_encryption.test @@ -1,4 +1,5 @@ -- source include/have_innodb.inc +-- source include/skip_innodb_log_archive.inc # innodb_encrypt_log -- source include/have_file_key_management_plugin.inc # test uses restart -- source include/not_embedded.inc diff --git a/mysql-test/suite/encryption/t/recovery_memory.test b/mysql-test/suite/encryption/t/recovery_memory.test index fc6f15f7ee27f..af539b3a36bec 100644 --- a/mysql-test/suite/encryption/t/recovery_memory.test +++ b/mysql-test/suite/encryption/t/recovery_memory.test @@ -1,5 +1,6 @@ --source include/have_debug.inc --source include/have_innodb.inc +--source include/skip_innodb_log_archive.inc # innodb_encrypt_log --source include/have_sequence.inc --source filekeys_plugin.inc diff --git a/mysql-test/suite/innodb/include/no_checkpoint_end.inc b/mysql-test/suite/innodb/include/no_checkpoint_end.inc index 61721650f329a..acb71c70ef933 100644 --- a/mysql-test/suite/innodb/include/no_checkpoint_end.inc +++ b/mysql-test/suite/innodb/include/no_checkpoint_end.inc @@ -4,32 +4,32 @@ if (!$no_checkpoint_kill) { --source include/kill_mysqld.inc } +--error 2 +--exec $MYSQLD_CMD --innodb --innodb-read-only --innodb-log-recovery-start=0 --innodb-invalid-option --innodb-page-size=$INNODB_PAGE_SIZE --innodb-buffer-pool-size=21m perl; -my $cp = $ENV{CHECKPOINT_LSN}; -$cp =~ s/^InnoDB\t\t//; -my $log = "$ENV{MYSQLD_DATADIR}ib_logfile0"; -open(LOG, "<$log") || die "Unable to open $log"; -seek(LOG, 4096, 0) || die "Unable to seek $log"; -die unless read(LOG, $_, 8) == 8; -my ($cp1hi,$cp1lo) = unpack("NN", $_); -seek(LOG, 8192, 0) || die "Unable to seek $log"; -die unless read(LOG, $_, 8) == 8; -my ($cp2hi,$cp2lo) = unpack("NN", $_); -close(LOG); - -my $cp1 = $cp1hi << 32 | $cp1lo; -my $cp2 = $cp2hi << 32 | $cp2lo; - +my $cp=0; +my $search_file= "$ENV{MYSQLTEST_VARDIR}/log/mysqld.1.err"; +open(FILE, '<', $search_file) || die("Can't open file $search_file: $!"); +while() +{ + if (/^CURRENT_TEST:/) + { + $cp=0; + } + elsif (!$cp && /\[Warning\] innodb_read_only prevents crash recovery between (\d+) and/o) + { + $cp=$1; + } +} +close(FILE); open(OUT, ">$ENV{MYSQLTEST_VARDIR}/log/check.txt") || die; - -if ($cp1 > $cp || $cp2 > $cp) { - print OUT "--source include/start_mysqld.inc\n" - unless $ENV{no_checkpoint_kill}; +if ($cp != $ENV{CHECKPOINT_LSN}) +{ + print OUT "--source include/start_mysqld.inc\n"; print OUT "$ENV{CLEANUP_IF_CHECKPOINT}\n"; - print OUT "--skip Extra checkpoint 1 after $cp ($cp1,$cp2)\n"; + print OUT "--skip Unexpected checkpoint $cp != $ENV{CHECKPOINT_LSN}\n"; } - close(OUT); EOF diff --git a/mysql-test/suite/innodb/include/no_checkpoint_prepare.inc b/mysql-test/suite/innodb/include/no_checkpoint_prepare.inc new file mode 100644 index 0000000000000..bf19bc6cc75c1 --- /dev/null +++ b/mysql-test/suite/innodb/include/no_checkpoint_prepare.inc @@ -0,0 +1,8 @@ +--disable_query_log +call mtr.add_suppression("InnoDB: innodb_read_only prevents crash recovery"); +call mtr.add_suppression("InnoDB: Plugin initialization aborted"); +call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); +call mtr.add_suppression("unknown option '--innodb-invalid-option'"); +call mtr.add_suppression("\\[ERROR\\] Aborting"); +let $INNODB_PAGE_SIZE=`SELECT @@GLOBAL.innodb_page_size`; +--enable_query_log diff --git a/mysql-test/suite/innodb/include/skip_innodb_log_archive.inc b/mysql-test/suite/innodb/include/skip_innodb_log_archive.inc new file mode 100644 index 0000000000000..984e5fefa369a --- /dev/null +++ b/mysql-test/suite/innodb/include/skip_innodb_log_archive.inc @@ -0,0 +1,4 @@ +if (`SELECT @@GLOBAL.innodb_log_archive`) +{ + --skip Test requires innodb_log_archive=OFF +} diff --git a/mysql-test/suite/innodb/r/101_compatibility.result b/mysql-test/suite/innodb/r/101_compatibility.result index 3782ea83baf39..9cdf88853e75c 100644 --- a/mysql-test/suite/innodb/r/101_compatibility.result +++ b/mysql-test/suite/innodb/r/101_compatibility.result @@ -36,7 +36,7 @@ test.tz check status OK test.tdd check status OK test.tp check status OK test.ti check status OK -# restart: --innodb-read-only +# restart: --innodb-read-only --innodb-log-recovery-start=0 CHECK TABLE tr,tc,td,tz,tdd,tp,ti; Table Op Msg_type Msg_text test.tr check status OK diff --git a/mysql-test/suite/innodb/r/alter_copy.result b/mysql-test/suite/innodb/r/alter_copy.result index bf6d2f910cb3f..ea21d0154b944 100644 --- a/mysql-test/suite/innodb/r/alter_copy.result +++ b/mysql-test/suite/innodb/r/alter_copy.result @@ -119,7 +119,7 @@ t1 CREATE TABLE `t1` ( CHECK TABLE t1; Table Op Msg_type Msg_text test.t1 check status OK -# restart: --innodb-read-only +# restart: --innodb-read-only --innodb-log-recovery-start=0 FTS_INDEX_1.ibd FTS_INDEX_2.ibd FTS_INDEX_3.ibd diff --git a/mysql-test/suite/innodb/r/alter_kill.result b/mysql-test/suite/innodb/r/alter_kill.result index d852734b5c085..d5a27f9357095 100644 --- a/mysql-test/suite/innodb/r/alter_kill.result +++ b/mysql-test/suite/innodb/r/alter_kill.result @@ -16,7 +16,7 @@ connection default; disconnect con1; # Corrupt FIL_PAGE_TYPE in bug16720368.ibd, # and recompute innodb_checksum_algorithm=crc32 -# restart: --innodb-flush-method=O_DIRECT +# restart: --innodb-flush-method=O_DIRECT --innodb-log-recovery-start=0 SELECT @@innodb_doublewrite; @@innodb_doublewrite OFF @@ -26,7 +26,7 @@ INSERT INTO bug16720368 VALUES(1); ERROR HY000: Table `test`.`bug16720368` is corrupted. Please drop the table and recreate. INSERT INTO bug16720368_1 VALUES(1); # Shut down the server to uncorrupt the data. -# restart +# restart: --innodb-log-recovery-start=0 INSERT INTO bug16720368 VALUES(9,1); SELECT COUNT(*) FROM bug16720368; COUNT(*) @@ -53,9 +53,9 @@ connection default; # Kill the server disconnect con1; # Attempt to start without an *.ibd file. -# restart +# restart: --innodb-log-recovery-start=0 FOUND 1 /\[ERROR\] InnoDB: Tablespace [0-9]+ was not found at .*test.bug16735660.ibd/ in mysqld.1.err -# restart +# restart: --innodb-log-recovery-start=0 SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; SELECT * FROM bug16735660; a diff --git a/mysql-test/suite/innodb/r/alter_missing_tablespace.result b/mysql-test/suite/innodb/r/alter_missing_tablespace.result index 65b01d89d91ce..cdd6e52aab5a0 100644 --- a/mysql-test/suite/innodb/r/alter_missing_tablespace.result +++ b/mysql-test/suite/innodb/r/alter_missing_tablespace.result @@ -6,7 +6,7 @@ CREATE TABLE t(a SERIAL)ENGINE=InnoDB; CREATE TABLE `x..d` (a INT PRIMARY KEY, b INT) ENGINE=InnoDB; CREATE TABLE t1(a SERIAL)ENGINE=InnoDB; INSERT INTO t1 VALUES(1),(2),(3); -# restart +# restart: --innodb-log-recovery-start=0 SELECT * FROM t; ERROR HY000: Got error 194 "Tablespace is missing for a table" from storage engine InnoDB ALTER TABLE t ADD INDEX (a), ALGORITHM=INPLACE; diff --git a/mysql-test/suite/innodb/r/autoinc_import.result b/mysql-test/suite/innodb/r/autoinc_import.result index b6a0f9d32f5e1..fd31824782a65 100644 --- a/mysql-test/suite/innodb/r/autoinc_import.result +++ b/mysql-test/suite/innodb/r/autoinc_import.result @@ -90,7 +90,7 @@ test.t10_1 check note Auto_increment will be checked on each open until CHECK TA test.t10_1 check status OK test.t10_1b check note Auto_increment will be checked on each open until CHECK TABLE FOR UPGRADE is executed test.t10_1b check status OK -# restart: --innodb-read-only --read-only +# restart: --innodb-read-only --read-only --innodb-log-recovery-start=0 CHECK TABLE t1, t1b, t1t, t1z, t5_7, t5_7b, t10_1, t10_1b; Table Op Msg_type Msg_text test.t1 check status OK @@ -119,7 +119,7 @@ test.t10_1 check note Auto_increment will be checked on each open until CHECK TA test.t10_1 check status OK test.t10_1b check note Auto_increment will be checked on each open until CHECK TABLE FOR UPGRADE is executed test.t10_1b check status OK -# restart: --innodb-read-only +# restart: --innodb-read-only --innodb-log-recovery-start=0 CHECK TABLE t1, t1b, t1t, t1z, t5_7, t5_7b, t10_1, t10_1b; Table Op Msg_type Msg_text test.t1 check status OK diff --git a/mysql-test/suite/innodb/r/corrupted_during_recovery.result b/mysql-test/suite/innodb/r/corrupted_during_recovery.result index 593943b4951ea..7aa097ee1bec2 100644 --- a/mysql-test/suite/innodb/r/corrupted_during_recovery.result +++ b/mysql-test/suite/innodb/r/corrupted_during_recovery.result @@ -1,14 +1,27 @@ -CREATE TABLE t1(a BIGINT PRIMARY KEY) ENGINE=InnoDB; +CREATE TABLE t1(a BIGINT PRIMARY KEY) ENGINE=InnoDB STATS_PERSISTENT=0; INSERT INTO t1 VALUES(1); +SET GLOBAL innodb_max_purge_lag_wait=0, innodb_log_checkpoint_now=ON; +connect stop_purge,localhost,root; +START TRANSACTION WITH CONSISTENT SNAPSHOT; +connection default; CREATE TABLE t2(a BIGINT PRIMARY KEY) ENGINE=InnoDB; INSERT INTO t1 VALUES(2); SET GLOBAL innodb_flush_log_at_trx_commit=1; INSERT INTO t2 VALUES(1); # Kill the server +disconnect stop_purge; +SELECT * FROM t2; +Got one of the listed errors +SELECT * FROM t2; +ERROR 42000: Unknown storage engine 'InnoDB' +FOUND 1 /InnoDB: cannot fulfill innodb_log_recovery_target.*/ in mysqld.1.err +DELETE FROM t1; +Got one of the listed errors +DELETE FROM t2; +Got one of the listed errors # Corrupt the pages SELECT * FROM t1; ERROR 42000: Unknown storage engine 'InnoDB' -FOUND 1 /InnoDB: Page \[page id: space=[1-9][0-9]*, page number=3\] log sequence number 1311768467463790320 is in the future!/ in mysqld.1.err SELECT * FROM t1; a 1 @@ -18,6 +31,7 @@ a CHECK TABLE t2; Table Op Msg_type Msg_text test.t2 check status OK +FOUND 1 /InnoDB: Page \[page id: space=[1-9][0-9]*, page number=3\] log sequence number 1311768467463790320 is in the future!.*/ in mysqld.1.err DROP TABLE t1, t2; CREATE TABLE t1(pk SERIAL) ENGINE=InnoDB; INSERT INTO t1 VALUES (1),(2),(3); @@ -31,6 +45,9 @@ DELETE FROM t1 WHERE pk=3; disconnect con1; # Corrupt the page SELECT * FROM t1; +ERROR 42000: Unknown storage engine 'InnoDB' +FOUND 1 /InnoDB: Did not find innodb_log_recovery_start=\d+ .*/ in mysqld.1.err +SELECT * FROM t1; pk 1 2 diff --git a/mysql-test/suite/innodb/r/doublewrite_debug.result b/mysql-test/suite/innodb/r/doublewrite_debug.result index e1d2b0137e1a2..23a4a44dc1947 100644 --- a/mysql-test/suite/innodb/r/doublewrite_debug.result +++ b/mysql-test/suite/innodb/r/doublewrite_debug.result @@ -23,7 +23,7 @@ commit work; # Test Begin: Test if recovery works if 1st page and 2nd page # of system tablespace is full of zeroes. SET GLOBAL innodb_fast_shutdown = 0; -# restart: --debug_dbug=+d,ib_log_checkpoint_avoid_hard --innodb_flush_sync=0 +# restart: --debug_dbug=+d,ib_log_checkpoint_avoid_hard --innodb_flush_sync=0 --innodb-log-recovery-start=0 begin; insert into t1 values (6, repeat('%', 400)); SET GLOBAL innodb_max_dirty_pages_pct_lwm=0, innodb_max_dirty_pages_pct=0; @@ -36,7 +36,7 @@ set global innodb_fil_make_page_dirty_debug = 0; # Kill the server # Make the 1st page (page_no=0) and 2nd page (page_no=1) # of the system tablespace all zeroes. -# restart +# restart: --innodb-log-recovery-start=0 FOUND 1 /InnoDB: Recovered page \[page id: space=0, page number=0\]/ in mysqld.1.err FOUND 1 /InnoDB: Recovered page \[page id: space=0, page number=1\]/ in mysqld.1.err check table t1; @@ -53,7 +53,7 @@ f1 f2 # --------------------------------------------------------------- # Test Begin: Test if recovery works if 1st page of # system tablespace is corrupted and 2nd page as corrupted. -# restart: --debug_dbug=+d,ib_log_checkpoint_avoid_hard --innodb_flush_sync=0 +# restart: --innodb-log-recovery-start=0 --debug_dbug=+d,ib_log_checkpoint_avoid_hard --innodb_flush_sync=0 begin; insert into t1 values (6, repeat('%', 400)); # Make the first page dirty for system tablespace @@ -65,7 +65,7 @@ set global innodb_fil_make_page_dirty_debug = 0; set global innodb_buf_flush_list_now = 1; # Kill the server # Corrupt the 1st page (page_no=0) and 2nd page of the system tablespace. -# restart +# restart: --innodb-log-recovery-start=0 FOUND 2 /InnoDB: Recovered page \[page id: space=0, page number=0\]/ in mysqld.1.err FOUND 2 /InnoDB: Recovered page \[page id: space=0, page number=1\]/ in mysqld.1.err check table t1; @@ -85,10 +85,10 @@ drop table t1; # MDEV-12600 crash during install_db with innodb_page_size=32K # and ibdata1=3M # -# restart: --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/doublewrite --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/doublewrite --innodb-undo-tablespaces=0 --innodb-data-file-path=ibdata1:1M;ibdata2:1M:autoextend +# restart: --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/doublewrite --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/doublewrite --innodb-undo-tablespaces=0 --innodb-log-recovery-start=0 --innodb-data-file-path=ibdata1:1M;ibdata2:1M:autoextend SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS FOUND 1 /\[ERROR\] InnoDB: Cannot create doublewrite buffer/ in mysqld.1.err -# restart +# restart: --innodb-log-recovery-start=0 diff --git a/mysql-test/suite/innodb/r/innodb-get-fk.result b/mysql-test/suite/innodb/r/innodb-get-fk.result index 5367c83545bfb..966be3969fe2f 100644 --- a/mysql-test/suite/innodb/r/innodb-get-fk.result +++ b/mysql-test/suite/innodb/r/innodb-get-fk.result @@ -26,7 +26,7 @@ KEY `fk_crewRoleAssigned_roleCode` (`role_code`), CONSTRAINT `fk_crewRoleAssigned_crewId` FOREIGN KEY (`crew_id`) REFERENCES `repro`.`crew` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, CONSTRAINT `fk_crewRoleAssigned_pilotId` FOREIGN KEY (`crew_id`) REFERENCES `repro`.`pilot` (`id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB COMMENT="This is a comment about tables"; -# restart: --innodb-read-only +# restart: --innodb-read-only --innodb-log-recovery-start=0 ALTER TABLE `repro`.`crew_role_assigned` COMMENT = 'innodb_read_only'; ERROR HY000: Table 'crew_role_assigned' is read only SHOW CREATE TABLE `repro`.`crew_role_assigned`; @@ -58,7 +58,7 @@ crew_role_assigned CREATE TABLE `crew_role_assigned` ( CONSTRAINT `fk_crewRoleAssigned_crewId` FOREIGN KEY (`crew_id`) REFERENCES `crew` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, CONSTRAINT `fk_crewRoleAssigned_pilotId` FOREIGN KEY (`crew_id`) REFERENCES `pilot` (`id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci COMMENT='This is a new comment about tables' -# restart: --innodb-read-only +# restart: --innodb-read-only --innodb-log-recovery-start=0 SHOW CREATE TABLE `repro`.`crew_role_assigned`; Table Create Table crew_role_assigned CREATE TABLE `crew_role_assigned` ( diff --git a/mysql-test/suite/innodb/r/innodb-wl5522,strict_crc32.rdiff b/mysql-test/suite/innodb/r/innodb-wl5522,strict_crc32.rdiff index 283bbe96aae97..6c31d9268be5f 100644 --- a/mysql-test/suite/innodb/r/innodb-wl5522,strict_crc32.rdiff +++ b/mysql-test/suite/innodb/r/innodb-wl5522,strict_crc32.rdiff @@ -1,6 +1,27 @@ --- innodb-wl5522.result +++ innodb-wl5522,strict_crc32.result~ -@@ -131,8 +131,7 @@ +@@ -1,9 +1,6 @@ + call mtr.add_suppression("InnoDB: Unable to import tablespace .* because it already exists. Please DISCARD the tablespace before IMPORT\\."); + call mtr.add_suppression("Index for table 't2' is corrupt; try to repair it"); + call mtr.add_suppression("InnoDB: Cannot save statistics for table `test`\\.`t1` because the \\.ibd file is missing"); +-call mtr.add_suppression("InnoDB: cannot fulfill innodb_log_recovery_target=102345<"); +-call mtr.add_suppression("InnoDB: Plugin initialization aborted"); +-call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed\\."); + FLUSH TABLES; + CREATE TABLE t1 + (a INT AUTO_INCREMENT PRIMARY KEY, +@@ -37,10 +34,6 @@ + t1.ibd + t2.frm + t2.ibd +-# restart: --innodb-log-recovery-target=102345 +-FOUND 1 /InnoDB: cannot fulfill innodb_log_recovery_target=1023454G"); +call mtr.add_suppression("InnoDB: innodb_log_archive=ON but .*/ib_logfile0 exists"); +call mtr.add_suppression("InnoDB: No matching file found for innodb_log_recovery_start="); +call mtr.add_suppression("InnoDB: File .*/ib_logfile0 was not found"); +call mtr.add_suppression("InnoDB: innodb_log_archive_start=\\d+ is after innodb_log_recovery_start=\\d+"); +call mtr.add_suppression("InnoDB: Did not find innodb_log_recovery_start=\\d+ "); SET GLOBAL innodb_log_file_size=4194304; SHOW VARIABLES LIKE 'innodb_log_file_size'; Variable_name Value @@ -11,7 +18,7 @@ a INT PRIMARY KEY AUTO_INCREMENT, b CHAR(255) NOT NULL) ENGINE=INNODB; INSERT INTO t SELECT NULL, REPEAT('a', 255) FROM seq_1_to_20000; -# restart: --innodb-log-file-size=4194304 +# restart: --innodb-log-file-size=4194304 --skip-innodb-log-archive SELECT COUNT(*) FROM t; COUNT(*) 20000 @@ -28,8 +35,24 @@ Got one of the listed errors connect con1,localhost,root; SET GLOBAL innodb_log_file_size=7340032; connection default; +SET GLOBAL innodb_log_archive=ON; +SET GLOBAL innodb_log_archive=OFF; KILL QUERY @id; connection con1; +SET GLOBAL innodb_log_archive=ON, innodb_log_file_size=10485760; +SELECT @@GLOBAL.innodb_log_file_size!=10485760; +@@GLOBAL.innodb_log_file_size!=10485760 +1 +SET GLOBAL innodb_log_file_size=4294971392; +ERROR HY000: Failed to create specific handler file +SELECT @@GLOBAL.innodb_log_file_size<=10485760; +@@GLOBAL.innodb_log_file_size<=10485760 +1 +SET GLOBAL innodb_log_file_size=4294967296; +SELECT @@GLOBAL.innodb_log_file_size<=10485760; +@@GLOBAL.innodb_log_file_size<=10485760 +1 +SET GLOBAL innodb_log_archive=OFF; connection default; SET GLOBAL innodb_log_file_size=5242880; connection con1; @@ -45,6 +68,18 @@ global_value connection con1; disconnect con1; connection default; +# restart: --innodb-log-recovery-start=12345 --innodb-log-archive-start=1234567 +$check_no_innodb; +ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS +FOUND 1 /InnoDB: innodb_log_archive_start=1234567 is after innodb_log_recovery_start=12345/ in mysqld.1.err +# restart: --innodb-log-archive --innodb-log-file-size=5g +$check_no_innodb; +ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS +FOUND 1 /InnoDB: innodb_log_archive=ON disallows innodb_log_file_size>4G/ in mysqld.1.err +# restart: --innodb-log-archive --innodb-log-file-size=4g +$check_no_innodb; +ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS +FOUND 1 /InnoDB: innodb_log_archive=ON but .*/ib_logfile0 exists/ in mysqld.1.err # restart SELECT * FROM t WHERE a<10; a b @@ -61,6 +96,23 @@ SELECT COUNT(*),LENGTH(b) FROM t GROUP BY b; COUNT(*) LENGTH(b) 9 0 19991 255 +SET GLOBAL innodb_log_archive=ON; +# restart: --innodb-log-recovery-start=12290 +$check_no_innodb; +ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS +FOUND 1 /InnoDB: No matching file found for innodb_log_recovery_start=12290/ in mysqld.1.err +# restart: with restart_parameters +$check_no_innodb; +ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS +NOT FOUND /InnoDB: File .*/ib_logfile0 as not found/ in mysqld.1.err +FOUND 2 /InnoDB: innodb_log_archive_start=\d+ is after innodb_log_recovery_start=\d+/ in mysqld.1.err +# restart: with restart_parameters +$check_no_innodb; +ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS +# restart: with restart_parameters +SET GLOBAL innodb_log_archive=OFF; +FOUND 1 /InnoDB: setting innodb_log_archive=0 at innodb_log_recovery_start=.*/ in mysqld.1.err +FOUND 1 /InnoDB: setting innodb_log_archive=1 at innodb_log_recovery_start=.*/ in mysqld.1.err SHOW VARIABLES LIKE 'innodb_log_file_size'; Variable_name Value innodb_log_file_size 5242880 diff --git a/mysql-test/suite/innodb/r/log_upgrade.result b/mysql-test/suite/innodb/r/log_upgrade.result index 4da83460f9364..d1f1867778c34 100644 --- a/mysql-test/suite/innodb/r/log_upgrade.result +++ b/mysql-test/suite/innodb/r/log_upgrade.result @@ -2,7 +2,7 @@ call mtr.add_suppression("InnoDB: The change buffer is corrupted"); # # MDEV-24412 InnoDB: Upgrade after a crash is not supported # -# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_upgrade --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_upgrade --innodb-force-recovery=5 --innodb-log-file-size=4m +# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_upgrade --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_upgrade --skip-innodb-log-archive --innodb-force-recovery=5 --innodb-log-file-size=4m SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); diff --git a/mysql-test/suite/innodb/r/log_upgrade_101_flags.result b/mysql-test/suite/innodb/r/log_upgrade_101_flags.result index 9bcb786170a63..1f5772c1d30f9 100644 --- a/mysql-test/suite/innodb/r/log_upgrade_101_flags.result +++ b/mysql-test/suite/innodb/r/log_upgrade_101_flags.result @@ -1,7 +1,7 @@ call mtr.add_suppression("InnoDB: The change buffer is corrupted"); call mtr.add_suppression("InnoDB: Tablespace size stored in header is 768 pages, but the sum of data file sizes is 384 pages"); call mtr.add_suppression("InnoDB: adjusting FSP_SPACE_FLAGS of file"); -# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_upgrade --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_upgrade --innodb-undo-directory=MYSQLTEST_VARDIR/tmp/log_upgrade --innodb-force-recovery=5 --innodb-log-file-size=4m --innodb_page_size=32k --innodb_buffer_pool_size=11M +# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_upgrade --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_upgrade --innodb-undo-directory=MYSQLTEST_VARDIR/tmp/log_upgrade --skip-innodb-log-archive --innodb-force-recovery=5 --innodb-log-file-size=4m --innodb_page_size=32k --innodb_buffer_pool_size=11M SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); diff --git a/mysql-test/suite/innodb/r/read_only_recover_committed.result b/mysql-test/suite/innodb/r/read_only_recover_committed.result index 0cdf4ec11183c..1a6f3bac40e0a 100644 --- a/mysql-test/suite/innodb/r/read_only_recover_committed.result +++ b/mysql-test/suite/innodb/r/read_only_recover_committed.result @@ -33,7 +33,7 @@ a 1 20 UPDATE t SET a=3 WHERE a=1; -# restart: --innodb-read-only +# restart: --innodb-read-only --innodb-log-recovery-start=0 SET GLOBAL innodb_status_output= @@GLOBAL.innodb_status_output; # Starting with MariaDB 10.2, innodb_read_only implies READ UNCOMMITTED. # In earlier versions, this would return the last committed version diff --git a/mysql-test/suite/innodb/r/read_only_recovery.result b/mysql-test/suite/innodb/r/read_only_recovery.result index 78815e563f733..7c15523dda381 100644 --- a/mysql-test/suite/innodb/r/read_only_recovery.result +++ b/mysql-test/suite/innodb/r/read_only_recovery.result @@ -25,7 +25,7 @@ SELECT * FROM t; a 1 UPDATE t SET a=3 WHERE a=1; -# restart: --innodb-read-only +# restart: --innodb-read-only --innodb-log-recovery-start=0 # Starting with MariaDB 10.2, innodb_read_only implies READ UNCOMMITTED. # In earlier versions, this would return the last committed version # (empty table)! diff --git a/mysql-test/suite/innodb/r/rename_table.result b/mysql-test/suite/innodb/r/rename_table.result index a3bf59101b39e..5842ca79ce860 100644 --- a/mysql-test/suite/innodb/r/rename_table.result +++ b/mysql-test/suite/innodb/r/rename_table.result @@ -1,4 +1,7 @@ call mtr.add_suppression("InnoDB: In RENAME TABLE table `test`.`t4` is referenced in foreign key constraints which are not compatible with the new table definition."); +call mtr.add_suppression("InnoDB: Plugin initialization aborted"); +call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); +call mtr.add_suppression("InnoDB: cannot fulfill innodb_log_recovery_target=20480<"); CREATE DATABASE test_jfg; CREATE DATABASE test_jfg2; CREATE TABLE test_jfg.test (a int unsigned PRIMARY KEY) ENGINE=InnoDB; @@ -8,6 +11,13 @@ FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESPACES WHERE filename LIKE '%test%'; path ./test_jfg2/test.ibd DROP DATABASE test_jfg; +# restart: --innodb-log-recovery-target=20480 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +COUNT(*) +0 +FOUND 1 /InnoDB: cannot fulfill innodb_log_recovery_target=20480 4 -# restart: --innodb_undo_tablespaces=4 --innodb_force_recovery=5 +# restart: --innodb_undo_tablespaces=4 --innodb_force_recovery=5 --innodb-log-recovery-start=0 # Display 2 undo tablespace SELECT @@global.innodb_undo_tablespaces; @@global.innodb_undo_tablespaces @@ -67,22 +67,22 @@ undo002 # system tablespace in buf_dblwr_t::init_or_load_pages() # SET GLOBAL innodb_fast_shutdown=0; -# restart: --innodb_undo_tablespaces=4 +# restart: --innodb_undo_tablespaces=4 --innodb-log-recovery-start=0 # Should list 4 undo log tablespaces undo001 undo002 undo003 undo004 set global innodb_fast_shutdown=0; -# restart: --innodb_read_only=1 +# restart: --innodb_read_only=1 --innodb-log-recovery-start=0 set global innodb_fast_shutdown=0; -# restart: --innodb-data-file-path=ibdata1:1M:autoextend --innodb_undo_directory=MYSQLTEST_VARDIR/tmp/undo_dir --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/bugdir --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/bugdir --innodb_undo_tablespaces=3 +# restart: --innodb-data-file-path=ibdata1:1M:autoextend --innodb_undo_directory=MYSQLTEST_VARDIR/tmp/undo_dir --innodb-log-recovery-start=0 --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/bugdir --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/bugdir --innodb_undo_tablespaces=3 # Should list 3 undo log tablespaces undo001 undo002 undo003 set global innodb_fast_shutdown=0; -# restart: --innodb-data-file-path=ibdata1:1M:autoextend --innodb_undo_directory=MYSQLTEST_VARDIR/tmp/undo_dir --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/bugdir_1 --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/bugdir_1 --innodb_undo_tablespaces=0 +# restart: --innodb-data-file-path=ibdata1:1M:autoextend --innodb_undo_directory=MYSQLTEST_VARDIR/tmp/undo_dir --innodb-log-recovery-start=0 --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/bugdir_1 --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/bugdir_1 --innodb_undo_tablespaces=0 # Shouldn't list 0 undo log tablespaces set global innodb_fast_shutdown=0; -# restart +# restart: --innodb-log-recovery-start=0 diff --git a/mysql-test/suite/innodb/r/undo_upgrade_debug.result b/mysql-test/suite/innodb/r/undo_upgrade_debug.result index eb606b54a020e..efa65ea9e4543 100644 --- a/mysql-test/suite/innodb/r/undo_upgrade_debug.result +++ b/mysql-test/suite/innodb/r/undo_upgrade_debug.result @@ -7,33 +7,33 @@ call mtr.add_suppression("Plugin 'InnoDB' init function returned error"); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); set global innodb_fast_shutdown=0; # case 1: Abort after resetting TRX_SYS page rollback segments -# restart: --innodb_undo_tablespaces=4 --debug_dbug=+d,after_rseg_reset_abort -# restart: --innodb_undo_tablespaces=4 +# restart: --innodb-log-recovery-start=0 --innodb_undo_tablespaces=4 --debug_dbug=+d,after_rseg_reset_abort +# restart: --innodb-log-recovery-start=0 --innodb_undo_tablespaces=4 # Should list 4 undo log tablespaces undo001 undo002 undo003 undo004 # case 2: Abort after deleting the old undo tablespaces -# restart: --innodb_undo_tablespaces=2 --debug_dbug=+d,after_deleting_old_undo_abort -# restart: --innodb_undo_tablespaces=2 +# restart: --innodb-log-recovery-start=0 --innodb_undo_tablespaces=2 --debug_dbug=+d,after_deleting_old_undo_abort +# restart: --innodb-log-recovery-start=0 --innodb_undo_tablespaces=2 # Should list 2 undo log tablespaces undo001 undo002 # case 3: Abort after successfully deleting the old undo tablespace -# restart: --innodb_undo_tablespaces=3 --debug_dbug=+d,after_deleting_old_undo_success -# restart: --innodb_undo_tablespaces=3 +# restart: --innodb-log-recovery-start=0 --innodb_undo_tablespaces=3 --debug_dbug=+d,after_deleting_old_undo_success +# restart: --innodb-log-recovery-start=0 --innodb_undo_tablespaces=3 # Should list 3 undo log tablespaces undo001 undo002 undo003 -# restart: --innodb_undo_tablespaces=4 +# restart: --innodb-log-recovery-start=0 --innodb_undo_tablespaces=4 # Should list 4 undo log tablespaces undo001 undo002 undo003 undo004 -# restart: --innodb_undo_tablespaces=2 +# restart: --innodb-log-recovery-start=0 --innodb_undo_tablespaces=2 # Should list 2 undo log tablespaces undo001 undo002 diff --git a/mysql-test/suite/innodb/t/101_compatibility.test b/mysql-test/suite/innodb/t/101_compatibility.test index 878daad275673..3d4bd491a38d0 100644 --- a/mysql-test/suite/innodb/t/101_compatibility.test +++ b/mysql-test/suite/innodb/t/101_compatibility.test @@ -99,7 +99,7 @@ convert_to_mariadb_101("$dd/test/ti.ibd", $ps); convert_to_mariadb_101("$ENV{MYSQL_TMP_DIR}/test/tdd.ibd", $ps); EOF ---let $restart_parameters=--innodb-read-only +--let $restart_parameters=--innodb-read-only --innodb-log-recovery-start=0 --source include/start_mysqld.inc CHECK TABLE tr,tc,td,tz,tdd,tp,ti; --source include/shutdown_mysqld.inc diff --git a/mysql-test/suite/innodb/t/alter_copy.test b/mysql-test/suite/innodb/t/alter_copy.test index 7592d6c37d61e..6f4348e6a8e7a 100644 --- a/mysql-test/suite/innodb/t/alter_copy.test +++ b/mysql-test/suite/innodb/t/alter_copy.test @@ -73,7 +73,7 @@ SELECT * FROM t1 WHERE MATCH(b,c) AGAINST ('column'); SHOW CREATE TABLE t1; CHECK TABLE t1; ---let $restart_parameters= --innodb-read-only +--let $restart_parameters= --innodb-read-only --innodb-log-recovery-start=0 --source include/restart_mysqld.inc --replace_regex /FTS_[0-9a-f]*_[0-9a-f]*/FTS/ diff --git a/mysql-test/suite/innodb/t/alter_kill.test b/mysql-test/suite/innodb/t/alter_kill.test index d18d3d36f292e..b305b1fc23056 100644 --- a/mysql-test/suite/innodb/t/alter_kill.test +++ b/mysql-test/suite/innodb/t/alter_kill.test @@ -3,6 +3,7 @@ -- source include/not_embedded.inc -- source include/no_valgrind_without_big.inc -- source include/innodb_checksum_algorithm.inc +--source include/no_checkpoint_prepare.inc let MYSQLD_DATADIR=`select @@datadir`; let PAGE_SIZE=`select @@innodb_page_size`; @@ -77,9 +78,9 @@ syswrite(FILE, $page, $ps)==$ps || die "Unable to write $file\n"; close(FILE) || die "Unable to close $file"; EOF --- let $restart_parameters=--innodb-flush-method=O_DIRECT +-- let $restart_parameters=--innodb-flush-method=O_DIRECT --innodb-log-recovery-start=0 -- source include/start_mysqld.inc --- let $restart_parameters= +-- let $restart_parameters=--innodb-log-recovery-start=0 SELECT @@innodb_doublewrite; --error ER_TABLE_CORRUPT @@ -161,6 +162,7 @@ XA END 'x'; XA PREPARE 'x'; --connection default --let CLEANUP_IF_CHECKPOINT=XA ROLLBACK 'x';DROP TABLE bug16735660; + --source ../include/no_checkpoint_end.inc -- disconnect con1 diff --git a/mysql-test/suite/innodb/t/alter_missing_tablespace.test b/mysql-test/suite/innodb/t/alter_missing_tablespace.test index ff93ec5f9c67c..a96bce101bf04 100644 --- a/mysql-test/suite/innodb/t/alter_missing_tablespace.test +++ b/mysql-test/suite/innodb/t/alter_missing_tablespace.test @@ -32,6 +32,7 @@ INSERT INTO t1 VALUES(1),(2),(3); --remove_file $MYSQLD_DATADIR/test/t.ibd --remove_file $MYSQLD_DATADIR/test/x@002e@002ed.ibd +--let $restart_parameters=--innodb-log-recovery-start=0 --source include/start_mysqld.inc # The table does exist, only the tablespace does not exist. diff --git a/mysql-test/suite/innodb/t/autoinc_import.test b/mysql-test/suite/innodb/t/autoinc_import.test index e137413dfb538..f3222be74cedf 100644 --- a/mysql-test/suite/innodb/t/autoinc_import.test +++ b/mysql-test/suite/innodb/t/autoinc_import.test @@ -120,13 +120,13 @@ ALTER TABLE t10_1b IMPORT TABLESPACE; CHECK TABLE t1, t1b, t1t, t1z, t5_7, t5_7b, t10_1, t10_1b; CHECK TABLE t1, t1b, t1t, t1z, t5_7, t5_7b, t10_1, t10_1b FOR UPGRADE; ---let $restart_parameters=--innodb-read-only --read-only +--let $restart_parameters=--innodb-read-only --read-only --innodb-log-recovery-start=0 --source include/restart_mysqld.inc CHECK TABLE t1, t1b, t1t, t1z, t5_7, t5_7b, t10_1, t10_1b; CHECK TABLE t1, t1b, t1t, t1z, t5_7, t5_7b, t10_1, t10_1b FOR UPGRADE; ---let $restart_parameters=--innodb-read-only +--let $restart_parameters=--innodb-read-only --innodb-log-recovery-start=0 --source include/restart_mysqld.inc CHECK TABLE t1, t1b, t1t, t1z, t5_7, t5_7b, t10_1, t10_1b; diff --git a/mysql-test/suite/innodb/t/corrupted_during_recovery.test b/mysql-test/suite/innodb/t/corrupted_during_recovery.test index 80db52e90edc1..d6b9fd16c5f44 100644 --- a/mysql-test/suite/innodb/t/corrupted_during_recovery.test +++ b/mysql-test/suite/innodb/t/corrupted_during_recovery.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/no_checkpoint_prepare.inc --disable_query_log call mtr.add_suppression("InnoDB: Plugin initialization aborted"); @@ -13,23 +14,60 @@ call mtr.add_suppression("InnoDB: Your database may be corrupt"); call mtr.add_suppression("InnoDB: MySQL-8\\.0 tablespace in .*test/t2\\.ibd"); call mtr.add_suppression("InnoDB: Restart in MySQL for migration/recovery\\."); call mtr.add_suppression("Fewer engines are enabled now than were before the crash"); +call mtr.add_suppression("InnoDB: Did not find innodb_log_recovery_start="); +call mtr.add_suppression("InnoDB: cannot fulfill innodb_log_recovery_target="); +# only issued for innodb_log_archive=OFF +call mtr.add_suppression("InnoDB: innodb_log_recovery_start=\\d+ is after innodb_log_recovery_target"); --enable_query_log let INNODB_PAGE_SIZE=`select @@innodb_page_size`; let ALGO=`select @@innodb_checksum_algorithm`; -CREATE TABLE t1(a BIGINT PRIMARY KEY) ENGINE=InnoDB; +let $before=`SELECT variable_value FROM information_schema.global_status +WHERE variable_name='innodb_lsn_current'`; +CREATE TABLE t1(a BIGINT PRIMARY KEY) ENGINE=InnoDB STATS_PERSISTENT=0; INSERT INTO t1 VALUES(1); # Force a redo log checkpoint. let $restart_noprint=2; ---source include/restart_mysqld.inc +SET GLOBAL innodb_max_purge_lag_wait=0, innodb_log_checkpoint_now=ON; --source ../include/no_checkpoint_start.inc +let $archived=`SELECT variable_value FROM information_schema.global_status +WHERE variable_name='innodb_lsn_archived'`; + +connect stop_purge,localhost,root; +START TRANSACTION WITH CONSISTENT SNAPSHOT; +connection default; + CREATE TABLE t2(a BIGINT PRIMARY KEY) ENGINE=InnoDB; INSERT INTO t1 VALUES(2); SET GLOBAL innodb_flush_log_at_trx_commit=1; +let $after=`SELECT variable_value FROM information_schema.global_status +WHERE variable_name='innodb_lsn_current'`; INSERT INTO t2 VALUES(1); +let $future=`SELECT variable_value+999999 FROM information_schema.global_status +WHERE variable_name='innodb_lsn_current'`; --let CLEANUP_IF_CHECKPOINT=DROP TABLE t1,t2; --source ../include/no_checkpoint_end.inc +disconnect stop_purge; + +--let $restart_parameters=--innodb-log-recovery-target=$before --innodb-log-recovery-start=0 +--source include/start_mysqld.inc +--error ER_UNKNOWN_STORAGE_ENGINE,ER_NO_SUCH_TABLE_IN_ENGINE +SELECT * FROM t2; +--let $restart_parameters=--innodb-log-recovery-target=$after --innodb-log-recovery-start=$before +--source include/restart_mysqld.inc +--error ER_UNKNOWN_STORAGE_ENGINE +SELECT * FROM t2; +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +let SEARCH_PATTERN=InnoDB: cannot fulfill innodb_log_recovery_target.*; +--source include/search_pattern_in_file.inc +--let $restart_parameters=--innodb-log-recovery-target=$before --innodb-log-recovery-start=$archived +--source include/restart_mysqld.inc +--error ER_OPEN_AS_READONLY,ER_UNKNOWN_STORAGE_ENGINE +DELETE FROM t1; +--error ER_OPEN_AS_READONLY,ER_NO_SUCH_TABLE_IN_ENGINE,ER_UNKNOWN_STORAGE_ENGINE +DELETE FROM t2; +--source include/shutdown_mysqld.inc --echo # Corrupt the pages @@ -76,11 +114,7 @@ EOF --error ER_UNKNOWN_STORAGE_ENGINE SELECT * FROM t1; -let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; -let SEARCH_PATTERN=InnoDB: Page \\[page id: space=[1-9][0-9]*, page number=3\\] log sequence number 1311768467463790320 is in the future!; ---source include/search_pattern_in_file.inc - -let $restart_parameters=--innodb_force_recovery=1; +let $restart_parameters=--innodb_force_recovery=1 --innodb-log-recovery-start=0; --source include/restart_mysqld.inc --error 0,ER_NO_SUCH_TABLE_IN_ENGINE @@ -88,6 +122,9 @@ SELECT * FROM t1; SELECT * FROM t2; CHECK TABLE t2; +let SEARCH_PATTERN=InnoDB: Page \\[page id: space=[1-9][0-9]*, page number=3\\] log sequence number 1311768467463790320 is in the future!.*; +--source include/search_pattern_in_file.inc + DROP TABLE t1, t2; # MDEV-21572 buf_page_get_gen() should apply buffered page @@ -137,6 +174,13 @@ sysseek(FILE, $ps * 3, SEEK_SET) or die "seek"; syswrite(FILE, $page); close FILE or die "close"; EOF +--let $restart_parameters=--innodb-log-recovery-start=$future --source include/start_mysqld.inc +--error ER_UNKNOWN_STORAGE_ENGINE +SELECT * FROM t1; +let SEARCH_PATTERN=InnoDB: Did not find innodb_log_recovery_start=\d+ .*; +--source include/search_pattern_in_file.inc +--let $restart_parameters= +--source include/restart_mysqld.inc SELECT * FROM t1; DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/doublewrite.test b/mysql-test/suite/innodb/t/doublewrite.test index da059bf80ea9a..a50a621f4b51e 100644 --- a/mysql-test/suite/innodb/t/doublewrite.test +++ b/mysql-test/suite/innodb/t/doublewrite.test @@ -5,6 +5,8 @@ --source include/innodb_page_size.inc --source include/not_embedded.inc +--source include/skip_innodb_log_archive.inc +--source include/no_checkpoint_prepare.inc --disable_query_log call mtr.add_suppression("InnoDB: Data file .* uses page size .* but the innodb_page_size start-up parameter is"); @@ -19,7 +21,7 @@ call mtr.add_suppression("InnoDB: Inconsistent tablespace ID in .*t1\\.ibd"); call mtr.add_suppression("\\[Warning\\] Found 1 prepared XA transactions"); call mtr.add_suppression("InnoDB: Header page consists of zero bytes in datafile:"); call mtr.add_suppression("InnoDB: Page \\[page id: space=[1-9][0-9]*, page number=3\\] log sequence number 18446744073709551615 is in the future!"); -call mtr.add_suppression("InnoDB: Your database may be corrupt or you may have copied the InnoDB tablespace but not the ib_logfile0"); +call mtr.add_suppression("InnoDB: Your database may be corrupt or you may have copied the InnoDB tablespaces but not the log"); call mtr.add_suppression("InnoDB: Plugin initialization aborted"); --enable_query_log diff --git a/mysql-test/suite/innodb/t/doublewrite_debug.test b/mysql-test/suite/innodb/t/doublewrite_debug.test index e31cf34dbc110..ce134752d098e 100644 --- a/mysql-test/suite/innodb/t/doublewrite_debug.test +++ b/mysql-test/suite/innodb/t/doublewrite_debug.test @@ -7,6 +7,7 @@ --source include/innodb_page_size.inc --source include/have_debug.inc --source include/not_embedded.inc +--source include/no_checkpoint_prepare.inc --disable_query_log call mtr.add_suppression("InnoDB: Data file .* uses page size .* but the innodb_page_size start-up parameter is"); call mtr.add_suppression("InnoDB: adjusting FSP_SPACE_FLAGS"); @@ -45,7 +46,7 @@ commit work; # Slow shutdown and restart to make sure ibuf merge is finished SET GLOBAL innodb_fast_shutdown = 0; let $shutdown_timeout=; -let $restart_parameters=--debug_dbug=+d,ib_log_checkpoint_avoid_hard --innodb_flush_sync=0; +let $restart_parameters=--debug_dbug=+d,ib_log_checkpoint_avoid_hard --innodb_flush_sync=0 --innodb-log-recovery-start=0; --source include/restart_mysqld.inc --source ../include/no_checkpoint_start.inc begin; @@ -84,7 +85,7 @@ print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'}); close FILE; EOF -let $restart_parameters=; +let $restart_parameters=--innodb-log-recovery-start=0; --source include/start_mysqld.inc let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=0, page number=0\\]; @@ -101,7 +102,7 @@ select f1, f2 from t1; --echo # Test Begin: Test if recovery works if 1st page of --echo # system tablespace is corrupted and 2nd page as corrupted. -let $restart_parameters=--debug_dbug=+d,ib_log_checkpoint_avoid_hard --innodb_flush_sync=0; +let $restart_parameters=--innodb-log-recovery-start=0 --debug_dbug=+d,ib_log_checkpoint_avoid_hard --innodb_flush_sync=0; --source include/restart_mysqld.inc --source ../include/no_checkpoint_start.inc begin; @@ -132,7 +133,7 @@ print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'}/2); close FILE; EOF -let $restart_parameters=; +let $restart_parameters=--innodb-log-recovery-start=0; --source include/start_mysqld.inc let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=0, page number=0\\]; @@ -159,7 +160,7 @@ WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); --let $ibp=--innodb-log-group-home-dir=$bugdir --innodb-data-home-dir=$bugdir ---let $ibp=$ibp --innodb-undo-tablespaces=0 +--let $ibp=$ibp --innodb-undo-tablespaces=0 --innodb-log-recovery-start=0 --let $ibp=$ibp --innodb-data-file-path=ibdata1:1M;ibdata2:1M:autoextend --let $restart_parameters= $ibp @@ -167,10 +168,9 @@ AND support IN ('YES', 'DEFAULT', 'ENABLED'); eval $check_no_innodb; --let SEARCH_PATTERN= \[ERROR\] InnoDB: Cannot create doublewrite buffer --source include/search_pattern_in_file.inc ---let $restart_parameters= +--let $restart_parameters=--innodb-log-recovery-start=0 --source include/restart_mysqld.inc --remove_file $bugdir/ibdata1 --remove_file $bugdir/ibdata2 ---remove_file $bugdir/ib_logfile0 --rmdir $bugdir diff --git a/mysql-test/suite/innodb/t/encryption_threads_shutdown.test b/mysql-test/suite/innodb/t/encryption_threads_shutdown.test index 16e2f318a4178..4ce1113c0cb80 100644 --- a/mysql-test/suite/innodb/t/encryption_threads_shutdown.test +++ b/mysql-test/suite/innodb/t/encryption_threads_shutdown.test @@ -1,5 +1,6 @@ # This test is for MDEV-24612 fix --source include/have_innodb.inc +--source include/skip_innodb_log_archive.inc --source include/not_embedded.inc call mtr.add_suppression("Creating system tablespace with existing redo log file is not recommended."); diff --git a/mysql-test/suite/innodb/t/innodb-get-fk.test b/mysql-test/suite/innodb/t/innodb-get-fk.test index b4b170b89f93f..a35bd3e83c90d 100644 --- a/mysql-test/suite/innodb/t/innodb-get-fk.test +++ b/mysql-test/suite/innodb/t/innodb-get-fk.test @@ -36,7 +36,7 @@ CONSTRAINT `fk_crewRoleAssigned_crewId` FOREIGN KEY (`crew_id`) REFERENCES `repr CONSTRAINT `fk_crewRoleAssigned_pilotId` FOREIGN KEY (`crew_id`) REFERENCES `repro`.`pilot` (`id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB COMMENT="This is a comment about tables"; --- let $restart_parameters=--innodb-read-only +-- let $restart_parameters=--innodb-read-only --innodb-log-recovery-start=0 -- source include/restart_mysqld.inc --error ER_OPEN_AS_READONLY @@ -54,7 +54,7 @@ SET GLOBAL innodb_buffer_pool_load_abort = ON; ALTER TABLE `repro`.`crew_role_assigned` COMMENT = "This is a new comment about tables"; SHOW CREATE TABLE `repro`.`crew_role_assigned`; --- let $restart_parameters=--innodb-read-only +-- let $restart_parameters=--innodb-read-only --innodb-log-recovery-start=0 -- source include/restart_mysqld.inc # diff --git a/mysql-test/suite/innodb/t/innodb-index.test b/mysql-test/suite/innodb/t/innodb-index.test index 3236267c03d7c..8df7dd21788b8 100644 --- a/mysql-test/suite/innodb/t/innodb-index.test +++ b/mysql-test/suite/innodb/t/innodb-index.test @@ -1,4 +1,5 @@ -- source include/have_innodb.inc +--source include/no_checkpoint_prepare.inc # Embedded server tests do not support restarting. -- source include/not_embedded.inc --source include/test_db_charset_latin1.inc diff --git a/mysql-test/suite/innodb/t/innodb-master.opt b/mysql-test/suite/innodb/t/innodb-master.opt index 2e71d62206dbd..5266978e4f0a7 100644 --- a/mysql-test/suite/innodb/t/innodb-master.opt +++ b/mysql-test/suite/innodb/t/innodb-master.opt @@ -2,5 +2,3 @@ --default-storage-engine=MyISAM --innodb-strict-mode=0 --innodb-file-per-table=0 ---loose-innodb-track-changed-pages ---loose-innodb-log-archive diff --git a/mysql-test/suite/innodb/t/innodb-wl5522.test b/mysql-test/suite/innodb/t/innodb-wl5522.test index 873cf6b3a23e9..5f0d2825f52f5 100644 --- a/mysql-test/suite/innodb/t/innodb-wl5522.test +++ b/mysql-test/suite/innodb/t/innodb-wl5522.test @@ -8,7 +8,11 @@ call mtr.add_suppression("InnoDB: Unable to import tablespace .* because it already exists. Please DISCARD the tablespace before IMPORT\\."); call mtr.add_suppression("Index for table 't2' is corrupt; try to repair it"); call mtr.add_suppression("InnoDB: Cannot save statistics for table `test`\\.`t1` because the \\.ibd file is missing"); - +if ($MTR_COMBINATION_STRICT_FULL_CRC32) { +call mtr.add_suppression("InnoDB: cannot fulfill innodb_log_recovery_target=102345<"); +call mtr.add_suppression("InnoDB: Plugin initialization aborted"); +call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed\\."); +} FLUSH TABLES; let $MYSQLD_DATADIR = `SELECT @@datadir`; @@ -36,6 +40,18 @@ SELECT * FROM t1 ORDER BY a DESC LIMIT 3; CREATE TABLE t2(a INT PRIMARY KEY) ENGINE=InnoDB ROW_FORMAT=COMPACT; --list_files $MYSQLD_DATADIR/test +if ($MTR_COMBINATION_STRICT_FULL_CRC32) { +--let $restart_parameters=--innodb-log-recovery-target=102345 +--let $shutdown_timeout=0 +--source include/restart_mysqld.inc +--let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err +--let SEARCH_PATTERN= InnoDB: cannot fulfill innodb_log_recovery_target=102345< +--source include/search_pattern_in_file.inc +--error ER_UNKNOWN_STORAGE_ENGINE,ER_NO_SUCH_TABLE_IN_ENGINE +FLUSH TABLE t1, t2 FOR EXPORT; +--let $restart_parameters= +--let $shutdown_timeout= +} --source include/restart_mysqld.inc FLUSH TABLE t1, t2 FOR EXPORT; --echo # List before copying files diff --git a/mysql-test/suite/innodb/t/innodb_bug14147491.test b/mysql-test/suite/innodb/t/innodb_bug14147491.test index cff1b0c78af56..b27d0a19e0ce2 100644 --- a/mysql-test/suite/innodb/t/innodb_bug14147491.test +++ b/mysql-test/suite/innodb/t/innodb_bug14147491.test @@ -64,6 +64,7 @@ while ($len = sysread IBD_FILE, $chunk, 1024) close IBD_FILE; EOF +--let $restart_parameters=--innodb-log-recovery-start=0 --source include/start_mysqld.inc --echo # Now t1 is corrupted but we should not crash diff --git a/mysql-test/suite/innodb/t/innodb_force_recovery.test b/mysql-test/suite/innodb/t/innodb_force_recovery.test index 4e11047a2d620..e0c71c6019c82 100644 --- a/mysql-test/suite/innodb/t/innodb_force_recovery.test +++ b/mysql-test/suite/innodb/t/innodb_force_recovery.test @@ -8,6 +8,9 @@ call mtr.add_suppression("InnoDB: Failed to find tablespace for table .* in the cache. Attempting to load the tablespace with space id"); call mtr.add_suppression("InnoDB: Allocated tablespace ID \\d+ for test.t[12], old maximum was"); call mtr.add_suppression("InnoDB: Allocated tablespace ID \\d+ for mysql.transaction_registry, old maximum was"); +call mtr.add_suppression("InnoDB: Plugin initialization aborted"); +call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); +call mtr.add_suppression("InnoDB: cannot fulfill innodb_log_recovery_target=20480<"); --enable_query_log create table t1(f1 int not null, f2 int not null, index idx(f2))engine=innodb; @@ -17,6 +20,16 @@ insert into t2 values(1, 2); SET GLOBAL innodb_fast_shutdown = 0; +--let $restart_parameters= --innodb-log-recovery-target=20480 +--source include/restart_mysqld.inc +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); + +--let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err +--let SEARCH_PATTERN= InnoDB: cannot fulfill innodb_log_recovery_target=20480<.* +--source include/search_pattern_in_file.inc + --let $restart_parameters= --innodb-force-recovery=4 --source include/restart_mysqld.inc let $status=`SHOW ENGINE INNODB STATUS`; diff --git a/mysql-test/suite/innodb/t/leaf_page_corrupted_during_recovery.test b/mysql-test/suite/innodb/t/leaf_page_corrupted_during_recovery.test index 02668a0207c2a..a8c1d8be3a1ad 100644 --- a/mysql-test/suite/innodb/t/leaf_page_corrupted_during_recovery.test +++ b/mysql-test/suite/innodb/t/leaf_page_corrupted_during_recovery.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/no_checkpoint_prepare.inc --source include/have_debug.inc --disable_query_log @@ -23,6 +24,7 @@ INSERT INTO t1 VALUES(1, 'sql'), (2, 'server'), (3, 'mariadb'), (12, 'test8'); let $restart_noprint=2; +let $restart_parameters=--innodb-log-recovery-start=0; --source include/restart_mysqld.inc let INNODB_PAGE_SIZE=`select @@innodb_page_size`; @@ -50,7 +52,7 @@ EOF SELECT * FROM t1 WHERE PK = 1; # Force recovery to ignore the corrupted page. -let $restart_parameters=--innodb-force-recovery=1; +let $restart_parameters=--innodb-log-recovery-start=0 --innodb-force-recovery=1; --source include/restart_mysqld.inc SELECT * FROM t1 WHERE PK = 1; --error ER_NOT_KEYFILE diff --git a/mysql-test/suite/innodb/t/log_archive.combinations b/mysql-test/suite/innodb/t/log_archive.combinations new file mode 100644 index 0000000000000..a5e27fd8c54d3 --- /dev/null +++ b/mysql-test/suite/innodb/t/log_archive.combinations @@ -0,0 +1,4 @@ +[mmap] +--innodb-log-file-mmap=ON +[pread] +--skip-innodb-log-file-mmap diff --git a/mysql-test/suite/innodb/t/log_archive.test b/mysql-test/suite/innodb/t/log_archive.test new file mode 100644 index 0000000000000..8358c47aa307e --- /dev/null +++ b/mysql-test/suite/innodb/t/log_archive.test @@ -0,0 +1,222 @@ +--source include/have_innodb.inc +--source include/not_embedded.inc + +call mtr.add_suppression("InnoDB: No matching file found for innodb_log_recovery_start=12288"); +call mtr.add_suppression("InnoDB: innodb_read_only prevents crash recovery between 18446744073705357279 and 18446744073705357311"); +call mtr.add_suppression("InnoDB: Did not find any checkpoint after LSN=18446744073701175296"); +call mtr.add_suppression("InnoDB: cannot fulfill innodb_log_recovery_target=18446744073705357290<18446744073705357306"); +call mtr.add_suppression("InnoDB: innodb_log_archive=ON but .*ib_logfile0 exists"); +call mtr.add_suppression("InnoDB: Renaming ib_ffffffffff803000\\.log to ib_logfile0"); +call mtr.add_suppression("InnoDB: ignoring .*ib_ffffffffffc00000\\.log"); +call mtr.add_suppression("InnoDB: Plugin initialization aborted"); +call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); + +let DATADIR= `select @@datadir`; +--source include/shutdown_mysqld.inc +--disable_result_log +--error 0,1 # fails if innodb_log_archive=ON +--move_file $DATADIR/ib_logfile0 $DATADIR/ib_logfile_hidden +--enable_result_log + +perl; +do "$ENV{MTR_SUITE_DIR}/include/crc32.pl"; +my $file= "$ENV{DATADIR}/ib_ffffffffff803000.log"; +my $file_size= 4<<20; +open(FILE, ">", $file) or die "Unable to open $file\n"; +binmode FILE; +print FILE pack("N", $file_size - 1); +seek FILE, $file_size - 33, 0 or die "Unable to seek $file\n"; +my $polynomial = 0x82f63b78; # CRC-32C +my $FILE_CHECKPOINT; +$FILE_CHECKPOINT=pack("CxxNN", 0xfa, 0xffffffff, 0xffbfffdf); +$FILE_CHECKPOINT .= pack("CN", 1, mycrc32($FILE_CHECKPOINT, 0, $polynomial)); +my $WRITE=pack("CxxCx[7]", 0x3a, 8); # write NULs to page 0:0 offset 8 +$WRITE .= pack("CN", 1, mycrc32($WRITE, 0, $polynomial)); +print FILE $FILE_CHECKPOINT, $WRITE; +$FILE_CHECKPOINT=pack("CxxNN", 0xfa, 0xffffffff, 0xffbfffff); +$FILE_CHECKPOINT .= pack("CN", 1, mycrc32($FILE_CHECKPOINT, 0, $polynomial)); +my $FILE_MODIFY=pack("CCxa*", 0xb9, 127, "a/b.ibd"); +$FILE_MODIFY .= pack("CN", 1, mycrc32($FILE_MODIFY, 0, $polynomial)); +print FILE substr($FILE_MODIFY, 0, 1); +close(FILE) or die "Unable to close $file\n"; +chmod 0444, $file or die "Unable to chmod 444 $file\n"; +$file= "$ENV{DATADIR}/ib_ffffffffffc00000.log"; +open(FILE, ">", $file) or die "Unable to open $file\n"; +binmode FILE; +seek FILE, 0x3000, 0 or die "Unable to seek $file\n"; +print FILE substr($FILE_MODIFY, 1), $FILE_MODIFY x 139819, $FILE_CHECKPOINT; +seek FILE, $file_size - 1, 0 or die "Unable to seek $file\n"; +print FILE chr(0); +close(FILE) or die "Unable to close $file\n"; +EOF + +--let $restart_parameters= --innodb-read-only --innodb-log-recovery-start=12288 +--source include/start_mysqld.inc +SELECT variable_name, variable_value FROM information_schema.global_status +WHERE variable_name LIKE 'INNODB_LSN%'; + +let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err; +let SEARCH_PATTERN = InnoDB: No matching file found for innodb_log_recovery_start=12288; +--source include/search_pattern_in_file.inc + +--let $restart_parameters= --innodb-read-only --innodb-log-recovery-start=18446744073705357279 +--source include/restart_mysqld.inc +SELECT variable_name, variable_value FROM information_schema.global_status +WHERE variable_name LIKE 'INNODB_LSN%'; + +let SEARCH_PATTERN = InnoDB: innodb_read_only prevents crash recovery between 18446744073705357279 and 18446744073705357311; +--source include/search_pattern_in_file.inc + +--let $restart_parameters= --innodb-read-only --innodb-log-recovery-start=18446744073705357311 +--source include/start_mysqld.inc +SELECT variable_name, variable_value FROM information_schema.global_status +WHERE variable_name LIKE 'INNODB_LSN%'; + +perl; +my $file= "$ENV{DATADIR}/ib_ffffffffff803000.log"; +chmod 0644, $file or die "Unable to chmod 644 $file\n"; +open(FILE, "+<", $file) or die "Unable to open $file\n"; +binmode FILE; +do "$ENV{MTR_SUITE_DIR}/include/crc32.pl"; +my $polynomial = 0x82f63b78; # CRC-32C +my $buf = pack("NNNNx[492]", 0x50687973, 0, 0xffffffff, 0xff803000); +$buf .= pack("N", mycrc32($buf, 0, $polynomial)); +print FILE $buf; +seek FILE, 0x1000, 0 or die "Unable to seek $file\n"; +# checkpoint buffer pointing to the FILE_MODIFY record +$buf = pack("NN", 0xffffffff, 0xffbfffdf) x 2 . chr(0) x 44; +$buf .= pack("N", mycrc32($buf, 0, $polynomial)); +print FILE $buf; +close(FILE) or die "Unable to close $file\n"; +EOF + +--let $restart_parameters= --innodb-log-recovery-start=0 --innodb-log-recovery-target=18446744073705357290 +--source include/restart_mysqld.inc +let SEARCH_PATTERN = InnoDB: Did not find any checkpoint after LSN=18446744073701175296; +--source include/search_pattern_in_file.inc + +--remove_file $DATADIR/ib_ffffffffffc00000.log +--source include/restart_mysqld.inc + +--source include/search_pattern_in_file.inc +let SEARCH_PATTERN = InnoDB: End of log at LSN=18446744073705357295; +--source include/search_pattern_in_file.inc +let SEARCH_PATTERN = InnoDB: cannot fulfill innodb_log_recovery_target=18446744073705357290<18446744073705357306; +--source include/search_pattern_in_file.inc + +--move_file $DATADIR/ib_ffffffffff803000.log $DATADIR/ib_logfile0 + +--let $restart_parameters= --innodb-log-recovery-start=0 --innodb-read-only --innodb-log-archive +--source include/restart_mysqld.inc +let SEARCH_PATTERN = InnoDB: innodb_log_archive=ON but .*ib_logfile0 exists; +--source include/search_pattern_in_file.inc + +--let $restart_parameters= --innodb-log-recovery-start=0 --innodb-read-only --skip-innodb-log-archive +--source include/restart_mysqld.inc +let SEARCH_PATTERN = InnoDB: innodb_read_only prevents crash recovery between 18446744073705357279 and 18446744073705357311; +--source include/search_pattern_in_file.inc + +--move_file $DATADIR/ib_logfile0 $DATADIR/ib_ffffffffff803000.log + +perl; +my $file= "$ENV{DATADIR}/ib_ffffffffff803000.log"; +my $file_size= 4<<20; +open(FILE, "+<", $file) or die "Unable to open $file\n"; +binmode FILE; +seek FILE, $file_size - 17, 0 or die "Unable to seek $file\n"; +print FILE chr(0); # trim the log after the FILE_CHECKPOINT +close(FILE) or die "Unable to close $file\n"; +EOF + +--let $restart_parameters= --innodb-log-recovery-start=0 --innodb-log-recovery-target=18446744073705357295 +--source include/restart_mysqld.inc + +SELECT variable_name, variable_value FROM information_schema.global_status +WHERE variable_name LIKE 'INNODB_LSN%'; + +let SEARCH_PATTERN = InnoDB: Renaming .*ffffffffff803000\\.log to ib_logfile0; +--source include/search_pattern_in_file.inc + +--error ER_CANT_CREATE_TABLE +CREATE TABLE t0 (a INT PRIMARY KEY) ENGINE=InnoDB; +--replace_result \\ / +--error ER_ERROR_ON_RENAME +RENAME TABLE mysql.innodb_table_stats TO mysql.intabdb_noble_stats; +--error ER_OPEN_AS_READONLY +DROP TABLE mysql.innodb_index_stats; +--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON +ALTER TABLE mysql.innodb_table_stats ROW_FORMAT=COMPACT, ALGORITHM=INPLACE; +--error ER_OPEN_AS_READONLY +TRUNCATE TABLE mysql.innodb_table_stats; + +--source include/shutdown_mysqld.inc +--move_file $DATADIR/ib_logfile0 $DATADIR/ib_ffffffffff803000.log + +--let $restart_parameters= --innodb-log-recovery-start=0 --innodb-log-recovery-target=18446744073705357290 +--source include/start_mysqld.inc + +SELECT variable_name, variable_value FROM information_schema.global_status +WHERE variable_name LIKE 'INNODB_LSN%'; + +--let $restart_parameters= --innodb-log-recovery-start=0 +--source include/shutdown_mysqld.inc + +perl; +my $file= "$ENV{DATADIR}/ib_ffffffffff803000.log"; +my $file_size= 4<<20; +open(FILE, "+<", $file) or die "Unable to open $file\n"; +binmode FILE; +print FILE pack("Nx[12284]", $file_size - 33); +seek FILE, $file_size - 17, 0 or die "Unable to seek $file\n"; +# WRITE to page 0:0 offset 0x7e +print FILE "0~\0\0~incomplete.."; +close(FILE) or die "Unable to close $file\n"; +EOF + +--let $restart_parameters= --innodb-log-recovery-start=18446744073705357279 +--source include/start_mysqld.inc + +SELECT variable_name, variable_value FROM information_schema.global_status +WHERE variable_name LIKE 'INNODB_LSN%'; + +--source include/shutdown_mysqld.inc + +perl; +do "$ENV{MTR_SUITE_DIR}/include/crc32.pl"; +my $file= "$ENV{DATADIR}/ib_ffffffffff803000.log"; +my $file_size= 4<<20; +open(FILE, "+<", $file) or die "Unable to open $file\n"; +binmode FILE; +print FILE pack("N", $file_size - 33) x 3072; +seek FILE, $file_size - 17, 0 or die "Unable to seek $file\n"; +my $polynomial = 0x82f63b78; # CRC-32C +my $FILE_MODIFY=pack("CCxa*", 0xb9, 127, "a/b.ibd"); +$FILE_MODIFY .= pack("CNxx", 1, mycrc32($FILE_MODIFY, 0, $polynomial)); +print FILE $FILE_MODIFY; +close(FILE) or die "Unable to close $file\n"; +EOF + +--write_file $DATADIR/ib_ffffffffffc00000.log +EOF + +--let $restart_parameters= --innodb-log-recovery-start=0 --innodb-log-buffer-size=5m +--source include/start_mysqld.inc + +SELECT variable_name, variable_value FROM information_schema.global_status +WHERE variable_name LIKE 'INNODB_LSN%'; +--let $restart_parameters= --innodb-log-recovery-start=0 +--source include/restart_mysqld.inc +SELECT variable_name, variable_value FROM information_schema.global_status +WHERE variable_name = 'INNODB_LSN_ARCHIVED'; + +let SEARCH_PATTERN = InnoDB: ignoring .*ffffffffffc00000\\.log; +--source include/search_pattern_in_file.inc + +--source include/shutdown_mysqld.inc +--remove_file $DATADIR/ib_ffffffffff803000.log +--remove_file $DATADIR/ib_ffffffffffc00000.log +--disable_result_log +--error 0,1 # fails if innodb_log_archive=ON +--move_file $DATADIR/ib_logfile_hidden $DATADIR/ib_logfile0 +--enable_result_log +--source include/start_mysqld.inc diff --git a/mysql-test/suite/innodb/t/log_corruption.test b/mysql-test/suite/innodb/t/log_corruption.test index 7c39ce02228f0..8b911b2ef62d0 100644 --- a/mysql-test/suite/innodb/t/log_corruption.test +++ b/mysql-test/suite/innodb/t/log_corruption.test @@ -1,6 +1,7 @@ --source include/have_innodb.inc --source include/have_innodb_16k.inc --source include/no_valgrind_without_big.inc +--source include/skip_innodb_log_archive.inc --disable_query_log call mtr.add_suppression("InnoDB: Upgrade after a crash is not supported"); @@ -19,6 +20,7 @@ call mtr.add_suppression("InnoDB: Obtaining redo log encryption key version 1 fa call mtr.add_suppression("InnoDB: Decrypting checkpoint failed"); call mtr.add_suppression("InnoDB: Log file .*ib_logfile1 is of different size 2097152 bytes than other log files (1048576|4194304) bytes!"); call mtr.add_suppression("InnoDB: The change buffer is corrupted"); +call mtr.add_suppression("InnoDB: cannot fulfill innodb_log_recovery_target=12345!="); --enable_query_log let bugdir= $MYSQLTEST_VARDIR/tmp/log_corruption; @@ -171,11 +173,19 @@ die unless seek(OUT, 0x800, 0); print OUT pack("NnnNx[496]N", 0x80000944, 12, 12, 0, 0xb2a); close OUT or die; EOF ---let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=4m +--let $restart_parameters= $dirs --innodb-log-recovery-target=12345 --source include/start_mysqld.inc SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); +--let SEARCH_PATTERN= InnoDB: cannot fulfill innodb_log_recovery_target=12345!= +--source include/search_pattern_in_file.inc + +--let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=4m +--source include/restart_mysqld.inc +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); --source include/shutdown_mysqld.inc --let SEARCH_PATTERN= InnoDB: Upgrading redo log: --source include/search_pattern_in_file.inc diff --git a/mysql-test/suite/innodb/t/log_corruption_recovery.test b/mysql-test/suite/innodb/t/log_corruption_recovery.test index 0a7603a5ee064..a030045b4a5b9 100644 --- a/mysql-test/suite/innodb/t/log_corruption_recovery.test +++ b/mysql-test/suite/innodb/t/log_corruption_recovery.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/skip_innodb_log_archive.inc --let DATADIR=`select @@datadir` let $check_no_innodb=SELECT * FROM INFORMATION_SCHEMA.ENGINES diff --git a/mysql-test/suite/innodb/t/log_data_file_size.test b/mysql-test/suite/innodb/t/log_data_file_size.test index fe75b9ab236a4..ce0cb3d7de61d 100644 --- a/mysql-test/suite/innodb/t/log_data_file_size.test +++ b/mysql-test/suite/innodb/t/log_data_file_size.test @@ -1,5 +1,6 @@ --source include/innodb_page_size.inc --source include/not_embedded.inc +--source include/no_checkpoint_prepare.inc let INNODB_PAGE_SIZE=`select @@innodb_page_size`; let MYSQLD_DATADIR=`select @@datadir`; diff --git a/mysql-test/suite/innodb/t/log_file.test b/mysql-test/suite/innodb/t/log_file.test index 7ff0de0fc013a..1f3516d632b5c 100644 --- a/mysql-test/suite/innodb/t/log_file.test +++ b/mysql-test/suite/innodb/t/log_file.test @@ -4,6 +4,7 @@ --source include/have_innodb.inc --source include/no_valgrind_without_big.inc +--source include/skip_innodb_log_archive.inc --disable_query_log call mtr.add_suppression("InnoDB: Could not create undo tablespace.*undo002"); @@ -188,10 +189,16 @@ let SEARCH_PATTERN=InnoDB: Failed to open the undo tablespace; --list_files $bugdir --source include/start_mysqld.inc eval $check_no_innodb; ---source include/shutdown_mysqld.inc -let SEARCH_PATTERN=redo log file .*ib_logfile0.* exists\. Creating system tablespace with existing redo log file is not recommended\. Please delete redo log file before creating new system tablespace\.; +let SEARCH_PATTERN=redo log file '.*ib_logfile0' exists\. Creating system tablespace with existing redo log file is not recommended\. Please delete redo log file before creating new system tablespace\.; --source include/search_pattern_in_file.inc +--move_file $bugdir/ib_logfile0 $bugdir/ib_0000000000003000.log +--source include/restart_mysqld.inc +let SEARCH_PATTERN=redo log file '.*ib_[?03]*\.log' exists\. Creating system tablespace with existing redo log file is not recommended\. Please delete redo log file before creating new system tablespace\.; +--source include/search_pattern_in_file.inc +--move_file $bugdir/ib_0000000000003000.log $bugdir/ib_logfile0 +--source include/shutdown_mysqld.inc + # clean up & Restore --source ../include/log_file_cleanup.inc diff --git a/mysql-test/suite/innodb/t/log_file_name.test b/mysql-test/suite/innodb/t/log_file_name.test index 0054ab20a1638..0958e51aef0a9 100644 --- a/mysql-test/suite/innodb/t/log_file_name.test +++ b/mysql-test/suite/innodb/t/log_file_name.test @@ -2,6 +2,7 @@ # Test the detection of duplicate tablespaces. --source include/have_innodb.inc +--source include/no_checkpoint_prepare.inc --source include/no_valgrind_without_big.inc # Embedded server does not support crashing @@ -12,6 +13,7 @@ FLUSH TABLES; CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB; +--let $restart_parameters=--innodb-log-recovery-start=0 --source include/restart_mysqld.inc --source include/no_checkpoint_start.inc CREATE TABLE t3(a INT PRIMARY KEY) ENGINE=InnoDB; @@ -188,7 +190,7 @@ EOF --copy_file $MYSQLD_DATADIR/test/u6.ibd $MYSQLD_DATADIR/test/u4.ibd ---let $restart_parameters= --innodb-force-recovery=1 +--let $restart_parameters= --innodb-force-recovery=1 --innodb-log-recovery-start=0 --source include/start_mysqld.inc DROP TABLE u1,u2,u3,u6; diff --git a/mysql-test/suite/innodb/t/log_file_overwrite.test b/mysql-test/suite/innodb/t/log_file_overwrite.test index 253cf2d611925..55e81da958e10 100644 --- a/mysql-test/suite/innodb/t/log_file_overwrite.test +++ b/mysql-test/suite/innodb/t/log_file_overwrite.test @@ -1,6 +1,7 @@ --source include/have_innodb.inc --source include/have_sequence.inc --source include/have_debug.inc +--source include/skip_innodb_log_archive.inc call mtr.add_suppression("InnoDB: Plugin initialization aborted"); call mtr.add_suppression("Fewer engines are enabled now than were before the crash"); diff --git a/mysql-test/suite/innodb/t/log_file_size.test b/mysql-test/suite/innodb/t/log_file_size.test index a4f85c2b93868..89813d81c22a5 100644 --- a/mysql-test/suite/innodb/t/log_file_size.test +++ b/mysql-test/suite/innodb/t/log_file_size.test @@ -1,5 +1,6 @@ # Test resizing the InnoDB redo log. --source include/innodb_page_size_small.inc +--source include/skip_innodb_log_archive.inc # Embedded server tests do not support restarting --source include/not_embedded.inc --source include/maybe_debug.inc diff --git a/mysql-test/suite/innodb/t/log_file_size_online.test b/mysql-test/suite/innodb/t/log_file_size_online.test index e97c7afe32556..dc45a1a408268 100644 --- a/mysql-test/suite/innodb/t/log_file_size_online.test +++ b/mysql-test/suite/innodb/t/log_file_size_online.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source ../../suite/encryption/include/skip_innodb_log_archive.inc --source include/have_sequence.inc --source include/no_valgrind_without_big.inc @@ -10,6 +11,17 @@ # files gets full # +call mtr.add_suppression("InnoDB: innodb_log_archive_start=1234567 is after innodb_log_recovery_start=12345"); +call mtr.add_suppression("InnoDB: innodb_log_archive=ON disallows innodb_log_file_size>4G"); +call mtr.add_suppression("InnoDB: innodb_log_archive=ON but .*/ib_logfile0 exists"); +call mtr.add_suppression("InnoDB: No matching file found for innodb_log_recovery_start="); +call mtr.add_suppression("InnoDB: File .*/ib_logfile0 was not found"); +call mtr.add_suppression("InnoDB: innodb_log_archive_start=\\d+ is after innodb_log_recovery_start=\\d+"); +call mtr.add_suppression("InnoDB: Did not find innodb_log_recovery_start=\\d+ "); +let $check_no_innodb=SELECT * FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); + let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err; SET GLOBAL innodb_log_file_size=4194304; @@ -24,7 +36,7 @@ ENGINE=INNODB; INSERT INTO t SELECT NULL, REPEAT('a', 255) FROM seq_1_to_20000; ---let $restart_parameters=--innodb-log-file-size=4194304 +--let $restart_parameters=--innodb-log-file-size=4194304 --skip-innodb-log-archive --source include/restart_mysqld.inc SELECT COUNT(*) FROM t; @@ -49,11 +61,28 @@ let $ID= `SELECT @id := CONNECTION_ID()`; send SET GLOBAL innodb_log_file_size=7340032; --connection default let $ignore= `SELECT @id := $ID`; +--error 0,ER_WRONG_USAGE +SET GLOBAL innodb_log_archive=ON; +--error 0,ER_WRONG_USAGE +SET GLOBAL innodb_log_archive=OFF; + KILL QUERY @id; --connection con1 --error 0,ER_QUERY_INTERRUPTED reap; +# When innodb_log_archive=ON, SET GLOBAL innodb_log_file_size is instantaneous +# but will not reflect the file size. +SET GLOBAL innodb_log_archive=ON, innodb_log_file_size=10485760; +SELECT @@GLOBAL.innodb_log_file_size!=10485760; +--error ER_CANT_CREATE_HANDLER_FILE +SET GLOBAL innodb_log_file_size=4294971392; +SELECT @@GLOBAL.innodb_log_file_size<=10485760; +SET GLOBAL innodb_log_file_size=4294967296; +SELECT @@GLOBAL.innodb_log_file_size<=10485760; + +SET GLOBAL innodb_log_archive=OFF; + --connection default send SET GLOBAL innodb_log_file_size=5242880; @@ -72,12 +101,74 @@ reap; --connection default --let $shutdown_timeout=0 +--let $restart_parameters=--innodb-log-recovery-start=12345 --innodb-log-archive-start=1234567 +--source include/restart_mysqld.inc +--let $shutdown_timeout= +evalp $check_no_innodb; + +let SEARCH_PATTERN = InnoDB: innodb_log_archive_start=1234567 is after innodb_log_recovery_start=12345; +--source include/search_pattern_in_file.inc + +--let $restart_parameters=--innodb-log-archive --innodb-log-file-size=5g +--source include/restart_mysqld.inc +evalp $check_no_innodb; + +let SEARCH_PATTERN = InnoDB: innodb_log_archive=ON disallows innodb_log_file_size>4G; +--source include/search_pattern_in_file.inc + +--let $restart_parameters=--innodb-log-archive --innodb-log-file-size=4g +--source include/restart_mysqld.inc +evalp $check_no_innodb; + +let SEARCH_PATTERN = InnoDB: innodb_log_archive=ON but .*/ib_logfile0 exists; +--source include/search_pattern_in_file.inc + --let $restart_parameters= --source include/restart_mysqld.inc SELECT * FROM t WHERE a<10; SELECT COUNT(*),LENGTH(b) FROM t GROUP BY b; +SET GLOBAL innodb_log_archive=ON; +let $archive_start=`SELECT variable_value FROM information_schema.global_status +WHERE variable_name='innodb_lsn_archived'`; +let $archive_start_1=`SELECT $archive_start-1`; + +--let $restart_parameters= --innodb-log-recovery-start=12290 +--source include/restart_mysqld.inc + +evalp $check_no_innodb; +let SEARCH_PATTERN = InnoDB: No matching file found for innodb_log_recovery_start=12290; +--source include/search_pattern_in_file.inc + +--let $restart_noprint=1 +--let $restart_parameters= --innodb-log-archive-start=$archive_start --innodb-log-recovery-start=$archive_start_1 +--source include/restart_mysqld.inc + +evalp $check_no_innodb; + +let SEARCH_PATTERN = InnoDB: File .*/ib_logfile0 as not found; +--source include/search_pattern_in_file.inc + +let SEARCH_PATTERN = InnoDB: innodb_log_archive_start=\\d+ is after innodb_log_recovery_start=\\d+; +--source include/search_pattern_in_file.inc + +--let $restart_parameters= --innodb-log-recovery-start=$archive_start_1 +--source include/restart_mysqld.inc + +evalp $check_no_innodb; + +--let $restart_parameters= --innodb-log-recovery-start=$archive_start +--source include/restart_mysqld.inc + +--let $restart_noprint= + +SET GLOBAL innodb_log_archive=OFF; +let SEARCH_PATTERN = InnoDB: setting innodb_log_archive=0 at innodb_log_recovery_start=.*; +--source include/search_pattern_in_file.inc +let SEARCH_PATTERN = InnoDB: setting innodb_log_archive=1 at innodb_log_recovery_start=.*; +--source include/search_pattern_in_file.inc + SHOW VARIABLES LIKE 'innodb_log_file_size'; SET GLOBAL innodb_log_file_size=6291456; SHOW VARIABLES LIKE 'innodb_log_file_size'; diff --git a/mysql-test/suite/innodb/t/log_upgrade.test b/mysql-test/suite/innodb/t/log_upgrade.test index a3d237875feac..d5a3fc7f3962b 100644 --- a/mysql-test/suite/innodb/t/log_upgrade.test +++ b/mysql-test/suite/innodb/t/log_upgrade.test @@ -16,6 +16,7 @@ let bugdir= $MYSQLTEST_VARDIR/tmp/log_upgrade; --let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err --let $dirs= --innodb-data-home-dir=$bugdir --innodb-log-group-home-dir=$bugdir +--let $dirs=$dirs --skip-innodb-log-archive --echo # --echo # MDEV-24412 InnoDB: Upgrade after a crash is not supported diff --git a/mysql-test/suite/innodb/t/log_upgrade_101_flags.test b/mysql-test/suite/innodb/t/log_upgrade_101_flags.test index 7b19986f73e8e..f37fe11e0b3cd 100644 --- a/mysql-test/suite/innodb/t/log_upgrade_101_flags.test +++ b/mysql-test/suite/innodb/t/log_upgrade_101_flags.test @@ -8,7 +8,7 @@ call mtr.add_suppression("InnoDB: adjusting FSP_SPACE_FLAGS of file"); let bugdir= $MYSQLTEST_VARDIR/tmp/log_upgrade; --mkdir $bugdir --let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err ---let $dirs= --innodb-data-home-dir=$bugdir --innodb-log-group-home-dir=$bugdir --innodb-undo-directory=$bugdir +--let $dirs= --innodb-data-home-dir=$bugdir --innodb-log-group-home-dir=$bugdir --innodb-undo-directory=$bugdir --skip-innodb-log-archive # Test case similar to log_upgrade.test perl; diff --git a/mysql-test/suite/innodb/t/missing_tablespaces.test b/mysql-test/suite/innodb/t/missing_tablespaces.test index 7aeb0a26f3042..5a5126ba8153e 100644 --- a/mysql-test/suite/innodb/t/missing_tablespaces.test +++ b/mysql-test/suite/innodb/t/missing_tablespaces.test @@ -18,6 +18,7 @@ CREATE TABLE `..................................................` (ID INT) ENGINE=INNODB; let $restart_noprint=2; +let $restart_parameters=--innodb-log-recovery-start=0; --source include/shutdown_mysqld.inc --remove_file $MYSQLTEST_VARDIR/mysqld.1/data/@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e/@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e@002e.ibd diff --git a/mysql-test/suite/innodb/t/read_only_recover_committed.test b/mysql-test/suite/innodb/t/read_only_recover_committed.test index 50863f0cc4a16..b082d2a1df49f 100644 --- a/mysql-test/suite/innodb/t/read_only_recover_committed.test +++ b/mysql-test/suite/innodb/t/read_only_recover_committed.test @@ -62,7 +62,7 @@ SELECT * FROM t; --enable_ps2_protocol # refused on MySQL 5.6, MariaDB 10.0, 10.1, but not MariaDB 10.2+ UPDATE t SET a=3 WHERE a=1; ---let $restart_parameters= --innodb-read-only +--let $restart_parameters= --innodb-read-only --innodb-log-recovery-start=0 --source include/restart_mysqld.inc SET GLOBAL innodb_status_output= @@GLOBAL.innodb_status_output; --echo # Starting with MariaDB 10.2, innodb_read_only implies READ UNCOMMITTED. diff --git a/mysql-test/suite/innodb/t/read_only_recovery.test b/mysql-test/suite/innodb/t/read_only_recovery.test index 75e9c53001759..b405e89aa49a7 100644 --- a/mysql-test/suite/innodb/t/read_only_recovery.test +++ b/mysql-test/suite/innodb/t/read_only_recovery.test @@ -30,7 +30,7 @@ SELECT * FROM t; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; SELECT * FROM t; UPDATE t SET a=3 WHERE a=1; ---let $restart_parameters= --innodb-read-only +--let $restart_parameters= --innodb-read-only --innodb-log-recovery-start=0 --source include/restart_mysqld.inc --echo # Starting with MariaDB 10.2, innodb_read_only implies READ UNCOMMITTED. --echo # In earlier versions, this would return the last committed version diff --git a/mysql-test/suite/innodb/t/recovery_memory.opt b/mysql-test/suite/innodb/t/recovery_memory.opt index 7207fd27a4212..b593fd3d5d9a3 100644 --- a/mysql-test/suite/innodb/t/recovery_memory.opt +++ b/mysql-test/suite/innodb/t/recovery_memory.opt @@ -1 +1,2 @@ --innodb_buffer_pool_size=1073741824 +--innodb-log-archive --skip-innodb-log-file-mmap diff --git a/mysql-test/suite/innodb/t/recovery_memory.test b/mysql-test/suite/innodb/t/recovery_memory.test index a1eeed13b07ac..b7737c5081ae6 100644 --- a/mysql-test/suite/innodb/t/recovery_memory.test +++ b/mysql-test/suite/innodb/t/recovery_memory.test @@ -34,7 +34,7 @@ DROP PROCEDURE dorepeat; --echo # if ($have_debug) { SET DEBUG_DBUG="+d,ib_log_checkpoint_avoid_hard"; -let $restart_parameters=--innodb_buffer_pool_size=6m --debug_dbug=+d,hdr_page_corrupt; +let $restart_parameters=--innodb_buffer_pool_size=6m --debug_dbug=+d,hdr_page_corrupt --innodb-log-recovery-target=18446744073709551615; } if (!$have_debug) { --echo SET DEBUG_DBUG="+d,ib_log_checkpoint_avoid_hard"; diff --git a/mysql-test/suite/innodb/t/rename_table.test b/mysql-test/suite/innodb/t/rename_table.test index a61813429b381..0551457372074 100644 --- a/mysql-test/suite/innodb/t/rename_table.test +++ b/mysql-test/suite/innodb/t/rename_table.test @@ -2,6 +2,9 @@ --source include/not_embedded.inc call mtr.add_suppression("InnoDB: In RENAME TABLE table `test`.`t4` is referenced in foreign key constraints which are not compatible with the new table definition."); +call mtr.add_suppression("InnoDB: Plugin initialization aborted"); +call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); +call mtr.add_suppression("InnoDB: cannot fulfill innodb_log_recovery_target=20480<"); CREATE DATABASE test_jfg; CREATE DATABASE test_jfg2; @@ -13,6 +16,18 @@ FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESPACES WHERE filename LIKE '%test%'; DROP DATABASE test_jfg; +--let $restart_parameters=--innodb-log-recovery-target=20480 +--source include/restart_mysqld.inc + +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); + +--let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err +--let SEARCH_PATTERN= InnoDB: cannot fulfill innodb_log_recovery_target=20480< +--source include/search_pattern_in_file.inc + +--let $restart_parameters= --source include/restart_mysqld.inc DROP DATABASE test_jfg2; diff --git a/mysql-test/suite/innodb/t/row_format_redundant.test b/mysql-test/suite/innodb/t/row_format_redundant.test index 9f85c45455b1a..2e6a1a1f38972 100644 --- a/mysql-test/suite/innodb/t/row_format_redundant.test +++ b/mysql-test/suite/innodb/t/row_format_redundant.test @@ -61,7 +61,7 @@ row_format=redundant; insert into t3 values(444, 'dddd', 'bbbbb', 'aaaaa'); insert into t3 values(555, 'eeee', 'ccccc', 'aaaaa'); ---let $restart_parameters= $d --innodb-read-only +--let $restart_parameters= $d --innodb-read-only --innodb-log-recovery-start=0 --source include/restart_mysqld.inc SELECT COUNT(*) FROM t1; @@ -75,7 +75,7 @@ TRUNCATE TABLE t2; --error ER_OPEN_AS_READONLY TRUNCATE TABLE t3; ---let $restart_parameters= $d --skip-innodb-fast-shutdown +--let $restart_parameters= $d --skip-innodb-fast-shutdown --innodb-log-recovery-start=0 --source include/restart_mysqld.inc TRUNCATE TABLE t1; @@ -153,6 +153,10 @@ DROP TABLE t2,t3; --let $restart_parameters= --source include/restart_mysqld.inc +if (!`select @@innodb_log_archive=0`) +{ +--replace_result ib_0000000000003000.log ib_logfile0 +} --list_files $bugdir --remove_files_wildcard $bugdir --rmdir $bugdir diff --git a/mysql-test/suite/innodb/t/shrink_cached_undo.test b/mysql-test/suite/innodb/t/shrink_cached_undo.test index 8274556d8fd57..83339faa5d63b 100644 --- a/mysql-test/suite/innodb/t/shrink_cached_undo.test +++ b/mysql-test/suite/innodb/t/shrink_cached_undo.test @@ -42,6 +42,6 @@ while ($c) DROP TABLE t2, t1; SET GLOBAL innodb_fast_shutdown=0; SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0; -let $restart_parameters=; +let $restart_parameters=--innodb-log-recovery-start=0; --source include/restart_mysqld.inc SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0; diff --git a/mysql-test/suite/innodb/t/sys_defragment.test b/mysql-test/suite/innodb/t/sys_defragment.test index a4e5a84450d0d..b880a9d09a826 100644 --- a/mysql-test/suite/innodb/t/sys_defragment.test +++ b/mysql-test/suite/innodb/t/sys_defragment.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/skip_innodb_log_archive.inc --source include/have_sequence.inc --source include/have_debug.inc diff --git a/mysql-test/suite/innodb/t/sys_defragment_fail.test b/mysql-test/suite/innodb/t/sys_defragment_fail.test index aca741b902163..a0c309350d58a 100644 --- a/mysql-test/suite/innodb/t/sys_defragment_fail.test +++ b/mysql-test/suite/innodb/t/sys_defragment_fail.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/skip_innodb_log_archive.inc --source include/have_debug.inc --source include/have_sequence.inc diff --git a/mysql-test/suite/innodb/t/sys_truncate_debug.test b/mysql-test/suite/innodb/t/sys_truncate_debug.test index 7dcb5ffde2aa9..c30e5f9340989 100644 --- a/mysql-test/suite/innodb/t/sys_truncate_debug.test +++ b/mysql-test/suite/innodb/t/sys_truncate_debug.test @@ -10,6 +10,7 @@ call mtr.add_suppression("InnoDB: Cannot shrink the system tablespace"); call mtr.add_suppression("InnoDB: Plugin initialization aborted"); call mtr.add_suppression("Plugin 'InnoDB' init function returned error"); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); +call mtr.add_suppression("InnoDB: A copy of page \\[page id: space=0, page number=.*\\] in the doublewrite buffer slot .* is beyond the end"); SET GLOBAL INNODB_LIMIT_OPTIMISTIC_INSERT_DEBUG=2; SET GLOBAL INNODB_FILE_PER_TABLE= 0; diff --git a/mysql-test/suite/innodb/t/sys_truncate_shutdown.test b/mysql-test/suite/innodb/t/sys_truncate_shutdown.test index 0f61049ba5ecd..76d4672001d24 100644 --- a/mysql-test/suite/innodb/t/sys_truncate_shutdown.test +++ b/mysql-test/suite/innodb/t/sys_truncate_shutdown.test @@ -1,6 +1,11 @@ --source include/have_innodb.inc --source include/have_sequence.inc SET GLOBAL innodb_fast_shutdown=0; +--disable_query_log +if (`SELECT @@GLOBAL.innodb_log_recovery_start`) { +call mtr.add_suppression("InnoDB: A copy of page \\[page id: space=0, page number=.*\\] in the doublewrite buffer slot .* is beyond the end"); +} +--enable_query_log --source include/restart_mysqld.inc SET GLOBAL INNODB_FILE_PER_TABLE= 0; SET UNIQUE_CHECKS=0, FOREIGN_KEY_CHECKS=0; diff --git a/mysql-test/suite/innodb/t/sys_truncate_shutdown_debug.test b/mysql-test/suite/innodb/t/sys_truncate_shutdown_debug.test index d819ebe42eef4..ba6efd4a196b9 100644 --- a/mysql-test/suite/innodb/t/sys_truncate_shutdown_debug.test +++ b/mysql-test/suite/innodb/t/sys_truncate_shutdown_debug.test @@ -9,6 +9,7 @@ call mtr.add_suppression("InnoDB: Cannot shrink the system tablespace"); call mtr.add_suppression("InnoDB: Plugin initialization aborted"); call mtr.add_suppression("Plugin 'InnoDB' init function returned error"); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); +call mtr.add_suppression("InnoDB: A copy of page \\[page id: space=0, page number=.*\\] in the doublewrite buffer slot .* is beyond the end"); SET GLOBAL innodb_fast_shutdown=0; --source include/restart_mysqld.inc diff --git a/mysql-test/suite/innodb/t/table_flags.test b/mysql-test/suite/innodb/t/table_flags.test index 40193b99c866d..4d9380506e4ff 100644 --- a/mysql-test/suite/innodb/t/table_flags.test +++ b/mysql-test/suite/innodb/t/table_flags.test @@ -36,7 +36,7 @@ let undodir= $MYSQLTEST_VARDIR/tmp/undo_dir; --let $d=--innodb-data-home-dir=$bugdir --innodb-log-group-home-dir=$bugdir --let $d=$d --innodb-data-file-path=ibdata1:1M:autoextend ---let $d=$d --innodb-undo-tablespaces=0 +--let $d=$d --innodb-undo-tablespaces=0 --innodb-log-recovery-start=0 --let $d=$d --skip-innodb-fast-shutdown --innodb_undo_directory=$undodir --let $restart_noprint=1 --let $restart_parameters=$d --innodb-stats-persistent=0 @@ -232,6 +232,10 @@ DROP TABLE tr,tc,td,tz,tp; --error 0,1 --remove_file $bugdir/ib_buffer_pool +if (!`select @@innodb_log_archive=0`) +{ +--replace_result ib_0000000000003000.log ib_logfile0 +} --list_files $bugdir --remove_files_wildcard $bugdir --rmdir $bugdir diff --git a/mysql-test/suite/innodb/t/temporary_table.test b/mysql-test/suite/innodb/t/temporary_table.test index 0a8e72969d559..2441c928ca125 100644 --- a/mysql-test/suite/innodb/t/temporary_table.test +++ b/mysql-test/suite/innodb/t/temporary_table.test @@ -111,7 +111,7 @@ call populate_t1(); drop procedure populate_t1; --echo # test read-only mode ---let $restart_parameters = --innodb-read-only +--let $restart_parameters = --innodb-read-only --innodb-log-recovery-start=0 --source include/restart_mysqld.inc --echo # files in MYSQL_DATA_DIR diff --git a/mysql-test/suite/innodb/t/truncate_missing.test b/mysql-test/suite/innodb/t/truncate_missing.test index d36a2de5cd954..9f347360766f7 100644 --- a/mysql-test/suite/innodb/t/truncate_missing.test +++ b/mysql-test/suite/innodb/t/truncate_missing.test @@ -13,6 +13,7 @@ let $datadir=`select @@datadir`; --source include/shutdown_mysqld.inc --remove_file $datadir/test/t.ibd +--let $restart_parameters=--innodb-log-recovery-start=0 --source include/start_mysqld.inc --error ER_GET_ERRNO diff --git a/mysql-test/suite/innodb/t/trx_id_future.test b/mysql-test/suite/innodb/t/trx_id_future.test index 049e8f2c6f32d..44f5ae0f5f14c 100644 --- a/mysql-test/suite/innodb/t/trx_id_future.test +++ b/mysql-test/suite/innodb/t/trx_id_future.test @@ -56,6 +56,7 @@ syswrite(FILE, $page, $ps)==$ps || die "Unable to write $file\n"; close(FILE) || die "Unable to close $file"; EOF +--let $restart_parameters=--innodb-log-recovery-start=0 --source include/start_mysqld.inc call mtr.add_suppression("\\[Warning\\] InnoDB: A transaction id in a record of table `test`\\.`t1` is newer than the system-wide maximum"); call mtr.add_suppression("\\[ERROR\\] InnoDB: We detected index corruption"); diff --git a/mysql-test/suite/innodb/t/undo_space_dblwr.test b/mysql-test/suite/innodb/t/undo_space_dblwr.test index 33e8ed9d65153..e0ca45cde7a17 100644 --- a/mysql-test/suite/innodb/t/undo_space_dblwr.test +++ b/mysql-test/suite/innodb/t/undo_space_dblwr.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/no_checkpoint_prepare.inc --source include/have_debug.inc --source include/not_embedded.inc call mtr.add_suppression("Checksum mismatch in the first page of file"); diff --git a/mysql-test/suite/innodb/t/undo_truncate.test b/mysql-test/suite/innodb/t/undo_truncate.test index 9abf08ff43631..dcf4a781d1c3e 100644 --- a/mysql-test/suite/innodb/t/undo_truncate.test +++ b/mysql-test/suite/innodb/t/undo_truncate.test @@ -13,7 +13,7 @@ call mtr.add_suppression("InnoDB: Trying to delete tablespace.*pending operation # Re-create the undo log tablespaces after slow shutdown SET GLOBAL innodb_fast_shutdown=0; -let $restart_parameters=--innodb_undo_tablespaces=2; +let $restart_parameters=--innodb_undo_tablespaces=2 --innodb-log-recovery-start=0; --source include/restart_mysqld.inc SET GLOBAL innodb_undo_log_truncate = 0; @@ -61,7 +61,7 @@ SET GLOBAL innodb_max_purge_lag_wait=0; SET GLOBAL innodb_max_undo_log_size=DEFAULT; SET GLOBAL innodb_max_purge_lag_wait=0; set global innodb_fast_shutdown=0; -let $restart_parameters=; +let $restart_parameters=--innodb-log-recovery-start=0; --source include/restart_mysqld.inc --replace_regex /.*Trx id counter ([0-9]+).*/\1/ let $trx_after= `SHOW ENGINE INNODB STATUS`; diff --git a/mysql-test/suite/innodb/t/undo_truncate_recover.test b/mysql-test/suite/innodb/t/undo_truncate_recover.test index 988b28f75b461..7069be26c6244 100644 --- a/mysql-test/suite/innodb/t/undo_truncate_recover.test +++ b/mysql-test/suite/innodb/t/undo_truncate_recover.test @@ -13,7 +13,7 @@ # Re-create the undo log tablespaces after slow shutdown SET GLOBAL innodb_fast_shutdown=0; -let $restart_parameters=--innodb_undo_tablespaces=2; +let $restart_parameters=--innodb_undo_tablespaces=2 --innodb-log-recovery-start=0; --source include/restart_mysqld.inc SET GLOBAL innodb_undo_log_truncate = 1; @@ -43,7 +43,7 @@ call mtr.add_suppression("InnoDB: innodb_undo_tablespaces=0 disables dedicated u SET GLOBAL innodb_fast_shutdown=0; --source include/shutdown_mysqld.inc --source include/search_pattern_in_file.inc ---let $restart_parameters= --innodb-undo-tablespaces=1 +--let $restart_parameters= --innodb-undo-tablespaces=1 --innodb-log-recovery-start=0 --let $restart_noprint=1 if ($checksum_algorithm == "strict_full_crc32") { diff --git a/mysql-test/suite/innodb/t/undo_upgrade.test b/mysql-test/suite/innodb/t/undo_upgrade.test index f27eae334d877..ee3e1e36a97dc 100644 --- a/mysql-test/suite/innodb/t/undo_upgrade.test +++ b/mysql-test/suite/innodb/t/undo_upgrade.test @@ -35,7 +35,7 @@ XA COMMIT 'zombie'; --echo # case 2: Successful innodb_undo_tablespace upgrade SET GLOBAL innodb_fast_shutdown=0; -let $restart_parameters=--innodb_undo_tablespaces=2; +let $restart_parameters=--innodb_undo_tablespaces=2 --innodb-log-recovery-start=0; --source include/restart_mysqld.inc --echo # Display 2 undo tablespaces @@ -48,7 +48,7 @@ DROP TABLE t1; --source include/wait_all_purged.inc --echo # case 3: Reduce the innodb_undo_tablespace to 0 -let $restart_parameters=--innodb_undo_tablespaces=0; +let $restart_parameters=--innodb_undo_tablespaces=0 --innodb-log-recovery-start=0; --source include/restart_mysqld.inc --echo # Display 0 undo tablespace @@ -58,7 +58,7 @@ SELECT @@global.innodb_undo_tablespaces; list_files $MYSQLD_DATADIR undo*; --echo # case 4: Change undo tablespace when force_recovery < 5 -let $restart_parameters=--innodb_undo_tablespaces=2 --innodb_force_recovery=4; +let $restart_parameters=--innodb_undo_tablespaces=2 --innodb_force_recovery=4 --innodb-log-recovery-start=0; --source include/restart_mysqld.inc --echo # Display 2 undo tablespace @@ -68,7 +68,7 @@ SELECT @@global.innodb_undo_tablespaces; list_files $MYSQLD_DATADIR undo*; --echo # case 5: Fail to change undo tablespace when force_recovery > 4 -let $restart_parameters=--innodb_undo_tablespaces=4 --innodb_force_recovery=5; +let $restart_parameters=--innodb_undo_tablespaces=4 --innodb_force_recovery=5 --innodb-log-recovery-start=0; --source include/restart_mysqld.inc --echo # Display 2 undo tablespace @@ -83,13 +83,13 @@ list_files $MYSQLD_DATADIR undo*; --echo # SET GLOBAL innodb_fast_shutdown=0; -let $restart_parameters=--innodb_undo_tablespaces=4; +let $restart_parameters=--innodb_undo_tablespaces=4 --innodb-log-recovery-start=0; --source include/restart_mysqld.inc --echo # Should list 4 undo log tablespaces list_files $MYSQLD_DATADIR undo*; set global innodb_fast_shutdown=0; -let $restart_parameters=--innodb_read_only=1; +let $restart_parameters=--innodb_read_only=1 --innodb-log-recovery-start=0; --source include/restart_mysqld.inc set global innodb_fast_shutdown=0; @@ -98,7 +98,7 @@ mkdir $bugdir; let undodir= $MYSQLTEST_VARDIR/tmp/undo_dir; mkdir $undodir; let $d= --innodb-data-file-path=ibdata1:1M:autoextend; -let $d=$d --innodb_undo_directory=$undodir; +let $d=$d --innodb_undo_directory=$undodir --innodb-log-recovery-start=0; let $restart_parameters= $d --innodb-data-home-dir=$bugdir --innodb-log-group-home-dir=$bugdir --innodb_undo_tablespaces=3; --source include/restart_mysqld.inc --echo # Should list 3 undo log tablespaces @@ -114,7 +114,7 @@ let $restart_parameters= $d --innodb-data-home-dir=$bugdir_1 --innodb-log-group- list_files $undodir undo*; set global innodb_fast_shutdown=0; -let $restart_parameters=; +let $restart_parameters=--innodb-log-recovery-start=0; --source include/restart_mysqld.inc rmdir $bugdir; rmdir $bugdir_1; diff --git a/mysql-test/suite/innodb/t/undo_upgrade_debug.test b/mysql-test/suite/innodb/t/undo_upgrade_debug.test index f8a40b22d977b..308057a150476 100644 --- a/mysql-test/suite/innodb/t/undo_upgrade_debug.test +++ b/mysql-test/suite/innodb/t/undo_upgrade_debug.test @@ -15,45 +15,45 @@ call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE faile set global innodb_fast_shutdown=0; --echo # case 1: Abort after resetting TRX_SYS page rollback segments -let $restart_parameters=--innodb_undo_tablespaces=4 --debug_dbug="+d,after_rseg_reset_abort"; +let $restart_parameters=--innodb-log-recovery-start=0 --innodb_undo_tablespaces=4 --debug_dbug="+d,after_rseg_reset_abort"; --source include/restart_mysqld.inc -let $restart_parameters=--innodb_undo_tablespaces=4; +let $restart_parameters=--innodb-log-recovery-start=0 --innodb_undo_tablespaces=4; --source include/restart_mysqld.inc --echo # Should list 4 undo log tablespaces list_files $MYSQLD_DATADIR undo*; --echo # case 2: Abort after deleting the old undo tablespaces -let $restart_parameters=--innodb_undo_tablespaces=2 --debug_dbug="+d,after_deleting_old_undo_abort"; +let $restart_parameters=--innodb-log-recovery-start=0 --innodb_undo_tablespaces=2 --debug_dbug="+d,after_deleting_old_undo_abort"; --source include/restart_mysqld.inc -let $restart_parameters=--innodb_undo_tablespaces=2; +let $restart_parameters=--innodb-log-recovery-start=0 --innodb_undo_tablespaces=2; --source include/restart_mysqld.inc --echo # Should list 2 undo log tablespaces list_files $MYSQLD_DATADIR undo*; --echo # case 3: Abort after successfully deleting the old undo tablespace -let $restart_parameters=--innodb_undo_tablespaces=3 --debug_dbug="+d,after_deleting_old_undo_success"; +let $restart_parameters=--innodb-log-recovery-start=0 --innodb_undo_tablespaces=3 --debug_dbug="+d,after_deleting_old_undo_success"; --source include/restart_mysqld.inc -let $restart_parameters=--innodb_undo_tablespaces=3; +let $restart_parameters=--innodb-log-recovery-start=0 --innodb_undo_tablespaces=3; --source include/restart_mysqld.inc --echo # Should list 3 undo log tablespaces list_files $MYSQLD_DATADIR undo*; -let $restart_parameters=--innodb_undo_tablespaces=4; +let $restart_parameters=--innodb-log-recovery-start=0 --innodb_undo_tablespaces=4; --source include/restart_mysqld.inc --echo # Should list 4 undo log tablespaces list_files $MYSQLD_DATADIR undo*; -let $restart_parameters=--innodb_undo_tablespaces=2; +let $restart_parameters=--innodb-log-recovery-start=0 --innodb_undo_tablespaces=2; --source include/restart_mysqld.inc --echo # Should list 2 undo log tablespaces diff --git a/mysql-test/suite/innodb_gis/t/rtree_split.test b/mysql-test/suite/innodb_gis/t/rtree_split.test index 00cbd71e644d6..9b2b0e301ce95 100644 --- a/mysql-test/suite/innodb_gis/t/rtree_split.test +++ b/mysql-test/suite/innodb_gis/t/rtree_split.test @@ -56,7 +56,7 @@ create spatial index idx2 on t1(c2); show create table t1; # test read only case -let $restart_parameters = --innodb-read-only; +let $restart_parameters = --innodb-read-only --innodb-log-recovery-start=0; --source include/restart_mysqld.inc set @g1 = ST_GeomFromText('Polygon((0 0,0 100,100 100,100 0,0 0))'); select count(*) from t1 where MBRWithin(t1.c2, @g1); diff --git a/mysql-test/suite/innodb_zip/r/restart.result b/mysql-test/suite/innodb_zip/r/restart.result index 4ce1c6ecbb91c..9fb40b08498b9 100644 --- a/mysql-test/suite/innodb_zip/r/restart.result +++ b/mysql-test/suite/innodb_zip/r/restart.result @@ -527,7 +527,7 @@ t7_restart#p#p1#sp#s3.ibd # # Start the server and show the tablespaces. # -# restart +# restart: --innodb-log-recovery-start=0 === information_schema.innodb_sys_tablespaces and innodb_sys_datafiles === Space_Name Page_Size Zip_Size Path innodb_undo001 DEFAULT DEFAULT MYSQLD_DATADIR//undo001 @@ -721,7 +721,7 @@ t77_restart#p#p1#sp#s3.ibd # # Restart the server # -# restart +# restart: --innodb-log-recovery-start=0 === information_schema.innodb_sys_tablespaces and innodb_sys_datafiles === Space_Name Page_Size Zip_Size Path innodb_undo001 DEFAULT DEFAULT MYSQLD_DATADIR//undo001 @@ -850,7 +850,7 @@ t77_restart#p#p1#sp#s3.ibd # # Start the server and check tablespaces. # -# restart +# restart: --innodb-log-recovery-start=0 === information_schema.innodb_sys_tablespaces and innodb_sys_datafiles === Space_Name Page_Size Zip_Size Path innodb_undo001 DEFAULT DEFAULT MYSQLD_DATADIR//undo001 @@ -983,7 +983,7 @@ t77_restart.par # # Start the server and check tablespaces. # -# restart +# restart: --innodb-log-recovery-start=0 === information_schema.innodb_sys_tablespaces and innodb_sys_datafiles === Space_Name Page_Size Zip_Size Path innodb_undo001 DEFAULT DEFAULT MYSQLD_DATADIR//undo001 diff --git a/mysql-test/suite/innodb_zip/r/wl6347_comp_indx_stat.result b/mysql-test/suite/innodb_zip/r/wl6347_comp_indx_stat.result index f46678340e2a7..e93e3ed8e2658 100644 --- a/mysql-test/suite/innodb_zip/r/wl6347_comp_indx_stat.result +++ b/mysql-test/suite/innodb_zip/r/wl6347_comp_indx_stat.result @@ -313,7 +313,7 @@ AND table_name='tab5' AND database_name='test' AND index_name like 'idx%' ; compress_stat 1 The size of the tab5.ibd file: 5242880 -# restart +# restart: --innodb-log-recovery-start=0 # set the flag on (default off) SET GLOBAL innodb_cmp_per_index_enabled=ON; # set the flags @@ -649,7 +649,7 @@ AND table_name='tab5' AND database_name='test' AND index_name like 'idx%' ; compress_stat 1 The size of the tab5.ibd file: 2097152 -# restart +# restart: --innodb-log-recovery-start=0 # set the flag on (default off) SET GLOBAL innodb_cmp_per_index_enabled=ON; # set the flags @@ -1918,7 +1918,7 @@ AND table_name='tab5' AND database_name='test' AND index_name like 'idx%' ; compress_stat 1 The size of the tab5.ibd file: 65536 -# restart +# restart: --innodb-log-recovery-start=0 # set the flag on (default off) SET GLOBAL innodb_cmp_per_index_enabled=ON; # set the flags @@ -2256,7 +2256,7 @@ AND table_name='tab5' AND database_name='test' AND index_name like 'idx%' ; compress_stat 1 The size of the tab5.ibd file: 65536 -# restart +# restart: --innodb-log-recovery-start=0 # set the flag on (default off) SET GLOBAL innodb_cmp_per_index_enabled=ON; # set the flags @@ -5009,7 +5009,7 @@ AND table_name='tab5' AND database_name='test' AND index_name like 'idx%' ; compress_stat 1 The size of the tab5.ibd file: 65536 -# restart +# restart: --innodb-log-recovery-start=0 # set the flag on (default off) SET GLOBAL innodb_cmp_per_index_enabled=ON; # set the flags @@ -6593,7 +6593,7 @@ AND table_name='tab5' AND database_name='test' AND index_name like 'idx%' ; compress_stat 1 The size of the tab5.ibd file: 65536 -# restart +# restart: --innodb-log-recovery-start=0 # set the flag on (default off) SET GLOBAL innodb_cmp_per_index_enabled=ON; # set the flags diff --git a/mysql-test/suite/innodb_zip/t/restart.test b/mysql-test/suite/innodb_zip/t/restart.test index d82475f8c1ed8..fbb2a16f5933f 100644 --- a/mysql-test/suite/innodb_zip/t/restart.test +++ b/mysql-test/suite/innodb_zip/t/restart.test @@ -249,6 +249,7 @@ SHOW CREATE TABLE t7_restart; --echo # Shutdown the server and make a backup of a tablespace --echo # --source include/shutdown_mysqld.inc +--let $restart_parameters=--innodb-log-recovery-start=0 --copy_file $MYSQL_TMP_DIR/alt_dir/test/t5_restart.ibd $MYSQL_TMP_DIR/alt_dir/test/t5_restart.ibd.bak --copy_file $MYSQL_DATA_DIR/test/t5_restart.isl $MYSQL_DATA_DIR/test/t5_restart.isl.bak @@ -557,6 +558,7 @@ SHOW CREATE TABLE t77_restart; --echo # --echo # Start the server and check tablespaces. + --echo # -- source include/start_mysqld.inc diff --git a/mysql-test/suite/innodb_zip/t/wl6347_comp_indx_stat.test b/mysql-test/suite/innodb_zip/t/wl6347_comp_indx_stat.test index ae32b03a78670..d52239e6246b8 100644 --- a/mysql-test/suite/innodb_zip/t/wl6347_comp_indx_stat.test +++ b/mysql-test/suite/innodb_zip/t/wl6347_comp_indx_stat.test @@ -63,6 +63,8 @@ SET @inl_val=2; SET @inl_val=2; --source suite/innodb_zip/include/innodb_stats_comp_index.inc +# Avoid crash recovery that would access compressed pages +--let $restart_parameters=--innodb-log-recovery-start=0 -- source include/restart_mysqld.inc --echo # set the flag on (default off) diff --git a/mysql-test/suite/perfschema/r/max_program_zero.result b/mysql-test/suite/perfschema/r/max_program_zero.result index 047643e06988d..a0e486b2af9c0 100644 --- a/mysql-test/suite/perfschema/r/max_program_zero.result +++ b/mysql-test/suite/perfschema/r/max_program_zero.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 1 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/ortho_iter.result b/mysql-test/suite/perfschema/r/ortho_iter.result index 56c22c8453d8e..589704f4056de 100644 --- a/mysql-test/suite/perfschema/r/ortho_iter.result +++ b/mysql-test/suite/perfschema/r/ortho_iter.result @@ -251,7 +251,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/privilege_table_io.result b/mysql-test/suite/perfschema/r/privilege_table_io.result index 0c82be9f05810..b518052613308 100644 --- a/mysql-test/suite/perfschema/r/privilege_table_io.result +++ b/mysql-test/suite/perfschema/r/privilege_table_io.result @@ -57,7 +57,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_idle.result b/mysql-test/suite/perfschema/r/start_server_disable_idle.result index d0665e3bf4c65..12b956d90769c 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_idle.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_idle.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_stages.result b/mysql-test/suite/perfschema/r/start_server_disable_stages.result index 2ef68328144ff..30ce9d12e56a0 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_stages.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_stages.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_statements.result b/mysql-test/suite/perfschema/r/start_server_disable_statements.result index 0ece2a0c52ed1..4bacdbdfedf68 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_statements.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_statements.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_transactions.result b/mysql-test/suite/perfschema/r/start_server_disable_transactions.result index ededc09aac95d..3a6831eb2c7ff 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_transactions.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_transactions.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_waits.result b/mysql-test/suite/perfschema/r/start_server_disable_waits.result index 23db9362161e4..a864576dfa979 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_waits.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_waits.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_innodb.result b/mysql-test/suite/perfschema/r/start_server_innodb.result index cff87457b3bef..09cba65c57ce0 100644 --- a/mysql-test/suite/perfschema/r/start_server_innodb.result +++ b/mysql-test/suite/perfschema/r/start_server_innodb.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_low_index.result b/mysql-test/suite/perfschema/r/start_server_low_index.result index 11cade0a2132f..dbd36d2eaa92d 100644 --- a/mysql-test/suite/perfschema/r/start_server_low_index.result +++ b/mysql-test/suite/perfschema/r/start_server_low_index.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_low_table_lock.result b/mysql-test/suite/perfschema/r/start_server_low_table_lock.result index 484550095202e..fab64f49d45e6 100644 --- a/mysql-test/suite/perfschema/r/start_server_low_table_lock.result +++ b/mysql-test/suite/perfschema/r/start_server_low_table_lock.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_account.result b/mysql-test/suite/perfschema/r/start_server_no_account.result index aab8d3eba9caa..e2cccdba19b43 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_account.result +++ b/mysql-test/suite/perfschema/r/start_server_no_account.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_cond_class.result b/mysql-test/suite/perfschema/r/start_server_no_cond_class.result index 4dfdab9de9f30..44b114013ace2 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_cond_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_cond_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result b/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result index a8d0cbeca3855..27ecb59a40f17 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_file_class.result b/mysql-test/suite/perfschema/r/start_server_no_file_class.result index fcc01880a7107..5189d36618b05 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_file_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_file_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_file_inst.result b/mysql-test/suite/perfschema/r/start_server_no_file_inst.result index c56201e7d0a80..533c02383c7b8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_file_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_file_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_host.result b/mysql-test/suite/perfschema/r/start_server_no_host.result index 662beb3b88a49..e328ea3d3c26a 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_host.result +++ b/mysql-test/suite/perfschema/r/start_server_no_host.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_index.result b/mysql-test/suite/perfschema/r/start_server_no_index.result index ccff0cb113faa..686f430cdc440 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_index.result +++ b/mysql-test/suite/perfschema/r/start_server_no_index.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mdl.result b/mysql-test/suite/perfschema/r/start_server_no_mdl.result index ebe64409deb0c..c2b6dc3d9eb74 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mdl.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mdl.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_memory_class.result b/mysql-test/suite/perfschema/r/start_server_no_memory_class.result index 01a217c534bfd..2a4cdcf8ddd37 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_memory_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_memory_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result b/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result index 1b3efda5210a9..9b77c7b897c47 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result b/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result index 599498915f3f1..3a62653efc18f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result b/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result index 73ac1acb9f145..7dd5842809135 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result +++ b/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result b/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result index 04aa037b960e3..3598c722b9453 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result b/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result index e8156711eb3f2..7a60805d2d03f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result b/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result index 2d17bd6f49203..76da2883ba6b2 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result +++ b/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result b/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result index 16afee28ee70b..58ac763394a96 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result +++ b/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_socket_class.result b/mysql-test/suite/perfschema/r/start_server_no_socket_class.result index 9de0006aa7abc..5a86d51a06231 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_socket_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_socket_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 0 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result b/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result index bcef0e5c01b05..75ab84f4b3cb8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 0 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stage_class.result b/mysql-test/suite/perfschema/r/start_server_no_stage_class.result index 1dda39dc79e92..bb750ebd34a26 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stage_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stage_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 0 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stages_history.result b/mysql-test/suite/perfschema/r/start_server_no_stages_history.result index 95584521eceb2..6cf3f740fb1be 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stages_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stages_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result index da6c54b6bbafa..1f2716410a9bc 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_statements_history.result b/mysql-test/suite/perfschema/r/start_server_no_statements_history.result index 09a9a544f5b13..f1078542f69ef 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_statements_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_statements_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result index 5ce9874799e8b..a6fc523df2c60 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result b/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result index a170fe097fd23..304732cb37927 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 0 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_inst.result b/mysql-test/suite/perfschema/r/start_server_no_table_inst.result index 3f009de021d1a..3ef43495d0f22 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 0 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_lock.result b/mysql-test/suite/perfschema/r/start_server_no_table_lock.result index db4fe3413106b..20cf1a531ca51 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_lock.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_lock.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_thread_class.result b/mysql-test/suite/perfschema/r/start_server_no_thread_class.result index b1b6e614c43ac..e8eb4589fce56 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_thread_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_thread_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result b/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result index e540f3ce78cbd..4dbf292fc3c10 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result b/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result index 80da238ba0ed7..09168c68f9fe8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result index f5d32f0100968..9ec63e991deda 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_user.result b/mysql-test/suite/perfschema/r/start_server_no_user.result index cb249b4e242d3..861b7d897c9c5 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_user.result +++ b/mysql-test/suite/perfschema/r/start_server_no_user.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_waits_history.result b/mysql-test/suite/perfschema/r/start_server_no_waits_history.result index c419aad2995f3..710cef232da58 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_waits_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_waits_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result index 0b2bf39ff874f..df4351a8f358f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_off.result b/mysql-test/suite/perfschema/r/start_server_off.result index 17db0395d894b..f3bf7bf7fb15e 100644 --- a/mysql-test/suite/perfschema/r/start_server_off.result +++ b/mysql-test/suite/perfschema/r/start_server_off.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_on.result b/mysql-test/suite/perfschema/r/start_server_on.result index cff87457b3bef..09cba65c57ce0 100644 --- a/mysql-test/suite/perfschema/r/start_server_on.result +++ b/mysql-test/suite/perfschema/r/start_server_on.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_variables.result b/mysql-test/suite/perfschema/r/start_server_variables.result index f25f9ab69d1c3..b86116e099cdd 100644 --- a/mysql-test/suite/perfschema/r/start_server_variables.result +++ b/mysql-test/suite/perfschema/r/start_server_variables.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/statement_program_lost_inst.result b/mysql-test/suite/perfschema/r/statement_program_lost_inst.result index 442e212557b55..3e8469b881801 100644 --- a/mysql-test/suite/perfschema/r/statement_program_lost_inst.result +++ b/mysql-test/suite/perfschema/r/statement_program_lost_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 2 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/plugins/r/compression,innodb-lz4.rdiff b/mysql-test/suite/plugins/r/compression,innodb-lz4.rdiff index 00abc19647ce7..61069bbdd5914 100644 --- a/mysql-test/suite/plugins/r/compression,innodb-lz4.rdiff +++ b/mysql-test/suite/plugins/r/compression,innodb-lz4.rdiff @@ -15,8 +15,8 @@ 0 abcabcabc 300 1 defdefdef 3000 2 ghighighi 30000 --# restart: --disable-provider-bzip2 -+# restart: --disable-provider-lz4 +-# restart: --disable-provider-bzip2 --innodb-log-recovery-start=0 ++# restart: --disable-provider-lz4 --innodb-log-recovery-start=0 select a, left(b, 9), length(b) from t1; ERROR HY000: Table `test`.`t1` is corrupted. Please drop the table and recreate. show warnings; diff --git a/mysql-test/suite/plugins/r/compression,innodb-lzma.rdiff b/mysql-test/suite/plugins/r/compression,innodb-lzma.rdiff index 0119adc74826b..72d7b4c420747 100644 --- a/mysql-test/suite/plugins/r/compression,innodb-lzma.rdiff +++ b/mysql-test/suite/plugins/r/compression,innodb-lzma.rdiff @@ -15,8 +15,8 @@ 0 abcabcabc 300 1 defdefdef 3000 2 ghighighi 30000 --# restart: --disable-provider-bzip2 -+# restart: --disable-provider-lzma +-# restart: --disable-provider-bzip2 --innodb-log-recovery-start=0 ++# restart: --disable-provider-lzma --innodb-log-recovery-start=0 select a, left(b, 9), length(b) from t1; ERROR HY000: Table `test`.`t1` is corrupted. Please drop the table and recreate. show warnings; diff --git a/mysql-test/suite/plugins/r/compression,innodb-lzo.rdiff b/mysql-test/suite/plugins/r/compression,innodb-lzo.rdiff index 52a794a99d795..098000aa04bbf 100644 --- a/mysql-test/suite/plugins/r/compression,innodb-lzo.rdiff +++ b/mysql-test/suite/plugins/r/compression,innodb-lzo.rdiff @@ -15,8 +15,8 @@ 0 abcabcabc 300 1 defdefdef 3000 2 ghighighi 30000 --# restart: --disable-provider-bzip2 -+# restart: --disable-provider-lzo +-# restart: --disable-provider-bzip2 --innodb-log-recovery-start=0 ++# restart: --disable-provider-lzo --innodb-log-recovery-start=0 select a, left(b, 9), length(b) from t1; ERROR HY000: Table `test`.`t1` is corrupted. Please drop the table and recreate. show warnings; diff --git a/mysql-test/suite/plugins/r/compression,innodb-snappy.rdiff b/mysql-test/suite/plugins/r/compression,innodb-snappy.rdiff index cbbd754aa1760..4fe6615964438 100644 --- a/mysql-test/suite/plugins/r/compression,innodb-snappy.rdiff +++ b/mysql-test/suite/plugins/r/compression,innodb-snappy.rdiff @@ -15,8 +15,8 @@ 0 abcabcabc 300 1 defdefdef 3000 2 ghighighi 30000 --# restart: --disable-provider-bzip2 -+# restart: --disable-provider-snappy +-# restart: --disable-provider-bzip2 --innodb-log-recovery-start=0 ++# restart: --disable-provider-snappy --innodb-log-recovery-start=0 select a, left(b, 9), length(b) from t1; ERROR HY000: Table `test`.`t1` is corrupted. Please drop the table and recreate. show warnings; diff --git a/mysql-test/suite/plugins/r/compression,mroonga-lz4.rdiff b/mysql-test/suite/plugins/r/compression,mroonga-lz4.rdiff index e48fb5eaa8f95..c5708fa6715da 100644 --- a/mysql-test/suite/plugins/r/compression,mroonga-lz4.rdiff +++ b/mysql-test/suite/plugins/r/compression,mroonga-lz4.rdiff @@ -21,8 +21,8 @@ 0 abcabcabc 300 1 defdefdef 3000 2 ghighighi 30000 --# restart: --disable-provider-bzip2 -+# restart: --disable-provider-lz4 +-# restart: --disable-provider-bzip2 --innodb-log-recovery-start=0 ++# restart: --disable-provider-lz4 --innodb-log-recovery-start=0 select a, left(b, 9), length(b) from t1; -ERROR HY000: Table `test`.`t1` is corrupted. Please drop the table and recreate. -show warnings; diff --git a/mysql-test/suite/plugins/r/compression.result b/mysql-test/suite/plugins/r/compression.result index 2319754960c59..eb2076ce8a32f 100644 --- a/mysql-test/suite/plugins/r/compression.result +++ b/mysql-test/suite/plugins/r/compression.result @@ -17,7 +17,7 @@ a left(b, 9) length(b) 0 abcabcabc 300 1 defdefdef 3000 2 ghighighi 30000 -# restart: --disable-provider-bzip2 +# restart: --disable-provider-bzip2 --innodb-log-recovery-start=0 select a, left(b, 9), length(b) from t1; ERROR HY000: Table `test`.`t1` is corrupted. Please drop the table and recreate. show warnings; diff --git a/mysql-test/suite/plugins/t/compression.test b/mysql-test/suite/plugins/t/compression.test index e05750f9e996e..5b46f8ce17595 100644 --- a/mysql-test/suite/plugins/t/compression.test +++ b/mysql-test/suite/plugins/t/compression.test @@ -37,7 +37,7 @@ insert t1 (a, b) values (1, repeat("def", 1000)); insert t1 (a, b) values (2, repeat("ghi", 10000)); select a, left(b, 9), length(b) from t1; -let $restart_parameters = --disable-provider-$alg; +let $restart_parameters = --disable-provider-$alg --innodb-log-recovery-start=0; source include/restart_mysqld.inc; if ($engine == "innodb") { diff --git a/mysql-test/suite/sql_sequence/read_only.result b/mysql-test/suite/sql_sequence/read_only.result index cd8d498bb814d..4abf909472790 100644 --- a/mysql-test/suite/sql_sequence/read_only.result +++ b/mysql-test/suite/sql_sequence/read_only.result @@ -1,5 +1,5 @@ create sequence s1 cache 2 engine=innodb; -# restart: --innodb-read-only +# restart: --innodb-read-only --innodb-log-recovery-start=0 connection default; show global variables like 'innodb_read_only'; Variable_name Value diff --git a/mysql-test/suite/sql_sequence/read_only.test b/mysql-test/suite/sql_sequence/read_only.test index 141a310c0401a..f4a51514149f7 100644 --- a/mysql-test/suite/sql_sequence/read_only.test +++ b/mysql-test/suite/sql_sequence/read_only.test @@ -7,7 +7,8 @@ create sequence s1 cache 2 engine=innodb; ---let $restart_parameters= --innodb-read-only +--let $restart_parameters= --innodb-read-only --innodb-log-recovery-start=0 + --source include/restart_mysqld.inc connection default; diff --git a/mysql-test/suite/sys_vars/r/innodb_buffer_pool_dump_abort_loads.result b/mysql-test/suite/sys_vars/r/innodb_buffer_pool_dump_abort_loads.result index 55dc9a9d57aab..6debc26fa67a8 100644 --- a/mysql-test/suite/sys_vars/r/innodb_buffer_pool_dump_abort_loads.result +++ b/mysql-test/suite/sys_vars/r/innodb_buffer_pool_dump_abort_loads.result @@ -40,7 +40,7 @@ INSERT INTO t1 VALUES (@a,@a,@a,@a,@a, SET GLOBAL innodb_buffer_pool_dump_now=1; # Restart server -# restart +# restart: --innodb-log-recovery-start=0 # Abort after 16 pages SET GLOBAL innodb_buffer_pool_load_pages_abort=16, @@ -55,7 +55,7 @@ INNODB_BUFFER_POOL_LOAD_INCOMPLETE ON INNODB_BUFFER_POOL_LOAD_STATUS Buffer pool(s) load aborted on request # Restart server -# restart +# restart: --innodb-log-recovery-start=0 # Load buffer pool SET GLOBAL innodb_buffer_pool_load_now=1; @@ -84,7 +84,7 @@ INNODB_BUFFER_POOL_DUMP_STATUS Buffer pool(s) dump completed at INNODB_BUFFER_POOL_LOAD_INCOMPLETE OFF # Restart server -# restart +# restart: --innodb-log-recovery-start=0 # Load buffer pool SET GLOBAL innodb_buffer_pool_load_now=1; diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 7ebab3dc8c7e9..1c99d71401803 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -945,6 +945,30 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED +VARIABLE_NAME INNODB_LOG_ARCHIVE +SESSION_VALUE NULL +DEFAULT_VALUE OFF +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BOOLEAN +VARIABLE_COMMENT Whether log archiving is desired +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST OFF,ON +READ_ONLY NO +COMMAND_LINE_ARGUMENT OPTIONAL +VARIABLE_NAME INNODB_LOG_ARCHIVE_START +SESSION_VALUE NULL +DEFAULT_VALUE 0 +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_COMMENT initial value of innodb_lsn_archived; 0=auto-detect +NUMERIC_MIN_VALUE 0 +NUMERIC_MAX_VALUE 18446744073709551615 +NUMERIC_BLOCK_SIZE 0 +ENUM_VALUE_LIST NULL +READ_ONLY YES +COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME INNODB_LOG_BUFFER_SIZE SESSION_VALUE NULL DEFAULT_VALUE 16777216 @@ -974,7 +998,7 @@ SESSION_VALUE NULL DEFAULT_VALUE ON VARIABLE_SCOPE GLOBAL VARIABLE_TYPE BOOLEAN -VARIABLE_COMMENT Whether ib_logfile0 resides in persistent memory (when supported) or should initially be memory-mapped +VARIABLE_COMMENT Whether the log resides in persistent memory (when supported) or should initially be memory-mapped NUMERIC_MIN_VALUE NULL NUMERIC_MAX_VALUE NULL NUMERIC_BLOCK_SIZE NULL @@ -986,7 +1010,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 100663296 VARIABLE_SCOPE GLOBAL VARIABLE_TYPE BIGINT UNSIGNED -VARIABLE_COMMENT Redo log size in bytes +VARIABLE_COMMENT Desired log file size in bytes NUMERIC_MIN_VALUE 4194304 NUMERIC_MAX_VALUE 18446744073709551615 NUMERIC_BLOCK_SIZE 4096 @@ -998,7 +1022,7 @@ SESSION_VALUE NULL DEFAULT_VALUE OFF VARIABLE_SCOPE GLOBAL VARIABLE_TYPE BOOLEAN -VARIABLE_COMMENT Whether each write to ib_logfile0 is write through +VARIABLE_COMMENT Whether each write to the log is write through NUMERIC_MIN_VALUE NULL NUMERIC_MAX_VALUE NULL NUMERIC_BLOCK_SIZE NULL @@ -1010,13 +1034,37 @@ SESSION_VALUE NULL DEFAULT_VALUE VARIABLE_SCOPE GLOBAL VARIABLE_TYPE VARCHAR -VARIABLE_COMMENT Path to ib_logfile0 +VARIABLE_COMMENT Path to ib_logfile0 or ib_*.log NUMERIC_MIN_VALUE NULL NUMERIC_MAX_VALUE NULL NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST NULL READ_ONLY YES COMMAND_LINE_ARGUMENT REQUIRED +VARIABLE_NAME INNODB_LOG_RECOVERY_START +SESSION_VALUE NULL +DEFAULT_VALUE 0 +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_COMMENT LSN to start recovery from (0=automatic) +NUMERIC_MIN_VALUE 0 +NUMERIC_MAX_VALUE 18446744073709551615 +NUMERIC_BLOCK_SIZE 0 +ENUM_VALUE_LIST NULL +READ_ONLY NO +COMMAND_LINE_ARGUMENT REQUIRED +VARIABLE_NAME INNODB_LOG_RECOVERY_TARGET +SESSION_VALUE NULL +DEFAULT_VALUE 0 +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_COMMENT recovery point objective (end LSN; 0=unlimited) +NUMERIC_MIN_VALUE 0 +NUMERIC_MAX_VALUE 18446744073709551615 +NUMERIC_BLOCK_SIZE 0 +ENUM_VALUE_LIST NULL +READ_ONLY YES +COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME INNODB_LOG_SPIN_WAIT_DELAY SESSION_VALUE NULL DEFAULT_VALUE 0 diff --git a/mysql-test/suite/sys_vars/t/innodb_buffer_pool_dump_abort_loads.test b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_dump_abort_loads.test index 089f3072205d0..f94c8b13fc431 100644 --- a/mysql-test/suite/sys_vars/t/innodb_buffer_pool_dump_abort_loads.test +++ b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_dump_abort_loads.test @@ -56,6 +56,8 @@ let $wait_condition = --echo --echo # Restart server +# Avoid any crash recovery that would load pages. +--let $restart_parameters=--innodb-log-recovery-start=0 --source include/restart_mysqld.inc --echo diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 32af809849be1..b77ecff4017d7 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -163,6 +163,7 @@ SET (SQL_SOURCE grant.cc sql_explain.cc sql_analyze_stmt.cc + sql_backup.cc sql_join_cache.cc create_options.cc multi_range_read.cc opt_histogram_json.cc diff --git a/sql/handler.h b/sql/handler.h index 354e5d6192d6a..2c96cdeab90fe 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -1892,9 +1892,41 @@ struct handlerton : public transaction_participant /********************************************************************* backup **********************************************************************/ + + /** BACKUP STAGE START */ void (*prepare_for_backup)(void); + /** BACKUP STAGE END */ void (*end_backup)(void); + /** + Start of BACKUP SERVER: collect all files to be backed up + @param thd current session + @param target target directory + @return error code + @retval 0 on success + */ + int (*backup_start)(THD *thd, IF_WIN(const char*,int) target); + /** + Process a file that was collected in backup_start(). + @param thd current session + @return number of files remaining, or negative on error + @retval 0 on completion + */ + int (*backup_step)(THD *thd); + /** + Finish copying and determine the logical time of the backup snapshot. + @param thd current sesssion + @param abort whether BACKUP SERVER was aborted + @return error code + @retval 0 on success + */ + int (*backup_end)(THD *thd, bool abort); + /** + After a successful backup_end(), finalize the backup. + @param thd current sesssion + */ + void (*backup_finalize)(THD *thd); + /********************************************************************** WSREP specific **********************************************************************/ diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 869adb1fc349b..4cdf3a5e322c0 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -3502,6 +3502,7 @@ SHOW_VAR com_status_vars[]= { {"assign_to_keycache", STMT_STATUS(SQLCOM_ASSIGN_TO_KEYCACHE)}, {"backup", STMT_STATUS(SQLCOM_BACKUP)}, {"backup_lock", STMT_STATUS(SQLCOM_BACKUP_LOCK)}, + {"backup_server", STMT_STATUS(SQLCOM_BACKUP_SERVER)}, {"begin", STMT_STATUS(SQLCOM_BEGIN)}, {"binlog", STMT_STATUS(SQLCOM_BINLOG_BASE64_EVENT)}, {"call_procedure", STMT_STATUS(SQLCOM_CALL)}, diff --git a/sql/sql_backup.cc b/sql/sql_backup.cc new file mode 100644 index 0000000000000..39254b45de1f8 --- /dev/null +++ b/sql/sql_backup.cc @@ -0,0 +1,101 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "my_global.h" +#include "mysys_err.h" +#include "sql_class.h" +#include "sql_backup.h" +#include "sql_parse.h" + +static my_bool backup_start(THD *thd, plugin_ref plugin, void *dst) noexcept +{ + handlerton *hton= plugin_hton(plugin); + if (hton->backup_start) + return hton->backup_start(thd, + IF_WIN(static_cast(dst), + int(reinterpret_cast(dst)))); + return false; +} + +static my_bool backup_end(THD *thd, plugin_ref plugin, void *arg) noexcept +{ + handlerton *hton= plugin_hton(plugin); + if (hton->backup_end) + return hton->backup_end(thd, arg != nullptr); + return false; +} + +static my_bool backup_step(THD *thd, plugin_ref plugin, void *) noexcept +{ + handlerton *hton= plugin_hton(plugin); + int res= 0; + if (hton->backup_step) + while ((res= hton->backup_step(thd))) + if (res < 0) + break; + return res != 0; +} + +static my_bool backup_finalize(THD *thd, plugin_ref plugin, void *) noexcept +{ + handlerton *hton= plugin_hton(plugin); + if (hton->backup_step) + hton->backup_finalize(thd); + return 0; +} + +bool Sql_cmd_backup::execute(THD *thd) +{ + if (check_global_access(thd, RELOAD_ACL) || + check_global_access(thd, SELECT_ACL) || + error_if_data_home_dir(target.str, "BACKUP SERVER TO")) + return true; + + if (my_mkdir(target.str, 0755, MYF(MY_WME))) + return true; + +#ifndef _WIN32 + int dir= open(target.str, O_DIRECTORY); + if (dir < 0) + { + my_error(EE_CANT_MKDIR, MYF(ME_BELL), target.str, errno); + return true; + } +#endif + + bool fail= plugin_foreach_with_mask(thd, backup_start, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + IF_WIN(const_cast(target.str), + reinterpret_cast(dir))); + if (!fail) + fail= plugin_foreach_with_mask(thd, backup_step, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, nullptr); + + plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + reinterpret_cast(fail)); + + plugin_foreach_with_mask(thd, backup_finalize, MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, nullptr); +#ifndef _WIN32 + close(dir); +#endif + + if (!fail) + my_ok(thd); + return fail; +} diff --git a/sql/sql_backup.h b/sql/sql_backup.h new file mode 100644 index 0000000000000..9aba2404dac58 --- /dev/null +++ b/sql/sql_backup.h @@ -0,0 +1,36 @@ +/***************************************************************************** +Copyright (c) 2026 MariaDB plc. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +#pragma once + +/** BACKUP SERVER */ +class Sql_cmd_backup : public Sql_cmd +{ + /** target directory */ + const LEX_CSTRING target; + +public: + explicit Sql_cmd_backup(LEX_CSTRING target) : target(target) {} + ~Sql_cmd_backup() = default; + + bool execute(THD *thd) override; + + enum_sql_command sql_command_code() const override + { + return SQLCOM_BACKUP_SERVER; + } +}; diff --git a/sql/sql_command.h b/sql/sql_command.h index 9c9166706a034..b8903399711f0 100644 --- a/sql/sql_command.h +++ b/sql/sql_command.h @@ -103,6 +103,7 @@ enum enum_sql_command { SQLCOM_SHOW_PACKAGE_BODY_CODE, SQLCOM_BACKUP, SQLCOM_BACKUP_LOCK, SQLCOM_SHOW_CREATE_SERVER, + SQLCOM_BACKUP_SERVER, /* When a command is added here, be sure it's also added in mysqld.cc diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 6250b450cc82e..bc35da5af1ce1 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -776,6 +776,7 @@ void init_update_queries(void) sql_command_flags[SQLCOM_DROP_SERVER]= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_BACKUP]= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_BACKUP_LOCK]= CF_AUTO_COMMIT_TRANS; + sql_command_flags[SQLCOM_BACKUP_SERVER]= CF_AUTO_COMMIT_TRANS; /* The following statements can deal with temporary tables, @@ -5899,6 +5900,7 @@ mysql_execute_command(THD *thd, bool is_called_from_prepared_stmt) case SQLCOM_CALL: case SQLCOM_REVOKE: case SQLCOM_GRANT: + case SQLCOM_BACKUP_SERVER: if (thd->variables.option_bits & OPTION_IF_EXISTS) lex->create_info.set(DDL_options_st::OPT_IF_EXISTS); DBUG_ASSERT(lex->m_sql_cmd != NULL); @@ -10246,7 +10248,7 @@ int test_if_data_home_dir(const char *dir) if (!dir) DBUG_RETURN(0); - (void) fn_format(path, dir, "", "", MY_RETURN_REAL_PATH); + (void) fn_format(path, dir, "", "", MY_RETURN_REAL_PATH|MY_RESOLVE_SYMLINKS); DBUG_RETURN(path_starts_from_data_home_dir(path)); } diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index c9826639eaaa6..ac78a64c5cd75 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -50,6 +50,7 @@ #include "sql_alter.h" // Sql_cmd_alter_table* #include "sql_truncate.h" // Sql_cmd_truncate_table #include "sql_admin.h" // Sql_cmd_analyze/Check..._table +#include "sql_backup.h" #include "sql_partition_admin.h" // Sql_cmd_alter_table_*_part. #include "sql_handler.h" // Sql_cmd_handler_* #include "sql_signal.h" @@ -15513,6 +15514,11 @@ backup_statements: /* Table list is empty for unlock */ Lex->sql_command= SQLCOM_BACKUP_LOCK; } + | SERVER_SYM TO_SYM TEXT_STRING_sys + { + Lex->sql_command= SQLCOM_BACKUP_SERVER; + Lex->m_sql_cmd= new (thd->mem_root) Sql_cmd_backup($3); + } ; opt_delete_gtid_domain: diff --git a/sql/sys_vars.inl b/sql/sys_vars.inl index 1c0984d596611..51c81bfceb397 100644 --- a/sql/sys_vars.inl +++ b/sql/sys_vars.inl @@ -2506,6 +2506,7 @@ public: bool session_update(THD *thd, set_var *var) override; }; +#ifdef HAVE_REPLICATION /* Class for replicate_events_marked_for_skip. We need a custom update function that ensures the slave is stopped when @@ -2639,6 +2640,7 @@ public: } const uchar *global_value_ptr(THD *thd, const LEX_CSTRING *base) const override; }; +#endif /* HAVE_REPLICATION */ /** diff --git a/sql/upgrade_conf_file.cc b/sql/upgrade_conf_file.cc index 0d7bc6034685c..f1fa9aac6ba3d 100644 --- a/sql/upgrade_conf_file.cc +++ b/sql/upgrade_conf_file.cc @@ -97,7 +97,6 @@ static const char *removed_variables[] = "innodb_locks_unsafe_for_binlog", "innodb_log_arch_dir", "innodb_log_arch_expire_sec", -"innodb_log_archive", "innodb_log_block_size", "innodb_log_checksum_algorithm", "innodb_log_checksums", diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 9e3a23b34ab46..d63751a16af08 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -185,6 +185,8 @@ SET(INNOBASE_SOURCES handler/handler0alter.cc handler/innodb_binlog.cc handler/i_s.cc + handler/backup_innodb.h + handler/backup_innodb.cc ibuf/ibuf0ibuf.cc include/btr0btr.h include/btr0btr.inl diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 828512c851831..efe9a923a3079 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -1238,7 +1238,8 @@ uint64_t btr_read_autoinc_with_fallback(const dict_table_t *table, const bool need_adjust{autoinc > max || autoinc < max_autoinc}; ut_ad(max_autoinc <= max); - if (UNIV_UNLIKELY(need_adjust) && !high_level_read_only && !opt_readonly) + if (UNIV_UNLIKELY(need_adjust) && !high_level_read_only && + !recv_sys.rpo && !opt_readonly) { sql_print_information("InnoDB: Resetting PAGE_ROOT_AUTO_INC from " UINT64PF " to " UINT64PF diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index ed078d91bed19..286758db0c989 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -605,7 +605,7 @@ static bool buf_page_check_lsn(bool check_lsn, const byte *read_buf) noexcept sql_print_error("InnoDB: Your database may be corrupt or" " you may have copied the InnoDB" - " tablespace but not the ib_logfile0. %s", + " tablespaces but not the log. %s", FORCE_RECOVERY_MSG); return true; diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index ab9dbe2897eb1..fb4a7bc5d9958 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -258,7 +258,7 @@ dberr_t buf_dblwr_t::init_or_load_pages(pfs_os_file_t file, const char *path) init(TRX_SYS_DOUBLEWRITE + read_buf); const bool upgrade_to_innodb_file_per_table= - !srv_read_only_mode && + !recv_sys.rpo && mach_read_from_4(TRX_SYS_DOUBLEWRITE_SPACE_ID_STORED + TRX_SYS_DOUBLEWRITE + read_buf) != TRX_SYS_DOUBLEWRITE_SPACE_ID_STORED_N; @@ -334,7 +334,8 @@ void buf_dblwr_t::recover() noexcept if (!is_created()) return; const lsn_t max_lsn{log_sys.get_flushed_lsn(std::memory_order_relaxed)}; - ut_ad(recv_sys.scanned_lsn == max_lsn); + ut_ad(recv_sys.scanned_lsn == max_lsn || + (recv_sys.rpo && recv_sys.rpo < max_lsn)); ut_ad(recv_sys.scanned_lsn >= recv_sys.lsn); uint32_t page_no_dblwr= 0; diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 78a0a85d43801..e522c0523d9b0 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -954,6 +954,13 @@ uint32_t fil_space_t::flush_freed(bool writable) noexcept mysql_mutex_assert_not_owner(&buf_pool.flush_list_mutex); mysql_mutex_assert_not_owner(&buf_pool.mutex); + /* Note: There is no need to invoke writing_start() or + writing_stop() here, because we are only overwriting freed (garbage) + pages. If backup reads a torn page, it will also have copied a + corresponding FREE_PAGE record, which would be applied on recovery. + Besides, the freed page should never be reachable from other pages + that are part of the snapshot. */ + const bool punch_hole= chain.start->punch_hole == 1; if (!punch_hole && !srv_immediate_scrub_data_uncompressed) return 0; @@ -1229,6 +1236,16 @@ ATTRIBUTE_COLD static size_t buf_flush_LRU_to_withdraw(size_t to_withdraw, return to_withdraw; } +/** Stop writing to a tablespace. +@param space tablespace +@return nullptr */ +static fil_space_t *writing_stop(fil_space_t *space) noexcept +{ + space->writing_stop(); + space->release(); + return nullptr; +} + /** Flush dirty blocks from the end buf_pool.LRU, and move clean blocks to buf_pool.free. @param max maximum number of blocks to flush @@ -1246,6 +1263,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, ? 0 : buf_pool.flush_neighbors; fil_space_t *space= nullptr; uint32_t last_space_id= FIL_NULL; + uint32_t backup_page_end= 0; static_assert(FIL_NULL > SRV_TMP_SPACE_ID, "consistency"); static_assert(FIL_NULL > SRV_SPACE_ID_UPPER_BOUND, "consistency"); @@ -1325,7 +1343,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, buf_pool.lru_hp.set(bpage); mysql_mutex_unlock(&buf_pool.mutex); if (space) - space->release(); + writing_stop(space); auto p= buf_flush_space(space_id); space= p.first; last_space_id= space_id; @@ -1334,6 +1352,10 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, mysql_mutex_lock(&buf_pool.mutex); goto no_space; } + + backup_page_end= space->writing_start() + ? space->backup_page_end() : 0; + mysql_mutex_lock(&buf_pool.mutex); buf_pool.stat.n_pages_written+= p.second; } @@ -1345,8 +1367,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, } else if (space->is_stopping_writes()) { - space->release(); - space= nullptr; + space= writing_stop(space); no_space: mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_flush_discard_page(bpage); @@ -1363,7 +1384,8 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, break; } - if (neighbors && space->is_rotational() && UNIV_LIKELY(!to_withdraw) && + if (neighbors && UNIV_LIKELY(!to_withdraw) && + UNIV_LIKELY(!backup_page_end) && space->is_rotational() && /* Skip neighbourhood flush from LRU list if we haven't yet reached half of the free page target. */ UT_LIST_GET_LEN(buf_pool.free) * 2 >= free_limit) @@ -1375,7 +1397,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, flush: if (UNIV_UNLIKELY(to_withdraw != 0)) to_withdraw= buf_flush_LRU_to_withdraw(to_withdraw, *bpage); - if (bpage->flush(space)) + if (bpage->id().page_no() >= backup_page_end && bpage->flush(space)) ++n->flushed; else continue; @@ -1391,7 +1413,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, buf_pool.lru_hp.set(nullptr); if (space) - space->release(); + writing_stop(space); if (scanned) { @@ -1438,6 +1460,7 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept ? 0 : buf_pool.flush_neighbors; fil_space_t *space= nullptr; uint32_t last_space_id= FIL_NULL; + uint32_t backup_page_end= 0; static_assert(FIL_NULL > SRV_TMP_SPACE_ID, "consistency"); static_assert(FIL_NULL > SRV_SPACE_ID_UPPER_BOUND, "consistency"); @@ -1509,10 +1532,12 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept mysql_mutex_unlock(&buf_pool.flush_list_mutex); mysql_mutex_unlock(&buf_pool.mutex); if (space) - space->release(); + writing_stop(space); auto p= buf_flush_space(space_id); space= p.first; last_space_id= space_id; + backup_page_end= space && space->writing_start() + ? space->backup_page_end() : 0; mysql_mutex_lock(&buf_pool.mutex); buf_pool.stat.n_pages_written+= p.second; mysql_mutex_lock(&buf_pool.flush_list_mutex); @@ -1521,10 +1546,7 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept ut_ad(!space); } else if (space->is_stopping_writes()) - { - space->release(); - space= nullptr; - } + space= writing_stop(space); if (!space) buf_flush_discard_page(bpage); @@ -1533,10 +1555,12 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept mysql_mutex_unlock(&buf_pool.flush_list_mutex); do { - if (neighbors && space->is_rotational()) + if (neighbors && UNIV_LIKELY(!backup_page_end) && + space->is_rotational()) count+= buf_flush_try_neighbors(space, page_id, bpage, neighbors == 1, count, max_n); - else if (bpage->flush(space)) + else if (bpage->id().page_no() >= backup_page_end && + bpage->flush(space)) ++count; else continue; @@ -1554,7 +1578,7 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept buf_pool.flush_hp.set(nullptr); if (space) - space->release(); + writing_stop(space); if (scanned) { @@ -1645,6 +1669,7 @@ bool buf_flush_list_space(fil_space_t *space, ulint *n_flushed) noexcept if (written) buf_pool.stat.n_pages_written+= written; } + mysql_mutex_lock(&buf_pool.flush_list_mutex); for (buf_page_t *bpage= UT_LIST_GET_LAST(buf_pool.flush_list); bpage; ) @@ -1687,17 +1712,28 @@ bool buf_flush_list_space(fil_space_t *space, ulint *n_flushed) noexcept acquired= false; goto was_freed; } + mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (bpage->flush(space)) + + if (UNIV_UNLIKELY(space->writing_start()) && + space->backup_page_end() < bpage->id().page_no()) + { + space->writing_stop(); + skip: + mysql_mutex_lock(&buf_pool.mutex); + mysql_mutex_lock(&buf_pool.flush_list_mutex); + may_have_skipped= true; + goto done; + } + + const bool written{bpage->flush(space)}; + space->writing_stop(); + + if (written) { ++n_flush; if (!--max_n_flush) - { - mysql_mutex_lock(&buf_pool.mutex); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - may_have_skipped= true; - goto done; - } + goto skip; mysql_mutex_lock(&buf_pool.mutex); } } @@ -1792,51 +1828,192 @@ static ulint buf_flush_LRU(ulint max_n) noexcept # include "cache.h" #endif - inline void log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); + ut_ad(archive + ? file_size <= ARCHIVE_FILE_SIZE_MAX + : checkpoint >= first_lsn); ut_ad(end_lsn >= checkpoint); ut_d(const lsn_t current_lsn{get_lsn()}); ut_ad(end_lsn <= current_lsn); ut_ad(end_lsn + SIZE_OF_FILE_CHECKPOINT + 8 * is_encrypted() <= current_lsn || srv_shutdown_state > SRV_SHUTDOWN_INITIATED); + ut_ad(this->end_lsn <= end_lsn); + ut_ad(checkpoint_buf != buf); + ut_ad(!checkpoint_buf || checkpoint_buf != resize_buf); + ut_ad(buf != resize_buf); + ut_ad(latch_have_wr()); - auto n= next_checkpoint_no; - const size_t offset{(n & 1) ? CHECKPOINT_2 : CHECKPOINT_1}; + size_t offset; static_assert(CPU_LEVEL1_DCACHE_LINESIZE >= 64, "efficiency"); static_assert(CPU_LEVEL1_DCACHE_LINESIZE <= 4096, "compatibility"); - byte* c= my_assume_aligned - (is_mmap() ? buf + offset : checkpoint_buf); - memset_aligned(c, 0, CPU_LEVEL1_DCACHE_LINESIZE); + lsn_t resizing{resize_lsn.load(std::memory_order_relaxed)}; + byte *c= checkpoint_buf; + bool archive_header_was_reset{false}; + + if (archive) + { + ut_ad(!resizing); + ut_ad(is_opened()); +#ifdef HAVE_PMEM + ut_ad(!resize_buf || !checkpoint_buf); + ut_ad(!resize_buf || resize_log.is_opened()); +#else + ut_ad(!resize_buf); +#endif + if (end_lsn >= first_lsn + ( +#ifdef HAVE_PMEM + c && is_mmap() ? 0 : +#endif + capacity())) + { +#ifdef HAVE_PMEM + if (resize_buf) + { + ut_ad(is_mmap()); + /* @see archived_mmap_switch_complete() */ + ut_ad(!c); + const lsn_t lsn{get_lsn()}; + ut_ad(lsn == current_lsn); + ut_ad(buf_size == capacity()); + persist(lsn); + checkpoint_buf= buf; + buf= resize_buf; + resize_buf= nullptr; + first_lsn+= capacity(); + file_size= resize_target; + goto unmap_old_checkpoint; + } + else if (c && is_mmap()) + { + unmap_old_checkpoint: + checkpoint_buf= nullptr; + my_munmap(c, lseek(resize_log.m_file, 0, SEEK_END)); + goto first_checkpoint_in_new_archive; + } + else +#endif + if (resize_log.is_opened()) + { +#ifdef HAVE_PMEM + ut_ad(!c == is_mmap()); + ut_ad(c || buf_size == capacity()); +#endif + first_lsn+= capacity(); + + file_size= resize_target; + +#ifdef HAVE_PMEM + if (!c) + { + first_checkpoint_in_new_archive: + c= buf; + ut_ad(!memcmp_aligned<512>(c, field_ref_zero, START_OFFSET)); + } + else +#endif + memset_aligned<512>(c, 0, write_size); + + ut_ad(current_lsn >= first_lsn); + ut_ad(current_lsn < first_lsn + capacity()); + circular_recovery_from_sequence_bit_0= false; + next_checkpoint_no= uint16_t(8 * is_encrypted()); + archive_header_was_reset= true; + + if (is_encrypted()) + log_crypt_write_header(c); + +#ifdef HAVE_PMEM + c= checkpoint_buf; +#endif + } + } + + ut_ad(end_lsn >= first_lsn); + offset= next_checkpoint_no * 4; + ut_ad(offset >= 8 * is_encrypted()); + if (UNIV_UNLIKELY(offset >= START_OFFSET)) + /* In case all slots are filled up, overwrite the last slot. */ + offset= START_OFFSET - 4, next_checkpoint_no= START_OFFSET / 4 - 1; + const lsn_t d{end_lsn - first_lsn + START_OFFSET}; + ut_ad(d <= lsn_t{~uint32_t{0}}); + ut_ad(c == checkpoint_buf); + +#ifdef HAVE_PMEM + if (!c) + { + ut_ad(is_mmap()); + c= buf; + goto archived_mmap; + } + else if (is_mmap()) + { + archived_mmap: + c+= offset; + ut_ad(next_checkpoint_no == uint16_t(8 * is_encrypted()) || + (mach_read_from_4(c - 4) && mach_read_from_4(c - 4) < d)); + ut_ad(!memcmp(c, field_ref_zero, 4) || offset == START_OFFSET - 4); + mach_write_to_4(my_assume_aligned<4>(c), uint32_t(d)); + c= reinterpret_cast(uintptr_t(c) & ~63); + goto persist_checkpoint; + } + else +#endif + { + const size_t o{offset & (write_size - 1)}; + offset&= ~size_t(write_size - 1); + if (!o) + memset_aligned<512>(c, 0, write_size); + else + ut_ad(next_checkpoint_no == uint16_t(8 * is_encrypted()) || + (mach_read_from_4(c + o - 4) && + mach_read_from_4(c + o - 4) < d)); + ut_ad(!memcmp(c + o, field_ref_zero, 4) || + offset + o == START_OFFSET - 4); + mach_write_to_4(my_assume_aligned<4>(c + o), uint32_t(d)); + if (resize_log.m_file == OS_FILE_CLOSED) + resize_log= log; /* Block concurrent set_archive() */ + goto write_checkpoint; + } + + goto wrote_checkpoint; + } + + static_assert(CHECKPOINT_1 << 1 == CHECKPOINT_2, ""); + offset= CHECKPOINT_1 << (next_checkpoint_no & 1); + c= is_mmap() ? buf + offset : checkpoint_buf; + circular_recovery_from_sequence_bit_0= !get_sequence_bit(checkpoint); + memset_aligned(c, 0, + CPU_LEVEL1_DCACHE_LINESIZE); mach_write_to_8(my_assume_aligned<8>(c), checkpoint); mach_write_to_8(my_assume_aligned<8>(c + 8), end_lsn); mach_write_to_4(my_assume_aligned<4>(c + 60), my_crc32c(0, c, 60)); - lsn_t resizing; - #ifdef HAVE_PMEM if (is_mmap()) { ut_ad(!is_opened()); - resizing= resize_lsn.load(std::memory_order_relaxed); - if (resizing > 1 && resizing <= checkpoint) { memcpy_aligned<64>(resize_buf + CHECKPOINT_1, c, 64); header_write(resize_buf, resizing, is_encrypted()); pmem_persist(resize_buf, resize_target); } + persist_checkpoint: pmem_persist(c, 64); } else #endif { + write_checkpoint: ut_ad(!is_mmap()); latch.wr_unlock(); log_write_and_flush_prepare(); resizing= resize_lsn.load(std::memory_order_relaxed); + ut_ad(!resizing || !archive); ut_ad(ut_is_2pow(write_size)); ut_ad(write_size >= 512); ut_ad(write_size <= 4096); @@ -1857,8 +2034,44 @@ inline void log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept resizing= resize_lsn.load(std::memory_order_relaxed); } + wrote_checkpoint: + ut_ad(!resizing || !archive); next_checkpoint_no++; last_checkpoint_lsn= checkpoint; + this->end_lsn= end_lsn; + if (!archive) + { + archived_checkpoint= checkpoint; + archived_lsn= end_lsn; + } + else if (archive_header_was_reset) + { + ut_ad(resize_log.m_file != log.m_file); + /* Make the previous archived log file read-only */ +#ifdef _WIN32 + resize_log.close(); + SetFileAttributesA(get_archive_path().c_str(), + FILE_ATTRIBUTE_READONLY | FILE_ATTRIBUTE_ARCHIVE); +#else + struct stat st; + if (!fstat(resize_log.m_file, &st)) + st.st_mode&= 0444; + else + st.st_mode= 0444; + if (fchmod(resize_log.m_file, st.st_mode)) + my_error(ER_ERROR_ON_CLOSE, MYF(ME_ERROR_LOG), + get_archive_path().c_str(), errno); + resize_log.close(); +#endif + } + else if (resize_log.m_file == log.m_file) + { + /* We may have assigned resize_log= log to keep set_archived() out. */ +#ifdef HAVE_PMEM + ut_ad(!is_mmap()); +#endif + resize_log.m_file= OS_FILE_CLOSED; + } DBUG_PRINT("ib_log", ("checkpoint ended at " LSN_PF ", flushed to " LSN_PF, checkpoint, get_flushed_lsn())); @@ -1875,6 +2088,7 @@ inline void log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept if (resizing > 1 && resizing <= checkpoint) { + ut_ad(!archive); ut_ad(is_mmap() == !resize_flush_buf); ut_ad(is_mmap() == !resize_log.is_opened()); @@ -1904,7 +2118,7 @@ inline void log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept ut_ad(!log.is_opened()); bool success; log.m_file= - os_file_create_func(get_log_file_path().c_str(), OS_FILE_OPEN, + os_file_create_func(get_circular_path().c_str(), OS_FILE_OPEN, OS_LOG_FILE, false, &success); ut_a(success); ut_a(log.is_opened()); @@ -1920,8 +2134,7 @@ inline void log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept ut_ad(!is_opened()); my_munmap(buf, file_size); buf= resize_buf; - buf_size= unsigned(std::min(resize_target - START_OFFSET, - buf_size_max)); + buf_size= unsigned(resize_target - START_OFFSET); } else #endif @@ -1965,6 +2178,7 @@ inline void log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept static void log_checkpoint_low(lsn_t oldest_lsn, lsn_t end_lsn) noexcept { ut_ad(!srv_read_only_mode); + ut_ad(!recv_no_log_write); ut_ad(log_sys.latch_have_wr()); ut_ad(oldest_lsn <= end_lsn); ut_ad(end_lsn == log_sys.get_lsn()); @@ -1974,15 +2188,16 @@ static void log_checkpoint_low(lsn_t oldest_lsn, lsn_t end_lsn) noexcept !log_sys.resize_in_progress() && oldest_lsn == log_sys.last_checkpoint_lsn + log_sys.is_encrypted() * 8 + SIZE_OF_FILE_CHECKPOINT)) - { - /* Do nothing, because nothing was logged (other than a - FILE_CHECKPOINT record) since the previous checkpoint. */ - log_sys.latch.wr_unlock(); - return; - } + if (oldest_lsn != log_sys.get_first_lsn()) + { + /* Do nothing, because nothing was logged (other than a + FILE_CHECKPOINT record) since the previous checkpoint. */ + log_sys.latch.wr_unlock(); + return; + } - ut_ad(!recv_no_log_write); - ut_ad(oldest_lsn > log_sys.last_checkpoint_lsn); + ut_ad(oldest_lsn > log_sys.last_checkpoint_lsn || + oldest_lsn == log_sys.get_first_lsn()); /* Repeat the FILE_MODIFY records after the checkpoint, in case some log records between the checkpoint and log_sys.lsn need them. Finally, write a FILE_CHECKPOINT record. Redo log apply expects to @@ -2011,7 +2226,8 @@ static void log_checkpoint_low(lsn_t oldest_lsn, lsn_t end_lsn) noexcept log_sys.latch.wr_lock(); } - ut_ad(oldest_lsn > log_sys.last_checkpoint_lsn); + ut_ad(oldest_lsn > log_sys.last_checkpoint_lsn || + oldest_lsn == log_sys.get_first_lsn()); ut_ad(log_sys.get_flushed_lsn() >= flush_lsn); log_sys.write_checkpoint(oldest_lsn, end_lsn); @@ -2057,6 +2273,9 @@ static void log_checkpoint() noexcept fil_flush_file_spaces(); binlog_write_up_to_now(); + if (UNIV_UNLIKELY(recv_sys.rpo != 0)) + return; + log_sys.latch.wr_lock(); const lsn_t end_lsn= log_sys.get_lsn(); mysql_mutex_lock(&buf_pool.flush_list_mutex); @@ -2074,6 +2293,7 @@ ATTRIBUTE_COLD void buf_flush_wait(lsn_t lsn, bool checkpoint) noexcept { mysql_mutex_assert_owner(&buf_pool.flush_list_mutex); ut_ad(log_sys.latch_have_wr()); + ut_ad(!checkpoint || !recv_sys.rpo); lsn_t oldest_lsn; if (!checkpoint); @@ -2134,8 +2354,10 @@ ATTRIBUTE_COLD void buf_flush_wait(lsn_t lsn, bool checkpoint) noexcept ATTRIBUTE_COLD void buf_flush_ahead(lsn_t lsn, bool furious) noexcept { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); - DBUG_EXECUTE_IF("ib_log_checkpoint_avoid_hard", return;); + DBUG_EXECUTE_IF("ib_log_checkpoint_avoid_hard", + if (!log_sys.archive) return;); Atomic_relaxed &limit= furious ? buf_flush_sync_lsn : buf_flush_async_lsn; @@ -2233,7 +2455,7 @@ static void buf_flush_sync_for_checkpoint(lsn_t lsn) noexcept const lsn_t checkpoint_lsn= measure ? measure : newest_lsn; // FIXME: limit checkpoint_lsn below binlog split write LSN - if (!recv_recovery_is_on() && + if (!recv_recovery_is_on() && !recv_sys.rpo && checkpoint_lsn > log_sys.last_checkpoint_lsn + SIZE_OF_FILE_CHECKPOINT + 8 * log_sys.is_encrypted()) { @@ -2246,6 +2468,7 @@ static void buf_flush_sync_for_checkpoint(lsn_t lsn) noexcept if (measure); else if (srv_shutdown_state < SRV_SHUTDOWN_CLEANUP || + recv_sys.rpo || log_sys.get_lsn() == log_sys.last_checkpoint_lsn + SIZE_OF_FILE_CHECKPOINT + 8 * log_sys.is_encrypted()) measure= LSN_MAX; @@ -2562,7 +2785,7 @@ static void buf_flush_page_cleaner() noexcept { if (recv_recovery_is_on()) continue; - IF_DBUG(if (log_sys.last_checkpoint_lsn && + IF_DBUG(if (log_sys.last_checkpoint_lsn > log_sys.get_first_lsn() && srv_shutdown_state < SRV_SHUTDOWN_CLEANUP && (_db_keyword_(nullptr, "ib_log_checkpoint_avoid", 1) || _db_keyword_(nullptr, "ib_log_checkpoint_avoid_hard", 1))) @@ -2751,7 +2974,7 @@ static void buf_flush_page_cleaner() noexcept } ut_ad(log_sys.get_lsn_approx() == log_sys.last_checkpoint_lsn + SIZE_OF_FILE_CHECKPOINT + 8 * log_sys.is_encrypted() || - srv_fast_shutdown == 2 || srv_read_only_mode || !srv_was_started); + srv_fast_shutdown == 2 || recv_sys.rpo || !srv_was_started); buf_page_cleaner_is_active= false; pthread_cond_broadcast(&buf_pool.done_flush_list); mysql_mutex_unlock(&buf_pool.flush_list_mutex); diff --git a/storage/innobase/dict/dict0crea.cc b/storage/innobase/dict/dict0crea.cc index c9651f423bfad..350d02863f475 100644 --- a/storage/innobase/dict/dict0crea.cc +++ b/storage/innobase/dict/dict0crea.cc @@ -1353,10 +1353,12 @@ bool dict_sys_t::load_sys_tables() noexcept dberr_t dict_sys_t::create_or_check_sys_tables() noexcept { + ut_ad(!srv_read_only_mode || recv_sys.rpo); + if (sys_tables_exist()) return DB_SUCCESS; - if (srv_read_only_mode || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO) + if (recv_sys.rpo || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO) return DB_READ_ONLY; if (load_sys_tables()) diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 2edc7068f17e3..1d3e5c0e0543a 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -917,6 +917,8 @@ void dict_sys_t::create() noexcept { ut_ad(this == &dict_sys); ut_ad(!is_initialised()); + ut_ad(!srv_read_only_mode || recv_sys.rpo); + m_initialised= true; UT_LIST_INIT(table_LRU, &dict_table_t::table_LRU); UT_LIST_INIT(table_non_LRU, &dict_table_t::table_LRU); @@ -930,7 +932,7 @@ void dict_sys_t::create() noexcept latch.SRW_LOCK_INIT(dict_operation_lock_key); - if (!srv_read_only_mode) + if (!recv_sys.rpo) { dict_foreign_err_file= os_file_create_tmpfile(); ut_a(dict_foreign_err_file); @@ -3165,6 +3167,8 @@ dict_foreign_parse_drop_constraints( bool if_exists = false; ut_a(trx->mysql_thd); + ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); cs = trx->mysql_thd->charset(); @@ -3207,7 +3211,7 @@ dict_foreign_parse_drop_constraints( if (!success) { syntax_error: - if (!srv_read_only_mode) { + { FILE* ef = dict_foreign_err_file; mysql_mutex_lock(&dict_foreign_err_mutex); @@ -3251,9 +3255,7 @@ dict_foreign_parse_drop_constraints( == table->foreign_set.end()) { if (if_exists) { goto loop; - } - - if (!srv_read_only_mode) { + } else { FILE* ef = dict_foreign_err_file; mysql_mutex_lock(&dict_foreign_err_mutex); @@ -3685,7 +3687,7 @@ void dict_set_corrupted(trx_t *trx, dict_index_t *index, const char *ctx) /* If this is read only mode, do not update SYS_INDEXES, just mark it as corrupted in memory */ - if (high_level_read_only) { + if (high_level_read_only || recv_sys.rpo) { index->type |= DICT_CORRUPT; goto func_exit; } diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index bfe02bf8cb435..08e0237756025 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -2866,7 +2866,7 @@ dberr_t dict_stats_save(dict_table_t* table, index_id_t index_id) ); #endif /* ENABLED_DEBUG_SYNC */ - if (high_level_read_only) { + if (high_level_read_only || recv_sys.rpo) { return DB_READ_ONLY; } diff --git a/storage/innobase/dict/dict0stats_bg.cc b/storage/innobase/dict/dict0stats_bg.cc index 135ffb6000580..40e70c185c5a5 100644 --- a/storage/innobase/dict/dict0stats_bg.cc +++ b/storage/innobase/dict/dict0stats_bg.cc @@ -76,6 +76,7 @@ thread de-initialization. */ static void dict_stats_recalc_pool_deinit() { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); recalc_pool.clear(); /* @@ -102,6 +103,7 @@ then it will be removed from the pool and skipped. */ static void dict_stats_recalc_pool_add(table_id_t id) { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); ut_ad(id); bool schedule = false; mysql_mutex_lock(&recalc_pool_mutex); @@ -199,6 +201,7 @@ no statistics are being updated on it. */ void dict_stats_recalc_pool_del(table_id_t id, bool have_mdl_exclusive) { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); ut_ad(id); mysql_mutex_lock(&recalc_pool_mutex); @@ -245,6 +248,7 @@ Must be called before dict_stats_thread() is started. */ void dict_stats_init() { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); mysql_mutex_init(recalc_pool_mutex_key, &recalc_pool_mutex, nullptr); pthread_cond_init(&recalc_pool_cond, nullptr); stats_initialised= true; @@ -260,6 +264,8 @@ void dict_stats_deinit() } ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); + stats_initialised = false; dict_stats_recalc_pool_deinit(); @@ -275,6 +281,7 @@ update its stats. static bool dict_stats_process_entry_from_recalc_pool(THD *thd) { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); table_id_t table_id; mysql_mutex_lock(&recalc_pool_mutex); next_table_id_with_mutex: diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 64bc6baaccaf4..bae59cb85a78c 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -334,7 +334,7 @@ void fil_space_destroy_crypt_data(fil_space_crypt_t **crypt_data) *crypt_data = NULL; mysql_mutex_unlock(&fil_crypt_threads_mutex); } else { - ut_ad(srv_read_only_mode || !srv_was_started); + ut_ad(recv_sys.rpo || !srv_was_started); c = *crypt_data; *crypt_data = NULL; } @@ -2113,7 +2113,9 @@ Adjust thread count for key rotation @param[in] enw_cnt Number of threads to be used */ void fil_crypt_set_thread_cnt(const uint new_cnt) { - if (srv_read_only_mode) + ut_ad(!srv_read_only_mode || recv_sys.rpo); + + if (recv_sys.rpo) return; if (!fil_crypt_threads_inited) { @@ -2270,6 +2272,7 @@ Init threads for key rotation */ void fil_crypt_threads_init() { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); if (!fil_crypt_threads_inited) { pthread_cond_init(&fil_crypt_cond, nullptr); diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 424d2ef488c93..bece5758743a6 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -3113,10 +3113,16 @@ void fil_names_dirty(fil_space_t *space) noexcept { ut_ad(log_sys.latch_have_wr()); ut_ad(recv_recovery_is_on()); + ut_ad(!srv_read_only_mode); ut_ad(log_sys.get_lsn() != 0); ut_ad(space->max_lsn == 0); ut_d(fil_space_validate_for_mtr_commit(space)); + if (UNIV_UNLIKELY(recv_sys.rpo != 0)) { + /* The log is read-only; do not write to it */ + return; + } + fil_system.named_spaces.push_back(*space); space->max_lsn = log_sys.get_lsn(); } @@ -3175,7 +3181,7 @@ ATTRIBUTE_COLD lsn_t fil_names_clear(lsn_t lsn) noexcept checkpoint LSN. Remove it from the list, so that if the tablespace is not going to be modified any more, subsequent checkpoints will - avoid calling fil_names_write() on it. */ + avoid calling fil_names_clear() on it. */ it->max_lsn = 0; fil_system.named_spaces.erase(it); } @@ -3183,7 +3189,7 @@ ATTRIBUTE_COLD lsn_t fil_names_clear(lsn_t lsn) noexcept /* max_lsn is the last LSN where fil_names_dirty_and_write() was called. If we kept track of "min_lsn" (the first LSN where max_lsn turned nonzero), we could avoid the - fil_names_write() call if min_lsn > lsn. */ + fil_names_clear() call if min_lsn > lsn. */ ut_ad(UT_LIST_GET_LEN((*it).chain) == 1); size_t s = mtr.log_file_op(FILE_MODIFY, (*it).id, name); ut_ad(s <= budget_left); diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 1e4a1040f6ee4..afc380e367cf9 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -2491,7 +2491,9 @@ fts_cmp_set_sync_doc_id( doc_id_t *doc_id, trx_t *trx=nullptr) { - if (srv_read_only_mode) { + ut_ad(!srv_read_only_mode || recv_sys.rpo); + + if (recv_sys.rpo) { return DB_READ_ONLY; } @@ -2579,7 +2581,9 @@ fts_update_sync_doc_id( fts_cache_t* cache = table->fts->cache; char fts_name[MAX_FULL_NAME_LEN]; - if (srv_read_only_mode) { + ut_ad(!srv_read_only_mode || recv_sys.rpo); + + if (recv_sys.rpo) { return DB_READ_ONLY; } @@ -2798,7 +2802,9 @@ fts_commit_table( /*=============*/ fts_trx_table_t* ftt) /*!< in: FTS table to commit*/ { - if (srv_read_only_mode) { + ut_ad(!srv_read_only_mode || recv_sys.rpo); + + if (recv_sys.rpo) { return DB_READ_ONLY; } @@ -4086,7 +4092,9 @@ fts_sync( bool unlock_cache, bool wait) { - if (srv_read_only_mode) { + ut_ad(!srv_read_only_mode || recv_sys.rpo); + + if (recv_sys.rpo) { return DB_READ_ONLY; } @@ -5609,7 +5617,8 @@ fts_load_stopword( trx->start_line = __LINE__; trx->start_file = __FILE__; #endif - trx_start_internal_low(trx, !high_level_read_only); + trx_start_internal_low(trx, !high_level_read_only + && !recv_sys.rpo); trx->op_info = "upload FTS stopword"; new_trx = TRUE; } diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index 76380b58d3b39..ea6ddab1b2458 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -2283,7 +2283,9 @@ fts_optimize_table( /*===============*/ dict_table_t* table) /*!< in: table to optimiza */ { - if (srv_read_only_mode) { + ut_ad(!srv_read_only_mode || recv_sys.rpo); + + if (recv_sys.rpo) { return DB_READ_ONLY; } @@ -2643,6 +2645,7 @@ Optimize all FTS tables. static void fts_optimize_callback(void *) { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); static ulint current; static bool done; @@ -2780,6 +2783,7 @@ fts_optimize_init(void) ib_alloc_t* heap_alloc; ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); /* For now we only support one optimize thread. */ ut_a(!fts_optimize_wq); @@ -2823,6 +2827,7 @@ void fts_optimize_shutdown() { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); /* If there is an ongoing activity on dictionary, such as srv_master_evict_from_table_cache(), wait for it */ diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc new file mode 100644 index 0000000000000..c5d46ae4861ed --- /dev/null +++ b/storage/innobase/handler/backup_innodb.cc @@ -0,0 +1,451 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "my_global.h" +#include "sql_class.h" +#include "backup_innodb.h" +#include "log0log.h" +#include "srw_lock.h" +#include "fil0fil.h" +#include + +#ifdef _WIN32 +#elif defined __APPLE__ +# include +# include +# include +#else +# ifdef __linux__ +# include +/* Copy a file to a stream or to a regular file. */ +static inline ssize_t +send_step(int in_fd, int out_fd, size_t count, off_t *offset) noexcept +{ + return sendfile(out_fd, in_fd, offset, count); +} +# endif +# if defined __linux__ || defined __FreeBSD__ +/* Copy between files in a single (type of) file system */ +static inline ssize_t +copy_step(int in_fd, int out_fd, size_t count, off_t *offset) noexcept +{ + return copy_file_range(in_fd, offset, out_fd, nullptr, count, 0); +} +# endif +# ifndef __linux__ +# include +/** Copy a file using a memory mapping. +@param in_fd source file +@param out_fd destination +@param count number of bytes to copy +@return error code +@retval 0 on success +@retval 1 if a memory mapping failed */ +static ssize_t mmap_copy(int in_fd, int out_fd, off_t count) +{ +#if SIZEOF_SIZE_T < 8 + if (count != ssize_t(count)) + return 1; +#endif +#if 1 + return 1; +#endif + void *p= mmap(nullptr, count, PROT_READ, MAP_SHARED, in_fd, 0); + if (p == MAP_FAILED) + return 1; + ssize_t ret; + size_t c= size_t(count); + for (const char *b= static_cast(p);; b+= ret) + { + ret= write(out_fd, b, std::min(c, size_t(INT_MAX >> 20 << 20))); + if (ret < 0) + break; + c-= ret; + if (!c) + { + ret= 0; + break; + } + if (!ret) + { + ret= -1; + break; + } + } + munmap(p, count); + return ret; +} + +static ssize_t pread_write(int in_fd, int out_fd, off_t count) noexcept +{ + constexpr size_t READ_WRITE_SIZE= 65536; + char *b= static_cast(aligned_malloc(READ_WRITE_SIZE, 4096)); + if (!b) + return -1; + ssize_t ret; + for (off_t o= 0;; o+= ret) + { + ret= pread(in_fd, b, ssize_t(std::min(count, off_t{READ_WRITE_SIZE})), o); + if (ret > 0) + ret= write(out_fd, b, ret); + if (ret < 0) + break; + count-= ret; + if (!count) + { + ret= 0; + break; + } + if (!ret) + { + ret= -1; + break; + } + } + aligned_free(b); + return ret; +} +# endif + +using copying_step= ssize_t(int,int,size_t,off_t*); +template +static ssize_t copy(int in_fd, int out_fd, off_t c) noexcept +{ + ssize_t ret; + for (off_t offset{0};;) + { + off_t count= c; + if (count > INT_MAX >> 20 << 20) + count = INT_MAX >> 20 << 20; + ret= step(in_fd, out_fd, size_t(count), &offset); + if (ret < 0) + break; + c-= ret; + if (!c) + return 0; + if (!ret) + return -1; + } + return ret; +} +#endif + +namespace +{ +class InnoDB_backup +{ + /** mutex protecting the queue */ + srw_mutex_impl mutex; + + /** the original innodb_log_file_size, or 0 if innodb_log_archive=ON */ + uint64_t old_size; + + /** collection of files to be copied */ + std::vector queue; + + /** target directory name or handle */ + IF_WIN(const char*,int) target; + + /** the checkpoint from which the backup starts */ + lsn_t checkpoint; + /** end_lsn of the checkpoint at the backup start */ + lsn_t checkpoint_end_lsn; +public: + /** + Start of BACKUP SERVER: collect all files to be backed up + @param thd current session + @param target target directory + @return error code + @retval 0 on success + */ + int init(THD *thd, IF_WIN(const char*,int) target) noexcept + { + mutex.init(); + mutex.wr_lock(); + const bool fail{log_sys.backup_start(&old_size, thd)}; + + if (!fail) + { + log_sys.latch.wr_lock(); + checkpoint= log_sys.archived_checkpoint; + checkpoint_end_lsn= log_sys.archived_lsn; + log_sys.latch.wr_unlock(); + + this->target= target; + /* Collect all tablespaces that have been created before our + start checkpoint. Newer tablespaces will be recovered by the + innodb_log_archive=ON recovery. + + If a tablespace is deleted before step() is invoked, the file + will not be copied, and a FILE_DELETE record in the log will + ensure correct recovery. + + If a tablespace is renamed between this and end(), the recovery + of a FILE_RENAME record will ensure the correct file name, + no matter which name was used by step(). */ + mysql_mutex_lock(&fil_system.mutex); + for (fil_space_t &space : fil_system.space_list) + if (space.id < SRV_SPACE_ID_UPPER_BOUND && + !space.is_being_imported() && + /* FIXME: how to initialize create_lsn for old files, to + have efficient incremental backup? + fil_node_t::read_page0() cannot assign it from + FIL_PAGE_LSN because that would not reflect the file + creation but for example allocating or freeing a page. + + The easy parts of initializing space->create_lsn are + as follows: + (1) In log_parse_file() when processing FILE_CREATE + (2) In deferred_spaces.create() */ + space.get_create_lsn() < checkpoint) + queue.emplace_back(space.id); + mysql_mutex_unlock(&fil_system.mutex); + } + mutex.wr_unlock(); + return fail; + } + + /** + Process a file that was collected at init(). + This may be invoked from multiple concurrent threads. + @param thd current session + @return number of files remaining, or negative on error + @retval 0 on completion + */ + int step(THD *thd) noexcept + { + uint32_t id= FIL_NULL; + mutex.wr_lock(); + size_t size{queue.size()}; + if (size) + { + size--; + id= queue.back(); + queue.pop_back(); + } + mutex.wr_unlock(); + + if (fil_space_t *space= fil_space_t::get(id)) + { + for (fil_node_t *node= UT_LIST_GET_FIRST(space->chain); node; + node= UT_LIST_GET_NEXT(chain, node)) + if (int res= backup(node)) + { + space->release(); + return res; + } + space->release(); + } + + size= std::min(size_t{std::numeric_limits::max()}, size); + + return int(size); + } + + /** + Finish copying and determine the logical time of the backup snapshot. + @param thd current session + @param abort whether BACKUP SERVER was aborted + @return error code + @retval 0 on success + */ + int end(THD *thd, bool abort) noexcept + { + mutex.wr_lock(); + if (abort) + queue.clear(); + ut_ad(queue.empty()); + mutex.wr_unlock(); + return 0; + } + + /** + After a successful end(), finalize the backup. + @param thd current session + */ + void fini(THD *thd) noexcept + { + ut_d(mutex.wr_lock()); + ut_ad(queue.empty()); + ut_d(mutex.wr_unlock()); + mutex.destroy(); + log_sys.backup_stop(old_size, thd); + } + +private: + /** Safely start backing up a tablespace file */ + static void backup_start(fil_space_t *space) noexcept + { + if (space->backup_start(space->size)) + os_aio_wait_until_no_pending_writes(false); + } + /* Stop backing up a tablespace */ + static void backup_stop(fil_space_t *space) noexcept + { space->backup_stop(); } + + /** + Back up a persistent InnoDB data file. + @param node InnoDB data file + */ + int backup(fil_node_t *node) noexcept + { + for (bool tried_mkdir{false};;) + { +#ifdef _WIN32 + backup_start(node->space); + std::string path{target}; + path.push_back('/'); + path.append(node->name); + bool ok= CopyFileExA(node->name, path.c_str(), nullptr, nullptr, nullptr, + COPY_FILE_NO_BUFFERING); + backup_stop(node->space); + if (!ok) + { + unsigned long err= GetLastError(); + if (err == ERROR_PATH_NOT_FOUND && !tried_mkdir && + node->space->id && !srv_is_undo_tablespace(node->space->id)) + { + tried_mkdir= true; + path= target; + const char *sep= strchr(node->name, '/'); + ut_ad(sep); + sep= strchr(sep + 1, '/'); + ut_ad(sep); + path.append(node->name, size_t(sep - node->name)); + if (CreateDirectory(path.c_str(), + my_dir_security_attributes.lpSecurityDescriptor + ? &my_dir_security_attributes : nullptr) || + (err= GetLastError()) == ERROR_ALREADY_EXISTS) + continue; + } + + my_osmaperr(err); + goto fail; + } + break; +#else + int f; +# ifdef __APPLE__ + backup_start(node->space); + f= fclonefileat(node->handle, target, node->name, 0); + backup_stop(node->space); + if (!f) + break; + switch (errno) { + case ENOENT: + goto try_mkdir; + case ENOTSUP: + break; + default: + goto fail; + } +# endif + f= openat(target, node->name, + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (f < 0) + { + if (errno == ENOENT) + { +# ifdef __APPLE__ + try_mkdir: +# endif + if (!tried_mkdir && node->space->id && + !srv_is_undo_tablespace(node->space->id)) + { + tried_mkdir= true; + const char *sep= strchr(node->name, '/'); + ut_ad(sep); + sep= strchr(sep + 1, '/'); + ut_ad(sep); + std::string dir{node->name, size_t(sep - node->name)}; + if (!mkdirat(target, dir.c_str(), 0777) || errno == EEXIST) + continue; + } + } + goto fail; + } +# ifdef __APPLE__ + backup_start(node->space); + int err= + fcopyfile(node->handle, f, nullptr, COPYFILE_ALL | COPYFILE_CLONE); + f= close(f) || err; + backup_stop(node->space); + if (f) + goto fail; +# else + do + { + /* FIXME: push down page-granularity locking */ + backup_start(node->space); + const off_t size= off_t{node->size} * node->space->physical_size(); +# if defined __linux__ || defined __FreeBSD__ + if (!copy(node->handle, f, size)) + continue; +# ifdef __linux__ + if (errno == EOPNOTSUPP && !copy(node->handle, f, size)) + continue; +# endif +# endif +# ifndef __linux__ // starting with Linux 2.6.33, we can rely on sendfile(2) + ssize_t err= mmap_copy(node->handle, f, size); + if (err == 1) + err= pread_write(node->handle, f, size); + if (!err) + continue; +# endif + backup_stop(node->space); + std::ignore= close(f); + goto fail; + } + while (false); + + backup_stop(node->space); + if (close(f)) + goto fail; +# endif + break; +#endif + } + sql_print_information("BACKUP SERVER: copy %s", node->name); + return 0; + fail: + my_error(ER_CANT_CREATE_FILE, MYF(0), node->name, errno); + return -1; + } +}; + +/** The backup context */ +static InnoDB_backup backup; +} + +int innodb_backup_start(THD *thd, IF_WIN(const char*,int) target) noexcept +{ + return backup.init(thd, target); +} + +int innodb_backup_step(THD *thd) noexcept +{ + return backup.step(thd); +} + +int innodb_backup_end(THD *thd, bool abort) noexcept +{ + return backup.end(thd, abort); +} + +void innodb_backup_finalize(THD *thd) noexcept +{ + backup.fini(thd); +} diff --git a/storage/innobase/handler/backup_innodb.h b/storage/innobase/handler/backup_innodb.h new file mode 100644 index 0000000000000..372167ba90439 --- /dev/null +++ b/storage/innobase/handler/backup_innodb.h @@ -0,0 +1,46 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/** + Start of BACKUP SERVER: collect all files to be backed up + @param thd current session + @param target target directory + @return error code + @retval 0 on success +*/ +int innodb_backup_start(THD *thd, IF_WIN(const char*,int) target) noexcept; + +/** + Process a file that was collected in backup_start(). + @param thd current session + @return number of files remaining, or negative on error + @retval 0 on completion +*/ +int innodb_backup_step(THD *thd) noexcept; + +/** + Finish copying and determine the logical time of the backup snapshot. + @param thd current session + @param abort whether BACKUP SERVER was aborted + @return error code + @retval 0 on success +*/ +int innodb_backup_end(THD *thd, bool abort) noexcept; + +/** + After a successful backup_end(), finalize the backup. + @param thd current session +*/ +void innodb_backup_finalize(THD *thd) noexcept; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 2e7d9b0b83cc4..7f2d808797dac 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -151,6 +151,7 @@ void close_thread_tables(THD* thd); #include "ha_innodb.h" #include "i_s.h" +#include "backup_innodb.h" #include #include @@ -234,7 +235,7 @@ static uint innodb_max_purge_lag_wait; static void innodb_max_purge_lag_wait_update(THD *thd, st_mysql_sys_var *, void *, const void *limit) { - if (high_level_read_only) + if (high_level_read_only || recv_sys.rpo) return; const uint l= *static_cast(limit); if (!trx_sys.history_exceeds(l)) @@ -994,6 +995,7 @@ static SHOW_VAR innodb_status_variables[]= { {"lsn_flushed", &export_vars.innodb_lsn_flushed, SHOW_ULONGLONG}, {"lsn_last_checkpoint", &export_vars.innodb_lsn_last_checkpoint, SHOW_ULONGLONG}, + {"lsn_archived", &export_vars.innodb_lsn_archived, SHOW_ULONGLONG}, {"master_thread_active_loops", &srv_main_active_loops, SHOW_SIZE_T}, {"master_thread_idle_loops", &srv_main_idle_loops, SHOW_SIZE_T}, {"max_trx_id", &export_vars.innodb_max_trx_id, SHOW_ULONGLONG}, @@ -1282,7 +1284,7 @@ tables that it was aware of, drop any leftover tables inside InnoDB. @param path database path */ static void innodb_drop_database(handlerton*, char *path) { - if (high_level_read_only) + if (high_level_read_only || recv_sys.rpo) return; ulint len= 0; @@ -1566,7 +1568,9 @@ innobase_start_trx_and_assign_read_view( @return false */ static bool innobase_flush_logs(handlerton*) { - if (!srv_read_only_mode) + ut_ad(!srv_read_only_mode || recv_sys.rpo); + + if (!recv_sys.rpo) /* Write and flush any outstanding redo log. */ log_buffer_flush_to_disk(true); return false; @@ -1997,12 +2001,14 @@ static void drop_garbage_tables_after_restore() static int innodb_ddl_recovery_done(handlerton*) { + ut_ad(!srv_read_only_mode || recv_sys.rpo); ut_ad(!ddl_recovery_done); ut_d(ddl_recovery_done= true); - if (!srv_read_only_mode && srv_operation <= SRV_OPERATION_EXPORT_RESTORED && + + if (!recv_sys.rpo && srv_operation <= SRV_OPERATION_EXPORT_RESTORED && srv_force_recovery < SRV_FORCE_NO_BACKGROUND) { - if (srv_start_after_restore && !high_level_read_only) + if (srv_start_after_restore) drop_garbage_tables_after_restore(); srv_init_purge_tasks(); } @@ -2805,7 +2811,7 @@ static int innobase_close_connection(THD *thd) noexcept static int innobase_rollback_by_xid(XID *xid) noexcept { DBUG_EXECUTE_IF("innobase_xa_fail", return XAER_RMFAIL;); - if (high_level_read_only) + if (high_level_read_only || recv_sys.rpo) return XAER_RMFAIL; if (trx_t *trx= trx_get_trx_by_xid(xid)) { @@ -3704,6 +3710,9 @@ compression_algorithm_is_not_loaded(ulong compression_algorithm, myf flags) return 1; } +/** Initial value of innodb_lsn_archived */ +static uint64_t innodb_log_archive_start; + /** Initialize, validate and normalize the InnoDB startup parameters. @return failure code @retval 0 on success @@ -4014,6 +4023,32 @@ static int innodb_init_params() skip_buffering_tweak: #endif + log_sys.archived_lsn= innodb_log_archive_start; + + if (recv_sys.recovery_start && + log_sys.archived_lsn > recv_sys.recovery_start) + { + sql_print_error("InnoDB: innodb_log_archive_start=" LSN_PF + " is after innodb_log_recovery_start=" LSN_PF, + log_sys.archived_lsn, recv_sys.recovery_start); + DBUG_RETURN(HA_ERR_INITIALIZATION); + } + + if (recv_sys.rpo && recv_sys.recovery_start > recv_sys.rpo) + { + sql_print_error("InnoDB: innodb_log_recovery_start=" LSN_PF + " is after innodb_log_recovery_target=" LSN_PF, + recv_sys.recovery_start, recv_sys.rpo); + DBUG_RETURN(HA_ERR_INITIALIZATION); + } + + if (log_sys.archive && srv_log_file_size > log_sys.ARCHIVE_FILE_SIZE_MAX) + { + sql_print_error("InnoDB: innodb_log_archive=ON" + " disallows innodb_log_file_size>4G"); + DBUG_RETURN(HA_ERR_INITIALIZATION); + } + if (!tpool::supports_native_aio()) srv_use_native_aio= FALSE; @@ -4148,6 +4183,10 @@ static int innodb_init(void* p) = innodb_prepare_commit_versioned; innobase_hton->update_optimizer_costs= innobase_update_optimizer_costs; + innobase_hton->backup_start = innodb_backup_start; + innobase_hton->backup_step = innodb_backup_step; + innobase_hton->backup_end = innodb_backup_end; + innobase_hton->backup_finalize = innodb_backup_finalize; innobase_hton->binlog_init= innodb_binlog_init; innobase_hton->set_binlog_max_size= ibb_set_max_size; innobase_hton->binlog_write_direct_ordered= @@ -4237,14 +4276,25 @@ static int innodb_init(void* p) dberr_t err = srv_sys_space.check_file_spec(&create_new_db, 5U << 20); if (err != DB_SUCCESS) { + init_abort: DBUG_RETURN(innodb_init_abort()); } + if (create_new_db && recv_sys.rpo) { + sql_print_error("InnoDB: Cannot fulfill" + " innodb_log_recovery_target" + " because no database exists!"); + goto init_abort; + } + err = srv_start(create_new_db); if (err != DB_SUCCESS) { + if (!recv_sys.rpo) { + recv_sys.rpo = srv_read_only_mode; + } innodb_shutdown(); - DBUG_RETURN(innodb_init_abort()); + goto init_abort; } srv_was_started = true; @@ -5750,6 +5800,7 @@ dberr_t ha_innobase::statistics_init(dict_table_t *table, bool recalc) { ut_ad(table->is_readable()); ut_ad(!table->stats_mutex_is_owner()); + ut_ad(!srv_read_only_mode || recv_sys.rpo); uint32_t stat= table->stat; dberr_t err= DB_SUCCESS; @@ -5759,7 +5810,7 @@ dberr_t ha_innobase::statistics_init(dict_table_t *table, bool recalc) dict_stats_empty_table(table); else { - if (dict_table_t::stats_is_persistent(stat) && !srv_read_only_mode + if (dict_table_t::stats_is_persistent(stat) && !recv_sys.rpo #ifdef WITH_WSREP && !wsrep_thd_skip_locking(m_user_thd) #endif @@ -7662,7 +7713,7 @@ int ha_innobase::is_valid_trx(bool altering_to_supported) const noexcept { ut_ad(m_prebuilt->trx == thd_to_trx(m_user_thd)); - if (high_level_read_only) + if (high_level_read_only || recv_sys.rpo) { ib_senderrf(m_user_thd, IB_LOG_LEVEL_WARN, ER_READ_ONLY_MODE); return HA_ERR_TABLE_READONLY; @@ -8023,6 +8074,7 @@ calc_row_difference( const bool skip_virtual = ha_innobase::omits_virtual_cols(*table->s); ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); clust_index = dict_table_get_first_index(prebuilt->table); auto_inc = 0; @@ -12063,7 +12115,7 @@ int create_table_info_t::prepare_create_table(const char* name, bool strict) DBUG_RETURN(HA_WRONG_CREATE_OPTION); } - if (high_level_read_only) { + if (high_level_read_only || recv_sys.rpo) { DBUG_RETURN(HA_ERR_TABLE_READONLY); } @@ -12089,6 +12141,17 @@ int create_table_info_t::prepare_create_table(const char* name, bool strict) DBUG_RETURN(parse_table_name(name)); } +/********************************************************************//** +Helper function to push warnings from InnoDB internals to SQL-layer. */ +static +void +ib_foreign_warn( + trx_t* trx, /*!< in: trx */ + dberr_t error, /*!< in: error code to push as warning */ + const char *table_name, + const char *format,/*!< in: warning message */ + ...); + /** Push warning message to SQL-layer based on foreign key constraint index match error. @param[in] trx Current transaction @@ -13488,7 +13551,7 @@ static bool delete_table_check_foreigns(const dict_table_t &table, int ha_innobase::delete_table(const char *name) { DBUG_ENTER("ha_innobase::delete_table"); - if (high_level_read_only) + if (high_level_read_only || recv_sys.rpo) DBUG_RETURN(HA_ERR_TABLE_READONLY); THD *thd= ha_thd(); @@ -13771,6 +13834,7 @@ static dberr_t innobase_rename_table(trx_t *trx, const char *from, DBUG_ASSERT(trx->dict_operation); ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); normalize_table_name(norm_to, sizeof(norm_to), to); normalize_table_name(norm_from, sizeof(norm_from), from); @@ -14108,7 +14172,7 @@ ha_innobase::rename_table( DBUG_ENTER("ha_innobase::rename_table"); - if (high_level_read_only) { + if (high_level_read_only || recv_sys.rpo) { ib_senderrf(thd, IB_LOG_LEVEL_WARN, ER_READ_ONLY_MODE); DBUG_RETURN(HA_ERR_TABLE_READONLY); } @@ -14764,6 +14828,8 @@ ha_innobase::info_low( DBUG_ENTER("info"); + ut_ad(!srv_read_only_mode || recv_sys.rpo); + DEBUG_SYNC_C("ha_innobase_info_low"); /* If we are forcing recovery at a high level, we will suppress @@ -14794,7 +14860,7 @@ ha_innobase::info_low( m_prebuilt->trx->op_info = "updating table statistics"; if (ib_table->stats_is_persistent() - && !srv_read_only_mode + && !recv_sys.rpo && dict_stats_persistent_storage_check(false) == SCHEMA_OK) { if (is_analyze) { @@ -15346,6 +15412,7 @@ ha_innobase::check( print_check_msg(thd, table->s->db.str, table->s->table_name.str, "check", "note", (opt_readonly || high_level_read_only + || recv_sys.rpo || !(check_opt->sql_flags & TT_FOR_UPGRADE)) ? "Auto_increment will be" " checked on each open until" @@ -15594,7 +15661,7 @@ int ha_innobase::check_for_upgrade(HA_CHECK_OPT *check_opt) if (m_prebuilt->table->get_index(*autoinc_col)) { check_opt->handler_flags= 1; - return (high_level_read_only && !opt_readonly) + return ((high_level_read_only || recv_sys.rpo) && !opt_readonly) ? HA_ADMIN_FAILED : HA_ADMIN_NEEDS_CHECK; } } @@ -16033,7 +16100,7 @@ ha_innobase::extra( trx->end_bulk_insert(*m_prebuilt->table); trx->clear_ddl_bulk(); if (!m_prebuilt->table->is_temporary() - && !high_level_read_only) { + && !high_level_read_only && !recv_sys.rpo) { /* During copy_data_between_tables(), InnoDB only updates transient statistics. */ if (!m_prebuilt->table->stats_is_persistent()) { @@ -16244,6 +16311,8 @@ ha_innobase::external_lock( DBUG_ENTER("ha_innobase::external_lock"); DBUG_PRINT("enter",("lock_type: %d", lock_type)); + ut_ad(!srv_read_only_mode || recv_sys.rpo); + update_thd(thd); trx_t* trx = m_prebuilt->trx; ut_ad(m_prebuilt->table); @@ -16305,7 +16374,7 @@ ha_innobase::external_lock( const auto sql_command = thd_sql_command(thd); /* Check for UPDATEs in read-only mode. */ - if (srv_read_only_mode) { + if (recv_sys.rpo) { switch (sql_command) { case SQLCOM_CREATE_TABLE: if (lock_type != F_WRLCK) { @@ -16358,7 +16427,7 @@ ha_innobase::external_lock( switch (m_prebuilt->table->quiesce) { case QUIESCE_START: /* Check for FLUSH TABLE t WITH READ LOCK; */ - if (!srv_read_only_mode + if (!recv_sys.rpo && sql_command == SQLCOM_FLUSH && lock_type == F_RDLCK) { @@ -16534,7 +16603,9 @@ innodb_show_status( /* We don't create the temp files or associated mutexes in read-only-mode */ - if (srv_read_only_mode) { + ut_ad(!srv_read_only_mode || recv_sys.rpo); + + if (recv_sys.rpo) { DBUG_RETURN(0); } @@ -16686,6 +16757,8 @@ ha_innobase::store_lock( 'lock'; this may also be TL_IGNORE */ { + ut_ad(!srv_read_only_mode || recv_sys.rpo); + /* Note that trx in this function is NOT necessarily m_prebuilt->trx because we call update_thd() later, in ::external_lock()! Failure to understand this caused a serious memory corruption bug in 5.1.11. */ @@ -16727,7 +16800,7 @@ ha_innobase::store_lock( const bool in_lock_tables = thd_in_lock_tables(thd); const enum enum_sql_command sql_command = thd_sql_command(thd); - if (srv_read_only_mode + if (recv_sys.rpo && (sql_command == SQLCOM_UPDATE || sql_command == SQLCOM_INSERT || sql_command == SQLCOM_REPLACE @@ -17383,7 +17456,7 @@ innobase_commit_by_xid( DBUG_EXECUTE_IF("innobase_xa_fail", return XAER_RMFAIL;); - if (high_level_read_only) { + if (high_level_read_only || recv_sys.rpo) { return(XAER_RMFAIL); } @@ -17460,7 +17533,7 @@ static int innobase_recover_rollback_by_xid(const XID *xid) static void innobase_tc_log_recovery_done() { - if (high_level_read_only) + if (high_level_read_only || recv_sys.rpo) return; /* Make durable any innobase_recover_rollback_by_xid(). */ @@ -17682,6 +17755,8 @@ fast_shutdown_validate( for update function */ struct st_mysql_value* value) /*!< in: incoming string */ { + ut_ad(!srv_read_only_mode || recv_sys.rpo); + if (check_sysvar_int(thd, var, save, value)) { return(1); } @@ -17689,7 +17764,7 @@ fast_shutdown_validate( uint new_val = *reinterpret_cast(save); if (srv_fast_shutdown && !new_val - && !srv_read_only_mode && abort_loop) { + && !recv_sys.rpo && abort_loop) { return(1); } @@ -18543,10 +18618,12 @@ static void checkpoint_now_set(THD* thd, st_mysql_sys_var*, void*, const void *save) { + ut_ad(!srv_read_only_mode || recv_sys.rpo); + if (!*static_cast(save)) return; - if (srv_read_only_mode) + if (recv_sys.rpo) { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, HA_ERR_UNSUPPORTED, @@ -18673,7 +18750,9 @@ buffer_pool_dump_now( const void* save) /*!< in: immediate result from check function */ { - if (*(my_bool*) save && !srv_read_only_mode) { + ut_ad(!srv_read_only_mode || recv_sys.rpo); + + if (*(my_bool*) save && !recv_sys.rpo) { mysql_mutex_unlock(&LOCK_global_system_variables); buf_dump_start(); mysql_mutex_lock(&LOCK_global_system_variables); @@ -18698,7 +18777,9 @@ buffer_pool_load_now( const void* save) /*!< in: immediate result from check function */ { - if (*(my_bool*) save && !srv_read_only_mode) { + ut_ad(!srv_read_only_mode || recv_sys.rpo); + + if (*(my_bool*) save && !recv_sys.rpo) { mysql_mutex_unlock(&LOCK_global_system_variables); buf_load_start(); mysql_mutex_lock(&LOCK_global_system_variables); @@ -18723,7 +18804,9 @@ buffer_pool_load_abort( const void* save) /*!< in: immediate result from check function */ { - if (*(my_bool*) save && !srv_read_only_mode) { + ut_ad(!srv_read_only_mode || recv_sys.rpo); + + if (*(my_bool*) save && !recv_sys.rpo) { mysql_mutex_unlock(&LOCK_global_system_variables); buf_load_abort(); mysql_mutex_lock(&LOCK_global_system_variables); @@ -18767,7 +18850,8 @@ static void innodb_data_file_write_through_update(THD *, st_mysql_sys_var*, static void innodb_doublewrite_update(THD *, st_mysql_sys_var*, void *, const void *save) { - if (!srv_read_only_mode) + ut_ad(!srv_read_only_mode || recv_sys.rpo); + if (!recv_sys.rpo) fil_system.set_use_doublewrite(*static_cast(save)); } @@ -18777,13 +18861,14 @@ static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, ut_ad(var == &srv_log_file_size); mysql_mutex_unlock(&LOCK_global_system_variables); - if (high_level_read_only) + if (high_level_read_only || recv_sys.rpo) ib_senderrf(thd, IB_LOG_LEVEL_ERROR, ER_READ_ONLY_MODE); else if ( #ifdef HAVE_PMEM - !log_sys.is_mmap() && + !log_sys.is_mmap() && #endif - *static_cast(save) < log_sys.buf_size) + *static_cast(save) < log_sys.buf_size + + log_t::START_OFFSET) my_printf_error(ER_WRONG_ARGUMENTS, "innodb_log_file_size must be at least" " innodb_log_buffer_size=%u", MYF(0), log_sys.buf_size); @@ -18801,39 +18886,7 @@ static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, ib_senderrf(thd, IB_LOG_LEVEL_ERROR, ER_CANT_CREATE_HANDLER_FILE); break; case log_t::RESIZE_STARTED: - for (timespec abstime;;) - { - if (thd_kill_level(thd)) - { - log_sys.resize_abort(thd); - break; - } - - set_timespec(abstime, 5); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - lsn_t resizing= log_sys.resize_in_progress(); - if (resizing > buf_pool.get_oldest_modification(0)) - { - buf_pool.page_cleaner_wakeup(true); - my_cond_timedwait(&buf_pool.done_flush_list, - &buf_pool.flush_list_mutex.m_mutex, &abstime); - resizing= log_sys.resize_in_progress(); - } - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (!resizing || !log_sys.resize_running(thd)) - break; - log_sys.latch.wr_lock(); - while (resizing > log_sys.get_lsn()) - { - ut_ad(!log_sys.is_mmap()); - /* The server is almost idle. Write dummy FILE_CHECKPOINT records - to ensure that the log resizing will complete. */ - mtr_t mtr{nullptr}; - mtr.start(); - mtr.commit_files(log_sys.last_checkpoint_lsn); - } - log_sys.latch.wr_unlock(); - } + log_sys.resize_finish(thd); } } mysql_mutex_lock(&LOCK_global_system_variables); @@ -19275,7 +19328,7 @@ static MYSQL_SYSVAR_ENUM(flush_method, innodb_flush_method, static MYSQL_SYSVAR_STR(log_group_home_dir, srv_log_group_home_dir, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "Path to ib_logfile0", NULL, NULL, NULL); + "Path to ib_logfile0 or ib_*.log", NULL, NULL, NULL); static MYSQL_SYSVAR_DOUBLE(max_dirty_pages_pct, srv_max_buf_pool_modified_pct, PLUGIN_VAR_RQCMDARG, @@ -19638,7 +19691,7 @@ static MYSQL_SYSVAR_UINT(log_buffer_size, log_sys.buf_size, NULL, NULL, 16U << 20, 2U << 20, log_sys.buf_size_max, 4096); static constexpr const char *innodb_log_file_mmap_description= - "Whether ib_logfile0" + "Whether the log" " resides in persistent memory (when supported) or" " should initially be memory-mapped"; static MYSQL_SYSVAR_BOOL(log_file_mmap, log_sys.log_mmap, @@ -19649,13 +19702,13 @@ static MYSQL_SYSVAR_BOOL(log_file_mmap, log_sys.log_mmap, #if defined __linux__ || defined _WIN32 static MYSQL_SYSVAR_BOOL(log_file_buffering, log_sys.log_buffered, PLUGIN_VAR_OPCMDARG, - "Whether the file system cache for ib_logfile0 is enabled", + "Whether the file system cache for ib_logfile0 or ib_*.log is enabled", nullptr, innodb_log_file_buffering_update, FALSE); #endif static MYSQL_SYSVAR_BOOL(log_file_write_through, log_sys.log_write_through, PLUGIN_VAR_OPCMDARG, - "Whether each write to ib_logfile0 is write through", + "Whether each write to the log is write through", nullptr, innodb_log_file_write_through_update, FALSE); static MYSQL_SYSVAR_BOOL(data_file_buffering, fil_system.buffered, @@ -19668,11 +19721,49 @@ static MYSQL_SYSVAR_BOOL(data_file_write_through, fil_system.write_through, "Whether each write to data files writes through", nullptr, innodb_data_file_write_through_update, FALSE); +static void innodb_log_archive_update(THD *thd, st_mysql_sys_var*, + void *, const void *save) noexcept +{ + mysql_mutex_unlock(&LOCK_global_system_variables); + log_sys.set_archive(*static_cast(save), thd); + mysql_mutex_lock(&LOCK_global_system_variables); +} + +static MYSQL_SYSVAR_BOOL(log_archive, log_sys.archive, + PLUGIN_VAR_OPCMDARG, + "Whether log archiving is desired", + nullptr, innodb_log_archive_update, FALSE); + +static MYSQL_SYSVAR_UINT64_T(log_archive_start, innodb_log_archive_start, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "initial value of innodb_lsn_archived; 0=auto-detect", + nullptr, nullptr, 0, 0, std::numeric_limits::max(), 0); + +static void innodb_log_recovery_start_update(THD *, st_mysql_sys_var*, + void *, const void *save) noexcept +{ + const lsn_t lsn{*static_cast(save)}; + recv_sys.recovery_start= lsn; + if (lsn && log_sys.archive) + log_sys.archived_checkpoint= lsn; +} + +static MYSQL_SYSVAR_UINT64_T(log_recovery_start, recv_sys.recovery_start, + PLUGIN_VAR_RQCMDARG, + "LSN to start recovery from (0=automatic)", + nullptr, innodb_log_recovery_start_update, + 0, 0, std::numeric_limits::max(), 0); + +static MYSQL_SYSVAR_UINT64_T(log_recovery_target, recv_sys.rpo, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "recovery point objective (end LSN; 0=unlimited)", + nullptr, nullptr, 0, 0, std::numeric_limits::max(), 0); + static MYSQL_SYSVAR_ULONGLONG(log_file_size, srv_log_file_size, PLUGIN_VAR_RQCMDARG, - "Redo log size in bytes", + "Desired log file size in bytes", nullptr, innodb_log_file_size_update, - 96 << 20, 4 << 20, std::numeric_limits::max(), 4096); + 96 << 20, log_t::FILE_SIZE_MIN, std::numeric_limits::max(), 4096); static uint innodb_log_spin_wait_delay; @@ -20110,6 +20201,10 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(log_file_write_through), MYSQL_SYSVAR(data_file_buffering), MYSQL_SYSVAR(data_file_write_through), + MYSQL_SYSVAR(log_archive), + MYSQL_SYSVAR(log_archive_start), + MYSQL_SYSVAR(log_recovery_start), + MYSQL_SYSVAR(log_recovery_target), MYSQL_SYSVAR(log_file_size), MYSQL_SYSVAR(log_write_ahead_size), MYSQL_SYSVAR(log_spin_wait_delay), diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index a3550491162ee..8be2b53225fef 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -7979,6 +7979,7 @@ ha_innobase::prepare_inplace_alter_table( DBUG_ASSERT(!ha_alter_info->handler_ctx); DBUG_ASSERT(ha_alter_info->create_info); DBUG_ASSERT(!srv_read_only_mode); + DBUG_ASSERT(!recv_sys.rpo); /* Init online ddl status variables */ onlineddl_rowlog_rows = 0; @@ -8820,6 +8821,7 @@ ha_innobase::inplace_alter_table( bool rebuild_templ = false; DBUG_ENTER("inplace_alter_table"); DBUG_ASSERT(!srv_read_only_mode); + DBUG_ASSERT(!recv_sys.rpo); DEBUG_SYNC(m_user_thd, "innodb_inplace_alter_table_enter"); @@ -11275,6 +11277,7 @@ ha_innobase::commit_inplace_alter_table( DBUG_ENTER("commit_inplace_alter_table"); DBUG_ASSERT(!srv_read_only_mode); + DBUG_ASSERT(!recv_sys.rpo); DBUG_ASSERT(!ctx0 || ctx0->prebuilt == m_prebuilt); DBUG_ASSERT(!ctx0 || ctx0->old_table == m_prebuilt->table); diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 31bcd6f863b2e..ab20e82c52f8f 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -891,11 +891,8 @@ static dberr_t ibuf_open(btr_cur_t *cur, mtr_t *mtr) ATTRIBUTE_COLD dberr_t ibuf_upgrade() { - if (srv_read_only_mode) - { - sql_print_error("InnoDB: innodb_read_only_mode prevents an upgrade"); - return DB_READ_ONLY; - } + ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); sql_print_information("InnoDB: Upgrading the change buffer"); @@ -1061,14 +1058,25 @@ dberr_t ibuf_upgrade_needed() err= DB_CORRUPTION; goto err_exit; } - else if (srv_read_only_mode) - { - sql_print_error("InnoDB: innodb_read_only=ON prevents an upgrade" - " of the change buffer"); - err= DB_READ_ONLY; - } - else if (srv_force_recovery != SRV_FORCE_NO_LOG_REDO) - err= DB_FAIL; + else + do + { + const char *reason; + if (srv_read_only_mode) + reason= "read_only=ON"; + else if (recv_sys.rpo) + reason= "log_recovery_target"; + else + { + if (srv_force_recovery != SRV_FORCE_NO_LOG_REDO) + err= DB_FAIL; + continue; + } + sql_print_error("InnoDB: innodb_%s prevents an upgrade" + " of the change buffer", reason); + err= DB_READ_ONLY; + } + while (false); goto func_exit; } diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 48697750c7dea..54f1a2bb2513c 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -408,6 +408,13 @@ struct fil_space_t final /** Whether any corruption of this tablespace has been reported */ mutable std::atomic_flag is_corrupted= ATOMIC_FLAG_INIT; + /** BACKUP SERVER flag in write_or_backup */ + static constexpr uint8_t BACKUP{128}; + /** whether there is a pending write or backup */ + std::atomic write_or_backup{0}; + /** first page number that is not being backed up */ + std::atomic backup_end{0}; + public: /** mutex to protect freed_ranges and last_freed_lsn */ std::mutex freed_range_mutex; @@ -1044,6 +1051,43 @@ struct fil_space_t final VALIDATE_IMPORT }; + /** Note that writes are being submitted to the tablespace. + @return whether a backup is pending */ + bool writing_start() noexcept + { + uint8_t wb{write_or_backup.fetch_add(1, std::memory_order_acq_rel)}; + ut_ad(~wb & (BACKUP - 1)); + return wb & BACKUP; + } + + /** Note that we there are no more pending writes to the tablespace. */ + void writing_stop() noexcept + { + ut_d(uint8_t wb=) write_or_backup.fetch_sub(1, std::memory_order_release); + ut_ad(wb & ~BACKUP); + } + + /** Note that we backing up some pages of the underlying files. + @param last_page the last page that is being backed up */ + bool backup_start(uint32_t last_page) noexcept + { + backup_end.store(last_page, std::memory_order_relaxed); + uint8_t wb{write_or_backup.fetch_add(BACKUP, std::memory_order_acq_rel)}; + ut_ad(!(wb & BACKUP)); + return wb & ~BACKUP; + } + /** Note that we are not currently backing up the underlying files. */ + void backup_stop() noexcept + { + backup_end.store(0, std::memory_order_relaxed); + ut_d(uint8_t wb=) + write_or_backup.fetch_sub(BACKUP, std::memory_order_release); + ut_ad(wb & BACKUP); + } + /** @return the first page number that is not being backed up */ + uint32_t backup_page_end() const noexcept + { return backup_end.load(std::memory_order_relaxed); } + /** Update the data structures on write completion */ void complete_write() noexcept; diff --git a/storage/innobase/include/ha_prototypes.h b/storage/innobase/include/ha_prototypes.h index 0cf8548341742..25ebda83f8df9 100644 --- a/storage/innobase/include/ha_prototypes.h +++ b/storage/innobase/include/ha_prototypes.h @@ -361,16 +361,6 @@ ATTRIBUTE_COLD void innodb_fk_error(const trx_t *trx, dberr_t err, const char *name, const dict_foreign_t& foreign); -/********************************************************************//** -Helper function to push warnings from InnoDB internals to SQL-layer. */ -void -ib_foreign_warn( - trx_t* trx, /*!< in: trx */ - dberr_t error, /*!< in: error code to push as warning */ - const char *table_name, - const char *format,/*!< in: warning message */ - ...); - /** Normalizes a table name string. A normalized name consists of the database name catenated to '/' and table name. For example: test/mytable. diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index e3a3c17773612..fd32b4f07abfb 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -25,6 +25,8 @@ Created 12/9/1995 Heikki Tuuri *******************************************************/ #pragma once +/** file name pattern of innodb_log_archive=ON */ +#define LOG_ARCHIVE_NAME "ib_%016" PRIx64 ".log" #include "log0types.h" #include "os0file.h" @@ -35,22 +37,6 @@ Created 12/9/1995 Heikki Tuuri using st_::span; -static const char LOG_FILE_NAME_PREFIX[] = "ib_logfile"; -static const char LOG_FILE_NAME[] = "ib_logfile0"; - -/** Composes full path for a redo log file -@param[in] filename name of the redo log file -@return path with log file name*/ -std::string get_log_file_path(const char *filename= LOG_FILE_NAME); - -/** Delete log file. -@param[in] suffix suffix of the file name */ -static inline void delete_log_file(const char* suffix) -{ - auto path = get_log_file_path(LOG_FILE_NAME_PREFIX).append(suffix); - os_file_delete_if_exists_func(path.c_str(), nullptr); -} - struct completion_callback; /** Ensure that the log has been written to the log file up to a given @@ -132,9 +118,12 @@ class log_file_t bool flush() const noexcept { return os_file_flush(m_file); } }; +struct recv_warp; + /** Redo log buffer */ struct log_t { + friend recv_warp; /** The maximum buf_size */ static constexpr unsigned buf_size_max= os_file_request_size_max; @@ -191,11 +180,10 @@ struct log_t public: /** innodb_log_buffer_size (usable append_prepare() size in bytes) */ unsigned buf_size; - /** set when there may be need to initiate a log checkpoint. - This must hold if lsn - last_checkpoint_lsn > max_checkpoint_age. */ - std::atomic need_checkpoint; - /** next checkpoint number (protected by latch.wr_lock()) */ - byte next_checkpoint_no; + /** current innodb_log_write_ahead_size */ + uint write_size; + /** maximum write_size */ + static constexpr uint WRITE_SIZE_MAX{4096}; /** log file size in bytes, including the header */ lsn_t file_size; @@ -221,6 +209,62 @@ struct log_t /** latch_have_wr() for checkpoint, latch_have_any() for append_prepare() */ log_rwlock latch; + /** format of the redo log: e.g., FORMAT_10_8 */ + uint32_t format; + /** the minimum log file size */ + static constexpr lsn_t FILE_SIZE_MIN{4 << 20}; + /** the maximum log file size in innodb_log_archive=ON format */ + static constexpr lsn_t ARCHIVE_FILE_SIZE_MAX{1ULL << 32}; +private: +#ifdef HAVE_PMEM + /** whether the memory-mapped interface is enabled for writing */ + bool is_pmem:1; +#endif + /** whether !archive log records may have been written with + get_sequence_bit()==0 */ + bool circular_recovery_from_sequence_bit_0:1; + /** whether we are between backup_start() and backup_stop() */ + bool backup:1; +public: + /** the default value of log_mmap */ + static constexpr bool log_mmap_default= +# if defined __linux__ /* MAP_POPULATE would enable read-ahead */ + true || +# elif defined __FreeBSD__ /* MAP_PREFAULT_READ would enable read-ahead */ + true || +# else /* an unnecessary read-ahead of a large ib_logfile0 is a risk */ +# endif + false; +#if defined __linux__ || defined _WIN32 +# ifdef _WIN32 + static constexpr bool log_maybe_unbuffered= true; +# else + /** whether file system caching may be disabled */ + bool log_maybe_unbuffered:1; +# endif + /** whether file system caching is enabled for the log */ + my_bool log_buffered; +#endif + /** whether each write to ib_logfile0 is durable (O_DSYNC) */ + my_bool log_write_through; + /** the current value of innodb_log_archive; protected by latch.wr_lock() */ + my_bool archive; + /** whether the memory-mapped interface is enabled for the log */ + my_bool log_mmap; + /** Set when there may be need for write_checkpoint(). + This must hold if lsn - last_checkpoint_lsn > max_checkpoint_age. */ + std::atomic need_checkpoint; + /** next available checkpoint number, + or UINT16_MAX in recv_sys_t::find_checkpoint_archived(); + protected by latch.wr_lock() */ + uint16_t next_checkpoint_no; +private: +#ifdef HAVE_PMEM + /** mutex protecting wrap-around in resize_write() */ + srw_mutex resize_wrap_mutex; +#endif +public: + /** log record buffer, written to by mtr_t::commit() */ alignas(CPU_LEVEL1_DCACHE_LINESIZE) byte *buf; @@ -248,15 +292,25 @@ struct log_t Atomic_relaxed last_checkpoint_lsn; /** The log writer (protected by latch.wr_lock()) */ lsn_t (*writer)() noexcept; + /** the earliest available checkpoint; protected by latch.wr_lock() */ + lsn_t archived_checkpoint; + /** end_lsn of archived_checkpoint; protected by latch.wr_lock() */ + lsn_t archived_lsn; /** Log file */ log_file_t log; private: - /** Log file being constructed during resizing; protected by latch */ + /** Log file being constructed during resizing, + or the previous archived log file; protected by latch */ log_file_t resize_log; - /** size of resize_log; protected by latch */ + /** size of resize_log, or the requested innodb_log_file_size + of the next file created if archive==TRUE; protected by latch */ lsn_t resize_target; - /** Buffer for writing to resize_log; @see buf */ + /** Buffer for writing to resize_log; @see buf + Also a spare buffer between archived_mmap_switch_prepare() + and archived_mmap_switch_complete(), + or archived_mmap_switch_recovery_prepare() + and archived_mmap_switch_recovery_complete(). */ byte *resize_buf; /** Buffer for writing to resize_log; @see flush_buf */ byte *resize_flush_buf; @@ -264,39 +318,11 @@ struct log_t /** log sequence number when log resizing was initiated; 0 if the log is not being resized, 1 if resize_start() is in progress */ std::atomic resize_lsn; - /** the log sequence number at the start of the log file */ + /** the log sequence number at the start of the current log file */ lsn_t first_lsn; + /** the log sequence number when the latest checkpoint was initiated */ + lsn_t end_lsn; public: - /** current innodb_log_write_ahead_size */ - uint write_size; - /** maximum write_size */ - static constexpr uint WRITE_SIZE_MAX{4096}; - /** format of the redo log: e.g., FORMAT_10_8 */ - uint32_t format; - /** whether the memory-mapped interface is enabled for the log */ - my_bool log_mmap; - /** the default value of log_mmap */ - static constexpr bool log_mmap_default= -# if defined __linux__ /* MAP_POPULATE would enable read-ahead */ - true || -# elif defined __FreeBSD__ /* MAP_PREFAULT_READ would enable read-ahead */ - true || -# else /* an unnecessary read-ahead of a large ib_logfile0 is a risk */ -# endif - false; -#if defined __linux__ || defined _WIN32 - /** whether file system caching is enabled for the log */ - my_bool log_buffered; -# ifdef _WIN32 - static constexpr bool log_maybe_unbuffered= true; -# else - /** whether file system caching may be disabled */ - bool log_maybe_unbuffered; -# endif -#endif - /** whether each write to ib_logfile0 is durable (O_DSYNC) */ - my_bool log_write_through; - /** Fields involved in checkpoints @{ */ lsn_t log_capacity; /*!< capacity of the log; if the checkpoint age exceeds this, it is @@ -314,17 +340,15 @@ struct log_t for lsn - last_checkpoint_lsn when a new query step is started */ - /** buffer for checkpoint header */ + /** buffer for checkpoint header; protected by latch; + nullptr if is_mmap() and at most one log file is open; + a pointer to the oldest open archive log file if is_mmap() */ byte *checkpoint_buf; /* @} */ private: - /** the thread that initiated resize_lsn() */ + /** the thread that initiated resize_start() */ Atomic_relaxed resize_initiator; -#ifdef HAVE_PMEM - /** mutex protecting wrap-around in resize_write() */ - srw_mutex resize_wrap_mutex; -#endif public: /** number of long append_prepare_wait(); protected by latch_have_wr() */ size_t waits; @@ -336,7 +360,7 @@ struct log_t bool is_mmap() const noexcept { return !flush_buf; } /** @return whether a handle to the log is open; - is_mmap() && !is_opened() holds for PMEM */ + is_mmap() && (is_opened() == archive) holds for PMEM */ bool is_opened() const noexcept { return log.is_opened(); } /** @return LSN at which log resizing was started and is still in progress @@ -350,11 +374,24 @@ struct log_t RESIZE_NO_CHANGE, RESIZE_IN_PROGRESS, RESIZE_STARTED, RESIZE_FAILED }; +private: /** Start resizing the log and release the exclusive latch. + @param size requested new file_size + @param thd the current thread identifier + @param backup whether the caller is backup_start() or backup_stop() + @return whether the resizing was started successfully */ + resize_start_status resize_start(uint64_t size, void *thd, bool backup) + noexcept; +public: + /** Start resizing the log. @param size requested new file_size @param thd the current thread identifier @return whether the resizing was started successfully */ - resize_start_status resize_start(os_offset_t size, void *thd) noexcept; + resize_start_status resize_start(uint64_t size, void *thd) noexcept + { return resize_start(size, thd, false); } + + /** Wait for the completion of resize_start() == RESIZE_STARTED */ + void resize_finish(THD *thd) noexcept; /** Abort a resize_start() that we started. @param thd thread identifier that had been passed to resize_start() */ @@ -365,41 +402,102 @@ struct log_t { return thd == resize_initiator; } /** Replicate a write to the log. + @tparam mmap whether the memory-mapped interface is enabled @param lsn start LSN @param end end of the mini-transaction @param len length of the mini-transaction @param seq offset of the sequence bit from the end */ + template inline void resize_write(lsn_t lsn, const byte *end, - size_t len, size_t seq) noexcept; + size_t len, size_t seq) noexcept + { + if (UNIV_LIKELY_NULL(resize_buf)) + resize_write_low(lsn, end, len, seq); + } private: + /** SET GLOBAL innodb_log_archive, or start/stop BACKUP SERVER + @param archive the new value of innodb_log_archive + @param thd SQL connection + @param backup whether the caller is backup_start() or backup_stop() + @return whether the operation failed */ + bool set_archive(my_bool archive, THD *thd, bool backup) noexcept; +public: + /** SET GLOBAL innodb_log_archive + @param archive the new value of innodb_log_archive + @param thd SQL connection + @return whether the operation failed */ + bool set_archive(my_bool archive, THD *thd) noexcept + { return set_archive(archive, thd, false); } + + /** Start BACKUP SERVER. + @param old_size the old file_size, or 0 on failure or when + already running innodb_log_archive=ON + @param thd SQL connection + @return whether the operation failed */ + bool backup_start(uint64_t *old_size, THD *thd) noexcept; + /** Stop BACKUP SERVER. + @param old_size the value returned by backup_start() + @param thd SQL connection */ + void backup_stop(uint64_t old_size, THD *thd) noexcept; + +private: + /** Replicate a write to the log. + @tparam mmap whether the memory-mapped interface is enabled + @param lsn start LSN + @param end end of the mini-transaction + @param len length of the mini-transaction + @param seq offset of the sequence bit from the end */ + template + ATTRIBUTE_COLD void resize_write_low(lsn_t lsn, const byte *end, + size_t len, size_t seq) noexcept; /** Write resize_buf to resize_log. @param b resize_buf or resize_flush_buf @param length the used length of b */ void resize_write_buf(const byte *b, size_t length) noexcept; + + /** Rename a log file. + @param old_name the old log file name + @return whether an error occurred */ + ATTRIBUTE_COLD static bool rename(const std::string &old_name) noexcept; public: /** Rename a log file after resizing. @return whether an error occurred */ - static bool resize_rename() noexcept; + ATTRIBUTE_COLD static bool resize_rename() noexcept; - /** @return pointer for writing to resize_buf - @retval nullptr if no is_mmap() based resizing is active */ - inline byte *resize_buf_begin(lsn_t lsn) const noexcept; - /** @return end of resize_buf */ - inline const byte *resize_buf_end() const noexcept - { return resize_buf + resize_target; } + /** During recovery, rename an archived log file to ib_logfile0. + @return whether an error occurred. */ + ATTRIBUTE_COLD bool archive_rename() noexcept; /** Initialise the redo log subsystem. */ void create() noexcept; + /** Access mode of a log file */ + enum log_access { +#ifdef HAVE_PMEM + /** read/write memory-mapping */ + PMEM= -1, +#endif + /** write via pwrite(); memory-mapped reads if log_sys.log_mmap */ + READ_WRITE= 0, + /** read via pread() or memory-mapping according to log_sys.log_mmap */ + READ_ONLY + }; + /** Attach a log file. @return whether the memory allocation succeeded */ - bool attach(log_file_t file, os_offset_t size) noexcept; + bool attach(log_file_t file, os_offset_t size, log_access access) noexcept; /** Disable memory-mapped access (update log_mmap) */ void clear_mmap() noexcept; void close_file(bool really_close= true) noexcept; + /** Stash a log archive file in multi-file recovery, + to allow quick switching between the last two files. */ + inline void stash_archive_file() noexcept; + /** Unstash a log archive file. */ + ATTRIBUTE_COLD void unstash_archive_file() noexcept; + #if defined __linux__ || defined _WIN32 /** Try to enable or disable file system caching (update log_buffered) */ void set_buffered(bool buffered) noexcept; @@ -416,6 +514,10 @@ struct log_t @param encrypted whether the log is encrypted */ static void header_write(byte *buf, lsn_t lsn, bool encrypted) noexcept; + /** Rewrite the log file header in set_archive() + @param archive the new value of innodb_log_archive */ + void header_rewrite(my_bool archive) noexcept; + /** @return an estimate of get_lsn(), using acquire-release ordering with write_buf() or persist(); an upper bound if said functions have updated only one of the fields, @@ -437,6 +539,13 @@ struct log_t (write_lsn_offset & (WRITE_BACKOFF - 1)); } + /** @return whether a back-off in a log write is in progress */ + bool is_backoff() const noexcept + { + ut_ad(latch_have_wr()); + return write_lsn_offset & WRITE_BACKOFF; + } + lsn_t get_flushed_lsn(std::memory_order order= std::memory_order_acquire) const noexcept { return flushed_to_disk_lsn.load(order); } @@ -444,21 +553,45 @@ struct log_t /** Initialize the LSN on initial log file creation. */ inline lsn_t init_lsn() noexcept; - void set_recovered_lsn(lsn_t lsn) noexcept - { - ut_ad(latch_have_wr()); - uint64_t lsn_offset= ((write_size - 1) & (lsn - first_lsn)); - write_lsn_offset= lsn_offset; - base_lsn.store(lsn - lsn_offset, std::memory_order_relaxed); - flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); - write_lsn= lsn; - } + /** Set the LSN up to which everything has been recovered from the log. */ + ATTRIBUTE_COLD void set_recovered_lsn(lsn_t lsn) noexcept; #ifdef HAVE_PMEM + /** @return whether the log is memory-mapped read-write */ + bool is_mmap_writeable() const noexcept { return is_pmem; } /** Persist the log. @param lsn desired new value of flushed_to_disk_lsn */ void persist(lsn_t lsn) noexcept; + /** @return the overflow buffer when ARCHIVED_MMAP is wrapping around */ + byte *get_archived_mmap_switch() const noexcept + { + ut_ad(archived_mmap_switch()); + return resize_buf + START_OFFSET; + } +#else + static constexpr bool is_mmap_writeable() { return false; } #endif + /** @return whether archived_mmap_switch_complete() needs to be called */ + bool archived_mmap_switch() const noexcept + { + ut_ad(latch_have_any()); + return UNIV_UNLIKELY(archive && resize_buf); + } + /** Create a new log file when the current one will fill up. + @param buf log records to append + @param length size of the log records, in bytes + @param offset log file offset */ + ATTRIBUTE_COLD void archive_new_write(const byte *buf, size_t length, + lsn_t offset) noexcept; + + /** Ensure that innodb_log_archive=ON will default to the current + innodb_log_file_size if no size has been specified. */ + void archive_set_size() noexcept + { + ut_ad(!resize_in_progress()); + if (!resize_target) + resize_target= file_size; + } bool check_for_checkpoint() const noexcept { @@ -492,13 +625,78 @@ struct log_t @param late whether the WRITE_BACKOFF flag had already been set @param ex whether log_sys.latch is exclusively locked */ ATTRIBUTE_COLD void append_prepare_wait(bool late, bool ex) noexcept; +#ifdef HAVE_PMEM + /** Wait in append_prepare() for buffer to become available + @param late whether the WRITE_BACKOFF flag had already been set + @param ex whether log_sys.latch is exclusively locked */ + ATTRIBUTE_COLD void archived_mmap_switch_prepare(bool late, bool ex) + noexcept; +#endif public: + /** Attempt to finish archived_mmap_switch_prepare(). + @return the current LSN in the new file + @retval 0 if no switch took place */ + ATTRIBUTE_COLD lsn_t archived_mmap_switch_complete() noexcept; + + /** Prepare for multi-file memory-mapped log recovery. */ + ATTRIBUTE_COLD void archived_mmap_switch_recovery_prepare() noexcept; + /** Finish archived_mmap_switch_recovery_prepare(). */ + ATTRIBUTE_COLD void archived_mmap_switch_recovery_complete() noexcept; + /** Prepare to switch archive log files on recovery. + @param lsn first sequence number of the file + @return whether a spare log file was opened */ + bool archived_switch_recovery_prepare(lsn_t lsn) noexcept; + /** Try to switch archive log files on recovery. + @return whether the log file was switched */ + bool archived_switch_recovery() noexcept; + /** Undo archived_switch_recovery() + in recv_sys_t::find_checkpoint_archived() */ + inline void archived_switch_recovery_rewind_checkpoint() noexcept; + /** Rewind the log during recovery. + @param lsn log sequence number to rewind to */ + ATTRIBUTE_COLD void recovery_rewind(lsn_t lsn) noexcept; + + /** How to write log */ + enum write { + /** normal writing !log_sys.is_mmap() */ + WRITE_NORMAL, + /** circular memory-mapped writing when log_sys.is_mmap() */ + CIRCULAR_MMAP, + /** memory-mapped log for log_sys.archive */ + ARCHIVED_MMAP + }; + + /** Get a name of a circular log file. + @param i log file number (0 to 101) + @return the path name of the log file */ + ATTRIBUTE_COLD static std::string get_circular_path(size_t i= 0); + + /** @return the name of the current log file */ + ATTRIBUTE_COLD std::string get_path() const; + + /** Append the archive log file base name to a string. + @param path directory name and separator + @param lsn first LSN stored in the file + @return path with the base file name appended */ + static ATTRIBUTE_COLD std::string &append_archive_name(std::string &path, + lsn_t lsn); + + /** Generate an archive log file name. + @param lsn first LSN stored in the file + @return archive log file name */ + ATTRIBUTE_COLD std::string get_archive_path(lsn_t lsn) const; + /** @return the current archive log file name */ + std::string get_archive_path() const { return get_archive_path(first_lsn); } + + /** @return the next archive log file name */ + ATTRIBUTE_COLD std::string get_next_archive_path() const; + /** Reserve space in the log buffer for appending data. - @tparam mmap log_sys.is_mmap() + @tparam mode how to write log @param size total length of the data to append(), in bytes @param ex whether log_sys.latch is exclusively locked @return the start LSN and the buffer position for append() */ - template + template std::pair append_prepare(size_t size, bool ex) noexcept; /** Append a string of bytes to the redo log. @@ -530,6 +728,18 @@ struct log_t /** @return the first LSN of the log file */ lsn_t get_first_lsn() const noexcept { return first_lsn; } + /** Set the recovered checkpoint. + @param lsn log sequence number of the checkpoint + @param end_lsn LSN passed to write_checkpoint() + @param number checkpoint number */ + void set_recovered_checkpoint(lsn_t lsn, lsn_t end_lsn, uint16_t number) + noexcept + { + last_checkpoint_lsn= lsn; + this->end_lsn= end_lsn; + next_checkpoint_no= number; + } + /** Determine the sequence bit at a log sequence number */ byte get_sequence_bit(lsn_t lsn) const noexcept { diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index 7de6a431329d1..7c38e7d95f42a 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -232,7 +232,9 @@ struct recv_sys_t Atomic_relaxed recovery_on= false; /** whether recv_recover_page(), invoked from buf_page_t::read_complete(), should apply log records*/ - bool apply_log_recs; + bool apply_log_recs:1; + /** whether a circular log was recovered with archive file name */ + bool was_archive:1; /** number of bytes in log_sys.buf */ size_t len; /** start offset of non-parsed log records in log_sys.buf */ @@ -247,9 +249,30 @@ struct recv_sys_t lsn_t scanned_lsn; /** log sequence number at the end of the FILE_CHECKPOINT record, or 0 */ lsn_t file_checkpoint; + /** recovery start checkpoint */ + lsn_t recovery_start; + /** recovery point objective (a limit for scanned_lsn) */ + lsn_t rpo; + /** the time when progress was last reported */ time_t progress_time; + /** an innodb_log_archive=ON file available for recovery */ + struct archive_log + { + /** the LSN that is past the end of the file; derived from + the start LSN and the file size */ + const lsn_t end; + /** READ_WRITE or READ_ONLY, initially derived from the file permissions. + In find_checkpoint(), all but the last file will be marked READ_ONLY. + Also the last file may be marked READ_ONLY if recv_sys.rpo is set. */ + log_t::log_access access; + }; + /** map of innodb_log_archive=ON files, indexed by the start LSN */ + using archive_map = std::map; + /** innodb_log_archive=ON files, with no gaps before the last file */ + archive_map log_archive; + using map = std::map, ut_allocator>>; @@ -415,6 +438,15 @@ struct recv_sys_t @return error code or DB_SUCCESS */ dberr_t find_checkpoint(); +private: + /** Find a checkpoint in an innodb_log_archive=ON file. + @param first_lsn the first LSN of the file + @param silent whether to silence error reporting + @return error code + @retval DB_SUCCESS if a suitable checkpoint was found */ + dberr_t find_checkpoint_archived(lsn_t first_lsn, bool silent); +public: + /** Register a redo log snippet for a page. @param it page iterator @param l redo log snippet diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index 78160acff54fb..1014089a850a2 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -701,13 +701,15 @@ struct mtr_t { ATTRIBUTE_NOINLINE size_t crc32c() noexcept; /** Commit the mini-transaction log. - @tparam pmem log_sys.is_mmap() + @tparam mmap log_sys.is_mmap() @param mtr mini-transaction @param lsns {start_lsn,flush_ahead_lsn} */ - template + template static void commit_log(mtr_t *mtr, std::pair lsns) noexcept; - /** Release log_sys.latch. */ + /** Release log_sys.latch. + @tparam mmap log_sys.is_mmap() */ + template void commit_log_release() noexcept; /** Append the redo log records to the redo log buffer. @@ -715,11 +717,11 @@ struct mtr_t { std::pair do_write() noexcept; /** Append the redo log records to the redo log buffer. - @tparam mmap log_sys.is_mmap() + @tparam how how to write @param mtr mini-transaction @param len number of bytes to write @return {start_lsn,flush_ahead_lsn} */ - template static + template static std::pair finish_writer(mtr_t *mtr, size_t len); /** The applicable variant of commit_log() */ diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 1eb429a94e16a..97cd183ed145d 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -597,6 +597,7 @@ struct export_var_t{ lsn_t innodb_lsn_current; lsn_t innodb_lsn_flushed; lsn_t innodb_lsn_last_checkpoint; + lsn_t innodb_lsn_archived; trx_id_t innodb_max_trx_id; #ifdef BTR_CUR_HASH_ADAPT ulint innodb_mem_adaptive_hash; diff --git a/storage/innobase/include/trx0sys.h b/storage/innobase/include/trx0sys.h index 8487996823690..cebaacb4e7b6a 100644 --- a/storage/innobase/include/trx0sys.h +++ b/storage/innobase/include/trx0sys.h @@ -416,7 +416,7 @@ class rw_trx_hash_t trx_state_eq(trx, TRX_STATE_PREPARED_RECOVERED) || (trx_state_eq(trx, TRX_STATE_ACTIVE) && (!srv_was_started || - srv_read_only_mode || + recv_sys.rpo || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO))); trx_free_at_shutdown(trx); } diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index bc77da288753c..0642ff58310e1 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -357,7 +357,7 @@ static bool lock_rec_validate_page(const buf_block_t *block, bool latched) lock_sys_t lock_sys; /** Only created if !srv_read_only_mode. Protected by lock_sys.latch. */ -static FILE *lock_latest_err_file; +FILE *lock_latest_err_file; /*********************************************************************//** Reports that a transaction id is insensible, i.e., in the future. */ @@ -418,6 +418,7 @@ void lock_sys_t::create(ulint n_cells) { ut_ad(this == &lock_sys); ut_ad(!is_initialised()); + ut_ad(srv_read_only_mode == !lock_latest_err_file); m_initialised= true; @@ -435,12 +436,6 @@ void lock_sys_t::create(ulint n_cells) rec_hash.create(n_cells); prdt_hash.create(n_cells); prdt_page_hash.create(n_cells); - - if (!srv_read_only_mode) - { - lock_latest_err_file= os_file_create_tmpfile(); - ut_a(lock_latest_err_file); - } } #ifdef UNIV_PFS_RWLOCK @@ -494,12 +489,6 @@ void lock_sys_t::close() if (!m_initialised) return; - if (lock_latest_err_file) - { - my_fclose(lock_latest_err_file, MYF(MY_WME)); - lock_latest_err_file= nullptr; - } - rec_hash.free(); prdt_hash.free(); prdt_page_hash.free(); diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 9891e1c8ec489..1f2e4cd2364f0 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -108,7 +108,9 @@ void log_t::create() noexcept resize_wrap_mutex.init(); #endif - last_checkpoint_lsn= FIRST_LSN; + last_checkpoint_lsn= 0; + first_lsn= FIRST_LSN; + end_lsn= FIRST_LSN; log_capacity= 0; max_modified_age_async= 0; max_checkpoint_age= 0; @@ -150,7 +152,7 @@ dberr_t log_file_t::read(os_offset_t offset, span buf) noexcept ut_a(size < buf.size()); } - sql_print_error("InnoDB: pread(\"ib_logfile0\") returned %zd," + sql_print_error("InnoDB: pread(log) returned %zd," " operating system error %u", s, unsigned(IF_WIN(GetLastError(), errno))); return DB_IO_ERROR; @@ -170,15 +172,17 @@ void log_file_t::write(os_offset_t offset, span buf) noexcept pwrite(m_file, data, size, offset)); if (UNIV_UNLIKELY(s <= 0)) break; + ut_ad(size >= size_t(s)); size-= size_t(s); if (!size) return; offset+= s; data+= s; - ut_a(size < buf.size()); + /* Out-of-bounds buffer access should make pwrite(2) report EFAULT */ + ut_ad(offset + size < buf.size()); } - sql_print_error("[FATAL] InnoDB: pwrite(\"ib_logfile0\") returned %zd," + sql_print_error("[FATAL] InnoDB: pwrite(log) returned %zd," " operating system error %u", s, unsigned(IF_WIN(GetLastError(), errno))); abort(); @@ -189,26 +193,25 @@ void log_file_t::write(os_offset_t offset, span buf) noexcept # endif /** Attempt to memory map a file. -@param file log file handle -@param size file size +@param file log file handle +@param size file size +@param access how to access the file @return pointer to memory mapping @retval MAP_FAILED if the memory cannot be mapped */ static void *log_mmap(os_file_t file, # ifdef HAVE_PMEM bool &is_pmem, /*!< whether the file is on pmem */ # endif - os_offset_t size) + os_offset_t size, log_t::log_access access) noexcept { #if SIZEOF_SIZE_T < 8 if (size != os_offset_t(size_t(size))) return MAP_FAILED; #endif - if (my_system_page_size > 4096) - return MAP_FAILED; -# ifndef HAVE_PMEM +#ifndef HAVE_PMEM if (!log_sys.log_mmap) /* If support for persistent memory (Linux: mount -o dax) is enabled, - we always attempt to open a MAP_SYNC memory mapping to ib_logfile0. + we always attempt to open a MAP_SYNC memory mapping to the log. This mapping will be read-only during crash recovery, and read-write during normal operation. @@ -216,7 +219,7 @@ static void *log_mmap(os_file_t file, innodb_log_file_mmap=ON. This may benefit mariadb-backup and crash recovery. */ return MAP_FAILED; -# endif +#endif /* For now, InnoDB does not support memory-mapped writes to a regular log file. @@ -226,43 +229,44 @@ static void *log_mmap(os_file_t file, The mapping will always be read-only if innodb_read_only=ON or if mariadb-backup is running in any other mode than --prepare --export. */ - const bool read_only= - srv_read_only_mode || srv_operation >= SRV_OPERATION_BACKUP; + ut_ad(access == log_t::READ_ONLY || + (!srv_read_only_mode && srv_operation < SRV_OPERATION_BACKUP)); -# ifdef _WIN32 - void *ptr= MAP_FAILED; - if (!read_only); +#ifdef _WIN32 + if (access < log_t::READ_ONLY); else if (HANDLE h= CreateFileMappingA(file, nullptr, PAGE_READONLY, DWORD(size >> 32), DWORD(size), nullptr)) { if (h != INVALID_HANDLE_VALUE) { - ptr= MapViewOfFileEx(h, FILE_MAP_READ, 0, 0, size, nullptr); + void *ptr= MapViewOfFileEx(h, FILE_MAP_READ, 0, 0, size, nullptr); CloseHandle(h); - if (!ptr) - ptr= MAP_FAILED; + if (ptr) + return ptr; } } -# else + return MAP_FAILED; +#else int flags= -# ifdef HAVE_PMEM +# ifdef HAVE_PMEM MAP_SHARED_VALIDATE | MAP_SYNC, -# else +# else MAP_SHARED, -# endif +# endif prot= PROT_READ; - if (!read_only) -# ifdef HAVE_PMEM +# ifdef HAVE_PMEM + if (access < log_t::READ_ONLY) prot= PROT_READ | PROT_WRITE; -# ifdef __linux__ /* On Linux, we pretend that /dev/shm is PMEM */ +# ifdef __linux__ /* On Linux, we pretend that /dev/shm is PMEM */ remap: -# endif -# else - return MAP_FAILED; # endif +# else + if (access < log_t::READ_ONLY) + return MAP_FAILED; +# endif void *ptr= my_mmap(0, size_t(size), prot, flags, file, 0); @@ -291,11 +295,11 @@ static void *log_mmap(os_file_t file, } } # endif /* __linux__ */ - if (read_only && log_sys.log_mmap) + if (access == log_t::READ_ONLY && log_sys.log_mmap) ptr= my_mmap(0, size_t(size), PROT_READ, MAP_SHARED, file, 0); # endif /* HAVE_PMEM */ -# endif return ptr; +# endif } #if defined __linux__ || defined _WIN32 @@ -317,43 +321,66 @@ ATTRIBUTE_COLD static void log_file_message() noexcept static inline void log_file_message() noexcept {} #endif -bool log_t::attach(log_file_t file, os_offset_t size) noexcept +bool log_t::attach(log_file_t file, os_offset_t size, + log_t::log_access access) noexcept { - log= file; + ut_ad(file.is_opened()); + ut_ad(!log.is_opened() || + (log.m_file == file.m_file && (archive || recv_sys.was_archive))); + ut_ad(archive || !resize_log.is_opened()); + ut_ad(archive || !buf); + ut_ad(archive || !resize_buf); + ut_ad(archive || !flush_buf); + ut_ad(archive || !resize_flush_buf); ut_ad(!size || size >= START_OFFSET + SIZE_OF_FILE_CHECKPOINT); - file_size= size; - - ut_ad(!buf); - ut_ad(!flush_buf); ut_ad(!writer); + + file_size= size; /* for HAVE_PMEM, clear_mmap() will assign buf_size later */ + if (size) { # ifdef HAVE_PMEM bool is_pmem; - void *ptr= ::log_mmap(log.m_file, is_pmem, size); + void *ptr= ::log_mmap(file.m_file, is_pmem, size, access); # else - void *ptr= ::log_mmap(log.m_file, size); + void *ptr= ::log_mmap(file.m_file, size, access); # endif if (ptr != MAP_FAILED) { + if (archive) + log= file; + else + file.close(); # ifdef HAVE_PMEM + this->is_pmem= is_pmem; if (is_pmem) { - log.close(); log_buffered= false; log_maybe_unbuffered= true; - IF_WIN(,mprotect(ptr, size_t(size), PROT_READ)); } # endif + IF_WIN(,mprotect(ptr, size_t(size), PROT_READ)); buf= static_cast(ptr); writer_update(false); # ifdef HAVE_PMEM if (is_pmem) - return true; + goto func_exit_no_message; # endif goto func_exit; } + + if (buf) + { + ut_ad(archive); + log= file; + log_mmap= false; + goto func_exit_no_message; + } } + else + ut_ad(!archive); + + log= file; log_mmap= false; buf= static_cast(ut_malloc_dontdump(buf_size, PSI_INSTRUMENT_ME)); if (!buf) @@ -380,6 +407,7 @@ bool log_t::attach(log_file_t file, os_offset_t size) noexcept checkpoint_buf= static_cast(aligned_malloc(write_size, write_size)); if (!checkpoint_buf) { + alloc_fail3: ut_free_dodump(flush_buf, buf_size); flush_buf= nullptr; goto alloc_fail2; @@ -387,11 +415,29 @@ bool log_t::attach(log_file_t file, os_offset_t size) noexcept TRASH_ALLOC(buf, buf_size); TRASH_ALLOC(flush_buf, buf_size); - writer_update(false); memset_aligned<512>(checkpoint_buf, 0, write_size); + if (archive) + { + const size_t offset= + std::min(size_t(START_OFFSET - 4), size_t(next_checkpoint_no * 4)); + const size_t bs{write_size}, bs_1{bs - 1}, tail{offset & bs_1}; + if (!tail); + else if (file.read(offset & ~bs_1, {checkpoint_buf, bs}) != DB_SUCCESS) + { + aligned_free(checkpoint_buf); + checkpoint_buf= nullptr; + goto alloc_fail3; + } + else + /* Clear any garbage that may have been left behind by a crash + during write_checkpoint() or set_archive(). See also clear_mmap(). */ + memset(checkpoint_buf + tail, 0, bs - tail); + } func_exit: log_file_message(); + func_exit_no_message: + writer_update(false); return true; } @@ -428,25 +474,31 @@ void log_t::create(lsn_t lsn) noexcept ut_ad(is_latest()); ut_ad(this == &log_sys); + next_checkpoint_no= 8 * is_encrypted(); write_lsn_offset= 0; base_lsn.store(lsn, std::memory_order_relaxed); flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); first_lsn= lsn; + end_lsn= lsn; write_lsn= lsn; + if (!archived_lsn) + archived_lsn= lsn; - last_checkpoint_lsn= 0; + last_checkpoint_lsn= lsn; DBUG_PRINT("ib_log", ("write header " LSN_PF, lsn)); #ifdef HAVE_PMEM if (is_mmap()) { - ut_ad(!is_opened()); + ut_ad(is_mmap_writeable()); + ut_ad(is_opened() == archive); mprotect(buf, size_t(file_size), PROT_READ | PROT_WRITE); + if (archive) + goto create_archive_header; memset_aligned<4096>(buf, 0, 4096); - log_sys.header_write(buf, lsn, is_encrypted()); + header_write(buf, lsn, is_encrypted()); pmem_persist(buf, 512); - buf_size= unsigned(std::min(capacity(), buf_size_max)); } else #endif @@ -454,15 +506,24 @@ void log_t::create(lsn_t lsn) noexcept ut_ad(!is_mmap()); memset_aligned<4096>(flush_buf, 0, buf_size); memset_aligned<4096>(buf, 0, buf_size); - log_sys.header_write(buf, lsn, is_encrypted()); - log.write(0, {buf, 4096}); - memset_aligned<512>(buf, 0, 512); + if (!archive) + { + header_write(buf, lsn, is_encrypted()); + log.write(0, {buf, 4096}); + memset_aligned<512>(buf, 0, 512); + } + else +#ifdef HAVE_PMEM + create_archive_header: +#endif + if (is_encrypted()) + log_crypt_write_header(buf); } } ATTRIBUTE_COLD static void log_close_failed(dberr_t err) noexcept { - ib::fatal() << "closing ib_logfile0 failed: " << err; + ib::fatal() << "closing log failed: " << err; } void log_t::close_file(bool really_close) noexcept @@ -525,11 +586,8 @@ static void log_resize_acquire() noexcept log_sys.latch.wr_lock(); } -/** Release the latches that protect the log. */ -void log_resize_release() noexcept +static void log_resize_release_locks() noexcept { - log_sys.latch.wr_unlock(); - #ifdef HAVE_PMEM if (!log_sys.is_mmap()) #endif @@ -541,6 +599,13 @@ void log_resize_release() noexcept } } +/** Release the latches that protect the log. */ +void log_resize_release() noexcept +{ + log_sys.latch.wr_unlock(); + log_resize_release_locks(); +} + #if defined __linux__ || defined _WIN32 /** Try to enable or disable file system caching (update log_buffered) */ void log_t::set_buffered(bool buffered) noexcept @@ -549,6 +614,7 @@ void log_t::set_buffered(bool buffered) noexcept #ifdef HAVE_PMEM is_mmap() || #endif + recv_sys.rpo || high_level_read_only) return; log_resize_acquire(); @@ -556,10 +622,9 @@ void log_t::set_buffered(bool buffered) noexcept { if (const dberr_t err= log.close()) log_close_failed(err); - std::string path{get_log_file_path()}; log_buffered= buffered; bool success; - log.m_file= os_file_create_func(path.c_str(), + log.m_file= os_file_create_func(get_path().c_str(), OS_FILE_OPEN, OS_LOG_FILE, false, &success); ut_a(log.m_file != OS_FILE_CLOSED); @@ -572,21 +637,20 @@ void log_t::set_buffered(bool buffered) noexcept /** Try to enable or disable durable writes (update log_write_through) */ void log_t::set_write_through(bool write_through) { - if (is_mmap() || high_level_read_only) + if (is_mmap() || high_level_read_only || recv_sys.rpo) return; log_resize_acquire(); if (!resize_in_progress() && is_opened() && bool(log_write_through) != write_through) { os_file_close_func(log.m_file); - log.m_file= OS_FILE_CLOSED; - std::string path{get_log_file_path()}; + log= OS_FILE_CLOSED; log_write_through= write_through; bool success; - log.m_file= os_file_create_func(path.c_str(), + log.m_file= os_file_create_func(get_path().c_str(), OS_FILE_OPEN, OS_LOG_FILE, false, &success); - ut_a(log.m_file != OS_FILE_CLOSED); + ut_a(log.is_opened()); sql_print_information(log_write_through ? "InnoDB: Log writes write through" : "InnoDB: Log writes may be cached"); @@ -594,18 +658,375 @@ void log_t::set_write_through(bool write_through) log_resize_release(); } -/** Start resizing the log and release the exclusive latch. -@param size requested new file_size -@param thd the current thread identifier +/** Rewrite the log file header in set_archive() +@param archive the new value of innodb_log_archive */ +void log_t::header_rewrite(my_bool archive) noexcept +{ + ut_ad(!resize_buf); + ut_ad(this->archive == !archive); + + /* We will rewrite the log file header while the file + name is not ib_logfile0. That is, the archived log file + recovery will accept both the circular and the archived + format for the last file. */ + sql_print_information("InnoDB: setting innodb_log_archive=%u" + " at innodb_log_recovery_start=" LSN_PF, + archive, end_lsn); + + byte* c= checkpoint_buf; + ut_ad(end_lsn >= first_lsn); + ut_ad(!archive || end_lsn <= first_lsn + ~0U); + ut_ad(format == (is_encrypted() ? FORMAT_ENC_11 : FORMAT_10_8)); +#ifdef HAVE_PMEM + if (!c) + { + ut_ad(is_mmap()); + ut_ad(is_mmap_writeable()); + if (!archive) + { + memset_aligned<512>(buf + 512, 0, START_OFFSET - 512); + c= buf + CHECKPOINT_1; + mach_write_to_8(my_assume_aligned<8>(c), last_checkpoint_lsn); + mach_write_to_8(my_assume_aligned<8>(c + 8), end_lsn); + mach_write_to_4(my_assume_aligned<4>(c + 60), my_crc32c(0, c, 60)); + pmem_persist(buf + 512, START_OFFSET - 512); + memset_aligned<512>(buf, 0, 512); + header_write(buf, first_lsn, is_encrypted()); + memset_aligned<512>(buf + 512, 0, CHECKPOINT_1 - 512); + pmem_persist(buf, CHECKPOINT_1); + } + else + { + next_checkpoint_no= uint16_t(8 * is_encrypted() + 1); + const uint32_t d= uint32_t(end_lsn - first_lsn + START_OFFSET); + if (!is_encrypted()) + { + mach_write_to_4(buf, d); + memset_aligned<4>(buf + 4, 0, 64 - 4); + } + else + { + log_crypt_write_header(buf); + mach_write_to_4(buf + 32, d); + memset_aligned<4>(buf + 36, 0, 64 - 36); + } + pmem_persist(buf, 64); + memset_aligned<64>(buf + 64, 0, START_OFFSET - 64); + pmem_persist(buf, START_OFFSET); + } + return; + } +#endif + memset_aligned<512>(c, 0, write_size); + + if (!archive) + { + mach_write_to_8(my_assume_aligned<8>(c), last_checkpoint_lsn); + mach_write_to_8(my_assume_aligned<8>(c + 8), end_lsn); + mach_write_to_4(my_assume_aligned<4>(c + 60), my_crc32c(0, c, 60)); + log.write(CHECKPOINT_1, {c, write_size}); + os_file_flush(log.m_file); + memset_aligned<512>(c, 0, write_size); + for (size_t offset= CHECKPOINT_1; (offset+= write_size) < START_OFFSET;) + log.write(offset, {c, write_size}); + header_write(c, first_lsn, is_encrypted()); + if (write_size > 512) + memset_aligned<512>(c + 512, 0, write_size - 512); + log.write(0, {c, write_size}); + os_file_flush(log.m_file); + memset_aligned<512>(c, 0, write_size); + for (size_t offset= 0; (offset+= write_size) < CHECKPOINT_1;) + log.write(offset, {c, write_size}); + } + else + { + next_checkpoint_no= uint16_t(8 * is_encrypted() + 1); + const uint32_t d= uint32_t(end_lsn - first_lsn + START_OFFSET); + if (!is_encrypted()) + mach_write_to_4(c, d); + else + { + log_crypt_write_header(c); + mach_write_to_4(c + 32, d); + } + log.write(0, {c, write_size}); + os_file_flush(log.m_file); + for (size_t offset= 0; (offset+= write_size) < START_OFFSET;) + log.write(offset, {field_ref_zero, write_size}); + } + + os_file_flush(log.m_file); +} + +/** SET GLOBAL innodb_log_archive +@param archive the new value of innodb_log_archive +@param thd SQL connection +@param backup whether the caller is backup_start() or backup_stop() +@return whether the operation failed */ +bool log_t::set_archive(my_bool archive, THD *thd, bool backup) noexcept +{ + bool fail= false; + thd_wait_begin(thd, THD_WAIT_DISKIO); + tpool::tpool_wait_begin(); + + for (;;) + { + IF_WIN(log_resize_acquire(), latch.wr_lock()); + if (resize_in_progress()) + { + my_printf_error(ER_WRONG_USAGE, + "SET GLOBAL innodb_log_file_size is in progress", + MYF(0)); + fail: + fail= true; + break; + } + if (archive && file_size > ARCHIVE_FILE_SIZE_MAX) + { + my_printf_error(ER_WRONG_USAGE, "innodb_log_file_size>4G", MYF(0)); + goto fail; + } + if (archive == this->archive) + break; + if ((!backup || archive) && thd_kill_level(thd)) + goto fail; + if (!backup && this->backup) + { + my_printf_error(ER_WRONG_USAGE, "BACKUP SERVER is in progress", MYF(0)); + goto fail; + } + + lsn_t wait_lsn; + + if (resize_log.is_opened()) + { + /* Prevent a race condition with write_checkpoint() */ +#ifdef HAVE_PMEM + retry: +#endif + wait_lsn= 0; + retry_after_checkpoint: + IF_WIN(log_resize_release_locks(),); + if (wait_lsn) + { + mysql_mutex_lock(&buf_pool.flush_list_mutex); + buf_flush_wait(wait_lsn, false); + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + } + latch.wr_unlock(); + continue; + } + + wait_lsn= get_lsn(); + const lsn_t checkpoint{last_checkpoint_lsn}; + + if (archive) + { + wait_lsn-= (wait_lsn - first_lsn) % capacity(); + /* We are in innodb_log_archive=OFF. If the file has wrapped + around between the checkpoint and the current position, we must + wait for a log checkpoint not before the desired first_lsn of + our innodb_log_archive=ON log file, because that format does not + allow any wrap-around. */ + if (checkpoint < wait_lsn) + goto retry_after_checkpoint; + } + else if (circular_recovery_from_sequence_bit_0) + { + /* When switching innodb_log_archive back and forth, it could be + the case that some records since the latest checkpoint were + written with get_sequence_bit() == 0. Let us wait for another + checkpoint. This guarantees that if the server crashes between + the completion of set_archive(false) and the next + write_checkpoint(), recovery will only encounter + get_sequence_bit() == 1, consistent with our first_lsn. */ + circular_recovery_from_sequence_bit_0= false; + goto retry_after_checkpoint; + } + else if (checkpoint < first_lsn) + { + /* write_checkpoint() switched files, but the latest FILE_CHECKPOINT + record points to a previous log file. In the innodb_log_archive=OFF + format the checkpoint must be within the only log file. */ + wait_lsn= first_lsn; + goto retry_after_checkpoint; + } + +#ifdef HAVE_PMEM + if (is_mmap()) + { + ut_ad(is_mmap_writeable()); + ut_ad(this->archive == log.is_opened()); + if (is_backoff()) + /* Prevent a race condition with append_prepare() */ + goto retry; + if (archive); + else if (resize_buf) + /* Wait for a call to archived_mmap_switch_complete() */ + goto retry; + else if (checkpoint_buf) + /* Wait for write_checkpoint() */ + goto retry; + else + log.close(); + } +#endif + + ut_ad(!resize_buf); + ut_ad(!resize_log.is_opened()); + + const lsn_t old_first_lsn{first_lsn}; + if (archive) + { + first_lsn= wait_lsn; + resize_target= file_size; + } + std::string normal_name{get_circular_path()}; + std::string arch_name{get_archive_path()}; + + const char *old_name= normal_name.c_str(); + const char *new_name= arch_name.c_str(); + if (!archive) + { + std::swap(old_name, new_name); + header_rewrite(archive); + } +#if defined HAVE_PMEM && !defined _WIN32 + else if (is_mmap()) + { + /* Open the file so that write_checkpoint() + will be able to flag it read-only */ + bool success; + log.m_file= + os_file_create_func(old_name, OS_FILE_OPEN, OS_LOG_FILE, + false, &success); + if (!log.is_opened()) + { + my_error(ER_ERROR_ON_READ, MYF(0), old_name, errno); + goto fail; + } + } +#endif + +#ifdef _WIN32 + /* On Microsoft Windows, there must be no open file handles to a + file that is being renamed. */ + if (const dberr_t err= log.close()) + log_close_failed(err); +#endif + int fail= my_rename(old_name, new_name, MY_SYNC_DIR); +#ifdef _WIN32 + { + bool success; + log.m_file= os_file_create_func(fail ? old_name : new_name, + OS_FILE_OPEN, OS_LOG_FILE, + false, &success); + ut_a(log.m_file != OS_FILE_CLOSED); + } +#endif + if (fail) + { + my_error(ER_ERROR_ON_RENAME, MYF(0), old_name, new_name, my_errno); + first_lsn= old_first_lsn; + goto fail; + } + + if (archive) + { + header_rewrite(archive); + archive_set_size(); + } + + archived_lsn= end_lsn; + this->archive= archive; + mtr_t::finisher_update(); + break; + } + + IF_WIN(log_resize_release(), latch.wr_unlock()); + tpool::tpool_wait_end(); + thd_wait_end(thd); + return fail; +} + +bool log_t::backup_start(uint64_t *old_size, THD *thd) noexcept +{ + latch.wr_lock(); + ut_ad(!backup); + backup= true; + const bool was_archived= bool(archive); + const uint64_t old_file_size{file_size}; + latch.wr_unlock(); + if (was_archived) + { + *old_size= 0; + return false; + } + if (old_file_size > ARCHIVE_FILE_SIZE_MAX) + { + if (resize_start(ARCHIVE_FILE_SIZE_MAX, thd, true) == RESIZE_STARTED) + resize_finish(thd); + latch.wr_lock(); + if (file_size > ARCHIVE_FILE_SIZE_MAX) + goto too_big; + latch.wr_unlock(); + } + if (!set_archive(true, thd, true)) + { + *old_size= old_file_size; + return false; + } + latch.wr_lock(); + too_big: + ut_ad(backup); + backup= false; + const uint64_t new_file_size{file_size}; + latch.wr_unlock(); + if (old_file_size != new_file_size && old_file_size && + resize_start(old_file_size, thd) == RESIZE_STARTED) + resize_finish(thd); + *old_size= 0; + return true; +} + +void log_t::backup_stop(uint64_t old_size, THD *thd) noexcept +{ + /* We will be invoked with old_size=0 after a failed backup_start(), + or if innodb_log_archive=ON held during a successful backup_start(). */ + if (UNIV_LIKELY(old_size != 0)) + { + ut_d(latch.wr_lock()); + ut_ad(backup); + ut_ad(!resize_in_progress()); + ut_ad(archive); + ut_d(latch.wr_unlock()); + ut_d(bool fail=) set_archive(false, thd, true); + ut_ad(!fail); + } + latch.wr_lock(); + ut_ad(!old_size || !resize_in_progress()); + ut_ad(!old_size || backup); + backup= false; + const uint64_t new_size{file_size}; + latch.wr_unlock(); + if (old_size && old_size != new_size && + resize_start(old_size, thd) == RESIZE_STARTED) + resize_finish(thd); +} + +/** Start resizing the log. +@param size requested new file_size +@param thd the current thread identifier +@param backup whether the caller is backup_start() or backup_stop() @return whether the resizing was started successfully */ -log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) - noexcept +log_t::resize_start_status log_t::resize_start(uint64_t size, void *thd, + bool backup) noexcept { ut_ad(size >= 4U << 20); ut_ad(!(size & 4095)); ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); ut_ad(thd); - log_resize_acquire(); resize_start_status status; @@ -614,6 +1035,27 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) status= RESIZE_NO_CHANGE; else if (resize_in_progress()) status= RESIZE_IN_PROGRESS; + else if (archive) + { + if (size > ARCHIVE_FILE_SIZE_MAX) + status= RESIZE_FAILED; + else if (resize_log.is_opened()) + /* The resize_target must not be changed after + archive_new_write() or archived_mmap_switch_prepare() and before + write_checkpoint() has been invoked. */ + status= RESIZE_IN_PROGRESS; + else + { + status= RESIZE_NO_CHANGE; + /* When the current log becomes full and a new archivable log file + is being created, it will be of this size. At that point we will assign + file_size= resize_target (and buf_size= capacity() for HAVE_PMEM). */ + resize_target= size; + } + } + else if (!backup && this->backup) + /* backup_start() or backup_stop() is running */ + status= RESIZE_FAILED; else { lsn_t start_lsn; @@ -622,7 +1064,7 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) ut_ad(!resize_buf); ut_ad(!resize_flush_buf); ut_ad(!resize_initiator); - std::string path{get_log_file_path("ib_logfile101")}; + const std::string path{get_circular_path(101)}; bool success; resize_initiator= thd; resize_lsn.store(1, std::memory_order_relaxed); @@ -642,11 +1084,13 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) #ifdef HAVE_PMEM else if (is_mmap()) { + ut_ad(is_mmap_writeable()); bool is_pmem{false}; - ptr= ::log_mmap(resize_log.m_file, is_pmem, size); + ptr= ::log_mmap(resize_log.m_file, is_pmem, size, READ_WRITE); if (ptr == MAP_FAILED) goto alloc_fail; + ut_ad(is_pmem == this->is_pmem); } #endif else @@ -713,6 +1157,44 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) return status; } +/** Wait for the completion of resize_start() == RESIZE_STARTED */ +void log_t::resize_finish(THD *thd) noexcept +{ + for (timespec abstime;;) + { + if (thd_kill_level(thd)) + { + resize_abort(thd); + break; + } + + set_timespec(abstime, 5); + mysql_mutex_lock(&buf_pool.flush_list_mutex); + lsn_t resizing= resize_in_progress(); + if (resizing > buf_pool.get_oldest_modification(0)) + { + buf_pool.page_cleaner_wakeup(true); + my_cond_timedwait(&buf_pool.done_flush_list, + &buf_pool.flush_list_mutex.m_mutex, &abstime); + resizing= resize_in_progress(); + } + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + if (!resizing || !resize_running(thd)) + break; + latch.wr_lock(); + while (resizing > get_lsn()) + { + ut_ad(!is_mmap()); + /* The server is almost idle. Write dummy FILE_CHECKPOINT records + to ensure that the log resizing will complete. */ + mtr_t mtr{nullptr}; + mtr.start(); + mtr.commit_files(last_checkpoint_lsn); + } + latch.wr_unlock(); + } +} + /** Abort a resize_start() that we started. */ void log_t::resize_abort(void *thd) noexcept { @@ -746,8 +1228,8 @@ void log_t::resize_abort(void *thd) noexcept resize_target= 0; resize_lsn.store(0, std::memory_order_relaxed); resize_initiator= nullptr; - std::string path{get_log_file_path("ib_logfile101")}; - IF_WIN(DeleteFile(path.c_str()), unlink(path.c_str())); + IF_WIN(DeleteFile(get_circular_path(101).c_str()), + unlink(get_circular_path(101).c_str())); writer_update(false); } @@ -755,10 +1237,13 @@ void log_t::resize_abort(void *thd) noexcept } /** Write an aligned buffer to ib_logfile0. -@param buf buffer to be written -@param length length of data to be written -@param offset log file offset */ -static void log_write_buf(const byte *buf, size_t length, lsn_t offset) +@param max_length the maximum length that can be written to the file +@param buf buffer to be written +@param length length of data to be written +@param offset log file offset */ +static void log_write_buf(lsn_t max_length, + const byte *buf, size_t length, lsn_t offset) + noexcept { ut_ad(write_lock.is_owner()); ut_ad(!recv_no_log_write); @@ -767,21 +1252,118 @@ static void log_write_buf(const byte *buf, size_t length, lsn_t offset) ut_ad(!(length & block_size_1)); ut_ad(!(size_t(buf) & block_size_1)); ut_ad(length); + ut_ad(max_length == log_sys.file_size - offset); - const lsn_t maximum_write_length{log_sys.file_size - offset}; - ut_ad(maximum_write_length <= log_sys.file_size - log_sys.START_OFFSET); - - if (UNIV_UNLIKELY(length > maximum_write_length)) + if (UNIV_UNLIKELY(length > max_length)) { - log_sys.log.write(offset, {buf, size_t(maximum_write_length)}); - length-= size_t(maximum_write_length); - buf+= size_t(maximum_write_length); + ut_ad(!log_sys.archive); + log_sys.log.write(offset, {buf, size_t(max_length)}); + length-= size_t(max_length); + buf+= size_t(max_length); ut_ad(log_sys.START_OFFSET + length < offset); offset= log_sys.START_OFFSET; } log_sys.log.write(offset, {buf, length}); } +ATTRIBUTE_COLD +std::string &log_t::append_archive_name(std::string &path, lsn_t lsn) +{ + path.append("ib_"); + for (int i= 16; i--; lsn<<= 4) + path.push_back("0123456789abcdef"[lsn >> 60]); + path.append(".log"); + return path; +} + +ATTRIBUTE_COLD std::string log_t::get_archive_path(lsn_t lsn) const +{ + size_t size= strlen(srv_log_group_home_dir); + retry: + switch (srv_log_group_home_dir[size - !!size]) { +#ifdef _WIN32 + case '\\': +#endif + case '/': + size--; + goto retry; + } + if (size == 1 && *srv_log_group_home_dir == '.') + size= 0; + std::string path; + path.reserve(size + sizeof "/ib_0000000000000000.log"); + path.assign(srv_log_group_home_dir, size); + if (size) + path.push_back('/'); + return append_archive_name(path, lsn); +} + +ATTRIBUTE_COLD std::string log_t::get_next_archive_path() const +{ return get_archive_path(first_lsn + capacity()); } + +ATTRIBUTE_COLD void log_t::archive_new_write(const byte *buf, size_t length, + lsn_t offset) noexcept +{ + ut_ad(latch_have_wr()); + ut_ad(write_lock.is_owner()); + ut_ad(archive); + ut_ad(!resize_buf); + ut_ad(!resize_in_progress()); + ut_ad(resize_target >= FILE_SIZE_MIN); + ut_ad(length); + ut_a(offset + length + START_OFFSET <= file_size + resize_target); + ut_ad(is_latest()); + + if (resize_log.is_opened() && resize_log.m_file != log.m_file) + { + /* We had already created a new log file in a previous invocation + of this function. The old log file (now pointed by resize_log) + will be closed in write_checkpoint() once the first checkpoint in + this new log file has been written. */ + ut_ad(offset >= file_size); + log.write(offset - file_size + START_OFFSET, {buf, length}); + return; + } + + if (const size_t first{size_t(file_size - offset)}) + { + log.write(offset, {buf, first}); + length-= first; + buf+= first; + } + + std::string path{get_next_archive_path()}; + bool success; + pfs_os_file_t file= + os_file_create_func(path.c_str(), OS_FILE_CREATE, OS_LOG_FILE, + false, &success); + ut_ad(success == (file != OS_FILE_CLOSED)); + if (file == OS_FILE_CLOSED) + { + file= os_file_create_func(path.c_str(), OS_FILE_OPEN, OS_LOG_FILE, + false, &success); + ut_ad(success == (file != OS_FILE_CLOSED)); + ut_ad(file == OS_FILE_CLOSED || os_file_get_size(file) < FILE_SIZE_MIN); + } + + if (file != OS_FILE_CLOSED) + { + if (os_file_set_size(path.c_str(), file, resize_target)) + { + resize_log= log; + log.m_file= file; + if (length) + log.write(START_OFFSET, {buf, length}); + return; + } + os_file_close(file); + IF_WIN(DeleteFile(path.c_str()), unlink(path.c_str())); + } + sql_print_error("[FATAL] InnoDB: Failed to create %s of %" PRIu64 + " bytes", path.c_str(), resize_target); + abort(); +} + /** Invoke commit_checkpoint_notify_ha() to notify that outstanding log writes have been completed. */ void log_flush_notify(lsn_t flush_lsn); @@ -908,16 +1490,137 @@ static size_t log_pad(lsn_t lsn, size_t pad, byte *begin, byte *extra) #endif #ifdef HAVE_PMEM +ATTRIBUTE_COLD +void log_t::archived_mmap_switch_prepare(bool late, bool ex) noexcept +{ + ut_ad(archive); + ut_ad(is_mmap()); + ut_ad(is_mmap_writeable()); + ut_ad(log.is_opened()); + ut_ad(!resize_log.is_opened()); + ut_ad(!resize_buf); + ut_ad(!checkpoint_buf); + ut_ad(!resize_in_progress()); + ut_ad(resize_target >= 4U << 20); + ut_ad(is_latest()); + + if (UNIV_LIKELY(!ex)) + { + latch.rd_unlock(); + if (!late) + { + /* Wait for all threads to back off. */ + latch.wr_lock(); + goto got_ex; + } + + const auto delay= my_cpu_relax_multiplier / 4 * srv_spin_wait_delay; + const auto rounds= srv_n_spin_wait_rounds; + + for (;;) + { + HMT_low(); + for (auto r= rounds + 1; r--; ) + { + if (write_lsn_offset.load(std::memory_order_relaxed) & WRITE_BACKOFF) + { + for (auto d= delay; d--; ) + MY_RELAX_CPU(); + } + else + { + HMT_medium(); + goto done; + } + } + HMT_medium(); + std::this_thread::sleep_for(std::chrono::microseconds(100)); + } + } + else + { + got_ex: + const uint64_t l= write_lsn_offset.load(std::memory_order_relaxed); + const lsn_t lsn= base_lsn.load(std::memory_order_relaxed) + + (l & (WRITE_BACKOFF - 1)); + waits++; + ut_ad(archive); + ut_ad(!resize_buf); + ut_ad(!resize_in_progress()); + ut_ad(resize_target >= 4U << 20); + ut_ad(is_latest()); + ut_ad(log.is_opened()); + ut_ad(!resize_log.is_opened()); + + do + { + std::string path{get_next_archive_path()}; + bool success; + os_file_t file= + os_file_create_func(path.c_str(), OS_FILE_CREATE, OS_LOG_FILE, + false, &success); + ut_ad(success == (file != OS_FILE_CLOSED)); + if (file == OS_FILE_CLOSED) + { + file= os_file_create_func(path.c_str(), OS_FILE_OPEN, OS_LOG_FILE, + false, &success); + ut_ad(success == (file != OS_FILE_CLOSED)); + ut_ad(file == OS_FILE_CLOSED || + os_file_get_size(file) < FILE_SIZE_MIN); + } + if (file != OS_FILE_CLOSED) + { + if (os_file_set_size(path.c_str(), file, resize_target)) + { + bool is_pmem; + resize_buf= + static_cast(::log_mmap(file, is_pmem, + resize_target, READ_WRITE)); + if (resize_buf == MAP_FAILED); + else if (!is_pmem) + my_munmap(resize_buf, resize_target); + else + { + /* Will be closed in write_checkpoint() */ + resize_log= log; + log= file; + continue; + } + resize_buf= nullptr; + os_file_close(file); + } + } + + IF_WIN(DeleteFile(path.c_str()), unlink(path.c_str())); + sql_print_error("[FATAL] InnoDB: Failed to create and map %s of %" PRIu64 + " bytes", path.c_str(), resize_target); + abort(); + } + while (false); + + ut_ad(lsn - get_flushed_lsn(std::memory_order_relaxed) < capacity()); + persist(lsn); + /* Above we cleared the WRITE_BACKOFF flag, + which our caller will recheck. */ + if (ex) + return; + latch.wr_unlock(); + } + +done: + latch.rd_lock(); +} + void log_t::persist(lsn_t lsn) noexcept { - ut_ad(!is_opened()); ut_ad(!write_lock.is_owner()); ut_ad(!flush_lock.is_owner()); ut_ad(latch_have_wr()); + ut_ad(is_opened() == archive); lsn_t old= flushed_to_disk_lsn.load(std::memory_order_relaxed); - if (old >= lsn) + if (old > lsn) return; const size_t start(calc_lsn_offset(old)); @@ -926,7 +1629,10 @@ void log_t::persist(lsn_t lsn) noexcept if (UNIV_UNLIKELY(end < start)) { pmem_persist(buf + start, file_size - start); - pmem_persist(buf + START_OFFSET, end - START_OFFSET); + byte *b= archive ? resize_buf : buf; + ut_ad(b || end == START_OFFSET); + /* pmem_persist(nullptr, 0) is a no-op */ + pmem_persist(b + START_OFFSET, end - START_OFFSET); } else pmem_persist(buf + start, end - start); @@ -1000,6 +1706,7 @@ lsn_t log_t::write_buf() noexcept ut_ad(!srv_read_only_mode); ut_ad(resizing == RETAIN_LATCH || (resizing == RESIZING) == (resize_in_progress() > 1)); + ut_ad(write_lsn >= first_lsn); const lsn_t lsn{get_lsn()}; @@ -1011,6 +1718,7 @@ lsn_t log_t::write_buf() noexcept } else { + ut_ad(!recv_sys.rpo); ut_ad(write_lock.is_owner()); ut_ad(!recv_no_log_write); write_lock.set_pending(lsn); @@ -1019,7 +1727,10 @@ lsn_t log_t::write_buf() noexcept ut_ad(ut_is_2pow(write_size)); lsn_t base= base_lsn.load(std::memory_order_relaxed); size_t length{size_t(lsn - base)}; - lsn_t offset{calc_lsn_offset(write_lsn)}; + lsn_t offset{write_lsn - first_lsn}; + if (resizing == RESIZING || !archive) + offset%= capacity(); + offset+= START_OFFSET; ut_ad(length >= (offset & write_size_1)); ut_ad(write_size_1 >= 511); @@ -1089,18 +1800,27 @@ lsn_t log_t::write_buf() noexcept ut_ad(base + (write_lsn_offset & (WRITE_TO_BUF - 1)) == lsn); write_to_log++; + DBUG_PRINT("ib_log", ("write " LSN_PF " to " LSN_PF " at " LSN_PF, + write_lsn, lsn, offset)); + + if (resizing != RESIZING && archive && offset + length > file_size) + { + archive_new_write(write_buf, length, offset); + if (resizing != RETAIN_LATCH) + latch.wr_unlock(); + goto written; + } if (resizing != RETAIN_LATCH) latch.wr_unlock(); - DBUG_PRINT("ib_log", ("write " LSN_PF " to " LSN_PF " at " LSN_PF, - write_lsn, lsn, offset)); - /* Do the write to the log file */ - log_write_buf(write_buf, length, offset); + ut_ad(file_size - offset <= capacity()); + log_write_buf(file_size - offset, write_buf, length, offset); if (UNIV_LIKELY_NULL(re_write_buf)) resize_write_buf(re_write_buf, length); + written: write_lsn= lsn; if (UNIV_UNLIKELY(srv_shutdown_state > SRV_SHUTDOWN_INITIATED)) @@ -1164,7 +1884,6 @@ void log_write_up_to(lsn_t lsn, bool durable, return; } #endif - ut_ad(!log_sys.is_mmap()); repeat: if (durable) @@ -1178,13 +1897,20 @@ void log_write_up_to(lsn_t lsn, bool durable, flush_lock.set_pending(lsn); } - lsn_t pending_write_lsn= 0, pending_flush_lsn= 0; + lsn_t pending_write_lsn= 0, pending_flush_lsn= 0, last_lsn= LSN_MAX; if (write_lock.acquire(lsn, durable ? nullptr : callback) == group_commit_lock::ACQUIRED) { ut_ad(!recv_no_log_write || srv_operation != SRV_OPERATION_NORMAL); log_sys.latch.wr_lock(); + ut_ad(!log_sys.is_mmap()); + if (log_sys.archive) + { + /* Prepare to check if log_t::archive_new_write() will be invoked. */ + lsn= log_sys.get_lsn(); + last_lsn= log_sys.get_first_lsn() + log_sys.capacity(); + } pending_write_lsn= write_lock.release(log_sys.writer()); } @@ -1193,6 +1919,11 @@ void log_write_up_to(lsn_t lsn, bool durable, pending_flush_lsn= log_flush(write_lock.value()); } + if (UNIV_UNLIKELY(lsn > last_lsn)) + /* Ensure that a call to log_t::archive_new_write() will be + completed by log_t::write_checkpoint(). */ + buf_flush_ahead(lsn, true); + if (pending_write_lsn || pending_flush_lsn) { /* There is no new group commit lead; some async waiters could stall. */ @@ -1202,6 +1933,23 @@ void log_write_up_to(lsn_t lsn, bool durable, } } +void log_t::set_recovered_lsn(lsn_t lsn) noexcept +{ + ut_ad(latch_have_wr()); + uint64_t lsn_offset= ((write_size - 1) & (lsn - first_lsn)); + write_lsn_offset= lsn_offset; + base_lsn.store(lsn - lsn_offset, std::memory_order_relaxed); + flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); + write_lsn= lsn; + latch.wr_unlock(); + /* Ensure that log_write_up_to(lsn) will be a no-op. */ + if (flush_lock.acquire(lsn, nullptr) == group_commit_lock::ACQUIRED) + flush_lock.release(lsn); + if (write_lock.acquire(lsn, nullptr) == group_commit_lock::ACQUIRED) + write_lock.release(lsn); + latch.wr_lock(); +} + static lsn_t log_writer() noexcept { return log_sys.write_buf(); @@ -1243,25 +1991,33 @@ ATTRIBUTE_COLD void log_write_and_flush_prepare() noexcept void log_t::clear_mmap() noexcept { - if (!is_mmap() || high_level_read_only) + ut_ad(latch_have_wr()); + ut_ad(is_mmap()); + ut_ad(!srv_read_only_mode || recv_sys.rpo); + + if (recv_sys.rpo && recv_sys.rpo < get_flushed_lsn()) return; + + latch.wr_unlock(); + log_resize_acquire(); + ut_ad(!resize_in_progress()); + ut_ad(get_lsn() == get_flushed_lsn(std::memory_order_relaxed)); #ifdef HAVE_PMEM - if (!is_opened()) + if (is_mmap_writeable() && !recv_sys.rpo) { - ut_d(latch.wr_lock()); - ut_ad(!resize_in_progress()); - ut_ad(get_lsn() == get_flushed_lsn(std::memory_order_relaxed)); - ut_d(latch.wr_unlock()); - return; + mprotect(buf, size_t(file_size), PROT_READ | PROT_WRITE); + ut_ad(next_checkpoint_no <= START_OFFSET / 4); + if (archive) + /* Clear any garbage that may have been left behind by a + crash during write_checkpoint() or set_archive(). */ + memset_aligned<4>(buf + next_checkpoint_no * 4, 0, + START_OFFSET - (next_checkpoint_no * 4)); } + else if (is_opened()) #endif - - log_resize_acquire(); - ut_ad(!resize_in_progress()); - ut_ad(write_lsn == get_lsn()); - ut_ad(write_lsn == get_flushed_lsn(std::memory_order_relaxed)); - { + ut_ad(write_lsn == get_lsn()); + if (buf) /* this may be invoked while creating a new database */ { alignas(16) byte log_block[log_t::WRITE_SIZE_MAX]; @@ -1269,7 +2025,7 @@ void log_t::clear_mmap() noexcept { ut_ad(write_lsn >= first_lsn); uint64_t bf{write_lsn - first_lsn}; - if (bf > capacity()) + if (!archive && bf > capacity()) bf%= capacity(); bf+= START_OFFSET; const size_t bs_1{bs - 1}; @@ -1281,19 +2037,23 @@ void log_t::clear_mmap() noexcept close_file(false); log_mmap= false; - ut_a(attach(log, file_size)); + buf_size= unsigned(std::min(uint64_t{buf_size}, capacity())); + ut_a(attach(log, file_size, READ_WRITE)); ut_ad(!is_mmap()); memcpy_aligned<16>(buf, log_block, bs); } } - log_resize_release(); + + write_lock.release(write_lock.value()); + flush_lock.release(flush_lock.value()); } /** Durably write the log up to log_sys.get_lsn(). */ ATTRIBUTE_COLD void log_write_and_flush() noexcept { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); #ifdef HAVE_PMEM if (log_sys.is_mmap()) log_sys.persist(log_sys.get_lsn()); @@ -1310,6 +2070,7 @@ ATTRIBUTE_COLD void log_t::checkpoint_margin() noexcept { ut_ad(this == &log_sys); ut_ad(!recv_no_log_write); + ut_ad(!recv_sys.rpo); thd_wait_begin(nullptr, THD_WAIT_DISKIO); tpool::tpool_wait_begin(); @@ -1327,21 +2088,24 @@ ATTRIBUTE_COLD void log_t::checkpoint_margin() noexcept } const lsn_t last{last_checkpoint_lsn}, max_age{max_checkpoint_age}; - lsn_t lsn{get_lsn()}; + lsn_t lsn{first_lsn}; - ut_ad(last >= first_lsn); + if (last < lsn); + else if (!archive) { - if (lsn - last <= max_age) + lsn= get_lsn(); + if (lsn - last <= max_age || DBUG_IF("ib_log_checkpoint_avoid_hard")) { -#ifndef DBUG_OFF - skip_checkpoint: -#endif + done: set_check_for_checkpoint(false); goto func_exit; } - DBUG_EXECUTE_IF("ib_log_checkpoint_avoid_hard", goto skip_checkpoint;); lsn-= max_age; } + else if (resize_log.is_opened()) + lsn+= file_size; + else + goto done; mysql_mutex_lock(&buf_pool.flush_list_mutex); @@ -1362,10 +2126,7 @@ void log_free_check() noexcept { ut_ad(!lock_sys.is_holder()); if (log_sys.check_for_checkpoint()) - { - ut_ad(!recv_no_log_write); log_sys.checkpoint_margin(); - } } #ifdef __linux__ @@ -1382,6 +2143,7 @@ ATTRIBUTE_COLD lsn_t logs_empty_and_mark_files_at_shutdown() noexcept sql_print_information("InnoDB: Starting shutdown..."); ut_ad(buf_pool.is_initialised() || !srv_was_started); + ut_ad(!srv_read_only_mode || recv_sys.rpo); /* Wait until the master task and all other operations are idle: our algorithm only works if the server is idle at shutdown */ @@ -1395,7 +2157,7 @@ ATTRIBUTE_COLD lsn_t logs_empty_and_mark_files_at_shutdown() noexcept srv_shutdown_state = SRV_SHUTDOWN_CLEANUP; if (srv_buffer_pool_dump_at_shutdown && - !srv_read_only_mode && srv_fast_shutdown < 2) { + !recv_sys.rpo && srv_fast_shutdown < 2) { buf_dump_start(); } srv_monitor_timer.reset(); @@ -1416,7 +2178,7 @@ ATTRIBUTE_COLD lsn_t logs_empty_and_mark_files_at_shutdown() noexcept shutdown, because the InnoDB layer may have committed or prepared transactions and we don't want to lose them. */ - if (ulint total_trx = srv_was_started && !srv_read_only_mode + if (ulint total_trx = srv_was_started && !recv_sys.rpo && srv_force_recovery < SRV_FORCE_NO_TRX_UNDO ? trx_sys.any_active_transactions() : 0) { @@ -1438,7 +2200,7 @@ ATTRIBUTE_COLD lsn_t logs_empty_and_mark_files_at_shutdown() noexcept const char* thread_name= nullptr; if (thread_name) { - ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); wait_suspend_loop: service_manager_extend_timeout( unsigned(COUNT_INTERVAL / 5), @@ -1463,7 +2225,7 @@ ATTRIBUTE_COLD lsn_t logs_empty_and_mark_files_at_shutdown() noexcept if (buf_pool.is_initialised()) { if (srv_fast_shutdown != 2 && !srv_read_only_mode && srv_was_started) { - log_make_checkpoint(); + buf_flush_sync_batch(0, !recv_sys.rpo); } buf_load_dump_end(); @@ -1478,7 +2240,7 @@ ATTRIBUTE_COLD lsn_t logs_empty_and_mark_files_at_shutdown() noexcept } mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (srv_fast_shutdown == 2 && !srv_read_only_mode) { + if (srv_fast_shutdown == 2 && !recv_sys.rpo) { sql_print_information( "InnoDB: Executing innodb_fast_shutdown=2." " Next startup will execute crash recovery!"); @@ -1506,6 +2268,7 @@ ATTRIBUTE_COLD lsn_t logs_empty_and_mark_files_at_shutdown() noexcept ut_d(mysql_mutex_unlock(&buf_pool.mutex)); ut_a(lsn == log_sys.last_checkpoint_lsn + SIZE_OF_FILE_CHECKPOINT + 8 * log_sys.is_encrypted() + || recv_sys.rpo || srv_force_recovery == SRV_FORCE_NO_LOG_REDO); ut_a(lsn >= recv_sys.lsn); return lsn; @@ -1556,10 +2319,10 @@ void log_t::close() recv_sys.close(); } -std::string get_log_file_path(const char *filename) +ATTRIBUTE_COLD std::string log_t::get_circular_path(size_t i) { - const size_t size= strlen(srv_log_group_home_dir) + /* path separator */ 1 + - strlen(filename) + /* longest suffix */ 3; + ut_ad(i <= 101); + const size_t size= strlen(srv_log_group_home_dir) + sizeof "/ib_logfile101"; std::string path; path.reserve(size); path.assign(srv_log_group_home_dir); @@ -1573,7 +2336,10 @@ std::string get_log_file_path(const char *filename) default: path.push_back('/'); } - path.append(filename); + return path.append("ib_logfile").append(std::to_string(i)); +} - return path; +ATTRIBUTE_COLD std::string log_t::get_path() const +{ + return archive ? get_archive_path() : get_circular_path(); } diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index c7dcb530504d5..19c1d79a2e7e9 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -59,6 +59,8 @@ Created 9/20/1997 Heikki Tuuri /** The recovery system */ recv_sys_t recv_sys; +/** 0 or the first LSN that would conflict with innodb_log_recovery_target */ +static lsn_t recv_sys_rpo_exceeded; /** TRUE when recv_init_crash_recovery() has been called. */ bool recv_needed_recovery; #ifdef UNIV_DEBUG @@ -1435,7 +1437,6 @@ void recv_sys_t::create() found_corrupt_log = false; found_corrupt_fs = false; file_checkpoint = 0; - progress_time = time(NULL); ut_ad(pages.empty()); pages_it = pages.end(); @@ -1488,10 +1489,10 @@ void recv_sys_t::debug_free() recv_needed_recovery= false; pages.clear(); pages_it= pages.end(); + log_archive.clear(); tmp_free(); mysql_mutex_unlock(&mutex); - log_sys.clear_mmap(); } /** Free a redo log snippet. @@ -1690,49 +1691,342 @@ static dberr_t recv_log_recover_10_5(lsn_t lsn_offset) return DB_SUCCESS; } +/** @return if the specified innodb_log_recovery_target is being violated */ +static bool recv_sys_invalid_rpo(lsn_t lsn) noexcept +{ + if (!recv_sys.rpo || recv_sys.rpo >= lsn) + return false; + sql_print_error("InnoDB: cannot fulfill innodb_log_recovery_target=%" + PRIu64 "<%" PRIu64, recv_sys.rpo, lsn); + log_sys.set_recovered_lsn(lsn); + return true; +} + +inline void log_t::stash_archive_file() noexcept +{ + ut_ad(log.is_opened()); + ut_ad(archive); + ut_ad(is_mmap() == !checkpoint_buf); + if (resize_log.is_opened()) + { + ut_ad(!is_mmap() == !resize_buf); + if (resize_buf) + my_munmap(resize_buf, size_t(resize_target)); + resize_log.close(); + } + if (is_mmap()) + { + resize_buf= buf; + buf= nullptr; + } + std::swap(log, resize_log); + resize_target= file_size; + writer= nullptr; +} + dberr_t recv_sys_t::find_checkpoint() { - bool wrong_size= false; byte *buf; + lsn_t first_lsn= 0; + bool read_only{srv_read_only_mode || srv_operation >= SRV_OPERATION_BACKUP}; + os_offset_t size= 0; ut_ad(pages.empty()); + ut_ad(log_archive.empty()); pages_it= pages.end(); if (files.empty()) { file_checkpoint= 0; - std::string path{get_log_file_path()}; + int archive= log_sys.archive; + retry: + std::string path{log_sys.get_circular_path()}; bool success; - os_file_t file{os_file_create_func(path.c_str(), - OS_FILE_OPEN, - OS_LOG_FILE, - srv_read_only_mode, &success)}; - if (file == OS_FILE_CLOSED) + os_file_t file{os_file_create_func(path.c_str(), archive < 0 + ? OS_FILE_OPEN : OS_FILE_OPEN_SILENT, + OS_LOG_FILE, read_only, &success)}; + if (file != OS_FILE_CLOSED) + { + if (archive > 0) + { + sql_print_error("InnoDB: innodb_log_archive=ON but %s exists", + path.c_str()); + return DB_ERROR; + } + } + else if (archive < 0 || srv_operation != SRV_OPERATION_NORMAL) return DB_ERROR; - const os_offset_t size{os_file_get_size(file)}; + else + { + path.reserve(strlen(srv_log_group_home_dir) + + sizeof "/ib_0000000000000000.log"); + if (!tmp_buf) + { + tmp_buf= static_cast + (ut_malloc_dontdump(tmp_buf_size, PSI_INSTRUMENT_ME)); + if (!tmp_buf) + return DB_OUT_OF_MEMORY; + } +#ifdef _WIN32 + WIN32_FIND_DATAA entry; + path.assign(srv_log_group_home_dir); + switch (path.back()) { + case '\\': case '/': + break; + default: + path.push_back('/'); + } + path.append("ib_????????????????.log"); + HANDLE d= FindFirstFileA(path.c_str(), &entry); + if (d != INVALID_HANDLE_VALUE) + goto readdir; +#else + DIR *d= opendir(srv_log_group_home_dir); + if (d) + goto readdir; +#endif + no_archive_found: + if (archive) + { + sql_print_error("InnoDB: innodb_log_archive files not found in '%s'", + srv_log_group_home_dir); + return DB_ERROR; + } + archive= -1; + goto retry; + + readdir: +#ifdef _WIN32 + do + { + if (entry.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + continue; + lsn_t lsn; + int n{0}; + const char *fn{entry.cFileName}; + if (1 != sscanf(fn, LOG_ARCHIVE_NAME "%n", &lsn, &n) || fn[n] || + lsn < log_t::FIRST_LSN) + continue; + LARGE_INTEGER filesize; + filesize.LowPart= entry.nFileSizeLow; + filesize.HighPart= entry.nFileSizeHigh; + if ((filesize.LowPart & 4095) || + lsn_t(filesize.QuadPart) > log_t::ARCHIVE_FILE_SIZE_MAX || + lsn_t(filesize.QuadPart) < log_t::FILE_SIZE_MIN) + { + sql_print_warning("InnoDB: ignoring %s", fn); + continue; + } + size= filesize.QuadPart; + log_archive.emplace + (lsn, archive_log{lsn - log_t::START_OFFSET + size, + log_t::log_access(entry.dwFileAttributes & + FILE_ATTRIBUTE_READONLY)}); + } + while (FindNextFile(d, &entry)); + FindClose(d); +#else + while (dirent *e= readdir(d)) + { + lsn_t lsn; + int n{0}; + const char *fn{e->d_name}; + if (1 != sscanf(fn, LOG_ARCHIVE_NAME "%n", &lsn, &n) || fn[n] || + lsn < log_t::FIRST_LSN) + continue; + path.assign(srv_log_group_home_dir); + path.push_back('/'); + struct stat st; + if (stat(log_sys.append_archive_name(path, lsn).c_str(), &st) || + (st.st_size & 4095) || + lsn_t(st.st_size) > log_t::ARCHIVE_FILE_SIZE_MAX || + lsn_t(st.st_size) < log_t::FILE_SIZE_MIN) + { + sql_print_warning("InnoDB: ignoring %s", path.c_str()); + continue; + } + size= st.st_size; + log_archive.emplace + (lsn, archive_log{lsn - log_t::START_OFFSET + size, + log_t::log_access(!(st.st_mode & 0200))}); + } + closedir(d); +#endif + + const archive_map::const_iterator end= log_archive.cend(); + archive_map::const_iterator found_recovery_start= end; + archive_map::iterator i= log_archive.begin(), start= i; + if (i == end) + goto no_archive_found; + log_sys.format= srv_encrypt_log + ? log_t::FORMAT_ENC_11 : log_t::FORMAT_10_8; + log_sys.archive= true; + for (;;) + { + const archive_map::iterator prev= i++; + const lsn_t last{prev->second.end}; + if (recovery_start >= prev->first && recovery_start < last) + found_recovery_start= prev; + if (i == end) + break; + if (last == i->first) + prev->second.access= log_t::READ_ONLY; + else + { + log_archive.erase(log_archive.begin(), start= i); + found_recovery_start= end; + } + } + + if (recovery_start && found_recovery_start == end) + { + sql_print_error("InnoDB: No matching file found for" + " innodb_log_recovery_start=" LSN_PF, recovery_start); + return DB_ERROR; + } + + i= log_archive.end(); + byte last_checkpoint_buf[log_t::WRITE_SIZE_MAX]; + for (uint16_t last_checkpoint_no= log_sys.next_checkpoint_no= UINT16_MAX; + i != start; ) + { + --i; + if (uint16_t(~last_checkpoint_no) && + recovery_start && i != found_recovery_start) + { + log_sys.unstash_archive_file(); + continue; + } + path.assign(srv_log_group_home_dir); + switch (path.back()) { +#ifdef _WIN32 + case '\\': +#endif + case '/': + break; + default: + path.push_back('/'); + } + read_only= bool(i->second.access); + ut_ad(log_t::log_access(read_only) == i->second.access); + const bool open_read_only{read_only || rpo || srv_read_only_mode}; + i->second.access= log_t::log_access(open_read_only); + static_assert(log_t::READ_WRITE == log_t::log_access(false), ""); + static_assert(log_t::READ_ONLY == log_t::log_access(true), ""); + file= + os_file_create_func(log_sys.append_archive_name(path, i->first). + c_str(), OS_FILE_OPEN, OS_LOG_FILE, + open_read_only, &success); + if (file == OS_FILE_CLOSED) + return DB_ERROR; + + if (UNIV_UNLIKELY(log_sys.buf_size > i->second.end - i->first)) + log_sys.buf_size= unsigned(i->second.end - i->first); + + if (!log_sys.attach(file, i->second.end - i->first + + log_t::START_OFFSET, log_t::READ_ONLY)) + { + os_file_close(file); + return DB_ERROR; + } + const dberr_t err= + find_checkpoint_archived(i->first, !read_only && i != start); + if (!uint16_t(~last_checkpoint_no)) + { + /* Determine the end of checkpoints in the last log file. */ + if (err == DB_SUCCESS) + { + last_checkpoint_no= log_sys.next_checkpoint_no; + ut_ad(last_checkpoint_no > uint16_t(8 * log_sys.is_encrypted())); + ut_ad(last_checkpoint_no <= uint16_t{log_t::START_OFFSET}); + if (!recovery_start || i == found_recovery_start) + return DB_SUCCESS; + log_sys.next_checkpoint_no= UINT16_MAX; + if (const byte *checkpoint_buf= log_sys.checkpoint_buf) + { + /* Clear any garbage that may have been left behind by a + crash during log_t::write_checkpoint() or log_t::set_archive(). + See also log_t::clear_mmap() and log_t::attach(). */ + const size_t bs{log_sys.write_size}; + memcpy(last_checkpoint_buf, checkpoint_buf, bs); + const size_t tail= (last_checkpoint_no * 4) & (bs - 1); + memset(last_checkpoint_buf + tail, 0, bs - tail); + } + goto next; + } + + const byte *buf= log_sys.checkpoint_buf; + if (!buf) + buf= log_sys.buf; + switch (mach_read_from_4(my_assume_aligned<4>(buf))) { + case log_t::FORMAT_10_8: + case log_t::FORMAT_ENC_11: + if (recv_check_log_block(buf)) + { + was_archive= true; +#ifdef HAVE_PMEM + if (log_sys.is_mmap_writeable()) + log_sys.log.close(); +#endif + log_sys.archive= false; + file= OS_FILE_CLOSED; + goto circular_log_recovery; + } + /* fall through */ + default: + last_checkpoint_no= uint16_t(8 * log_sys.is_encrypted()); + } + } + else if (err == DB_SUCCESS) + { + ut_ad(!recovery_start || i == found_recovery_start); + /* Restore the checkpoint number of the last log file. */ + log_sys.next_checkpoint_no= last_checkpoint_no; + ut_ad(log_sys.is_mmap() == !log_sys.checkpoint_buf); + if (byte *checkpoint_buf= log_sys.checkpoint_buf) + memcpy(checkpoint_buf, last_checkpoint_buf, log_sys.write_size); + return DB_SUCCESS; + } + + if ((recovery_start ? i == found_recovery_start : read_only) || + i == start) + return err; + next: + log_sys.stash_archive_file(); + } + } + + ut_ad(!log_sys.archive); + size= os_file_get_size(file); if (!size) { if (srv_operation != SRV_OPERATION_NORMAL) goto too_small; } - else if (size < log_t::START_OFFSET + SIZE_OF_FILE_CHECKPOINT) + else { - too_small: - sql_print_error("InnoDB: File %.*s is too small", - int(path.size()), path.data()); - err_exit: - os_file_close(file); - return DB_ERROR; + if (size < log_t::START_OFFSET + SIZE_OF_FILE_CHECKPOINT) + { + too_small: + sql_print_error("InnoDB: File %s is too small", path.c_str()); + err_exit: + os_file_close(file); + return DB_ERROR; + } + else if (!log_sys.attach(file, size, log_t::log_access(read_only))) + goto err_exit; + else + file= OS_FILE_CLOSED; } - else if (!log_sys.attach(file, size)) - goto err_exit; - else - file= OS_FILE_CLOSED; - recv_sys.files.emplace_back(file); + circular_log_recovery: + ut_ad(!log_sys.archive); + ut_ad(!log_sys.is_mmap_writeable() || !log_sys.is_opened()); + files.emplace_back(file); + for (int i= 1; i < 101; i++) { - path= get_log_file_path(LOG_FILE_NAME_PREFIX).append(std::to_string(i)); + path= log_sys.get_circular_path(i); file= os_file_create_func(path.c_str(), OS_FILE_OPEN_SILENT, OS_LOG_FILE, true, &success); @@ -1744,14 +2038,14 @@ dberr_t recv_sys_t::find_checkpoint() sql_print_error("InnoDB: Log file %.*s is of different size " UINT64PF " bytes than other log files " UINT64PF " bytes!", int(path.size()), path.data(), sz, size); - wrong_size= true; + first_lsn= LSN_MAX; } - recv_sys.files.emplace_back(file); + files.emplace_back(file); } if (!size) { - if (wrong_size) + if (first_lsn == LSN_MAX) return DB_CORRUPTION; lsn= log_sys.last_checkpoint_lsn; log_sys.format= log_t::FORMAT_3_23; @@ -1760,7 +2054,10 @@ dberr_t recv_sys_t::find_checkpoint() } else ut_ad(srv_operation == SRV_OPERATION_BACKUP); - log_sys.last_checkpoint_lsn= 0; + + ut_ad(!log_sys.last_checkpoint_lsn); + ut_ad(!log_sys.archive); + ut_ad(!file_checkpoint); lsn= 0; buf= my_assume_aligned<4096>(log_sys.buf); if (!log_sys.is_mmap()) @@ -1771,7 +2068,7 @@ dberr_t recv_sys_t::find_checkpoint() log_sys.format= mach_read_from_4(buf + LOG_HEADER_FORMAT); if (log_sys.format == log_t::FORMAT_3_23) { - if (wrong_size) + if (first_lsn == LSN_MAX) return DB_CORRUPTION; if (dberr_t err= recv_log_recover_pre_10_2()) return err; @@ -1779,7 +2076,14 @@ dberr_t recv_sys_t::find_checkpoint() memset_aligned<4096>(const_cast(field_ref_zero), 0, 4096); /* Mark the redo log for upgrading. */ lsn= file_checkpoint= log_sys.last_checkpoint_lsn; + log_sys.archived_checkpoint= lsn; log_sys.set_recovered_lsn(lsn); + if (rpo && rpo != lsn) + { + sql_print_error("InnoDB: cannot fulfill innodb_log_recovery_target=%" + PRIu64 "!=%" PRIu64, rpo, lsn); + return DB_CORRUPTION; + } if (UNIV_LIKELY(lsn != 0)) scanned_lsn= lsn; log_sys.next_checkpoint_no= 0; @@ -1792,7 +2096,7 @@ dberr_t recv_sys_t::find_checkpoint() return DB_CORRUPTION; } - const lsn_t first_lsn{mach_read_from_8(buf + LOG_HEADER_START_LSN)}; + first_lsn= mach_read_from_8(buf + LOG_HEADER_START_LSN); log_sys.set_first_lsn(first_lsn); char creator[LOG_HEADER_CREATOR_END - LOG_HEADER_CREATOR + 1]; memcpy(creator, buf + LOG_HEADER_CREATOR, sizeof creator); @@ -1848,15 +2152,19 @@ dberr_t recv_sys_t::find_checkpoint() } if (checkpoint_lsn >= log_sys.last_checkpoint_lsn) - { - log_sys.last_checkpoint_lsn= checkpoint_lsn; - log_sys.next_checkpoint_no= field == log_t::CHECKPOINT_1; - lsn= end_lsn; - } + log_sys.set_recovered_checkpoint(checkpoint_lsn, lsn= end_lsn, + field == log_t::CHECKPOINT_1); } - if (!log_sys.last_checkpoint_lsn) + log_sys.archived_checkpoint= log_sys.last_checkpoint_lsn; + if (!log_sys.archived_checkpoint) goto got_no_checkpoint; - if (!memcmp(creator, "Backup ", 7)) + else if (!log_sys.archived_lsn) + log_sys.archived_lsn= lsn; + if (recv_sys_invalid_rpo(lsn)) + return DB_READ_ONLY; + + if (!memcmp(creator, "Backup ", 7) && + !recv_sys.rpo && !high_level_read_only) srv_start_after_restore= true; if (!tmp_buf) @@ -1922,7 +2230,7 @@ dberr_t recv_sys_t::find_checkpoint() return DB_ERROR; } - if (wrong_size) + if (first_lsn == LSN_MAX) return DB_CORRUPTION; if (dberr_t err= recv_log_recover_10_5(lsn_offset)) @@ -2218,10 +2526,16 @@ struct recv_buf constexpr bool operator==(const recv_buf other) const { return ptr == other.ptr; } - static const byte *end() { return &log_sys.buf[recv_sys.len]; } + static const byte *end() noexcept { return &log_sys.buf[recv_sys.len]; } - static constexpr bool may_wrap() { return false; } - static constexpr bool is_wrapped(const recv_buf&) { return false; } + size_t get_offset() const noexcept + { + size_t offset= size_t(ptr - log_sys.buf); + ut_ad(offset < recv_sys.len); + return offset; + } + + static constexpr bool is_split(const recv_buf&) noexcept { return false; } bool is_eof(size_t len= 0) const noexcept { return ptr + len >= end(); } byte operator*() const noexcept { return *ptr; } @@ -2242,6 +2556,9 @@ struct recv_buf { return my_crc32c(crc, start.ptr, ptr - start.ptr); } + + void *memcpy(void *buf, size_t size) const noexcept + { return ::memcpy(buf, ptr, size); } }; /** Ring buffer wrapper for log_sys.buf[]; recv_sys.len == log_sys.file_size */ @@ -2249,8 +2566,20 @@ struct recv_ring : public recv_buf { constexpr recv_ring(const byte *ptr) : recv_buf(ptr) {} - static constexpr bool may_wrap() { return true; } - bool is_wrapped(const recv_ring &end) const { return end.ptr < ptr; } + size_t get_offset() const noexcept + { + size_t offset= size_t(ptr - log_sys.buf); + if (offset == log_sys.file_size) + { + ut_ad(log_sys.is_mmap()); + offset= log_sys.START_OFFSET; + } + else + ut_ad(offset < log_sys.file_size); + return offset; + } + + bool is_split(const recv_ring &end) const noexcept { return end.ptr < ptr; } constexpr static bool is_eof() { return false; } constexpr static bool is_eof(size_t) { return false; } @@ -2320,6 +2649,131 @@ struct recv_ring : public recv_buf &log_sys.buf[log_sys.START_OFFSET], ptr - &log_sys.buf[log_sys.START_OFFSET]); } + + void *memcpy(void *buf, size_t size) const noexcept + { + size_t wrap_size(end() - ptr); + ut_ad(wrap_size < size); + ::memcpy(buf, ptr, wrap_size); + ::memcpy(static_cast(buf) + wrap_size, + &log_sys.buf[log_sys.START_OFFSET], size - wrap_size); + return buf; + } +}; + +/** Buffer wrapper for memory-mapped log_sys.archive, +with the capability to warp from log_sys.buf to log_sys.resize_buf */ +struct recv_warp : public recv_buf +{ + constexpr recv_warp(const byte *ptr) : recv_buf(ptr) {} + + size_t get_offset() const noexcept + { + size_t offset= size_t(ptr - log_sys.buf); + if (offset < recv_sys.len) + return offset; + offset= size_t(ptr - &log_sys.resize_buf[log_sys.START_OFFSET]); + ut_ad(offset < log_sys.resize_target); + return recv_sys.len + offset; + } + + bool is_split(const recv_warp &end) const noexcept + { + const size_t len{recv_sys.len}; + const byte *buf{log_sys.buf}; + int db= (size_t(end.ptr - buf) < len) - (size_t(ptr - buf) < len); + ut_ad(db <= 0); + return db != 0; + } + + constexpr static bool is_eof() { return false; } + constexpr static bool is_eof(size_t) { return false; } + + byte operator*() const noexcept + { + ut_ad((ptr >= &log_sys.buf[log_sys.START_OFFSET] && ptr < end()) || + (ptr >= &log_sys.resize_buf[log_sys.START_OFFSET] && + ptr < &log_sys.resize_buf[log_sys.resize_target])); + return *ptr; + } + recv_warp operator+(size_t len) const noexcept + { recv_warp r{*this}; return r+= len; } + recv_warp &operator++() noexcept { return *this+= 1; } + recv_warp &operator+=(size_t len) noexcept + { + ut_ad(len < recv_sys.MTR_SIZE_MAX * 2); + const byte *const e{end()}; + const bool first{ptr < e && ptr >= log_sys.buf}; + ut_ad(!first || ptr >= &log_sys.buf[log_sys.START_OFFSET]); + ptr+= len; + if (first) + { + if (ptr < e) + return *this; + ptr= &log_sys.resize_buf[log_sys.START_OFFSET + (ptr - e)]; + } + ut_ad(ptr >= &log_sys.resize_buf[log_sys.START_OFFSET]); + ut_ad(ptr < &log_sys.resize_buf[log_sys.resize_target]); + return *this; + } + size_t operator-(const recv_warp &start) const noexcept + { + return start.is_split(*this) + ? size_t((ptr - &log_sys.resize_buf[log_sys.START_OFFSET]) + + (end() - start.ptr)) + : size_t(ptr - start.ptr); + } + + uint32_t decode_varint() const noexcept + { + recv_warp log{*this}; + uint32_t i{*log}; + if (i < MIN_2BYTE) + return i; + uint32_t j{*++log}; + if (i < 0xc0) + return MIN_2BYTE + ((i & ~0xc0) << 8 | j); + j<<= 8; + j|= *++log; + if (i < 0xe0) + return MIN_3BYTE + ((i & ~0xe0) << 16 | j); + j<<= 8; + j|= *++log; + if (i < 0xf0) + return MIN_4BYTE + ((i & ~0xf0) << 24 | j); + if (i == 0xf0) + { + j<<= 8; + j|= *++log; + if (j <= ~MIN_5BYTE) + return MIN_5BYTE + j; + } + return MLOG_DECODE_ERROR; + } + + uint32_t crc32c(uint32_t crc, const recv_warp &start) const noexcept + { + return start.is_split(*this) + ? my_crc32c(my_crc32c(crc, start.ptr, end() - start.ptr), + &log_sys.resize_buf[log_sys.START_OFFSET], + ptr - &log_sys.resize_buf[log_sys.START_OFFSET]) + : my_crc32c(crc, start.ptr, ptr - start.ptr); + } + + void *memcpy(void *buf, size_t size) const noexcept + { + if (is_split(*this + size)) + { + size_t wrap_size(end() - ptr); + ut_ad(wrap_size < size); + ::memcpy(buf, ptr, wrap_size); + ::memcpy(static_cast(buf) + wrap_size, + &log_sys.resize_buf[log_sys.START_OFFSET], size - wrap_size); + return buf; + } + else + return ::memcpy(buf, ptr, size); + } }; ATTRIBUTE_COLD @@ -2428,7 +2882,11 @@ recv_sys_t::parse_mtr_result log_parse_start(source &l, unsigned nonce) return recv_sys_t::PREMATURE_EOF; eom_found: - if (*l != log_sys.get_sequence_bit((l - begin) + recv_sys.lsn)) + const lsn_t end_lsn{(l - begin) + recv_sys.lsn}; + + /* The sequence bit only matters for detecting log file wrap-around, + which is only possible in the innodb_log_archive=OFF format. */ + if (!log_sys.archive && *l != log_sys.get_sequence_bit(end_lsn)) return recv_sys_t::GOT_EOF; if (l.is_eof(5 + nonce)) @@ -2441,19 +2899,24 @@ recv_sys_t::parse_mtr_result log_parse_start(source &l, unsigned nonce) crc32c= crc.crc32c(crc32c, l + 1); ut_ad(!crc.is_eof(3)); - if (!crc.is_wrapped(crc + 3)) + if (!crc.is_split(crc + 3)) stored_crc32c= mach_read_from_4(crc.ptr); else { byte b[4]; - size_t wrap_size(crc.end() - crc.ptr); - ut_ad(wrap_size < 4); - memcpy(b, crc.ptr, wrap_size); - memcpy(b + wrap_size, &log_sys.buf[log_sys.START_OFFSET], 4 - wrap_size); - stored_crc32c= mach_read_from_4(b); + stored_crc32c= mach_read_from_4(static_cast(crc.memcpy(b, 4))); } - return crc32c == stored_crc32c ? recv_sys_t::OK : recv_sys_t::GOT_EOF; + if (UNIV_UNLIKELY(crc32c != stored_crc32c)) + return recv_sys_t::GOT_EOF; + + if (recv_sys.rpo && recv_sys.rpo < end_lsn) + { + recv_sys_rpo_exceeded= end_lsn; + return recv_sys_t::GOT_EOF; + } + + return recv_sys_t::OK; } /** Parse and register one log_t::FORMAT_10_8 mini-transaction. @@ -2475,7 +2938,7 @@ recv_sys_t::parse_mtr_result recv_sys_t::parse(source l, bool if_exists) (srv_operation == SRV_OPERATION_BACKUP || srv_operation == SRV_OPERATION_BACKUP_NO_DEFER)); mysql_mutex_assert_owner(&mutex); - ut_ad(log_sys.last_checkpoint_lsn); + ut_ad(log_sys.last_checkpoint_lsn || log_sys.archive); ut_ad(log_sys.is_recoverable()); ut_ad(log_sys.format == format); @@ -2498,33 +2961,21 @@ recv_sys_t::parse_mtr_result recv_sys_t::parse(source l, bool if_exists) const size_t s{l - begin}; ut_ad(s + 9 <= tmp_buf_size); constexpr size_t tail_size{format == log_t::FORMAT_10_8 ? 1 + 4 : 1 + 8 + 4}; - const bool is_wrapped{begin.is_wrapped(l + (tail_size - 5))}; + const bool is_split{begin.is_split(l + (tail_size - 5))}; l+= tail_size; start_offset= begin.ptr - log_sys.buf; - offset= l.ptr - log_sys.buf; - if (l.may_wrap() && offset == log_sys.file_size) - { - ut_ad(log_sys.is_mmap()); - offset= log_sys.START_OFFSET; - } - ut_ad(offset < log_sys.file_size); + offset= l.get_offset(); const byte *ptr; - if (format == log_t::FORMAT_ENC_11 || is_wrapped) + if (format == log_t::FORMAT_ENC_11 || is_split) { byte *tmp= tmp_buf; ptr= tmp; - if (format == log_t::FORMAT_ENC_11 && !is_wrapped) + if (format == log_t::FORMAT_ENC_11 && !is_split) memcpy(tmp, begin.ptr, s + 9); else - { - size_t wrap_size(begin.end() - begin.ptr); - ut_ad(wrap_size < s + 9); - memcpy(tmp, begin.ptr, wrap_size); - memcpy(tmp + wrap_size, &log_sys.buf[log_sys.START_OFFSET], - s + 9 - wrap_size); - } + begin.memcpy(tmp, s + 9); if (format == log_t::FORMAT_ENC_11) log_decrypt_mtr(tmp, ptr + s); } @@ -2751,14 +3202,14 @@ log_parse_file(const page_id_t id, bool if_exists, ? "ignored" : recv_sys.file_checkpoint ? "reread" : "read", recv_sys.lsn); - if (c == log_sys.last_checkpoint_lsn) + /* There can be multiple FILE_CHECKPOINT for the same LSN. */ + if (!recv_sys.file_checkpoint) { - /* There can be multiple FILE_CHECKPOINT for the same LSN. */ - if (!recv_sys.file_checkpoint) - { - recv_sys.file_checkpoint= recv_sys.lsn; - return recv_sys_t::GOT_EOF; - } + ut_ad(log_sys.last_checkpoint_lsn || log_sys.archive); + if (!log_sys.last_checkpoint_lsn) + log_sys.last_checkpoint_lsn= c; + recv_sys.file_checkpoint= recv_sys.lsn; + return recv_sys_t::GOT_EOF; } } break; @@ -2803,6 +3254,18 @@ log_parse_file(const page_id_t id, bool if_exists, break; } + if (!log_sys.last_checkpoint_lsn) + { + /* We are currently validating checkpoints in + recv_log_t::find_checkpoint_archived(). We must not open and + validate data files until we actually start recovery from a + checkpoint, because there could be lots of FILE_MODIFY and + FILE_CHECKPOINT log records to be parsed. */ + ut_ad(!recv_sys.file_checkpoint); + ut_ad(log_sys.archive); + return recv_sys_t::OK; + } + fil_name_process(reinterpret_cast(l), fnend - l, space_id, fn2 ? FILE_MODIFY : mfile_type_t(b & 0xf0), recv_sys.start_lsn, if_exists); @@ -2850,7 +3313,7 @@ log_page_modify(uint32_t space_id, uint32_t page_no) noexcept FILE_CHECKPOINT. There should be a FILE_DELETE or FILE_MODIFY for this tablespace later, to be handled in fil_name_process(). */ recv_spaces.emplace_hint(i, space_id, file_name_t("", false)); - else + else if (!srv_read_only_mode) { if (!srv_force_recovery) { @@ -3241,6 +3704,147 @@ recv_sys_t::parse_mtr_result recv_sys_t::parse_mtr(bool if_exists) return recv_sys.parse(s, if_exists); } +inline void log_t::archived_switch_recovery_rewind_checkpoint() noexcept +{ + ut_ad(latch_have_wr()); + ut_ad(archive); + ut_ad(log.is_opened()); + ut_ad(resize_log.is_opened()); + ut_ad(!uint16_t(~next_checkpoint_no)); + ut_ad(is_mmap() == !!resize_buf); + + if (is_mmap()) + std::swap(buf, resize_buf); + + std::swap(log, resize_log); + std::swap(file_size, resize_target); + + first_lsn-= capacity(); +} + +bool log_t::archived_switch_recovery_prepare(lsn_t lsn) noexcept +{ + mysql_mutex_assert_owner(&recv_sys.mutex); + ut_ad(latch_have_wr()); + ut_ad(archive); + ut_ad(!resize_in_progress()); + ut_ad(!resize_flush_buf); + + if (resize_log.is_opened()) + return true; + + ut_ad(!resize_buf); + + recv_sys_t::archive_map::const_iterator i{recv_sys.log_archive.find(lsn)}; + if (i == recv_sys.log_archive.cend()) + return false; + resize_target= i->second.end - lsn + START_OFFSET; + bool success; + std::string path_name{get_archive_path(lsn)}; + const char *const fn= path_name.c_str(); +#ifdef HAVE_PMEM + static_assert(int{PMEM} == -1, ""); +#endif + static_assert(int{READ_WRITE} == 0, ""); + static_assert(int{READ_ONLY} == 1, ""); + resize_log.m_file= os_file_create_func(fn, OS_FILE_OPEN, OS_LOG_FILE, + int{i->second.access} > 0, &success); + ut_ad(success == (resize_log.m_file != OS_FILE_CLOSED)); + if (resize_log.m_file == OS_FILE_CLOSED) + { + fail: + recv_sys.set_corrupt_log(); + return false; + } + + if (is_mmap()) + { +#ifdef _WIN32 + void *ptr= nullptr; + HANDLE h= + CreateFileMappingA(resize_log.m_file, nullptr, PAGE_READONLY, + DWORD(resize_target >> 32), DWORD(resize_target), + nullptr); + if (h != INVALID_HANDLE_VALUE) + { + ptr= MapViewOfFileEx(h, FILE_MAP_READ, 0, 0, resize_target, nullptr); + CloseHandle(h); + } + if (!ptr) +#else + void *ptr= my_mmap(0, size_t(resize_target), PROT_READ, MAP_SHARED, + resize_log.m_file, 0); + if (ptr == MAP_FAILED) +#endif + { + resize_log.close(); + sql_print_error("InnoDB: Unable to map %s (" LSN_PF " bytes)", fn, + resize_target); + goto fail; + } + + resize_buf= static_cast(ptr); + } + + return true; +} + +bool log_t::archived_switch_recovery() noexcept +{ + if (!archived_switch_recovery_prepare(first_lsn + capacity())) + return false; + + if (uint16_t(~next_checkpoint_no)) + { + /* We have completed recv_sys_t::find_checkpoint_archived(), + and archived_switch_recovery_rewind_checkpoint() will not be called. */ + if (is_mmap()) + { + my_munmap(buf, size_t(file_size)); + buf= nullptr; + } + log.close(); + } + + if (is_mmap()) + std::swap(buf, resize_buf); + + std::swap(log, resize_log); + + first_lsn+= capacity(); + std::swap(file_size, resize_target); + + return true; +} + +ATTRIBUTE_COLD void log_t::archived_mmap_switch_recovery_complete() noexcept +{ + ut_ad(archived_mmap_switch()); + ut_ad(buf); + ut_ad(!checkpoint_buf); + ut_ad(recv_sys.offset > capacity()); + + if (uint16_t(~next_checkpoint_no)) + { + my_munmap(buf, size_t(file_size)); + buf= resize_buf; + resize_buf= nullptr; + std::swap(log, resize_log); + resize_log.close(); + } + else + { + std::swap(buf, resize_buf); + std::swap(log, resize_log); + } + + const size_t jump= size_t(capacity()); + recv_sys.offset-= jump; + first_lsn+= jump; + std::swap(file_size, resize_target); + recv_sys.len= size_t(file_size); +} + template recv_sys_t::parse_mtr_result recv_sys_t::parse_mmap(bool if_exists) { @@ -3250,6 +3854,21 @@ recv_sys_t::parse_mtr_result recv_sys_t::parse_mmap(bool if_exists) ut_ad(recv_sys.len == log_sys.file_size); ut_ad(recv_sys.offset >= log_sys.START_OFFSET); ut_ad(recv_sys.offset <= recv_sys.len); + if (log_sys.archive) + { + if (!log_sys.archived_mmap_switch()) + { + ut_d(auto i= recv_sys.log_archive.find(log_sys.get_first_lsn())); + ut_ad(i != recv_sys.log_archive.end()); + ut_d(i++); + ut_ad(i == recv_sys.log_archive.end()); + return GOT_EOF; + } + recv_warp s{&log_sys.buf[recv_sys.offset]}; + auto r= recv_sys.parse(s,if_exists); + log_sys.archived_mmap_switch_recovery_complete(); + return r; + } recv_ring s {recv_sys.offset == recv_sys.len ? &log_sys.buf[log_sys.START_OFFSET] @@ -4236,22 +4855,167 @@ void recv_sys_t::apply(bool last_batch) in ascending order of buf_page_t::oldest_modification. */ log_sort_flush_list(); -#ifdef HAVE_PMEM - if (last_batch && log_sys.is_mmap() && !log_sys.is_opened()) - mprotect(log_sys.buf, len, PROT_READ | PROT_WRITE); -#endif - mysql_mutex_lock(&mutex); ut_d(after_apply= true); clear(); } +/** Find the end of the circular log when multi-batch recovery is needed. */ +static ATTRIBUTE_COLD recv_sys_t::parse_mtr_result +recv_scan_log_circular_skip_the_rest(const recv_sys_t::parser &parser) +{ + ut_ad(!log_sys.archive); + recv_sys_t::parse_mtr_result r; + while ((r= parser(false)) == recv_sys_t::OK); + return r; +} + +/** Find the end of the archived log when multi-batch recovery is needed. */ +static ATTRIBUTE_COLD recv_sys_t::parse_mtr_result +recv_scan_log_archive_skip_the_rest(const recv_sys_t::parser &parser) +{ + ut_ad(log_sys.archive); + recv_sys_t::parse_mtr_result r; + lsn_t first_lsn{log_sys.get_first_lsn()}; + log_sys.archived_switch_recovery_prepare(first_lsn + log_sys.capacity()); + + while ((r= parser(false)) == recv_sys_t::OK) + { + const lsn_t new_first_lsn{log_sys.get_first_lsn()}; + if (UNIV_UNLIKELY(first_lsn != new_first_lsn) && + log_sys.archived_switch_recovery_prepare(new_first_lsn + + log_sys.capacity())) + first_lsn= new_first_lsn; + } + + return r; +} + +static recv_sys_t::parse_mtr_result (*recv_scan_skip_the_rest[2]) +(const recv_sys_t::parser&)= { + recv_scan_log_circular_skip_the_rest, + recv_scan_log_archive_skip_the_rest +}; + +/** Report progress when recovery is taking a long time. */ +static ATTRIBUTE_COLD void recv_report_progress() +{ + if (recv_sys.report(time(nullptr))) + { + const size_t n= recv_sys.pages.size(); + sql_print_information("InnoDB: Parsed redo log up to LSN=" LSN_PF + "; to recover: %zu pages", recv_sys.lsn, n); + service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL, + "Parsed redo log up to LSN=" LSN_PF + "; to recover: %zu pages", recv_sys.lsn, n); + } +} + +/** Parse and store records in a circular log file. */ +static ATTRIBUTE_COLD recv_sys_t::parse_mtr_result +recv_scan_circular_store(const recv_sys_t::parser &parser, bool last_phase) +{ + ut_ad(!log_sys.archive); + recv_sys_t::parse_mtr_result r; + uint16_t count= 0; + while ((r= parser(last_phase)) == recv_sys_t::OK) + if (!++count) + recv_report_progress(); + return r; +} + +/** Parse and store records in the archived log. */ +static ATTRIBUTE_COLD recv_sys_t::parse_mtr_result +recv_scan_archive_store(const recv_sys_t::parser &parser, bool last_phase) +{ + ut_ad(log_sys.archive); + lsn_t first_lsn{log_sys.get_first_lsn()}; + log_sys.archived_switch_recovery_prepare(first_lsn + log_sys.capacity()); + recv_sys_t::parse_mtr_result r; + uint16_t count= 0; + while ((r= parser(last_phase)) == recv_sys_t::OK) + { + const lsn_t new_first_lsn{log_sys.get_first_lsn()}; + if (UNIV_UNLIKELY(first_lsn != new_first_lsn) && + log_sys.archived_switch_recovery_prepare(new_first_lsn + + log_sys.capacity())) + first_lsn= new_first_lsn; + if (!++count) + recv_report_progress(); + } + return r; +} + +static recv_sys_t::parse_mtr_result (*recv_scan_store[2]) +(const recv_sys_t::parser&,bool)= { + recv_scan_circular_store, recv_scan_archive_store +}; + +ATTRIBUTE_COLD void log_t::unstash_archive_file() noexcept +{ + ut_ad(latch_have_wr()); + ut_ad(archive); + ut_ad(is_mmap() == !checkpoint_buf); + + if (resize_log.is_opened()) + { + ut_ad(!is_mmap() == !resize_buf); + if (resize_buf) + { + my_munmap(resize_buf, size_t(resize_target)); + resize_buf= nullptr; + } + resize_log.close(); + } +} + +ATTRIBUTE_COLD void log_t::recovery_rewind(lsn_t lsn) noexcept +{ + mysql_mutex_assert_owner(&recv_sys.mutex); + ut_ad(latch_have_wr()); + ut_ad(uint16_t(~next_checkpoint_no) || !archive); + ut_ad(!resize_in_progress()); + ut_ad(!resize_flush_buf); + ut_ad(recv_sys.file_checkpoint); + ut_ad(recv_sys.lsn >= lsn); + recv_sys.lsn= lsn; + + if (lsn >= first_lsn) + return; + ut_ad(archive); + ut_ad(log.is_opened()); + if (!archive) + return; /* safety */ + + unstash_archive_file(); + recv_sys_t::archive_map::const_iterator i= + recv_sys.log_archive.upper_bound(lsn); + if (i == recv_sys.log_archive.cbegin() || + !archived_switch_recovery_prepare((--i)->first)) + recv_sys.set_corrupt_log(); + else + { + ut_ad(lsn_t(i->second.end - i->first + START_OFFSET) == resize_target); + first_lsn= i->first; + log.close(); + if (is_mmap()) + { + my_munmap(buf, size_t(file_size)); + buf= resize_buf; + resize_buf= nullptr; + } + std::swap(log, resize_log); + std::swap(file_size, resize_target); + } +} + /** Scan log and store records to the parsing buffer. @param last_phase whether changes can be applied to the tablespaces @param parser log parsers for store::NO and store::YES @return whether rescan is needed (not everything was stored) */ -static bool recv_scan_log(bool last_phase, const recv_sys_t::parser *parser) +static bool +recv_scan_log(const bool last_phase, const recv_sys_t::parser *parser) noexcept { DBUG_ENTER("recv_scan_log"); @@ -4264,7 +5028,8 @@ static bool recv_scan_log(bool last_phase, const recv_sys_t::parser *parser) else ut_ad(recv_sys.file_checkpoint); - bool store{recv_sys.file_checkpoint != 0}; + bool store{recv_sys.file_checkpoint != 0}, archive_exhausted{false}; + ut_ad(store || !last_phase); size_t buf_size= log_sys.buf_size; if (log_sys.is_mmap()) { @@ -4279,6 +5044,9 @@ static bool recv_scan_log(bool last_phase, const recv_sys_t::parser *parser) recv_sys.len= 0; } + if (log_sys.archive && log_sys.next_checkpoint_no) + log_sys.archived_switch_recovery_prepare(log_sys.get_first_lsn() + + log_sys.capacity()); lsn_t rewound_lsn= 0; for (ut_d(lsn_t source_offset= 0);;) { @@ -4288,19 +5056,34 @@ static bool recv_scan_log(bool last_phase, const recv_sys_t::parser *parser) #endif if (size_t size= buf_size - recv_sys.len) { + ut_ad(!log_sys.is_mmap()); + const lsn_t end_lsn{recv_sys.lsn + recv_sys.len - recv_sys.offset}; #ifndef UNIV_DEBUG lsn_t #endif - source_offset= - log_sys.calc_lsn_offset(recv_sys.lsn + recv_sys.len - recv_sys.offset); + source_offset= log_sys.calc_lsn_offset(end_lsn); ut_ad(!wrap || source_offset == log_t::START_OFFSET); source_offset&= ~block_size_1; if (source_offset + size > log_sys.file_size) size= static_cast(log_sys.file_size - source_offset); - if (dberr_t err= log_sys.log.read(source_offset, - {log_sys.buf + recv_sys.len, size})) + if (log_sys.archive && + end_lsn >= log_sys.get_first_lsn() + log_sys.capacity() && + !log_sys.archived_switch_recovery()) + { + ut_ad(!archive_exhausted); + archive_exhausted= true; + ut_ad(end_lsn == log_sys.get_first_lsn() + log_sys.capacity()); + ut_d(auto i= recv_sys.log_archive.find(log_sys.get_first_lsn())); + ut_ad(i != recv_sys.log_archive.end()); + ut_ad(i->second.end == end_lsn); + ut_d(i++); + ut_ad(i == recv_sys.log_archive.end()); + } + else if (dberr_t err= + log_sys.log.read(source_offset, + {log_sys.buf + recv_sys.len, size})) { sql_print_error("InnoDB: Failed to read log at %" PRIu64 ": %s", source_offset, ut_strerr(err)); @@ -4331,7 +5114,7 @@ static bool recv_scan_log(bool last_phase, const recv_sys_t::parser *parser) ut_ad(!recv_sys.file_checkpoint); for (;;) { - const byte& b{log_sys.buf[recv_sys.offset]}; + const byte b{log_sys.buf[recv_sys.offset]}; r= parser[false](false); switch (r) { case recv_sys_t::PREMATURE_EOF: @@ -4348,7 +5131,7 @@ static bool recv_scan_log(bool last_phase, const recv_sys_t::parser *parser) ut_ad(!end || end == recv_sys.lsn); bool corrupt_fs= recv_sys.is_corrupt_fs(); - if (!end && !corrupt_fs) + if (!end && !corrupt_fs && !log_sys.archive) { recv_sys.set_corrupt_log(); sql_print_error("InnoDB: Missing FILE_CHECKPOINT(" LSN_PF @@ -4382,23 +5165,9 @@ static bool recv_scan_log(bool last_phase, const recv_sys_t::parser *parser) } } - if (!store) - skip_the_rest: - while ((r= parser[false](false)) == recv_sys_t::OK); - else + if (store) { - uint16_t count= 0; - while ((r= parser[true](last_phase)) == recv_sys_t::OK) - if (!++count && recv_sys.report(time(nullptr))) - { - const size_t n= recv_sys.pages.size(); - sql_print_information("InnoDB: Parsed redo log up to LSN=" LSN_PF - "; to recover: %zu pages", recv_sys.lsn, n); - service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL, - "Parsed redo log up to LSN=" LSN_PF - "; to recover: %zu pages", - recv_sys.lsn, n); - } + r= recv_scan_store[bool(log_sys.archive)](parser[true], last_phase); if (r == recv_sys_t::GOT_OOM) { ut_ad(!last_phase); @@ -4407,9 +5176,12 @@ static bool recv_scan_log(bool last_phase, const recv_sys_t::parser *parser) if (recv_sys.scanned_lsn <= 1) goto skip_the_rest; ut_ad(recv_sys.file_checkpoint); - goto func_exit; + goto rewind_exit; } } + else + skip_the_rest: + r= recv_scan_skip_the_rest[bool(log_sys.archive)](parser[false]); if (r != recv_sys_t::PREMATURE_EOF) { @@ -4438,12 +5210,15 @@ static bool recv_scan_log(bool last_phase, const recv_sys_t::parser *parser) recv_sys.lsn == recv_sys.scanned_lsn) goto got_eof; + if (archive_exhausted) + goto got_eof; + if (recv_sys.offset > buf_size / 4 || (recv_sys.offset > block_size_1 && recv_sys.len >= buf_size - recv_sys.MTR_SIZE_MAX)) { const size_t ofs{recv_sys.offset & ~block_size_1}; - memmove_aligned<64>(log_sys.buf, log_sys.buf + ofs, recv_sys.len - ofs); + memmove_aligned<512>(log_sys.buf, log_sys.buf + ofs, recv_sys.len - ofs); recv_sys.len-= ofs; recv_sys.offset&= block_size_1; } @@ -4454,18 +5229,18 @@ static bool recv_scan_log(bool last_phase, const recv_sys_t::parser *parser) ut_ad(!rewound_lsn); ut_ad(recv_sys.lsn >= recv_sys.file_checkpoint); log_sys.set_recovered_lsn(recv_sys.lsn); + recover_end: binlog_recover_end(recv_sys.lsn); } else if (rewound_lsn) { + rewind_exit: ut_ad(!store); - ut_ad(recv_sys.file_checkpoint); - recv_sys.lsn= rewound_lsn; + log_sys.recovery_rewind(rewound_lsn); } else if (store) - binlog_recover_end(recv_sys.lsn); + goto recover_end; -func_exit: ut_d(recv_sys.after_apply= last_phase); mysql_mutex_unlock(&recv_sys.mutex); DBUG_RETURN(!store); @@ -4759,19 +5534,52 @@ static dberr_t recv_rename_files() inline void log_t::set_recovered() noexcept { ut_ad(get_flushed_lsn() == get_lsn()); - ut_ad(recv_sys.lsn == get_flushed_lsn()); - if (!is_mmap()) - { - const size_t bs{log_sys.write_size}, bs_1{bs - 1}; - memmove_aligned<512>(buf, buf + (recv_sys.offset & ~bs_1), bs); - } -#ifdef HAVE_PMEM - else + ut_ad(recv_sys.lsn == get_flushed_lsn() || + (recv_sys.rpo && recv_sys.rpo < get_flushed_lsn())); + ut_ad(!resize_log.is_opened()); + ut_ad(!resize_buf); + ut_ad(!resize_flush_buf); + circular_recovery_from_sequence_bit_0= !archive && + !get_sequence_bit(last_checkpoint_lsn); + ut_ad(write_size >= 512); + ut_ad(ut_is_2pow(write_size)); + + if (is_mmap()) + clear_mmap(); + else if (const size_t offset= recv_sys.offset & ~size_t(write_size - 1)) + memmove_aligned<512>(buf, buf + offset, write_size); +} + +/** During recovery, rename an archived log file to ib_logfile0. */ +ATTRIBUTE_COLD bool log_t::archive_rename() noexcept +{ + ut_ad(!archive); + ut_ad(!srv_read_only_mode); + /* Only rename if we successfully applied all the log. */ + bool success= recv_sys.rpo && recv_sys.rpo < get_flushed_lsn(); + if (!success) { - buf_size= unsigned(std::min(capacity(), buf_size_max)); - mprotect(buf, size_t(file_size), PROT_READ | PROT_WRITE); - } + { + const std::string old_name{get_archive_path(get_first_lsn())}; + sql_print_warning("InnoDB: Renaming %.*s to ib_logfile0", + int(old_name.size()), old_name.data()); +#ifdef _WIN32 + /* On Windows, open files cannot be renamed. */ + ut_ad(!is_mmap()); + log.close(); #endif + success= !rename(old_name); + } +#ifdef _WIN32 + if (success) + { + log.m_file= os_file_create_func(get_path().c_str(), OS_FILE_OPEN, + OS_LOG_FILE, false, &success); + ut_ad(success == log.is_opened()); + } +#endif + } + return success; } inline bool recv_sys_t::validate_checkpoint() const noexcept @@ -4806,6 +5614,156 @@ static recv_sys_t::parser get_parse_mmap() noexcept ut_error; } +dberr_t recv_sys_t::find_checkpoint_archived(lsn_t first_lsn, bool silent) +{ + ut_ad(log_sys.archive); + ut_ad(!log_sys.checkpoint_buf == log_sys.is_mmap()); + ut_ad(!uint16_t(~log_sys.next_checkpoint_no)); + const byte *buf; + if (byte *c= log_sys.checkpoint_buf) + { + buf= c; + if (dberr_t err= log_sys.log.read(0, {c, log_sys.write_size})) + return err; + } + else + buf= log_sys.buf; + uint16_t n_checkpoint= 0; + { + const uint32_t format{mach_read_from_4(my_assume_aligned<4>(buf))}; + if (srv_encrypt_log ? format != 1 : format < log_t::START_OFFSET) + { + /* If we are able to start recovery from a previous file and + recover up to the end of the last file, the subsequent call to + log_t::write_checkpoint() will fix the last file header. */ + if (!silent) + sql_print_error((format == 1 || format >= log_t::START_OFFSET) + ? "InnoDB: " LOG_ARCHIVE_NAME + " does not match innodb_encrypt_log" + : "InnoDB: " LOG_ARCHIVE_NAME + " is in unrecognized format", + first_lsn); + return DB_ERROR; + } + + if (!srv_encrypt_log); + else if (!log_crypt_read_header(buf)) + return DB_ERROR; + else + { + buf+= 32/*log_crypt_read_header()*/, n_checkpoint= 8/* 32/4 */; + if (!tmp_buf) + { + tmp_buf= static_cast + (ut_malloc_dontdump(tmp_buf_size, PSI_INSTRUMENT_ME)); + if (!tmp_buf) + return DB_OUT_OF_MEMORY; + } + } + } + + log_sys.last_checkpoint_lsn= 0; + log_sys.set_first_lsn(first_lsn); + lsn= 0; + /* Validate the checkpoints */ + lsn_t end_lsn{0}, checkpoint{0}, recovery_start_checkpoint{0}; + const recv_sys_t::parser parser[2] { + get_parse_mmap(), get_parse_mmap() + }; + ut_ad(recv_spaces.empty()); + while (n_checkpoint < log_sys.START_OFFSET / 4) + { + const uint32_t d{mach_read_from_4(my_assume_aligned<4>(buf))}; + if (d < log_sys.START_OFFSET || d >= log_sys.file_size) + break; + const lsn_t parse_start{first_lsn + d - log_sys.START_OFFSET}; + if (parse_start < end_lsn) + break; + lsn= parse_start; + file_checkpoint= 0; + log_sys.last_checkpoint_lsn= 0; + ut_d(const bool rescan=) recv_scan_log(false, parser); + ut_ad(rescan); + ut_ad(recv_spaces.empty()); + if (!file_checkpoint) + { + found_corrupt_log= false; + break; + } + ut_ad(file_checkpoint == lsn); + checkpoint= log_sys.last_checkpoint_lsn; + ut_ad(checkpoint); + ut_ad(checkpoint < lsn); + if (!log_sys.archived_checkpoint) + log_sys.archived_checkpoint= checkpoint; + if (!log_sys.archived_lsn) + log_sys.archived_lsn= parse_start; + end_lsn= parse_start; + if (end_lsn == recovery_start) + recovery_start_checkpoint= checkpoint; + n_checkpoint++; + + if (first_lsn != log_sys.get_first_lsn()) + { + /* This checkpoint spanned two files, and it therefore should be + the last valid checkpoint in the file. Either + log_t::archived_switch_recovery() switched log_sys.log to point + to the second file, or log_t::archived_mmap_switch_recovery_complete() + switched both log_sys.log and log_sys.buf. */ + log_sys.archived_switch_recovery_rewind_checkpoint(); + break; + } + + buf+= 4; + if (byte *c= log_sys.checkpoint_buf) + { + uint offset(n_checkpoint * 4); + if (offset & (log_sys.write_size - 1) || offset == log_sys.START_OFFSET) + continue; + buf= c; + if (dberr_t err= log_sys.log.read(offset, {c, log_sys.write_size})) + return err; + } + } + + if (recv_sys_invalid_rpo(recv_sys_rpo_exceeded)) + return DB_READ_ONLY; + + if (!checkpoint) + { + if (!silent) + sql_print_error("InnoDB: Did not find any checkpoint after LSN=" LSN_PF, + first_lsn); + return DB_CORRUPTION; + } + + if (!recovery_start); + else if (recovery_start_checkpoint) + checkpoint= recovery_start_checkpoint, end_lsn= recovery_start; + else + { + end_lsn= lsn= recovery_start; + file_checkpoint= 0; + log_sys.last_checkpoint_lsn= 0; + ut_d(const bool rescan=) recv_scan_log(false, parser); + ut_ad(rescan); + ut_ad(recv_spaces.empty()); + if (!file_checkpoint) + { + ut_ad(!silent); + sql_print_error("InnoDB: Did not find innodb_log_recovery_start=" LSN_PF + " in " LOG_ARCHIVE_NAME, + recovery_start, first_lsn); + return DB_CORRUPTION; + } + checkpoint= log_sys.last_checkpoint_lsn; + } + + file_checkpoint= 0; + log_sys.set_recovered_checkpoint(checkpoint, lsn= end_lsn, n_checkpoint); + return DB_SUCCESS; +} + /** Start recovering from a redo log checkpoint. of first system tablespace page @return error code or DB_SUCCESS */ @@ -4825,10 +5783,12 @@ dberr_t recv_recovery_from_checkpoint_start() if (srv_force_recovery >= SRV_FORCE_NO_LOG_REDO) { sql_print_information("InnoDB: innodb_force_recovery=6" " skips redo log apply"); + recv_sys.rpo = LSN_MAX; return err; } recv_sys.recovery_on = true; + recv_sys_rpo_exceeded = 0; log_sys.latch.wr_lock(); log_sys.set_capacity(); @@ -4846,6 +5806,19 @@ dberr_t recv_recovery_from_checkpoint_start() recv_sys_t::parser parser[2]; if (log_sys.is_recoverable()) { + if (recv_sys.recovery_start > recv_sys.lsn) { + /* recv_sys_t::find_checkpoint_archived() + already checked this */ + ut_ad(!log_sys.archive); + sql_print_error("InnoDB: Did not find" + " innodb_log_recovery_start=%" PRIu64 + " between %" PRIu64 " and %" PRIu64, + recv_sys.recovery_start, + log_sys.last_checkpoint_lsn.load(), + recv_sys.lsn); + err = DB_CORRUPTION; + goto func_exit; + } const bool rewind = recv_sys.lsn != log_sys.last_checkpoint_lsn; parser[false] = get_parse_mmap(); @@ -4854,8 +5827,12 @@ dberr_t recv_recovery_from_checkpoint_start() if (recv_needed_recovery) { read_only_recovery: sql_print_warning("InnoDB: innodb_read_only" - " prevents crash recovery"); + " prevents crash recovery between " LSN_PF + " and " LSN_PF, + log_sys.last_checkpoint_lsn.load(), + recv_sys.lsn); err = DB_READ_ONLY; + recv_sys.rpo = recv_sys.scanned_lsn; goto func_exit; } if (recv_sys.is_corrupt_log()) { @@ -4869,14 +5846,19 @@ dberr_t recv_recovery_from_checkpoint_start() ut_ad(recv_sys.file_checkpoint); ut_ad(log_sys.get_flushed_lsn() >= recv_sys.scanned_lsn); if (rewind) { - recv_sys.lsn = log_sys.last_checkpoint_lsn; + mysql_mutex_lock(&recv_sys.mutex); recv_sys.offset = 0; recv_sys.len = 0; + log_sys.recovery_rewind(log_sys.last_checkpoint_lsn); + mysql_mutex_unlock(&recv_sys.mutex); } rescan = recv_scan_log(false, parser); - if (srv_read_only_mode && recv_needed_recovery) { - goto read_only_recovery; + if (srv_read_only_mode) { + recv_sys.rpo = recv_sys.scanned_lsn; + if (recv_needed_recovery) { + goto read_only_recovery; + } } if ((recv_sys.is_corrupt_log() && !srv_force_recovery) @@ -4885,7 +5867,11 @@ dberr_t recv_recovery_from_checkpoint_start() } } - log_sys.set_recovered_lsn(recv_sys.scanned_lsn); + if (recv_sys_invalid_rpo(recv_sys_rpo_exceeded)) { + high_level_read_only = true; + } else { + log_sys.set_recovered_lsn(recv_sys.scanned_lsn); + } if (recv_needed_recovery) { bool missing_tablespace = false; @@ -4931,7 +5917,7 @@ dberr_t recv_recovery_from_checkpoint_start() mysql_mutex_lock(&recv_sys.mutex); ut_ad(log_sys.get_flushed_lsn() >= recv_sys.lsn); recv_sys.clear(); - recv_sys.lsn = log_sys.last_checkpoint_lsn; + log_sys.recovery_rewind(log_sys.last_checkpoint_lsn); mysql_mutex_unlock(&recv_sys.mutex); } @@ -4962,15 +5948,20 @@ dberr_t recv_recovery_from_checkpoint_start() ut_ad(recv_sys.pages.empty()); } - if (!log_sys.is_recoverable()) { - } else if (recv_sys.validate_checkpoint()) { + if (log_sys.is_recoverable()) { + if (recv_sys_rpo_exceeded || recv_sys.validate_checkpoint()) { err_exit: - err = DB_ERROR; - goto func_exit; - } + err = DB_ERROR; + goto func_exit; + } - if (!srv_read_only_mode && log_sys.is_recoverable()) { - log_sys.set_recovered(); + if (!srv_read_only_mode) { + log_sys.set_recovered(); + if (UNIV_UNLIKELY(recv_sys.was_archive) + && !log_sys.archive_rename()) { + goto err_exit; + } + } } DBUG_EXECUTE_IF("before_final_redo_apply", goto err_exit;); diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 68467d9cefaf3..fdcb42a0a6b0a 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -52,12 +52,14 @@ void mtr_t::finisher_update() if (log_sys.is_mmap()) { commit_logger= mtr_t::commit_log; - finisher= mtr_t::finish_writer; + finisher= log_sys.archive + ? mtr_t::finish_writer + : mtr_t::finish_writer; return; } commit_logger= mtr_t::commit_log; #endif - finisher= mtr_t::finish_writer; + finisher= mtr_t::finish_writer; } void mtr_memo_slot_t::release() const @@ -342,7 +344,56 @@ void mtr_t::release() m_memo.clear(); } -ATTRIBUTE_NOINLINE void mtr_t::commit_log_release() noexcept +#ifdef HAVE_PMEM +ATTRIBUTE_COLD lsn_t log_t::archived_mmap_switch_complete() noexcept +{ + ut_ad(latch_have_wr()); + if (!archive) + return 0; + ut_ad(file_size <= ARCHIVE_FILE_SIZE_MAX); + ut_ad(resize_target <= ARCHIVE_FILE_SIZE_MAX); + if (!resize_buf) + return 0; + const lsn_t lsn{get_lsn()}, last_lsn{first_lsn + capacity()}; + if (lsn < last_lsn) + return 0; + ut_a(!checkpoint_buf); + persist(last_lsn); + checkpoint_buf= buf; + buf= resize_buf; + resize_buf= nullptr; + first_lsn= last_lsn; + file_size= resize_target; + return lsn; +} + +template<> +ATTRIBUTE_NOINLINE void mtr_t::commit_log_release() noexcept +{ + if (m_latch_ex) + { + completed: + const lsn_t lsn{log_sys.archived_mmap_switch_complete()}; + log_sys.latch.wr_unlock(); + m_latch_ex= false; + if (lsn) + buf_flush_ahead(lsn, true); + } + else + { + const bool retry{log_sys.archived_mmap_switch()}; + log_sys.latch.rd_unlock(); + if (retry) + { + log_sys.latch.wr_lock(); + goto completed; + } + } +} +#endif + +template<> +ATTRIBUTE_NOINLINE void mtr_t::commit_log_release() noexcept { if (m_latch_ex) { @@ -401,12 +452,12 @@ void mtr_t::commit_log(mtr_t *mtr, std::pair lsns) noexcept buf_pool.page_cleaner_wakeup(); mysql_mutex_unlock(&buf_pool.flush_list_mutex); - mtr->commit_log_release(); + mtr->commit_log_release(); mtr->release(); } else { - mtr->commit_log_release(); + mtr->commit_log_release(); for (auto it= mtr->m_memo.rbegin(); it != mtr->m_memo.rend(); ) { @@ -494,6 +545,7 @@ void mtr_t::commit() } ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); std::pair lsns{do_write()}; process_freed_pages(); #ifdef HAVE_PMEM @@ -554,6 +606,7 @@ void mtr_t::commit_shrink(fil_space_t &space, uint32_t size) { ut_ad(is_active()); ut_ad(!high_level_read_only); + ut_ad(!recv_sys.rpo); ut_ad(m_modifications); ut_ad(!m_memo.empty()); ut_ad(!recv_recovery_is_on()); @@ -666,6 +719,7 @@ bool mtr_t::commit_file(fil_space_t &space, const char *name) { ut_ad(is_active()); ut_ad(!high_level_read_only); + ut_ad(!recv_sys.rpo); ut_ad(m_modifications); ut_ad(!m_made_dirty); ut_ad(!recv_recovery_is_on()); @@ -695,6 +749,11 @@ bool mtr_t::commit_file(fil_space_t &space, const char *name) /* Durably write the log for the file system operation. */ log_write_and_flush(); +#ifdef HAVE_PMEM + const lsn_t wait_lsn= log_sys.archived_mmap_switch() + ? log_sys.get_first_lsn() + log_sys.capacity() : 0; +#endif + log_sys.latch.wr_unlock(); m_latch_ex= false; @@ -717,6 +776,11 @@ bool mtr_t::commit_file(fil_space_t &space, const char *name) mysql_mutex_unlock(&buf_pool.flush_list_mutex); release_resources(); +#ifdef HAVE_PMEM + if (wait_lsn) + buf_flush_ahead(wait_lsn, true); +#endif + return success; } @@ -747,6 +811,7 @@ ATTRIBUTE_COLD lsn_t mtr_t::commit_files(lsn_t checkpoint_lsn) ut_ad(!m_made_dirty); ut_ad(m_memo.empty()); ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); ut_ad(!m_freed_space); ut_ad(!m_freed_pages); ut_ad(!m_user_space); @@ -858,7 +923,9 @@ static time_t log_close_warn_time; making the server crash-unsafe. */ ATTRIBUTE_COLD static void log_overwrite_warning(lsn_t lsn) { - if (log_sys.overwrite_warned) + ut_ad(!log_sys.archive); /* we hope that this is unreachable */ + + if (log_sys.overwrite_warned || log_sys.archive) return; time_t t= time(nullptr); @@ -878,6 +945,40 @@ ATTRIBUTE_COLD static void log_overwrite_warning(lsn_t lsn) ? ". Shutdown is in progress" : ""); } + +#ifdef HAVE_PMEM +template<> +inline std::pair +log_t::append_prepare(size_t size, bool ex) noexcept +{ + ut_ad(ex ? latch_have_wr() : latch_have_rd()); + ut_ad(is_mmap()); + ut_ad(is_mmap_writeable()); + ut_ad(archive); + ut_ad(archived_lsn); + + uint64_t l, lsn; + static_assert(WRITE_TO_BUF == WRITE_BACKOFF << 1, ""); + while (UNIV_UNLIKELY((l= write_lsn_offset.fetch_add(size + WRITE_TO_BUF) & + (WRITE_TO_BUF - 1)) + + (lsn= base_lsn.load(std::memory_order_relaxed)) + + size + START_OFFSET >= first_lsn + file_size) && + !resize_buf && !checkpoint_buf) + { + /* The following is inlined here instead of being part of + archive_mmap_switch_prepare() below, in order to increase the + locality of reference and to expedite setting the WRITE_BACKOFF flag. */ + bool late(write_lsn_offset.fetch_or(WRITE_BACKOFF) & WRITE_BACKOFF); + /* Subtract our LSN overshoot. */ + write_lsn_offset.fetch_sub(size); + archived_mmap_switch_prepare(late, ex); + } + + lsn+= l; + return {lsn, buf + FIRST_LSN + (lsn - first_lsn)}; +} +#endif + ATTRIBUTE_COLD void log_t::append_prepare_wait(bool late, bool ex) noexcept { if (UNIV_LIKELY(!ex)) @@ -924,6 +1025,8 @@ ATTRIBUTE_COLD void log_t::append_prepare_wait(bool late, bool ex) noexcept const bool is_pmem{is_mmap()}; if (is_pmem) { + ut_ad(is_mmap_writeable()); + ut_ad(!archive); ut_ad(lsn - get_flushed_lsn(std::memory_order_relaxed) < capacity() || overwrite_warned); persist(lsn); @@ -948,22 +1051,25 @@ ATTRIBUTE_COLD void log_t::append_prepare_wait(bool late, bool ex) noexcept } /** Reserve space in the log buffer for appending data. -@tparam mmap log_sys.is_mmap() +@tparam mode how to write log @param size total length of the data to append(), in bytes @param ex whether log_sys.latch is exclusively locked @return the start LSN and the buffer position for append() */ -template +template inline std::pair log_t::append_prepare(size_t size, bool ex) noexcept { ut_ad(ex ? latch_have_wr() : latch_have_rd()); - ut_ad(mmap == is_mmap()); - ut_ad(!mmap || buf_size == std::min(capacity(), buf_size_max)); - const size_t buf_size{this->buf_size - size}; + static_assert(!bool(WRITE_NORMAL), ""); + static_assert(bool(CIRCULAR_MMAP), ""); + static_assert(mode == WRITE_NORMAL || mode == CIRCULAR_MMAP, ""); + ut_ad(bool(mode) == is_mmap()); + ut_ad(is_mmap() == is_mmap_writeable()); uint64_t l; static_assert(WRITE_TO_BUF == WRITE_BACKOFF << 1, ""); while (UNIV_UNLIKELY((l= write_lsn_offset.fetch_add(size + WRITE_TO_BUF) & - (WRITE_TO_BUF - 1)) >= buf_size)) + (WRITE_TO_BUF - 1)) >= + (mode ? capacity() : buf_size) - size)) { /* The following is inlined here instead of being part of append_prepare_wait(), in order to increase the locality of reference @@ -974,14 +1080,14 @@ std::pair log_t::append_prepare(size_t size, bool ex) noexcept append_prepare_wait(late, ex); } - const lsn_t lsn{l + base_lsn.load(std::memory_order_relaxed)}, - end_lsn{lsn + size}; + const lsn_t lsn{l + base_lsn.load(std::memory_order_relaxed)}; - if (UNIV_UNLIKELY(end_lsn >= last_checkpoint_lsn + log_capacity)) + if (UNIV_UNLIKELY(lsn + size >= last_checkpoint_lsn + log_capacity) && + (mode == CIRCULAR_MMAP || !archive)) set_check_for_checkpoint(true); return {lsn, - buf + size_t(mmap ? FIRST_LSN + (lsn - first_lsn) % capacity() : l)}; + buf + size_t(mode ? FIRST_LSN + (lsn - first_lsn) % capacity() : l)}; } /** Finish appending data to the log. @@ -1005,7 +1111,7 @@ static lsn_t log_close(lsn_t lsn) noexcept /* The last checkpoint is too old. Let us set an appropriate checkpoint age target, that is, a checkpoint LSN target that is the current LSN minus the maximum age. Let us see if are exceeding the - log_checkpoint_margin() limit that will involve a synchronous wait + log_t::checkpoint_margin() limit that will involve a synchronous wait in each write operation. */ const bool furious{checkpoint_age >= log_sys.max_checkpoint_age}; @@ -1017,7 +1123,7 @@ static lsn_t log_close(lsn_t lsn) noexcept The aim of the more aggressive target is that mtr_flush_ahead() will request more progress in buf_flush_page_cleaner() sooner, so that it will be less likely that several threads will end up waiting in - log_checkpoint_margin(). That function will use the less aggressive + log_t::checkpoint_margin(). That function will use the less aggressive limit (lsn - log_sys.max_checkpoint_age) in order to minimize the synchronous wait time. */ if (furious) @@ -1117,12 +1223,14 @@ std::pair mtr_t::do_write() noexcept return finish_write(len); } -inline void log_t::resize_write(lsn_t lsn, const byte *end, size_t len, - size_t seq) noexcept +template +ATTRIBUTE_COLD +void log_t::resize_write_low(lsn_t lsn, const byte *end, + size_t len, size_t seq) noexcept { ut_ad(latch_have_any()); + ut_ad(resize_buf); - if (UNIV_LIKELY_NULL(resize_buf)) { ut_ad(end >= buf); end-= len; @@ -1214,92 +1322,155 @@ inline void log_t::resize_write(lsn_t lsn, const byte *end, size_t len, inline void log_t::append(byte *&d, const void *s, size_t size) noexcept { ut_ad(log_sys.latch_have_any()); - ut_ad(d + size <= log_sys.buf + - (log_sys.is_mmap() ? log_sys.file_size : log_sys.buf_size)); + ut_ad(log_sys.is_mmap() + ? ((d >= log_sys.buf && d + size <= log_sys.buf + log_sys.file_size) || + (log_sys.archive && + d >= log_sys.resize_buf && + d + size <= log_sys.resize_buf + log_sys.resize_target)) + : (d >= log_sys.buf && d + size <= log_sys.buf + log_sys.buf_size)); memcpy(d, s, size); d+= size; } -template -std::pair mtr_t::finish_writer(mtr_t *mtr, size_t len) +template +std::pair +mtr_t::finish_writer(mtr_t *mtr, size_t len) { ut_ad(log_sys.is_latest()); ut_ad(!recv_no_log_write); + ut_ad(!recv_sys.rpo); ut_ad(mtr->is_logged()); ut_ad(mtr->m_latch_ex ? log_sys.latch_have_wr() : log_sys.latch_have_rd()); ut_ad(len < recv_sys.MTR_SIZE_MAX); + ut_ad(mode == log_t::WRITE_NORMAL || + log_sys.archive == (mode == log_t::ARCHIVED_MMAP)); const size_t size{mtr->m_commit_lsn ? 5U + 8U : 5U}; std::pair start= - log_sys.append_prepare(len, mtr->m_latch_ex); + log_sys.append_prepare(len, mtr->m_latch_ex); - if (!mmap) - { + if (mode == log_t::WRITE_NORMAL) +#ifdef HAVE_PMEM + write_normal: +#endif for (const mtr_buf_t::block_t &b : mtr->m_log) log_sys.append(start.second, b.begin(), b.used()); - - write_trailer: - *start.second++= log_sys.get_sequence_bit(start.first + len - size); - if (mtr->m_commit_lsn) - { - mach_write_to_8(start.second, mtr->m_commit_lsn); - mtr->m_crc= my_crc32c(mtr->m_crc, start.second, 8); - start.second+= 8; - } - mach_write_to_4(start.second, mtr->m_crc); - start.second+= 4; - } +#ifdef HAVE_PMEM else { - if (UNIV_LIKELY(start.second + len <= &log_sys.buf[log_sys.file_size])) + const size_t file_size= log_sys.file_size; + byte *const buf{log_sys.buf}; + byte *end= &buf[file_size]; + if (UNIV_LIKELY(start.second + len <= end)) + goto write_normal; + byte *const begin= mode == log_t::ARCHIVED_MMAP + ? log_sys.get_archived_mmap_switch() + : buf + log_sys.START_OFFSET; + if (mode == log_t::ARCHIVED_MMAP && start.second > end) { - for (const mtr_buf_t::block_t &b : mtr->m_log) - log_sys.append(start.second, b.begin(), b.used()); - goto write_trailer; + /* Our mini-transaction will not span two log files. We are + somewhere between log_t::archived_mmap_switch_prepare() and + log_t::archived_mmap_switch_complete(), and our entire log must + be written to the new file. */ + start.second= begin + (start.second - end); + goto write_normal; } + for (const mtr_buf_t::block_t &b : mtr->m_log) { size_t size{b.used()}; - const size_t size_left(&log_sys.buf[log_sys.file_size] - start.second); + const size_t size_left(end - start.second); const byte *src= b.begin(); if (size > size_left) { ::memcpy(start.second, src, size_left); - start.second= &log_sys.buf[log_sys.START_OFFSET]; + start.second= begin; + if (mode == log_t::ARCHIVED_MMAP) + /* An approximation; the minimum innodb_log_file_size + always exceeds the maximum mtr->get_log_size() */ + end= begin + file_size; src+= size_left; size-= size_left; } ::memcpy(start.second, src, size); start.second+= size; } - const size_t size_left(&log_sys.buf[log_sys.file_size] - start.second); - if (size_left > size) - goto write_trailer; + const size_t size_left(end - start.second); + if (size_left <= size) + { + byte tail[5 + 8]; + tail[0]= + (mode == log_t::WRITE_NORMAL + ? log_sys.archive : mode == log_t::ARCHIVED_MMAP) || + log_sys.get_sequence_bit(start.first + len - size); - byte tail[5 + 8]; - tail[0]= log_sys.get_sequence_bit(start.first + len - size); + if (mtr->m_commit_lsn) + { + mach_write_to_8(tail + 1, mtr->m_commit_lsn); + mtr->m_crc= my_crc32c(mtr->m_crc, tail + 1, 8); + mach_write_to_4(tail + 9, mtr->m_crc); + } + else + mach_write_to_4(tail + 1, mtr->m_crc); - if (mtr->m_commit_lsn) - { - mach_write_to_8(tail + 1, mtr->m_commit_lsn); - mtr->m_crc= my_crc32c(mtr->m_crc, tail + 1, 8); - mach_write_to_4(tail + 9, mtr->m_crc); + ::memcpy(start.second, tail, size_left); + ::memcpy(begin, tail + size_left, size - size_left); + start.second= ((size >= size_left) ? begin : end) + (size - size_left); + goto wrote_trailer; } - else - mach_write_to_4(tail + 1, mtr->m_crc); - - ::memcpy(start.second, tail, size_left); - ::memcpy(log_sys.buf + log_sys.START_OFFSET, tail + size_left, - size - size_left); - start.second= log_sys.buf + - ((size >= size_left) ? log_sys.START_OFFSET : log_sys.file_size) + - (size - size_left); } +#endif + + *start.second++= + (mode == log_t::WRITE_NORMAL + ? log_sys.archive : mode == log_t::ARCHIVED_MMAP) || + log_sys.get_sequence_bit(start.first + len - size); + + if (mtr->m_commit_lsn) + { + mach_write_to_8(start.second, mtr->m_commit_lsn); + mtr->m_crc= my_crc32c(mtr->m_crc, start.second, 8); + start.second+= 8; + } + mach_write_to_4(start.second, mtr->m_crc); + start.second+= 4; - log_sys.resize_write(start.first, start.second, len, size); +#ifdef HAVE_PMEM +wrote_trailer: +#else + static_assert(mode == log_t::WRITE_NORMAL, ""); +#endif mtr->m_commit_lsn= start.first + len; - return {start.first, log_close(mtr->m_commit_lsn)}; + + switch (mode) { + case log_t::ARCHIVED_MMAP: + ut_ad(!log_sys.resize_in_progress()); + return {start.first, (log_sys.get_first_lsn() > log_sys.last_checkpoint_lsn + ? log_sys.get_first_lsn() : 0)}; + case log_t::CIRCULAR_MMAP: + log_sys.resize_write(start.first, start.second, len, size); + return {start.first, log_close(mtr->m_commit_lsn)}; + case log_t::WRITE_NORMAL: + log_sys.resize_write(start.first, start.second, len, size); + } + + lsn_t wait_lsn; + + if (!log_sys.archive) + wait_lsn= log_close(mtr->m_commit_lsn); + else + { + wait_lsn= log_sys.get_first_lsn(); + if (UNIV_LIKELY(wait_lsn <= log_sys.last_checkpoint_lsn)) + { + wait_lsn+= log_sys.capacity(); + if (UNIV_LIKELY(wait_lsn >= mtr->m_commit_lsn)) + wait_lsn= 0; + } + } + + return {start.first, wait_lsn}; } bool mtr_t::have_x_latch(const buf_block_t &block) const diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index d8d967997274c..fb139aa87fcef 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -4027,6 +4027,7 @@ dberr_t FetchIndexRootPages::run(const fil_iterator_t& iter, const bool full_crc32 = fil_space_t::full_crc32(m_space_flags); bool skip_checksum_check = false; ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); if (!page_compress_buf) return DB_OUT_OF_MEMORY; @@ -4127,6 +4128,7 @@ static dberr_t fil_iterate( byte* page_compress_buf= static_cast(malloc(get_buf_size())); ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); if (!page_compress_buf) { return DB_OUT_OF_MEMORY; @@ -4455,6 +4457,7 @@ fil_tablespace_iterate( ut_a(n_io_buffers > 0); ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); DBUG_EXECUTE_IF("ib_import_trigger_corruption_1", return(DB_CORRUPTION);); @@ -4733,6 +4736,7 @@ row_import_for_mysql( /* The caller assured that this is not read_only_mode and that no temporary tablespace is being imported. */ ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); ut_ad(!table->is_temporary()); ut_ad(table->space_id); diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index af72252f5f719..38c47fc3b75ee 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -679,6 +679,7 @@ row_ins_set_detailed( dict_foreign_t* foreign) /*!< in: foreign key constraint */ { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); mysql_mutex_lock(&srv_misc_tmpfile_mutex); rewind(srv_misc_tmpfile); @@ -713,6 +714,7 @@ row_ins_foreign_trx_print( ulint heap_size; ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); { TMLockMutexGuard g{SRW_LOCK_CALL}; @@ -751,10 +753,6 @@ row_ins_foreign_report_err( { std::string fk_str; - if (srv_read_only_mode) { - return; - } - FILE* ef = dict_foreign_err_file; trx_t* trx = thr_get_trx(thr); @@ -807,11 +805,6 @@ row_ins_foreign_report_add_err( child table */ { std::string fk_str; - - if (srv_read_only_mode) { - return; - } - FILE* ef = dict_foreign_err_file; row_ins_set_detailed(trx, foreign); diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 6c83de220f750..2dc11e4503e18 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -3713,6 +3713,7 @@ row_merge_insert_index_tuples( DBUG_ENTER("row_merge_insert_index_tuples"); ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); ut_ad(!(index->type & DICT_FTS)); ut_ad(!dict_index_is_spatial(index)); @@ -3886,6 +3887,7 @@ row_merge_drop_index_dict( pars_info_t* info; ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); ut_ad(trx->dict_operation_lock_mode); ut_ad(trx->dict_operation); ut_ad(dict_sys.locked()); @@ -3949,6 +3951,7 @@ row_merge_drop_indexes_dict( pars_info_t* info; ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); ut_ad(trx->dict_operation_lock_mode); ut_ad(trx->dict_operation); ut_ad(dict_sys.locked()); @@ -4023,6 +4026,7 @@ row_merge_drop_indexes( dict_index_t* next_index; ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); ut_ad(trx->dict_operation_lock_mode); ut_ad(trx->dict_operation); ut_ad(dict_sys.locked()); @@ -4426,6 +4430,7 @@ row_merge_file_destroy( merge_file_t* merge_file) /*!< in/out: merge file structure */ { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); if (merge_file->fd != OS_FILE_CLOSED) { row_merge_file_destroy_low(merge_file->fd); @@ -4503,6 +4508,7 @@ row_merge_create_index( DBUG_ENTER("row_merge_create_index"); ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); /* Create the index prototype, using the passed in def, this is not a persistent operation. We pass 0 as the space id, and determine at @@ -4639,6 +4645,7 @@ row_merge_build_indexes( DBUG_ENTER("row_merge_build_indexes"); ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); ut_ad((old_table == new_table) == !col_map); ut_ad(!defaults || col_map); diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 0a749743f16de..b4c9fb8c6dfd4 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1240,6 +1240,8 @@ row_insert_for_mysql( mem_heap_t* blob_heap = NULL; ut_ad(trx); + ut_ad(!high_level_read_only); + ut_ad(!recv_sys.rpo); ut_a(prebuilt->magic_n == ROW_PREBUILT_ALLOCATED); ut_a(prebuilt->magic_n2 == ROW_PREBUILT_ALLOCATED); @@ -1251,8 +1253,6 @@ row_insert_for_mysql( return(DB_TABLESPACE_DELETED); } else if (!table->is_readable()) { return row_mysql_get_table_error(trx, table); - } else if (high_level_read_only) { - return(DB_READ_ONLY); } else if (UNIV_UNLIKELY(table->corrupted) || dict_table_get_first_index(table)->is_corrupted()) { return DB_TABLE_CORRUPT; @@ -1611,6 +1611,8 @@ row_update_for_mysql(row_prebuilt_t* prebuilt) DBUG_ENTER("row_update_for_mysql"); ut_ad(trx); + ut_ad(!high_level_read_only); + ut_ad(!recv_sys.rpo); ut_a(prebuilt->magic_n == ROW_PREBUILT_ALLOCATED); ut_a(prebuilt->magic_n2 == ROW_PREBUILT_ALLOCATED); ut_a(prebuilt->template_type == ROW_MYSQL_WHOLE_ROW); @@ -1620,10 +1622,6 @@ row_update_for_mysql(row_prebuilt_t* prebuilt) return row_mysql_get_table_error(trx, table); } - if (high_level_read_only) { - return(DB_READ_ONLY); - } - DEBUG_SYNC_C("innodb_row_update_for_mysql_begin"); trx->op_info = "updating or deleting"; @@ -2263,8 +2261,10 @@ row_discard_tablespace_foreign_key_checks( const trx_t* trx, /*!< in: transaction handle */ const dict_table_t* table) /*!< in: table to be discarded */ { + ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); - if (srv_read_only_mode || !trx->check_foreigns) { + if (!trx->check_foreigns) { return(DB_SUCCESS); } @@ -2561,10 +2561,8 @@ row_rename_table_for_mysql( ut_a(new_name != NULL); ut_ad(trx->state == TRX_STATE_ACTIVE); ut_ad(trx->dict_operation_lock_mode); - - if (high_level_read_only) { - return(DB_READ_ONLY); - } + ut_ad(!high_level_read_only); + ut_ad(!recv_sys.rpo); trx->op_info = "renaming table"; diff --git a/storage/innobase/row/row0quiesce.cc b/storage/innobase/row/row0quiesce.cc index 8d4394a12f60c..15d5d65f6cd1d 100644 --- a/storage/innobase/row/row0quiesce.cc +++ b/storage/innobase/row/row0quiesce.cc @@ -504,6 +504,7 @@ row_quiesce_table_start( ut_a(trx->mysql_thd != 0); ut_a(srv_n_purge_threads > 0); ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); ut_a(trx->mysql_thd != 0); @@ -602,8 +603,9 @@ row_quiesce_set_state( trx_t* trx) /*!< in/out: transaction */ { ut_a(srv_n_purge_threads > 0); + ut_ad(!srv_read_only_mode || recv_sys.rpo); - if (srv_read_only_mode) { + if (recv_sys.rpo) { ib_senderrf(trx->mysql_thd, IB_LOG_LEVEL_WARN, ER_READ_ONLY_MODE); diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index e304f0c8613fc..2188f84fd5aa1 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -693,7 +693,7 @@ srv_printf_innodb_monitor( mysql_mutex_lock(&dict_foreign_err_mutex); - if (!srv_read_only_mode && ftell(dict_foreign_err_file) != 0L) { + if (!recv_sys.rpo && ftell(dict_foreign_err_file) != 0L) { fputs("------------------------\n" "LATEST FOREIGN KEY ERROR\n" "------------------------\n", file); @@ -815,9 +815,11 @@ void srv_export_innodb_status(void) /*==========================*/ { + ut_ad(!srv_read_only_mode || recv_sys.rpo); + fil_crypt_stat_t crypt_stat; - if (!srv_read_only_mode) { + if (!recv_sys.rpo) { fil_crypt_total_stat(&crypt_stat); } innodb_io_slots_stats(tpool::aio_opcode::AIO_PREAD, @@ -932,7 +934,7 @@ srv_export_innodb_status(void) export_vars.innodb_onlineddl_rowlog_pct_used = onlineddl_rowlog_pct_used; export_vars.innodb_onlineddl_pct_progress = onlineddl_pct_progress; - if (!srv_read_only_mode) { + if (!recv_sys.rpo) { export_vars.innodb_encryption_rotation_pages_read_from_cache = crypt_stat.pages_read_from_cache; export_vars.innodb_encryption_rotation_pages_read_from_disk = @@ -953,6 +955,7 @@ srv_export_innodb_status(void) export_vars.innodb_lsn_current = log_sys.get_lsn(); export_vars.innodb_lsn_flushed = log_sys.get_flushed_lsn(); export_vars.innodb_lsn_last_checkpoint = log_sys.last_checkpoint_lsn; + export_vars.innodb_lsn_archived = log_sys.archived_lsn; export_vars.innodb_checkpoint_max_age = static_cast( log_sys.max_checkpoint_age); log_sys.latch.wr_unlock(); @@ -1012,7 +1015,7 @@ static void srv_monitor() /* We don't create the temp files or associated mutexes in read-only-mode */ - if (!srv_read_only_mode && srv_innodb_status) { + if (!recv_sys.rpo && srv_innodb_status) { mysql_mutex_lock(&srv_monitor_file_mutex); rewind(srv_monitor_file); if (!srv_printf_innodb_monitor(srv_monitor_file, @@ -1090,6 +1093,7 @@ bool srv_any_background_activity() if (purge_sys.enabled() || srv_master_timer.get()) { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); return true; } return false; @@ -1183,6 +1187,7 @@ void purge_sys_t::resume() return; } ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); ut_ad(srv_force_recovery < SRV_FORCE_NO_BACKGROUND); purge_coordinator_task.enable(); latch.wr_lock(SRW_LOCK_CALL); @@ -1287,11 +1292,15 @@ void srv_master_callback(void*) ut_a(srv_shutdown_state <= SRV_SHUTDOWN_INITIATED); MONITOR_INC(MONITOR_MASTER_THREAD_SLEEP); - purge_sys.wake_if_not_active(); + if (!recv_sys.rpo) + purge_sys.wake_if_not_active(); ulonglong counter_time= microsecond_interval_timer(); - srv_sync_log_buffer_in_background(); - MONITOR_INC_TIME_IN_MICRO_SECS(MONITOR_SRV_LOG_FLUSH_MICROSECOND, - counter_time); + if (!recv_sys.rpo) + { + srv_sync_log_buffer_in_background(); + MONITOR_INC_TIME_IN_MICRO_SECS(MONITOR_SRV_LOG_FLUSH_MICROSECOND, + counter_time); + } if (srv_check_activity(&old_activity_count)) srv_master_do_active_tasks(counter_time); @@ -1344,6 +1353,7 @@ Fetch and execute a task from the work queue. static bool srv_task_execute(THD *thd) { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); ut_ad(srv_force_recovery < SRV_FORCE_NO_BACKGROUND); mysql_mutex_lock(&srv_sys.tasks_mutex); @@ -1378,6 +1388,7 @@ void srv_update_purge_thread_count(uint n) inline void purge_coordinator_state::do_purge(trx_t *trx) { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); if (!purge_sys.enabled() || purge_sys.paused()) return; @@ -1493,6 +1504,7 @@ static void purge_worker_callback(void*) { ut_ad(!current_thd); ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); ut_ad(srv_force_recovery < SRV_FORCE_NO_BACKGROUND); void *ctx; THD *thd= acquire_thd(&ctx); @@ -1543,6 +1555,7 @@ srv_que_task_enqueue_low( que_thr_t* thr) /*!< in: query thread */ { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); mysql_mutex_lock(&srv_sys.tasks_mutex); UT_LIST_ADD_LAST(srv_sys.tasks, thr); @@ -1557,6 +1570,7 @@ ulint srv_get_task_queue_length() ulint n_tasks; ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); mysql_mutex_lock(&srv_sys.tasks_mutex); @@ -1588,7 +1602,8 @@ void srv_purge_shutdown() } purge_sys.coordinator_shutdown(); srv_shutdown_purge_tasks(); - if (!srv_fast_shutdown && !high_level_read_only && srv_was_started && + if (!srv_fast_shutdown && !high_level_read_only && + !recv_sys.rpo && srv_was_started && !opt_bootstrap && srv_operation == SRV_OPERATION_NORMAL && !srv_sys_space.is_shrink_fail()) fsp_system_tablespace_truncate(true); diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index cee2b78256177..08488adcb09fd 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -139,6 +139,9 @@ std::unique_ptr srv_master_timer; mysql_pfs_key_t thread_pool_thread_key; #endif /* UNIV_PFS_THREAD */ +/** Only created if !srv_read_only_mode. Protected by lock_sys.latch. */ +extern FILE *lock_latest_err_file; + #ifdef HAVE_PSI_STAGE_INTERFACE /** Array of all InnoDB stage events for monitoring activities via performance schema. */ @@ -158,18 +161,18 @@ static PSI_stage_info* srv_stages[] = static void delete_log_files() { for (size_t i= 1; i < 102; i++) - delete_log_file(std::to_string(i).c_str()); + os_file_delete_if_exists_func(log_sys.get_circular_path(i).c_str(), nullptr); } /** Creates log file. @param create_new_db whether the database is being initialized @param lsn log sequence number -@param logfile0 name of the log file @return DB_SUCCESS or error code */ static dberr_t create_log_file(bool create_new_db, lsn_t lsn) { ut_ad(log_sys.latch_have_wr()); ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); ut_ad(!buf_pool.get_oldest_modification(0)); /* We will retain ib_logfile0 until we have written a new logically @@ -188,7 +191,7 @@ static dberr_t create_log_file(bool create_new_db, lsn_t lsn) log_sys.set_capacity(); - std::string logfile0{get_log_file_path("ib_logfile101")}; + const std::string logfile0{log_sys.get_circular_path(101)}; bool ret; os_file_t file{ os_file_create_func(logfile0.c_str(), @@ -215,7 +218,7 @@ static dberr_t create_log_file(bool create_new_db, lsn_t lsn) } log_sys.set_latest_format(srv_encrypt_log); - if (!log_sys.attach(file, srv_log_file_size)) { + if (!log_sys.attach(file, srv_log_file_size, log_t::READ_WRITE)) { goto close_and_exit; } @@ -252,14 +255,20 @@ static dberr_t create_log_file(bool create_new_db, lsn_t lsn) /** Rename the redo log file after resizing. @return whether an error occurred */ -bool log_t::resize_rename() noexcept +ATTRIBUTE_COLD bool log_t::resize_rename() noexcept { - std::string old_name{get_log_file_path("ib_logfile101")}; - std::string new_name{get_log_file_path()}; + return rename(get_circular_path(101)); +} + +/** Rename the redo log file after resizing. +@return whether an error occurred */ +ATTRIBUTE_COLD bool log_t::rename(const std::string &old_name) noexcept +{ + const std::string new_name{log_sys.get_path()}; if (IF_WIN(MoveFileEx(old_name.c_str(), new_name.c_str(), MOVEFILE_REPLACE_EXISTING), - !rename(old_name.c_str(), new_name.c_str()))) + !::rename(old_name.c_str(), new_name.c_str()))) return false; sql_print_error("InnoDB: Failed to rename log from %.*s to %.*s (error %d)", @@ -780,7 +789,7 @@ srv_check_undo_redo_logs_exists() } /* Check if redo log file exists */ - auto logfilename = get_log_file_path(); + std::string logfilename{log_sys.get_circular_path()}; fh = os_file_create_func(logfilename.c_str(), OS_FILE_OPEN_RETRY_SILENT, @@ -789,6 +798,7 @@ srv_check_undo_redo_logs_exists() if (ret) { os_file_close_func(fh); +found_log: ib::error() << "redo log file '" << logfilename << "' exists. Creating system tablespace with" " existing redo log file is not recommended." @@ -797,6 +807,41 @@ srv_check_undo_redo_logs_exists() return DB_ERROR; } +#ifdef _WIN32 + logfilename.assign(srv_log_group_home_dir); + switch (logfilename.back()) { + case '\\': case '/': + break; + default: + logfilename.push_back('/'); + } + logfilename.append("ib_????????????????.log"); + WIN32_FIND_DATAA entry; + HANDLE d = FindFirstFileA(logfilename.c_str(), &entry); + if (d != INVALID_HANDLE_VALUE) { + FindClose(d); + goto found_log; + } +#else + if (DIR* d = opendir(srv_log_group_home_dir)) { + while (dirent* e = readdir(d)) { + lsn_t lsn; + int n{0}; + const char* fn{e->d_name}; + if (1 != sscanf(fn, LOG_ARCHIVE_NAME "%n", &lsn, &n) + || fn[n] || lsn < log_t::FIRST_LSN) { + continue; + } + logfilename.assign(srv_log_group_home_dir); + logfilename.push_back('/'); + log_sys.append_archive_name(logfilename, lsn); + closedir(d); + goto found_log; + } + closedir(d); + } +#endif + return(DB_SUCCESS); } @@ -1009,6 +1054,7 @@ static void srv_shutdown_bg_undo_sources() if (srv_undo_sources) { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); fts_optimize_shutdown(); dict_stats_shutdown(); srv_undo_sources= false; @@ -1154,12 +1200,16 @@ ATTRIBUTE_COLD static dberr_t srv_log_rebuild() noexcept /** Rebuild the redo log if needed. */ static dberr_t srv_log_rebuild_if_needed() noexcept { + ut_ad(!srv_read_only_mode || recv_sys.rpo); + if (srv_force_recovery == SRV_FORCE_NO_LOG_REDO) /* Completely ignore the redo log. */ return DB_SUCCESS; - if (srv_read_only_mode) + if (recv_sys.rpo) /* Leave the redo log alone. */ return DB_SUCCESS; + if (log_sys.archive) + return DB_SUCCESS; /* Never rebuild archived log files. */ if (log_sys.file_size == srv_log_file_size && log_sys.format == @@ -1316,60 +1366,12 @@ dberr_t srv_start(bool create_new_db) ib::info() << my_crc32c_implementation(); - if (!srv_read_only_mode) { - mysql_mutex_init(srv_monitor_file_mutex_key, - &srv_monitor_file_mutex, nullptr); - mysql_mutex_init(srv_misc_tmpfile_mutex_key, - &srv_misc_tmpfile_mutex, nullptr); - } - - if (!srv_read_only_mode) { - if (srv_innodb_status) { - const size_t len = strlen(fil_path_to_mysql_datadir) + - 20 + sizeof "/innodb_status."; - srv_monitor_file_name = static_cast( - ut_malloc_nokey(len)); - - snprintf(srv_monitor_file_name, len, - "%s/innodb_status." ULINTPF, - fil_path_to_mysql_datadir, - static_cast - (IF_WIN(GetCurrentProcessId(), getpid()))); - - srv_monitor_file = my_fopen(srv_monitor_file_name, - O_RDWR|O_TRUNC|O_CREAT, - MYF(MY_WME)); - - if (!srv_monitor_file) { - ib::error() << "Unable to create " - << srv_monitor_file_name << ": " - << strerror(errno); - if (err == DB_SUCCESS) { - err = DB_ERROR; - } - } - } else { - - srv_monitor_file_name = NULL; - srv_monitor_file = os_file_create_tmpfile(); - - if (!srv_monitor_file && err == DB_SUCCESS) { - err = DB_ERROR; - } - } - - srv_misc_tmpfile = os_file_create_tmpfile(); - - if (!srv_misc_tmpfile && err == DB_SUCCESS) { - err = DB_ERROR; - } - } - - if (err != DB_SUCCESS) { - return(srv_init_abort(err)); + if (os_aio_init()) { + return(srv_init_abort(DB_ERROR)); } - if (os_aio_init()) { + if (!srv_read_only_mode + && !(lock_latest_err_file = os_file_create_tmpfile())) { return(srv_init_abort(DB_ERROR)); } @@ -1407,6 +1409,10 @@ dberr_t srv_start(bool create_new_db) || srv_operation == SRV_OPERATION_RESTORE || srv_operation == SRV_OPERATION_RESTORE_EXPORT); ut_ad(!recv_sys.recovery_on); + /* Suppress warnings in fil_space_t::create() for files + that are being read before dict_boot() has recovered + DICT_HDR_MAX_SPACE_ID. */ + fil_system.space_id_reuse_warned = true; if (srv_force_recovery >= SRV_FORCE_NO_LOG_REDO) { sql_print_information("InnoDB: innodb_force_recovery=6" @@ -1474,7 +1480,6 @@ dberr_t srv_start(bool create_new_db) ? DB_SUCCESS : DB_ERROR; mysql_mutex_unlock(&recv_sys.mutex); - if (err == DB_SUCCESS) { mtr.start(); err= srv_undo_tablespaces_init(create_new_db, &mtr); @@ -1493,12 +1498,6 @@ dberr_t srv_start(bool create_new_db) return(srv_init_abort(err)); } - /* Initialize objects used by dict stats gathering thread, which - can also be used by recovery if it tries to drop some table */ - if (!srv_read_only_mode) { - dict_stats_init(); - } - trx_sys.create(); if (create_new_db) { @@ -1556,12 +1555,9 @@ dberr_t srv_start(bool create_new_db) if (log_sys.resize_rename()) { return(srv_init_abort(DB_ERROR)); } - } else { - /* Suppress warnings in fil_space_t::create() for files - that are being read before dict_boot() has recovered - DICT_HDR_MAX_SPACE_ID. */ - fil_system.space_id_reuse_warned = true; + if (log_sys.archive) log_sys.archive_set_size(); + } else { /* We always try to do a recovery, even if the database had been shut down normally: this is the normal startup path */ @@ -1659,25 +1655,23 @@ dberr_t srv_start(bool create_new_db) fil_system.space_id_reuse_warned = false; - if (srv_operation > SRV_OPERATION_EXPORT_RESTORED) { - ut_ad(srv_operation == SRV_OPERATION_RESTORE_EXPORT - || srv_operation == SRV_OPERATION_RESTORE); - return(err); - } - /* Upgrade or resize or rebuild the redo logs before generating any dirty pages, so that the old redo log file will not be written to. */ - err = srv_log_rebuild_if_needed(); + if (srv_operation <= SRV_OPERATION_EXPORT_RESTORED) { + err = srv_log_rebuild_if_needed(); - if (err != DB_SUCCESS) { - return srv_init_abort(err); + if (err != DB_SUCCESS) { + return srv_init_abort(err); + } } recv_sys.debug_free(); - if (!srv_read_only_mode) { + if (log_sys.archive) log_sys.archive_set_size(); + + if (!recv_sys.rpo) { const uint32_t flags = FSP_FLAGS_PAGE_SSIZE(); for (uint32_t id = srv_undo_space_id_start; id <= srv_undo_tablespaces; id++) { @@ -1766,6 +1760,55 @@ dberr_t srv_start(bool create_new_db) ut_ad(err == DB_SUCCESS); ut_a(sum_of_new_sizes != ULINT_UNDEFINED); + if (!recv_sys.rpo && srv_operation != SRV_OPERATION_RESTORE) { + if (srv_innodb_status) { + const size_t len = strlen(fil_path_to_mysql_datadir) + + 20 + sizeof "/innodb_status."; + srv_monitor_file_name = static_cast( + ut_malloc_nokey(len)); + + snprintf(srv_monitor_file_name, len, + "%s/innodb_status." ULINTPF, + fil_path_to_mysql_datadir, + static_cast + (IF_WIN(GetCurrentProcessId(), getpid()))); + + srv_monitor_file = my_fopen(srv_monitor_file_name, + O_RDWR|O_TRUNC|O_CREAT, + MYF(MY_WME)); + + if (!srv_monitor_file) { + ib::error() << "Unable to create " + << srv_monitor_file_name << ": " + << strerror(errno); + } + } else { + srv_monitor_file_name = NULL; + srv_monitor_file = os_file_create_tmpfile(); + } + + if (!srv_monitor_file) { + return srv_init_abort(DB_ERROR); + } else { + mysql_mutex_init(srv_monitor_file_mutex_key, + &srv_monitor_file_mutex, nullptr); + } + + if (!(srv_misc_tmpfile = os_file_create_tmpfile())) { + return srv_init_abort(DB_ERROR); + } + + mysql_mutex_init(srv_misc_tmpfile_mutex_key, + &srv_misc_tmpfile_mutex, nullptr); + + } + + if (srv_operation > SRV_OPERATION_EXPORT_RESTORED) { + ut_ad(srv_operation == SRV_OPERATION_RESTORE_EXPORT + || srv_operation == SRV_OPERATION_RESTORE); + return(DB_SUCCESS); + } + /* Create the doublewrite buffer to a new tablespace */ if (!srv_read_only_mode && srv_force_recovery < SRV_FORCE_NO_TRX_UNDO && !buf_dblwr.create()) { @@ -1773,7 +1816,7 @@ dberr_t srv_start(bool create_new_db) } /* Recreate the undo tablespaces */ - if (!high_level_read_only) { + if (!high_level_read_only && !recv_sys.rpo) { err = srv_undo_tablespaces_reinitialize(); if (err) { return srv_init_abort(err); @@ -1802,7 +1845,7 @@ dberr_t srv_start(bool create_new_db) ut_ad(high_level_read_only || srv_force_recovery < SRV_FORCE_NO_UNDO_LOG_SCAN); - if (!high_level_read_only + if (!high_level_read_only && !recv_sys.rpo && srv_sys_space.can_auto_shrink()) { fsp_system_tablespace_truncate(false); DBUG_EXECUTE_IF("crash_after_sys_truncate", @@ -1811,7 +1854,7 @@ dberr_t srv_start(bool create_new_db) /* Validate a few system page types that were left uninitialized before MySQL or MariaDB 5.5. */ - if (!high_level_read_only + if (!high_level_read_only && !recv_sys.rpo && !fil_system.sys_space->full_crc32()) { buf_block_t* block; mtr.start(); @@ -1860,7 +1903,7 @@ dberr_t srv_start(bool create_new_db) be free of any locks. The data dictionary latch should guarantee that there is at most one data dictionary transaction active at a time. */ - if (!high_level_read_only + if (!high_level_read_only && !recv_sys.rpo && srv_force_recovery <= SRV_FORCE_NO_TRX_UNDO) { /* If the following call is ever removed, the first-time ha_innobase::open() must hold (or @@ -1880,7 +1923,7 @@ dberr_t srv_start(bool create_new_db) } if (srv_force_recovery < SRV_FORCE_NO_TRX_UNDO - && !srv_read_only_mode) { + && !recv_sys.rpo) { /* Drop partially created indexes. */ row_merge_drop_temp_indexes(); #ifdef EMBEDDED_LIBRARY @@ -1905,7 +1948,8 @@ dberr_t srv_start(bool create_new_db) ut_ad(srv_force_recovery >= SRV_FORCE_NO_UNDO_LOG_SCAN || !purge_sys.enabled()); - if (srv_force_recovery < SRV_FORCE_NO_BACKGROUND) { + if (srv_force_recovery < SRV_FORCE_NO_BACKGROUND + && !recv_sys.rpo) { srv_undo_sources = true; /* Create the dict stats gathering task */ dict_stats_start(); @@ -1965,7 +2009,9 @@ dberr_t srv_start(bool create_new_db) trx_sys.get_max_trx_id()); } - if (!srv_read_only_mode) { + ut_ad(!srv_read_only_mode || recv_sys.rpo); + + if (!recv_sys.rpo) { if (create_new_db) { srv_buffer_pool_load_at_startup = FALSE; } @@ -1989,6 +2035,8 @@ dberr_t srv_start(bool create_new_db) } #endif /* WITH_WSREP */ + dict_stats_init(); + /* Create thread(s) that handles key rotation. This is needed already here as log_preflush_pool_modified_pages will flush dirty pages and that might need e.g. @@ -2009,12 +2057,13 @@ PRAGMA_REENABLE_CHECK_STACK_FRAME */ void innodb_preshutdown() { + ut_ad(!srv_read_only_mode || recv_sys.rpo); static bool first_time= true; if (!first_time) return; first_time= false; - if (srv_read_only_mode) + if (recv_sys.rpo) return; #ifndef EMBEDDED_LIBRARY @@ -2045,6 +2094,7 @@ void innodb_preshutdown() /** Shut down InnoDB. */ void innodb_shutdown() { + ut_ad(!srv_read_only_mode || recv_sys.rpo); innodb_preshutdown(); ut_ad(!srv_undo_sources); const lsn_t lsn{logs_empty_and_mark_files_at_shutdown()}; @@ -2055,35 +2105,21 @@ void innodb_shutdown() ut_ad(!buf_page_cleaner_is_active); srv_shutdown_threads(); - if (srv_monitor_file) { - my_fclose(srv_monitor_file, MYF(MY_WME)); - srv_monitor_file = 0; - if (srv_monitor_file_name) { - unlink(srv_monitor_file_name); - ut_free(srv_monitor_file_name); - } - } - - if (srv_misc_tmpfile) { - my_fclose(srv_misc_tmpfile, MYF(MY_WME)); - srv_misc_tmpfile = 0; - } - ut_ad(dict_sys.is_initialised() || !srv_was_started); ut_ad(trx_sys.is_initialised() || !srv_was_started); ut_ad(buf_dblwr.is_created() || !srv_was_started - || srv_read_only_mode + || recv_sys.rpo || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); ut_ad(lock_sys.is_initialised() || !srv_was_started); ut_ad(log_sys.is_initialised() || !srv_was_started); - dict_stats_deinit(); - if (srv_started_redo) { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); /* srv_shutdown_bg_undo_sources() already invoked fts_optimize_shutdown(); dict_stats_shutdown(); */ + dict_stats_deinit(); fil_crypt_threads_cleanup(); } @@ -2102,11 +2138,34 @@ void innodb_shutdown() lock_sys.close(); trx_pool_close(); - if (!srv_read_only_mode) { - mysql_mutex_destroy(&srv_monitor_file_mutex); - mysql_mutex_destroy(&srv_misc_tmpfile_mutex); + if (lock_latest_err_file) { + my_fclose(lock_latest_err_file, MYF(MY_WME)); + lock_latest_err_file = nullptr; + } + + if (!recv_sys.rpo && srv_operation != SRV_OPERATION_RESTORE) { + if (srv_monitor_file) { + my_fclose(srv_monitor_file, MYF(MY_WME)); + srv_monitor_file = nullptr; + if (srv_monitor_file_name) { + unlink(srv_monitor_file_name); + ut_free(srv_monitor_file_name); + srv_monitor_file_name = nullptr; + } + mysql_mutex_destroy(&srv_monitor_file_mutex); + } + + if (srv_misc_tmpfile) { + my_fclose(srv_misc_tmpfile, MYF(MY_WME)); + srv_misc_tmpfile = nullptr; + mysql_mutex_destroy(&srv_misc_tmpfile_mutex); + } } + ut_ad(!srv_monitor_file_name); + ut_ad(!srv_monitor_file); + ut_ad(!srv_misc_tmpfile); + dict_sys.close(); btr_search_sys_free(); srv_free(); diff --git a/storage/innobase/trx/trx0rec.cc b/storage/innobase/trx/trx0rec.cc index cfe42f77e4cf8..0f91ea888f9ed 100644 --- a/storage/innobase/trx/trx0rec.cc +++ b/storage/innobase/trx/trx0rec.cc @@ -1857,6 +1857,7 @@ trx_undo_report_row_operation( ut_ad(!update || rec); ut_ad(!rec || rec_offs_validate(rec, index, offsets)); ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo || index->table->is_temporary()); trx = thr_get_trx(thr); /* This function must not be invoked during rollback diff --git a/storage/innobase/trx/trx0roll.cc b/storage/innobase/trx/trx0roll.cc index 0d07e5ad27f50..11d411d7a42f2 100644 --- a/storage/innobase/trx/trx0roll.cc +++ b/storage/innobase/trx/trx0roll.cc @@ -491,6 +491,7 @@ Note: this is done in a background thread. */ void trx_rollback_all_recovered(void*) { ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); if (trx_sys.rw_trx_hash.size()) { ib::info() << "Starting in background the rollback of" diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index a6406971c8ab0..640352a0aacc3 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -523,13 +523,15 @@ inline void trx_t::release_locks() TRANSACTIONAL_TARGET void trx_free_at_shutdown(trx_t *trx) { ut_ad(trx->is_recovered); + ut_ad(!srv_read_only_mode || recv_sys.rpo); + ut_a(trx_state_eq(trx, TRX_STATE_PREPARED) || trx_state_eq(trx, TRX_STATE_PREPARED_RECOVERED) || (trx_state_eq(trx, TRX_STATE_ACTIVE) && (!srv_was_started || srv_operation == SRV_OPERATION_RESTORE || srv_operation == SRV_OPERATION_RESTORE_EXPORT - || srv_read_only_mode + || recv_sys.rpo || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO || (!srv_is_being_started && !srv_undo_sources && srv_fast_shutdown)))); @@ -965,7 +967,7 @@ trx_start_low( && (!trx->mysql_thd || read_write || trx->dict_operation)) { /* Temporary rseg is assigned only if the transaction updates a temporary table */ - if (!high_level_read_only) { + if (!high_level_read_only && !recv_sys.rpo) { trx_assign_rseg_low(trx); } } else { @@ -976,7 +978,7 @@ trx_start_low( to write to the temporary table. */ if (read_write) { - ut_ad(!srv_read_only_mode); + ut_ad(!recv_sys.rpo); trx_sys.register_rw(trx); } } else { @@ -2267,7 +2269,7 @@ trx_set_rw_mode( ut_ad(!trx->read_only); ut_ad(trx->id == 0); - if (high_level_read_only) { + if (high_level_read_only || recv_sys.rpo) { return; } diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc index be1815ab60285..ff4d93fccc6a1 100644 --- a/storage/innobase/trx/trx0undo.cc +++ b/storage/innobase/trx/trx0undo.cc @@ -1496,6 +1496,8 @@ void trx_undo_set_state_at_prepare(trx_undo_t *undo, bool rollback, mtr_t *mtr) /** At shutdown, frees the undo logs of a transaction. */ void trx_undo_free_at_shutdown(trx_t *trx) { + ut_ad(!srv_read_only_mode || recv_sys.rpo); + if (trx_undo_t*& undo = trx->rsegs.m_redo.undo) { switch (undo->state) { case TRX_UNDO_PREPARED: @@ -1509,7 +1511,7 @@ void trx_undo_free_at_shutdown(trx_t *trx) /* trx_t::commit_state() assigns trx->state = TRX_STATE_COMMITTED_IN_MEMORY. */ ut_a(!srv_was_started - || srv_read_only_mode + || recv_sys.rpo || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO || srv_fast_shutdown); break;