MyRocks built-in percona(8.0.13-
我在将RocksDB引擎作为内置引擎编入percona的经历
更新（2019.10.21）
按照原文的修改方式的确引起了mtr单元测试log_components_filter故障，在之前基础上的bug fix patch如下：
diff --git a/include/mysql/components/services/log_builtins.h b/include/mysql/components/services/log_builtins.h
index 0000941..1877519 100644
--- a/include/mysql/components/services/log_builtins.h
+++ b/include/mysql/components/services/log_builtins.h
@@ -1422,15 +1422,7 @@ inline bool init_logging_service_for_plugin(
return false;
}
-/* We define the following two interfaces empty because:
- 1. rocksdb is built-in engine now;
- 2. other built-in engines, like innodb, myisam, etc., do not call
- the following two interfaces;
- 3. we keep the code where rocksdb engine calls the following
- interfaces, instead of remove it, for futural unexpection.
-*/
-#elif defined(EXTRA_CODE_FOR_UNIT_TESTING) || defined(ROCKSDB_PLATFORM_POSIX)
+#elif defined(EXTRA_CODE_FOR_UNIT_TESTING)
/**
Method is used by unit tests.
diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc
index e38c15e..3e2c968 100644
--- a/storage/rocksdb/ha_rocksdb.cc
+++ b/storage/rocksdb/ha_rocksdb.cc
@@ -82,11 +82,6 @@
#include "./rdb_psi.h"
#include "./rdb_threads.h"
-// MySQL 8.0 logger service interface
-static SERVICE_TYPE(registry) *reg_srv = nullptr;
-SERVICE_TYPE(log_builtins) *log_bi = nullptr;
-SERVICE_TYPE(log_builtins_string) *log_bs = nullptr;
-
#include <sys/eventfd.h>
#include <boost/thread/thread.hpp>
#include <boost/lockfree/queue.hpp>
@@ -2450,11 +2445,6 @@ static uint rocksdb_partition_flags() { return (HA_CANNOT_PARTITION_FK); }
static int rocksdb_init_func(void *const p) {
- // Initialize error logging service.
- if (init_logging_service_for_plugin(®_srv, &log_bi, &log_bs)) {
- return HA_EXIT_FAILURE;
- }
-
if (rdb_check_rocksdb_corruption()) {
LogPluginErrMsg(ERROR_LEVEL, 0,
"There was corruption detected in the RockDB data files. "
@@ -2586,7 +2576,6 @@ static int rocksdb_init_func(void *const p) {
LogPluginErrMsg(
ERROR_LEVEL, 0,
"Can't enable both use_direct_reads and allow_mmap_reads\n");
- deinit_logging_service_for_plugin(®_srv, &log_bi, &log_bs);
return HA_EXIT_FAILURE;
}
@@ -2625,7 +2614,6 @@ static int rocksdb_init_func(void *const p) {
LogPluginErrMsg(ERROR_LEVEL, 0,
"Can't enable both use_direct_io_for_flush_and_compaction "
"and allow_mmap_writes\n");
- deinit_logging_service_for_plugin(®_srv, &log_bi, &log_bs);
return HA_EXIT_FAILURE;
}
@@ -2634,7 +2622,6 @@ static int rocksdb_init_func(void *const p) {
LogPluginErrMsg(ERROR_LEVEL, 0,
"rocksdb_flush_log_at_trx_commit needs to be 0 to use "
"allow_mmap_writes");
- deinit_logging_service_for_plugin(®_srv, &log_bi, &log_bs);
return HA_EXIT_FAILURE;
}
@@ -2718,14 +2705,12 @@ static int rocksdb_init_func(void *const p) {
if (!status.ok()) {
LogPluginErrMsg(ERROR_LEVEL, 0, "Persistent cache returned error: (%s)",
status.getState());
- deinit_logging_service_for_plugin(®_srv, &log_bi, &log_bs);
return HA_EXIT_FAILURE;
}
rocksdb_tbl_options->persistent_cache = pcache;
} else if (strlen(rocksdb_persistent_cache_path)) {
LogPluginErrMsg(ERROR_LEVEL, 0,
"ust specify rocksdb_persistent_cache_size_mb");
- deinit_logging_service_for_plugin(®_srv, &log_bi, &log_bs);
return HA_EXIT_FAILURE;
}
@@ -2734,7 +2719,6 @@ static int rocksdb_init_func(void *const p) {
rocksdb_default_cf_options,
rocksdb_override_cf_options)) {
LogPluginErrMsg(ERROR_LEVEL, 0, "Failed to initialize CF options map.");
- deinit_logging_service_for_plugin(®_srv, &log_bi, &log_bs);
return HA_EXIT_FAILURE;
}
@@ -2799,7 +2783,6 @@ static int rocksdb_init_func(void *const p) {
if (dict_manager.init(rdb, &cf_manager)) {
LogPluginErrMsg(ERROR_LEVEL, 0, "Failed to initialize data dictionary.");
- deinit_logging_service_for_plugin(®_srv, &log_bi, &log_bs);
return HA_EXIT_FAILURE;
}
@@ -2810,7 +2793,6 @@ static int rocksdb_init_func(void *const p) {
#endif // defined(ROCKSDB_INCLUDE_VALIDATE_TABLES) &&
// ROCKSDB_INCLUDE_VALIDATE_TABLES
LogPluginErrMsg(ERROR_LEVEL, 0, "Failed to initialize DDL manager.");
- deinit_logging_service_for_plugin(®_srv, &log_bi, &log_bs);
return HA_EXIT_FAILURE;
}
@@ -2852,7 +2834,6 @@ static int rocksdb_init_func(void *const p) {
if (err != 0) {
LogPluginErrMsg(ERROR_LEVEL, 0,
"Couldn't start the background thread: (errno=%d)", err);
- deinit_logging_service_for_plugin(®_srv, &log_bi, &log_bs);
return HA_EXIT_FAILURE;
}
@@ -2865,7 +2846,6 @@ static int rocksdb_init_func(void *const p) {
if (err != 0) {
LogPluginErrMsg(ERROR_LEVEL, 0,
"Couldn't start the drop index thread: (errno=%d)", err);
- deinit_logging_service_for_plugin(®_srv, &log_bi, &log_bs);
return HA_EXIT_FAILURE;
}
@@ -2879,7 +2859,6 @@ static int rocksdb_init_func(void *const p) {
HA_ERR_ROCKSDB_LAST);
if (err != 0) {
LogPluginErrMsg(ERROR_LEVEL, 0, "Couldn't initialize error messages");
- deinit_logging_service_for_plugin(®_srv, &log_bi, &log_bs);
return HA_EXIT_FAILURE;
}
@@ -2985,8 +2964,6 @@ static int rocksdb_done_func(void *const p) {
my_error_unregister(HA_ERR_ROCKSDB_FIRST, HA_ERR_ROCKSDB_LAST);
- deinit_logging_service_for_plugin(®_srv, &log_bi, &log_bs);
-
// clear the initialized flag and unlock
rdb_get_hton_init_state()->set_initialized(false);
出发点
使用MySQL（本文中的MySQL和percona混用）时，我们常在配置文件中指定参数默认值，如binlog_format、default-storage-engine。
percona提供存储引擎MyRocks（本文中MyRocks和RocksDB都指Percona中的RocksDB存储引擎），以动态加载的方式供用户安装。我们或许想在配置文件中也制定RocksDB的相关参数，比如rocksdb_deadlock_detect等。但由于RocksDB作为动态链接库存在，MySQL启动时并无RocksDB任何信息（只有当MySQL启动完毕后才能加载，比如percona官方给出的ps-admin --enable-rocksdb，或手动安装install plugin rocksdb soname 'ha_rocksdb.so'。），配置文件中的rocksdb_xxx参数被认为是unkown variable，导致无法启动MySQL Server。
所以，我们考虑把RocksDB作为percona的内置引擎编译，就像MyISAM一样。
Patch
先给出结论：给出针对percona 8.0.13-4版本的patch，使得MyRocks built-in percona
diff --git a/include/mysql/components/services/log_builtins.h b/include/mysql/components/services/log_builtins.h
index 7e61b0c..5d56d91 100644
--- a/include/mysql/components/services/log_builtins.h
+++ b/include/mysql/components/services/log_builtins.h
@@ -1385,7 +1385,15 @@ inline bool init_logging_service_for_plugin(
return false;
}
-#elif defined(EXTRA_CODE_FOR_UNIT_TESTING)
+/*
+ We define the following two interfaces empty because:
+ 1. rocksdb is built-in engine now;
+ 2. other built-in engines, like innodb, myisam, etc., do not call
+ the following two interfaces;
+ 3. we keep the code where rocksdb engine calls the following
+ interfaces, instead of remove it, for futural unexpection.
+*/
+#elif defined(EXTRA_CODE_FOR_UNIT_TESTING) || defined(ROCKSDB_PLATFORM_POSIX)
/**
Method is used by unit tests.
diff --git a/storage/rocksdb/CMakeLists.txt b/storage/rocksdb/CMakeLists.txt
index 8e9086d..a823887 100644
--- a/storage/rocksdb/CMakeLists.txt
+++ b/storage/rocksdb/CMakeLists.txt
@@ -150,6 +150,11 @@ INCLUDE_DIRECTORIES(
${CMAKE_CURRENT_SOURCE_DIR}/third_party/zstd/lib/dictBuilder
)
+SET(PARTITION_BASE_DIR ${CMAKE_SOURCE_DIR}/sql/partitioning)
+SET(PARTITION_SOURCE ${PARTITION_BASE_DIR}/partition_base.cc)
+ADD_CONVENIENCE_LIBRARY(partition_base ${PARTITION_SOURCE})
+ADD_DEPENDENCIES(partition_base GenError)
+
ADD_DEFINITIONS(-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DZLIB -DLZ4
-DZSTD -DROCKSDB_SUPPORT_THREAD_LOCAL)
@@ -186,8 +191,16 @@ SET(ROCKSDB_SOURCES
SET(rocksdb_static_libs ${rocksdb_static_libs} ${ZLIB_LIBRARY} regex "-lrt")
-MYSQL_ADD_PLUGIN(rocksdb ${ROCKSDB_SOURCES} STORAGE_ENGINE DEFAULT MODULE_ONLY
- LINK_LIBRARIES ${rocksdb_static_libs}
+#MYSQL_ADD_PLUGIN(rocksdb ${ROCKSDB_SOURCES} STORAGE_ENGINE DEFAULT MODULE_ONLY
+# LINK_LIBRARIES ${rocksdb_static_libs}
+#)
+
+MYSQL_ADD_PLUGIN(rocksdb_se ${ROCKSDB_SOURCES}
+ STORAGE_ENGINE
+ STATIC_ONLY
+ MANDATORY
+ MODULE_OUTPUT_NAME ha_rocksdb
+ LINK_LIBRARIES ${rocksdb_static_libs} partition_base
)
# TODO: read this file list from src.mk:TOOL_SOURCES
patch解读
1. CMakeList.txt
MYSQL_ADD_PLUGIN(rocksdb_se ${ROCKSDB_SOURCES} #引擎名取rocksdb_se，下一节说明为何不是rocksdb或其他
STORAGE_ENGINE
STATIC_ONLY #静态编译成librocksdb_se.a，MODULE_ONLY动态编译为ha_rocksdb.so
MANDATORY #此关键字也在后面说明必要性
MODULE_OUTPUT_NAME ha_rocksdb #参考innodb和myisam的写法，未验证必要性
LINK_LIBRARIES ${rocksdb_static_libs} partition_base #partition_base后文说明
)
2. Why not rocksdb, but rocksdb_se ?
上一节我们看到，CMakeList中我们给引擎起名为rocksdb_se而非rocksdb（参考myisam和innodb，都是直呼myisam或innodb），那么为什么rocksdb引擎要增加_se？
我们尝试MYSQL_ADD_PLUGIN(rocksdb ${ROCKSDB_SOURCES} ...)的方式构建，但编译遇到错误：
... sql_builtin.cc:madatory_plugins: error: undefined reference to builtin_rocksdb_plugin
builtin_rocksdb_plugin是什么，我们到构建目录查看sql_builtin.cc一探究竟：
//in sql_builtin.cc
builtin_plugin builtin_perfschema_plugin, builtin_csv_plugin, builtin_innobase_plugin, ... //一些引擎名
struct st_mysql_plugin *mysql_optional_plugins[] = {builtin_archive_plugin, builtin_blackhole_plugin, ...} //一些引擎名，看似是optional的，不是我们关注的重点
struct st_mysql_plugin *mysql_mandatory_plugins[] = {builtin_binlog_plugin, ..., builtin_innobase_plugin, builtin_rocksdb_plugin, ...} //一些引擎名，我们发现builtin_rocksdb_plugin“混迹其中”
mysql_mandatory_plugins是一个数组，元素是一些mandatory的引擎，和CMakeList.txt中的MANDATORY对应，我们推测此处builtin_rocksdb_plugin和CMakeList.txt中MYSQL_ADD_PLUGIN(rocksdb ...)相对应（在plugin.cmake文件所定义的MYSQL_ADD_PLUGIN宏定义得以印证）。
mysql_mandatory_plugins使用了一个未定义的元素builtin_rocksdb_plugin，那么一定其他代码声明了builtin_rocksdb_plugin，哪个文件声明了builtin_rocksdb_plugin呢？
我们在构建目录下正则匹配全部含有builtin_rocksdb_plugin关键字的文件（grep -a 'builtin_rocksdb_plugin' -R -n *），发现在librocksdb.a文件中，零零散散有builtin_rocksdb_plugin的字样（毕竟是二进制文件），同时还有builtin_rocksdb_se_plugin_interface_version、builtin_rocksdb_se_sizeof_struct_st_plugin这样的字眼，这些又是什么呢？看起来这些名字是非常统一，格式化的。
我们想参考下MyISAM是如何的（因为我们期待RocksDB和MyISAM效果一样地被内置进去，又不作为默认引擎，这也是不参考innodb的理由），查看libmyisam.a检索builtin_myisam_plugin，我们也发现了builtin_myisam_plugin_interface_version等字样。
于是我们可以推测，builtin_xxx_plugin, builtin_xxx_plugin_interface_version等应该是一套机制生成的，存储引擎被声明的时候经由这套机制生成builtin_xxx_plugin注册到mysql_mandatory_plugins和mysql_optional_plugins数组当中。
继续检索builtin、plugin_interface_version等字眼，我们定位到这样一个宏定义：
#define mysql_declare_plugin(NAME) \
__MYSQL_DECLARE_PLUGIN(NAME, builtin_##NAME##_plugin_interface_version, \
builtin_##NAME##_sizeof_struct_st_plugin, \
builtin_##NAME##_plugin)
根据名字推测，这个宏定义声明了plugin，那么mysql_declare_plugin被哪里调用了？
//in ha_rocksdb.cc
mysql_declare_plugin(rocksdb_se){
MYSQL_STORAGE_ENGINE_PLUGIN, /* Plugin Type */
&rocksdb_storage_engine, /* Plugin Descriptor */
"ROCKSDB", /* Plugin Name */
... //还有其他构造属性
} ...
//in ha_myisam.cc
mysql_declare_plugin(myisam){
MYSQL_STORAGE_ENGINE_PLUGIN,
&myisam_storage_engine,
"MyISAM",
...
} ...
我们已经找到了源头，源文件在声明rocksdb作为存储引擎时，赋名为rocksdb_se，因此宏定义生成了builtin_rocksdb_se_plugin。
最后，我们在CMakeList中将库名写为rocksdb_se即可。
3. What is partition_base ?
CMakeList.txt中，我们增加了一个依赖partition_base，原因是什么呢？
解决了rocksdb_se的问题后，编译不再报错“undefined reference to builtin_rocksdb_plugin”，但有新错误产生：
undefined reference to native_part::Partition_base::Partition_base
我们在ha_rocksdb.cc中包含了partition_base.h，而且错误为undefined reference而非not declared，也就是说，头文件的声明已经获取到，源文件的函数定义未获取到。
所以，我们在CMakeList中增加一库partition_base，只包含partition_base.cc，作为rocksdb的依赖。
4. log_builtins.h？
构建报错：
init_logging_service_for_plugin is not declared in this scope
deinit_logging_service_for_plugin is not declared in this scope
然而，我们在ha_rocksdb.cc中已#include <log_builtins.h>，但为什么还会报错呢？
仔细阅读源码后，我们发现代码结构简化如下：
#if defined (MYSQL_DYNAMIC_PLUGIN) //动态链接库声明定义了init和deinit接口
declare init_logging_service_for_plugin
declare deinit_logging_service_for_plugin
#elif defined(EXTRA_CODE_FOR_UNIT_TESTING) //单元测试情况下也定义init和deinit接口
declare init_logging_service_for_plugin
declare deinit_logging_service_for_plugin
#endif //rocksdb现在既不是动态库，也不满足单元测试，于是没有声明init和deinit接口
此处因为缩进问题，困扰好久才得以发现
我们最终在rocksdb情况下（可以参考CMakeList.txt发现ROCKSDB_PLATFORM_POSIX），把这两个接口定义为空，出于：
- 将rocksdb与MYSQL_DYNAMIC_PLUGIN下的接口做相同定义，存在级联的仅在not declared错误
- myisam/innodb/blackhole等内置引擎，是没有调用init/deinit（缩写）两个接口的，所以rocksdb也应向已有内置引擎靠拢
- 为什么不删除rocksdb调用init/deinit的代码？a) 可能存在未知的影响，b) 也为后续可能迁回动态rocksdb链接库留有余地。
总结
在cmake、静态、动态链接上比较菜，所以问题拖了三天解决，期间得到帮助才得以解决。
不知为何，percona不将rocksdb作为内置引擎？（我们发现，增加rocksdb作为内置引擎，percona启动感觉慢了些）
