http://blog.csdn.net/acceptedxukai/article/details/18136903
http://blog.csdn.net/acceptedxukai/article/details/18181563
本文所引用的源码全部来自Redis2.8.2版本。
Redis AOF数据持久化机制的实现相关代码是redis.c, redis.h, aof.c, bio.c, rio.c,
config.c
在阅读本文之前请先阅读Redis数据持久化机制AOF原理分析之配置详解文章,了解AOF相关参数的解析,文章链接
http://blog.csdn.net/acceptedxukai/article/details/18135219
转载请注明,文章出自http://blog.csdn.net/acceptedxukai/article/details/18136903
下面将介绍AOF数据持久化机制的实现
Server启动加载AOF文件数据
Server启动加载AOF文件数据的执行步骤为:main() -> initServerConfig() ->
loadServerConfig() -> initServer() ->
loadDataFromDisk()。initServerConfig()主要为初始化默认的AOF参数配置;loadServerConfig()加载配置文件redis.conf中AOF的参数配置,覆盖Server的默认AOF参数配置,如果配置appendonly
on,那么AOF数据持久化功能将被激活,server.aof_state参数被设置为REDIS_AOF_ON;loadDataFromDisk()判断server.aof_state
==
REDIS_AOF_ON,结果为True就调用loadAppendOnlyFile函数加载AOF文件中的数据,加载的方法就是读取AOF文件中数据,由于AOF文件中存储的数据与客户端发送的请求格式相同完全符合Redis的通信协议,因此Server创建伪客户端fakeClient,将解析后的AOF文件数据像客户端请求一样调用各种指令,cmd->proc(fakeClient),将AOF文件中的数据重现到Redis
Server数据库中。
-
- void loadDataFromDisk(void) {
- long long start = ustime();
- if (server.aof_state == REDIS_AOF_ON) {
- if (loadAppendOnlyFile(server.aof_filename) == REDIS_OK)
- redisLog(REDIS_NOTICE,"DB loaded from append only file: %.3f seconds",(float)(ustime()-start)/1000000);
- } else {
- if (rdbLoad(server.rdb_filename) == REDIS_OK) {
- redisLog(REDIS_NOTICE,"DB loaded from disk: %.3f seconds",
- (float)(ustime()-start)/1000000);
- } else if (errno != ENOENT) {
- redisLog(REDIS_WARNING,"Fatal error loading the DB: %s. Exiting.",strerror(errno));
- exit(1);
- }
- }
- }
Server首先判断加载AOF文件是因为AOF文件中的数据要比RDB文件中的数据要新。
- int loadAppendOnlyFile(char *filename) {
- struct redisClient *fakeClient;
- FILE *fp = fopen(filename,"r");
- struct redis_stat sb;
- int old_aof_state = server.aof_state;
- long loops = 0;
-
-
-
- if (fp && redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0) {
- server.aof_current_size = 0;
- fclose(fp);
- return REDIS_ERR;
- }
-
- if (fp == NULL) {
- redisLog(REDIS_WARNING,"Fatal error: can‘t open the append log file for reading: %s",strerror(errno));
- exit(1);
- }
-
-
-
- server.aof_state = REDIS_AOF_OFF;
-
- fakeClient = createFakeClient();
- startLoading(fp);
-
- while(1) {
- int argc, j;
- unsigned long len;
- robj **argv;
- char buf[128];
- sds argsds;
- struct redisCommand *cmd;
-
-
-
- if (!(loops++ % 1000)) {
- loadingProgress(ftello(fp));
- aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT);
- }
-
- if (fgets(buf,sizeof(buf),fp) == NULL) {
- if (feof(fp))
- break;
- else
- goto readerr;
- }
-
- if (buf[0] != ‘*‘) goto fmterr;
- argc = atoi(buf+1);
- if (argc < 1) goto fmterr;
-
- argv = zmalloc(sizeof(robj*)*argc);
- for (j = 0; j < argc; j++) {
- if (fgets(buf,sizeof(buf),fp) == NULL) goto readerr;
- if (buf[0] != ‘$‘) goto fmterr;
- len = strtol(buf+1,NULL,10);
- argsds = sdsnewlen(NULL,len);
-
- if (len && fread(argsds,len,1,fp) == 0) goto fmterr;
- argv[j] = createObject(REDIS_STRING,argsds);
- if (fread(buf,2,1,fp) == 0) goto fmterr;
- }
-
-
- cmd = lookupCommand(argv[0]->ptr);
- if (!cmd) {
- redisLog(REDIS_WARNING,"Unknown command ‘%s‘ reading the append only file", (char*)argv[0]->ptr);
- exit(1);
- }
-
- fakeClient->argc = argc;
- fakeClient->argv = argv;
- cmd->proc(fakeClient);
-
-
- redisAssert(fakeClient->bufpos == 0 && listLength(fakeClient->reply) == 0);
-
- redisAssert((fakeClient->flags & REDIS_BLOCKED) == 0);
-
-
-
- for (j = 0; j < fakeClient->argc; j++)
- decrRefCount(fakeClient->argv[j]);
- zfree(fakeClient->argv);
- }
-
-
-
- if (fakeClient->flags & REDIS_MULTI) goto readerr;
-
- fclose(fp);
- freeFakeClient(fakeClient);
- server.aof_state = old_aof_state;
- stopLoading();
- aofUpdateCurrentSize();
- server.aof_rewrite_base_size = server.aof_current_size;
- return REDIS_OK;
- …………
- }
在前面一篇关于AOF参数配置的博客遗留了一个问题,server.aof_current_size参数的初始化,下面解决这个疑问。
- void aofUpdateCurrentSize(void) {
- struct redis_stat sb;
-
- if (redis_fstat(server.aof_fd,&sb) == -1) {
- redisLog(REDIS_WARNING,"Unable to obtain the AOF file length. stat: %s",
- strerror(errno));
- } else {
- server.aof_current_size = sb.st_size;
- }
- }
redis_fstat是作者对Linux中fstat64函数的重命名,该还是就是获取文件相关的参数信息,具体可以Google之,sb.st_size就是当前AOF文件的大小。这里需要知道server.aof_fd即AOF文件描述符,该参数的初始化在initServer()函数中
-
- if (server.aof_state == REDIS_AOF_ON) {
- server.aof_fd = open(server.aof_filename,O_WRONLY|O_APPEND|O_CREAT,0644);
- if (server.aof_fd == -1) {
- redisLog(REDIS_WARNING, "Can‘t open the append-only file: %s",strerror(errno));
- exit(1);
- }
- }
至此,Redis Server启动加载硬盘中AOF文件数据的操作就成功结束了。
Server数据库产生新数据如何持久化到硬盘
当客户端执行Set等修改数据库中字段的指令时就会造成Server数据库中数据被修改,这些修改的数据应该被实时更新到AOF文件中,并且也要按照一定的fsync机制刷新到硬盘中,保证数据不会丢失。
在上一篇博客中,提到了三种fsync方式:appendfsync always, appendfsync
everysec, appendfsync no. 具体体现在server.aof_fsync参数中。
首先看当客户端请求的指令造成数据被修改,Redis是如何将修改数据的指令添加到server.aof_buf中的。
call() -> propagate()
-> feedAppendOnlyFile(),call()函数判断执行指令后是否造成数据被修改。
feedAppendOnlyFile函数首先会判断Server是否开启了AOF,如果开启AOF,那么根据Redis通讯协议将修改数据的指令重现成请求的字符串,注意在超时设置的处理方式,接着将字符串append到server.aof_buf中即可。该函数最后两行代码需要注意,这才是重点,如果server.aof_child_pid
!= -1那么表明此时Server正在重写rewrite
AOF文件,需要将被修改的数据追加到server.aof_rewrite_buf_blocks链表中,等待rewrite结束后,追加到AOF文件中。具体见下面代码的注释。
-
-
-
-
-
-
-
-
- void propagate(struct redisCommand *cmd, int dbid, robj **argv, int argc,
- int flags)
- {
-
- if (server.aof_state != REDIS_AOF_OFF && flags & REDIS_PROPAGATE_AOF)
- feedAppendOnlyFile(cmd,dbid,argv,argc);
- if (flags & REDIS_PROPAGATE_REPL)
- replicationFeedSlaves(server.slaves,dbid,argv,argc);
- }
-
- void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int argc) {
- sds buf = sdsempty();
- robj *tmpargv[3];
-
-
-
-
- if (dictid != server.aof_selected_db) {
- char seldb[64];
-
- snprintf(seldb,sizeof(seldb),"%d",dictid);
- buf = sdscatprintf(buf,"*2\r\n$6\r\nSELECT\r\n$%lu\r\n%s\r\n",
- (unsigned long)strlen(seldb),seldb);
- server.aof_selected_db = dictid;
- }
-
-
- if (cmd->proc == expireCommand || cmd->proc == pexpireCommand ||
- cmd->proc == expireatCommand) {
-
- buf = catAppendOnlyExpireAtCommand(buf,cmd,argv[1],argv[2]);
- }
- else if (cmd->proc == setexCommand || cmd->proc == psetexCommand) {
-
- tmpargv[0] = createStringObject("SET",3);
- tmpargv[1] = argv[1];
- tmpargv[2] = argv[3];
- buf = catAppendOnlyGenericCommand(buf,3,tmpargv);
- decrRefCount(tmpargv[0]);
- buf = catAppendOnlyExpireAtCommand(buf,cmd,argv[1],argv[2]);
- } else {
-
-
-
- buf = catAppendOnlyGenericCommand(buf,argc,argv);
- }
-
-
-
-
-
- if (server.aof_state == REDIS_AOF_ON)
- server.aof_buf = sdscatlen(server.aof_buf,buf,sdslen(buf));
-
-
-
-
-
-
-
-
-
-
- if (server.aof_child_pid != -1)
- aofRewriteBufferAppend((unsigned char*)buf,sdslen(buf));
-
-
-
-
-
-
-
-
-
-
- sdsfree(buf);
- }
Server在每次事件循环之前会调用一次beforeSleep函数,下面看看这个函数做了什么工作?
-
-
-
- void beforeSleep(struct aeEventLoop *eventLoop) {
- REDIS_NOTUSED(eventLoop);
- listNode *ln;
- redisClient *c;
-
-
-
- if (server.active_expire_enabled && server.masterhost == NULL)
- activeExpireCycle(ACTIVE_EXPIRE_CYCLE_FAST);
-
-
- while (listLength(server.unblocked_clients)) {
- ln = listFirst(server.unblocked_clients);
- redisAssert(ln != NULL);
- c = ln->value;
- listDelNode(server.unblocked_clients,ln);
- c->flags &= ~REDIS_UNBLOCKED;
-
-
-
- if (c->querybuf && sdslen(c->querybuf) > 0) {
- server.current_client = c;
- processInputBuffer(c);
- server.current_client = NULL;
- }
- }
-
-
-
- flushAppendOnlyFile(0);
- }
通过上面的代码及注释可以发现,beforeSleep函数做了三件事:1、处理过期键,2、处理阻塞期间的客户端请求,3、将server.aof_buf中的数据追加到AOF文件中并fsync刷新到硬盘上,flushAppendOnlyFile函数给定了一个参数force,表示是否强制写入AOF文件,0表示非强制即支持延迟写,1表示强制写入。
- void flushAppendOnlyFile(int force) {
- ssize_t nwritten;
- int sync_in_progress = 0;
- if (sdslen(server.aof_buf) == 0) return;
-
- if (server.aof_fsync == AOF_FSYNC_EVERYSEC)
- sync_in_progress = bioPendingJobsOfType(REDIS_BIO_AOF_FSYNC) != 0;
-
-
- if (server.aof_fsync == AOF_FSYNC_EVERYSEC && !force) {
-
-
-
-
- if (sync_in_progress) {
-
- if (server.aof_flush_postponed_start == 0) {
-
-
- server.aof_flush_postponed_start = server.unixtime;
- return;
- } else if (server.unixtime - server.aof_flush_postponed_start < 2) {
-
-
-
- return;
- }
-
-
- server.aof_delayed_fsync++;
- redisLog(REDIS_NOTICE,"Asynchronous AOF fsync is taking too long (disk is busy?). Writing the AOF buffer without waiting for fsync to complete, this may slow down Redis.");
- }
- }
-
-
- server.aof_flush_postponed_start = 0;
-
-
-
-
-
-
-
- nwritten = write(server.aof_fd,server.aof_buf,sdslen(server.aof_buf));
- if (nwritten != (signed)sdslen(server.aof_buf)) {
-
-
-
- if (nwritten == -1) {
- redisLog(REDIS_WARNING,"Exiting on error writing to the append-only file: %s",strerror(errno));
- } else {
- redisLog(REDIS_WARNING,"Exiting on short write while writing to "
- "the append-only file: %s (nwritten=%ld, "
- "expected=%ld)",
- strerror(errno),
- (long)nwritten,
- (long)sdslen(server.aof_buf));
-
- if (ftruncate(server.aof_fd, server.aof_current_size) == -1) {
- redisLog(REDIS_WARNING, "Could not remove short write "
- "from the append-only file. Redis may refuse "
- "to load the AOF the next time it starts. "
- "ftruncate: %s", strerror(errno));
- }
- }
- exit(1);
- }
- server.aof_current_size += nwritten;
-
-
-
-
- if ((sdslen(server.aof_buf)+sdsavail(server.aof_buf)) < 4000) {
- sdsclear(server.aof_buf);
- } else {
- sdsfree(server.aof_buf);
- server.aof_buf = sdsempty();
- }
-
-
-
-
-
- if (server.aof_no_fsync_on_rewrite &&
- (server.aof_child_pid != -1 || server.rdb_child_pid != -1))
- return;
-
-
- if (server.aof_fsync == AOF_FSYNC_ALWAYS) {
-
-
- aof_fsync(server.aof_fd);
- server.aof_last_fsync = server.unixtime;
- } else if ((server.aof_fsync == AOF_FSYNC_EVERYSEC &&
- server.unixtime > server.aof_last_fsync)) {
- if (!sync_in_progress) aof_background_fsync(server.aof_fd);
- server.aof_last_fsync = server.unixtime;
- }
- }
上述代码中请关注server.aof_fsync参数,即设置Redis fsync
AOF文件到硬盘的策略,如果设置为AOF_FSYNC_ALWAYS,那么直接在主进程中fsync,如果设置为AOF_FSYNC_EVERYSEC,那么放入后台线程中fsync,后台线程的代码在bio.c中。
小结
文章写到这,已经解决的了Redis
Server启动加载AOF文件和如何将客户端请求产生的新的数据追加到AOF文件中,对于追加数据到AOF文件中,根据fsync的配置策略如何将写入到AOF文件中的新数据刷新到硬盘中,直接在主进程中fsync或是在后台线程fsync。
至此,AOF数据持久化还剩下如何rewrite
AOF,接受客户端发送的BGREWRITEAOF请求,此部分内容待下篇博客中解析。
感谢此篇博客给我在理解Redis AOF数据持久化方面的巨大帮助,http://chenzhenianqing.cn/articles/786.html
本人Redis-2.8.2的源码注释已经放到Github中,有需要的读者可以下载,我也会在后续的时间中更新,https://github.com/xkeyideal/annotated-redis-2.8.2
本人不怎么会使用Git,望有人能教我一下。
--------------------------------------------------------------------------------------------------------------------------------------------------------------
本文所引用的源码全部来自Redis2.8.2版本。
Redis AOF数据持久化机制的实现相关代码是redis.c, redis.h, aof.c, bio.c, rio.c,
config.c
在阅读本文之前请先阅读Redis数据持久化机制AOF原理分析之配置详解文章,了解AOF相关参数的解析,文章链接
http://blog.csdn.net/acceptedxukai/article/details/18135219
接着上一篇文章,本文将介绍Redis是如何实现AOF rewrite的。
转载请注明,文章出自http://blog.csdn.net/acceptedxukai/article/details/18181563
AOF rewrite的触发机制
如果Redis只是将客户端修改数据库的指令重现存储在AOF文件中,那么AOF文件的大小会不断的增加,因为AOF文件只是简单的重现存储了客户端的指令,而并没有进行合并。对于该问题最简单的处理方式,即当AOF文件满足一定条件时就对AOF进行rewrite,rewrite是根据当前内存数据库中的数据进行遍历写到一个临时的AOF文件,待写完后替换掉原来的AOF文件即可。
Redis触发AOF rewrite机制有三种:
1、Redis
Server接收到客户端发送的BGREWRITEAOF指令请求,如果当前AOF/RDB数据持久化没有在执行,那么执行,反之,等当前AOF/RDB数据持久化结束后执行AOF
rewrite
2、在Redis配置文件redis.conf中,用户设置了auto-aof-rewrite-percentage和auto-aof-rewrite-min-size参数,并且当前AOF文件大小server.aof_current_size大于auto-aof-rewrite-min-size(server.aof_rewrite_min_size),同时AOF文件大小的增长率大于auto-aof-rewrite-percentage(server.aof_rewrite_perc)时,会自动触发AOF
rewrite
3、用户设置“config set appendonly
yes”开启AOF的时,调用startAppendOnly函数会触发rewrite
下面分别介绍上述三种机制的处理.
接收到BGREWRITEAOF指令
- <span style="font-size:12px;">void bgrewriteaofCommand(redisClient *c) {
-
- if (server.aof_child_pid != -1) {
- addReplyError(c,"Background append only file rewriting already in progress");
- } else if (server.rdb_child_pid != -1) {
-
-
- server.aof_rewrite_scheduled = 1;
- addReplyStatus(c,"Background append only file rewriting scheduled");
- } else if (rewriteAppendOnlyFileBackground() == REDIS_OK) {
-
- addReplyStatus(c,"Background append only file rewriting started");
- } else {
- addReply(c,shared.err);
- }
- }</span>
当AOF
rewrite请求被挂起时,在serverCron函数中,会处理。
-
-
-
-
-
- if (server.rdb_child_pid == -1 && server.aof_child_pid == -1 &&
- server.aof_rewrite_scheduled)
- {
- rewriteAppendOnlyFileBackground();
- }
Server自动对AOF进行rewrite
在serverCron函数中会周期性判断
-
-
- if (server.rdb_child_pid == -1 &&
- server.aof_child_pid == -1 &&
- server.aof_rewrite_perc &&
- server.aof_current_size > server.aof_rewrite_min_size)
- {
- long long base = server.aof_rewrite_base_size ?
- server.aof_rewrite_base_size : 1;
- long long growth = (server.aof_current_size*100/base) - 100;
- if (growth >= server.aof_rewrite_perc) {
- redisLog(REDIS_NOTICE,"Starting automatic rewriting of AOF on %lld%% growth",growth);
- rewriteAppendOnlyFileBackground();
- }
- }
config set appendonly yes
当客户端发送该指令时,config.c中的configSetCommand函数会做出响应,startAppendOnly函数会执行AOF
rewrite
- if (!strcasecmp(c->argv[2]->ptr,"appendonly")) {
- int enable = yesnotoi(o->ptr);
-
- if (enable == -1) goto badfmt;
- if (enable == 0 && server.aof_state != REDIS_AOF_OFF) {
- stopAppendOnly();
- } else if (enable && server.aof_state == REDIS_AOF_OFF) {
- if (startAppendOnly() == REDIS_ERR) {
- addReplyError(c,
- "Unable to turn on AOF. Check server logs.");
- return;
- }
- }
- }
- int startAppendOnly(void) {
- server.aof_last_fsync = server.unixtime;
- server.aof_fd = open(server.aof_filename,O_WRONLY|O_APPEND|O_CREAT,0644);
- redisAssert(server.aof_state == REDIS_AOF_OFF);
- if (server.aof_fd == -1) {
- redisLog(REDIS_WARNING,"Redis needs to enable the AOF but can‘t open the append only file: %s",strerror(errno));
- return REDIS_ERR;
- }
- if (rewriteAppendOnlyFileBackground() == REDIS_ERR) {
- close(server.aof_fd);
- redisLog(REDIS_WARNING,"Redis needs to enable the AOF but can‘t trigger a background AOF rewrite operation. Check the above logs for more info about the error.");
- return REDIS_ERR;
- }
-
-
- server.aof_state = REDIS_AOF_WAIT_REWRITE;
- return REDIS_OK;
- }
Redis AOF
rewrite机制的实现
从上述分析可以看出rewrite的实现全部依靠rewriteAppendOnlyFileBackground函数,下面分析该函数,通过下面的代码可以看出,Redis是fork出一个子进程来操作AOF
rewrite,然后子进程调用rewriteAppendOnlyFile函数,将数据写到一个临时文件temp-rewriteaof-bg-%d.aof中。如果子进程完成会通过exit(0)函数通知父进程rewrite结束,在serverCron函数中使用wait3函数接收子进程退出状态,然后执行后续的AOF
rewrite的收尾工作,后面将会分析。
父进程的工作主要包括清楚server.aof_rewrite_scheduled标志,记录子进程IDserver.aof_child_pid
= childpid,记录rewrite的开始时间server.aof_rewrite_time_start =
time(NULL)等。
- int rewriteAppendOnlyFileBackground(void) {
- pid_t childpid;
- long long start;
-
-
- if (server.aof_child_pid != -1) return REDIS_ERR;
- start = ustime();
- if ((childpid = fork()) == 0) {
- char tmpfile[256];
-
-
- closeListeningSockets(0);
- redisSetProcTitle("redis-aof-rewrite");
- snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
- if (rewriteAppendOnlyFile(tmpfile) == REDIS_OK) {
- size_t private_dirty = zmalloc_get_private_dirty();
-
- if (private_dirty) {
- redisLog(REDIS_NOTICE,
- "AOF rewrite: %zu MB of memory used by copy-on-write",
- private_dirty/(1024*1024));
- }
- exitFromChild(0);
- } else {
- exitFromChild(1);
- }
- } else {
-
- server.stat_fork_time = ustime()-start;
- if (childpid == -1) {
- redisLog(REDIS_WARNING,
- "Can‘t rewrite append only file in background: fork: %s",
- strerror(errno));
- return REDIS_ERR;
- }
- redisLog(REDIS_NOTICE,
- "Background append only file rewriting started by pid %d",childpid);
- server.aof_rewrite_scheduled = 0;
- server.aof_rewrite_time_start = time(NULL);
- server.aof_child_pid = childpid;
- updateDictResizePolicy();
-
-
-
-
- server.aof_selected_db = -1;
- replicationScriptCacheFlush();
- return REDIS_OK;
- }
- return REDIS_OK;
- }
接下来介绍rewriteAppendOnlyFile函数,该函数的主要工作为:遍历所有数据库中的数据,将其写入到临时文件temp-rewriteaof-%d.aof中,写入函数定义在rio.c中,比较简单,然后将数据刷新到硬盘中,然后将文件名rename为其调用者给定的临时文件名,注意仔细看代码,这里并没有修改为正式的AOF文件名。
在写入文件时如果设置server.aof_rewrite_incremental_fsync参数,那么在rioWrite函数中fwrite部分数据就会将数据fsync到硬盘中,来保证数据的正确性。
- int rewriteAppendOnlyFile(char *filename) {
- dictIterator *di = NULL;
- dictEntry *de;
- rio aof;
- FILE *fp;
- char tmpfile[256];
- int j;
- long long now = mstime();
-
-
-
- snprintf(tmpfile,256,"temp-rewriteaof-%d.aof", (int) getpid());
- fp = fopen(tmpfile,"w");
- if (!fp) {
- redisLog(REDIS_WARNING, "Opening the temp file for AOF rewrite in rewriteAppendOnlyFile(): %s", strerror(errno));
- return REDIS_ERR;
- }
-
- rioInitWithFile(&aof,fp);
-
- if (server.aof_rewrite_incremental_fsync)
- rioSetAutoSync(&aof,REDIS_AOF_AUTOSYNC_BYTES);
- for (j = 0; j < server.dbnum; j++) {
- char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n";
- redisDb *db = server.db+j;
- dict *d = db->dict;
- if (dictSize(d) == 0) continue;
- di = dictGetSafeIterator(d);
- if (!di) {
- fclose(fp);
- return REDIS_ERR;
- }
-
-
- if (rioWrite(&aof,selectcmd,sizeof(selectcmd)-1) == 0) goto werr;
- if (rioWriteBulkLongLong(&aof,j) == 0) goto werr;
-
-
- while((de = dictNext(di)) != NULL) {
- sds keystr;
- robj key, *o;
- long long expiretime;
-
- keystr = dictGetKey(de);
- o = dictGetVal(de);
- initStaticStringObject(key,keystr);
-
- expiretime = getExpire(db,&key);
-
-
- if (expiretime != -1 && expiretime < now) continue;
-
-
- if (o->type == REDIS_STRING) {
-
- char cmd[]="*3\r\n$3\r\nSET\r\n";
- if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
-
- if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
- if (rioWriteBulkObject(&aof,o) == 0) goto werr;
- } else if (o->type == REDIS_LIST) {
- if (rewriteListObject(&aof,&key,o) == 0) goto werr;
- } else if (o->type == REDIS_SET) {
- if (rewriteSetObject(&aof,&key,o) == 0) goto werr;
- } else if (o->type == REDIS_ZSET) {
- if (rewriteSortedSetObject(&aof,&key,o) == 0) goto werr;
- } else if (o->type == REDIS_HASH) {
- if (rewriteHashObject(&aof,&key,o) == 0) goto werr;
- } else {
- redisPanic("Unknown object type");
- }
-
- if (expiretime != -1) {
- char cmd[]="*3\r\n$9\r\nPEXPIREAT\r\n";
- if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
- if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
- if (rioWriteBulkLongLong(&aof,expiretime) == 0) goto werr;
- }
- }
- dictReleaseIterator(di);
- }
-
-
- fflush(fp);
- aof_fsync(fileno(fp));
- fclose(fp);
-
-
-
- if (rename(tmpfile,filename) == -1) {
- redisLog(REDIS_WARNING,"Error moving temp append only file on the final destination: %s", strerror(errno));
- unlink(tmpfile);
- return REDIS_ERR;
- }
- redisLog(REDIS_NOTICE,"SYNC append only file rewrite performed");
- return REDIS_OK;
-
- werr:
- fclose(fp);
- unlink(tmpfile);
- redisLog(REDIS_WARNING,"Write error writing append only file on disk: %s", strerror(errno));
- if (di) dictReleaseIterator(di);
- return REDIS_ERR;
- }
AOF
rewrite工作到这里已经结束一半,上一篇文章提到如果server.aof_state !=
REDIS_AOF_OFF,那么就会将客户端请求指令修改的数据通过feedAppendOnlyFile函数追加到AOF文件中,那么此时AOF已经rewrite了,必须要处理此时出现的差异数据,记得在feedAppendOnlyFile函数中有这么一段代码
- if (server.aof_child_pid != -1)
- aofRewriteBufferAppend((unsigned char*)buf,sdslen(buf));
如果AOF
rewrite正在进行,那么就将修改数据的指令字符串存储到server.aof_rewrite_buf_blocks链表中,等待AOF
rewrite子进程结束后处理,处理此部分数据的代码在serverCron函数中。需要指出的是wait3函数我不了解,可能下面注释会有点问题。
-
-
- if (server.rdb_child_pid != -1 || server.aof_child_pid != -1) {
- int statloc;
- pid_t pid;
-
- if ((pid = wait3(&statloc,WNOHANG,NULL)) != 0) {
- int exitcode = WEXITSTATUS(statloc);
- int bysignal = 0;
-
- if (WIFSIGNALED(statloc)) bysignal = WTERMSIG(statloc);
-
- if (pid == server.rdb_child_pid) {
- backgroundSaveDoneHandler(exitcode,bysignal);
- } else if (pid == server.aof_child_pid) {
- backgroundRewriteDoneHandler(exitcode,bysignal);
- } else {
- redisLog(REDIS_WARNING,
- "Warning, detected child with unmatched pid: %ld",
- (long)pid);
- }
-
- updateDictResizePolicy();
- }
- }
对于AOF
rewrite期间出现的差异数据,Server通过backgroundSaveDoneHandler函数将
server.aof_rewrite_buf_blocks链表中数据追加到新的AOF文件中。
backgroundSaveDoneHandler函数执行步骤:
1、通过判断子进程的退出状态,正确的退出状态为exit(0),即exitcode为0,bysignal我不清楚具体意义,如果退出状态正确,backgroundSaveDoneHandler函数才会开始处理
2、通过对rewriteAppendOnlyFileBackground函数的分析,可以知道rewrite后的AOF临时文件名为temp-rewriteaof-bg-%d.aof(%d=server.aof_child_pid)中,接着需要打开此临时文件
3、调用aofRewriteBufferWrite函数将server.aof_rewrite_buf_blocks中差异数据写到该临时文件中
4、如果旧的AOF文件未打开,那么打开旧的AOF文件,将文件描述符赋值给临时变量oldfd
5、将临时的AOF文件名rename为正常的AOF文件名
6、如果旧的AOF文件未打开,那么此时只需要关闭新的AOF文件,此时的server.aof_rewrite_buf_blocks数据应该为空;如果旧的AOF是打开的,那么将server.aof_fd指向newfd,然后根据相应的fsync策略将数据刷新到硬盘上
7、调用aofUpdateCurrentSize函数统计AOF文件的大小,更新server.aof_rewrite_base_size,为serverCron中自动AOF
rewrite做相应判断
8、如果之前是REDIS_AOF_WAIT_REWRITE状态,则设置server.aof_state为REDIS_AOF_ON,因为只有“config
set appendonly
yes”指令才会设置这个状态,也就是需要写完快照后,立即打开AOF;而BGREWRITEAOF不需要打开AOF
9、调用后台线程去关闭旧的AOF文件
下面是backgroundSaveDoneHandler函数的注释代码
-
-
- void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
- if (!bysignal && exitcode == 0) {
- int newfd, oldfd;
- char tmpfile[256];
- long long now = ustime();
-
- redisLog(REDIS_NOTICE,
- "Background AOF rewrite terminated with success");
-
-
-
- snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof",
- (int)server.aof_child_pid);
- newfd = open(tmpfile,O_WRONLY|O_APPEND);
- if (newfd == -1) {
- redisLog(REDIS_WARNING,
- "Unable to open the temporary AOF produced by the child: %s", strerror(errno));
- goto cleanup;
- }
-
- if (aofRewriteBufferWrite(newfd) == -1) {
- redisLog(REDIS_WARNING,
- "Error trying to flush the parent diff to the rewritten AOF: %s", strerror(errno));
- close(newfd);
- goto cleanup;
- }
-
- redisLog(REDIS_NOTICE,
- "Parent diff successfully flushed to the rewritten AOF (%lu bytes)", aofRewriteBufferSize());
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- if (server.aof_fd == -1) {
-
-
-
-
-
- oldfd = open(server.aof_filename,O_RDONLY|O_NONBLOCK);
- } else {
-
- oldfd = -1;
- }
-
-
-
-
-
- if (rename(tmpfile,server.aof_filename) == -1) {
- redisLog(REDIS_WARNING,
- "Error trying to rename the temporary AOF file: %s", strerror(errno));
- close(newfd);
- if (oldfd != -1) close(oldfd);
- goto cleanup;
- }
-
-
-
- if (server.aof_fd == -1) {
-
-
- close(newfd);
- } else {
-
- oldfd = server.aof_fd;
-
- server.aof_fd = newfd;
-
- if (server.aof_fsync == AOF_FSYNC_ALWAYS)
- aof_fsync(newfd);
- else if (server.aof_fsync == AOF_FSYNC_EVERYSEC)
- aof_background_fsync(newfd);
- server.aof_selected_db = -1;
- aofUpdateCurrentSize();
- server.aof_rewrite_base_size = server.aof_current_size;
-
-
-
-
- sdsfree(server.aof_buf);
- server.aof_buf = sdsempty();
- }
-
- server.aof_lastbgrewrite_status = REDIS_OK;
-
- redisLog(REDIS_NOTICE, "Background AOF rewrite finished successfully");
-
-
- if (server.aof_state == REDIS_AOF_WAIT_REWRITE)
- server.aof_state = REDIS_AOF_ON;
-
-
-
- if (oldfd != -1) bioCreateBackgroundJob(REDIS_BIO_CLOSE_FILE,(void*)(long)oldfd,NULL,NULL);
-
- redisLog(REDIS_VERBOSE,
- "Background AOF rewrite signal handler took %lldus", ustime()-now);
- } else if (!bysignal && exitcode != 0) {
- server.aof_lastbgrewrite_status = REDIS_ERR;
-
- redisLog(REDIS_WARNING,
- "Background AOF rewrite terminated with error");
- } else {
- server.aof_lastbgrewrite_status = REDIS_ERR;
-
- redisLog(REDIS_WARNING,
- "Background AOF rewrite terminated by signal %d", bysignal);
- }
-
- cleanup:
- aofRewriteBufferReset();
- aofRemoveTempFile(server.aof_child_pid);
- server.aof_child_pid = -1;
- server.aof_rewrite_time_last = time(NULL)-server.aof_rewrite_time_start;
- server.aof_rewrite_time_start = -1;
-
- if (server.aof_state == REDIS_AOF_WAIT_REWRITE)
- server.aof_rewrite_scheduled = 1;
- }
至此,AOF数据持久化已经全部结束了,剩下的就是一些细节的处理,以及一些Linux库函数的理解,对于rename、unlink、wait3等库
函数的深入认识就去问Google吧。
小结
Redis
AOF数据持久化的实现机制通过三篇文章基本上比较详细的分析了,但这只是从代码层面去看AOF,对于AOF持久化的优缺点网上有很多分析,Redis的官方网站也有英文介绍,Redis的数据持久化还有一种方法叫RDB,更多RDB的内容等下次再分析。
Redis数据持久化机制AOF原理分析一---转
原文:http://www.cnblogs.com/davidwang456/p/3521182.html