csync+lsync 实时双向批量同步小文件

cysnc+lsync配置安装

环境 ubuntu 12.04

1） 两台机器安装配置：

    apt-get install xinetd csync2 lsyncd

    cat /etc/xinetd.d/csync2         
        service csync2
            {
                disable = no
                port = 30865
                socket_type = stream
                protocol = tcp
                wait = no
                user = root
                passenv = PATH
                server = /usr/sbin/csync2
                server_args = -i
            }

2） node2配置：

在主机node2上面执行csync2 -k /etc/csync2.key (时间长， 耐心等待)  

在主机node2上面csync配置下面三个文件：

cat /etc/csync2.cfg
    nossl * *;
    group WEB
    {
        host node2.wenjun.com;
        host node3.wenjun.com;
        key /etc/csync2.key;
        include /home/ubuntu;
        exclude *~ .*;
        auto none;
    }

cat /etc/csync2_web1.cfg
    nossl * *;
    group web1
    {
            host node2.wenjun.com;
            host (node3.wenjun.com);    #（）向这台推送数据
            key /etc/csync2.key;
            include /home/ubuntu;
            exclude *~ .*;
            auto none;
    }

cat /etc/csync2_web2.cfg
    nossl * *;
    group web2
    {
            host (node2.wenjun.com);
            host node3.wenjun.com;
            key /etc/csync2.key;
            include /home/ubuntu;
            exclude *~ .*;
            auto none;
    }

lsyncd node2配置

cat /etc/lsyncd/lsyncd.conf.lua
    settings = {
            logident        = "lsyncd",
            logfacility     = "user",
        logfile = "/var/log/lsyncd.log",
        statusFile = "/var/log/lsyncd.stat",
        statusInterval = 1,
    }
    
    initSync = {
            delay = 1,
            maxProcesses = 1,
            action = function(inlet)
                    local config = inlet.getConfig()
                    local elist = inlet.getEvents(function(event)
                            return event.etype ~= "Blanket"
                    end)
                    local directory = string.sub(config.source, 1, -2)
                    local paths = elist.getPaths(function(etype, path)
                            return "\t" .. config.syncid .. ":" .. directory .. path
                    end)
                    log("Normal", "Processing syncing list:\n", table.concat(paths, "\n"))
                    spawn(elist, "/usr/sbin/csync2", "-C", config.syncid, "-x")
            end,
            collect = function(agent, exitcode)
                    local config = agent.config
                    if not agent.isList and agent.etype == "Blanket" then
                            if exitcode == 0 then
                                    log("Normal", "Startup of ‘", config.syncid, "‘ instance finished.")
                            elseif config.exitcodes and config.exitcodes[exitcode] == "again" then
                                    log("Normal", "Retrying startup of ‘", config.syncid, "‘ instance.")
                                    return "again"
                            else
                                    log("Error", "Failure on startup of ‘", config.syncid, "‘ instance.")
                                    terminate(-1)
                            end
                            return
                    end
                    local rc = config.exitcodes and config.exitcodes[exitcode]
                    if rc == "die" then
                            return rc
                    end
                    if agent.isList then
                            if rc == "again" then
                                    log("Normal", "Retrying events list on exitcode = ", exitcode)
                            else
                                    log("Normal", "Finished events list = ", exitcode)
                            end
                    else
                            if rc == "again" then
                                    log("Normal", "Retrying ", agent.etype, " on ", agent.sourcePath, " = ", exitcode)
                            else
                                    log("Normal", "Finished ", agent.etype, " on ", agent.sourcePath, " = ", exitcode)
                            end
                    end
                    return rc
            end,
            init = function(inlet)
                    local config = inlet.getConfig()
                    local event = inlet.createBlanketEvent()
                    log("Normal", "Recursive startup sync: ", config.syncid, ":", config.source)
                    spawn(event, "/usr/sbin/csync2", "-C", config.syncid, "-x")
            end,
            prepare = function(config)
                    if not config.syncid then
                            error("Missing ‘syncid‘ parameter.", 4)
                    end
                    local c = "csync2_" .. config.syncid .. ".cfg"
                    local f, err = io.open("/etc/" .. c, "r")
                    if not f then
                            error("Invalid ‘syncid‘ parameter: " .. err, 4)
                    end
                    f:close()
            end
    }
    local sources = {
            ["/home/ubuntu"] = "web1"    
    }
    for key, value in pairs(sources) do
            sync {initSync, source=key, syncid=value}
    end

3） node3配置

同步node2 csync配置到node3

    scp node2:/etc/csync.* /etc/

同步node2 lsyncd 配置到node3

    scp -r node2:/etc/lsyncd  /etc/

    修改下面部分：

        local sources = {        
                ["/home/ubuntu"] = "web2"    
        }

4） 两台机器都执行

    csync2 -xv            # 初始化csync2 的sqlite数据库
    csync2 -vvv -T        # 测试csync2
    
    /etc/init.d/lsyncd start        #启动lsyncd

    tail -f /var/log/lsyncd.log     #查看日志输出显示如下为正常

        Tue Jul 29 17:25:23 2014 Normal: Recursive startup sync: web1:/home/ubuntu/wenjun1/        
        Tue Jul 29 17:25:23 2014 Normal: Startup of ‘web1‘ instance finished.
        Tue Jul 29 17:25:33 2014 Normal: Processing syncing list:

        Wed Jul 30 10:56:28 2014 Normal: Recursive startup sync: web2:/home/ubuntu/wenjun1/
        Wed Jul 30 10:56:28 2014 Normal: Startup of ‘web2‘ instance finished.
        Wed Jul 30 10:56:39 2014 Normal: Processing syncing list:

5） 测试同步

    测试1：

    #!/bin/sh
    for ((i=0; i<100; i++)); do
            touch /home/ubuntu/${i}.html
    done

    测试2：

    #!/bin/bash    
    for ((i=0;i<10;i++))
        do 
            number=$((1000000 + ($(od -An -N2 -i /dev/random)) % (10000 + 1000))) 
            for ((j=0; j<1000; j++))
        do 
                    touch /home/ubuntu/${i}-${j}.html 
            done 
            sleep ${number}  
    done

6） 适用环境

    此方案也可做成链式同步;N台相同资源机器同步

参考： http://oss.linbit.com/csync2/paper.pdf

本文出自 “willard_SA” 博客，请务必保留此出处http://374400.blog.51cto.com/364400/1532772

csync+lsync 实时双向批量同步小文件,布布扣,bubuko.com

csync+lsync 实时双向批量同步小文件

原文：http://374400.blog.51cto.com/364400/1532772