MongoDB 集群终极方案

查看硬盘分配情况

 1[root@dscn022 mongodb]# lsblk  
 2NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT  
 3sda 8:0 0 278.5G 0 disk  
 4├─sda1 8:1 0 500M 0 part /boot  
 5└─sda2 8:2 0 224.9G 0 part  
 6├─rhel-root 253:0 0 50G 0 lvm /  
 7├─rhel-swap 253:1 0 4G 0 lvm [SWAP]  
 8├─rhel-home 253:2 0 100.9G 0 lvm /home  
 9├─rhel-var 253:3 0 20G 0 lvm /var  
10└─rhel-var_log 253:4 0 50G 0 lvm /var/log  
11sdb 8:16 0 3.7T 0 disk  
12└─sdb1 8:17 0 3.7T 0 part /home/hadmin/logs  
13sdc 8:32 0 3.7T 0 disk  
14└─sdc1 8:33 0 3.7T 0 part  
15sdd 8:48 0 3.7T 0 disk  
16└─sdd1 8:49 0 3.7T 0 part  
17sde 8:64 0 3.7T 0 disk  
18└─sde1 8:65 0 3.7T 0 part  
19sdf 8:80 0 3.7T 0 disk  
20└─sdf1 8:81 0 3.7T 0 part /home/hadmin/data/mongodb/prev  
21sdg 8:96 0 3.7T 0 disk  
22└─sdg1 8:97 0 3.7T 0 part /home/hadmin/data/mongodb/curr  
23sdh 8:112 0 3.7T 0 disk  
24└─sdh1 8:113 0 3.7T 0 part /home/hadmin/data/mongodb/next  
25sdi 8:128 0 3.7T 0 disk  
26└─sdi1 8:129 0 3.7T 0 part  
27sdj 8:144 0 3.7T 0 disk  
28└─sdj1 8:145 0 3.7T 0 part  
29sdk 8:160 0 3.7T 0 disk  
30└─sdk1 8:161 0 3.7T 0 part  
31sdl 8:176 0 3.7T 0 disk  
32└─sdl1 8:177 0 3.7T 0 part  
33sdm 8:192 0 3.7T 0 disk  
34└─sdm1 8:193 0 3.7T 0 part  
35loop0 7:0 0 3.8G 0 loop /mnt  

进行配置

分配mongoDB 目录

1mkdir -p /home/hadmin/data/mongodb/prev  
2mkdir -p /home/hadmin/data/mongodb/curr  
3mkdir -p /home/hadmin/data/mongodb/next  

进行mount

1mount /dev/sdf1 /home/hadmin/data/mongodb/prev  
2mount /dev/sdg1 /home/hadmin/data/mongodb/curr  
3mount /dev/sdh1 /home/hadmin/data/mongodb/next  
4  

查看硬盘大小

 1  
 2[root@dscn022 mongodb]# df -h  
 3Filesystem Size Used Avail Use% Mounted on  
 4/dev/mapper/rhel-root 50G 5.1G 45G 11% /  
 5devtmpfs 126G 0 126G 0% /dev  
 6tmpfs 126G 0 126G 0% /dev/shm  
 7tmpfs 126G 50M 126G 1% /run  
 8tmpfs 126G 0 126G 0% /sys/fs/cgroup  
 9/dev/sda1 494M 127M 368M 26% /boot  
10/dev/mapper/rhel-var 20G 101M 20G 1% /var  
11/dev/mapper/rhel-home 101G 1.5G 100G 2% /home  
12/dev/mapper/rhel-var_log 50G 58M 50G 1% /var/log  
13/dev/loop0 3.8G 3.8G 0 100% /mnt  
14tmpfs 126G 64K 126G 1% /tmp  
15tmpfs 26G 0 26G 0% /run/user/0  
16/dev/sdb1 3.6T 89M 3.4T 1% /home/hadmin/logs  
17/dev/sdf1 3.6T 390M 3.4T 1% /home/hadmin/data/mongodb/prev  
18/dev/sdg1 3.6T 390M 3.4T 1% /home/hadmin/data/mongodb/curr  
19/dev/sdh1 3.6T 390M 3.4T 1% /home/hadmin/data/mongodb/next  
20  

脚本 （傻瓜模式）

• 重要 numactl
• yum install -y numactl
• 配置Mongodb用到的文件夹都要mkdir
• common.sh

 1#!/bin/bash  
 2  
 3# $1就是shard几  
 4# $2目录，重新规划磁盘后可以修改此处  
 5# $3是关闭位，启动不传，关闭传 --shutdown  
 6  
 7INDEX=$1  
 8  
 9PORT=$(( 27018 + $INDEX ))  
10HOSTNAME=`cat /etc/hostname`  
11SHARD="TOD-Shard-$INDEX"  
12DB_PATH=" /home/hadmin/data/mongodb/"$2  
13PID_FILE="/home/hadmin/data/pids/mongodb/$SHARD.pid"  
14LOG_FILE="/home/hadmin/data/logs/mongodb/$SHARD.log"  
15  
16numactl --interleave=all /home/hadmin/mongodb/bin/mongod --logpath $LOG_FILE --logappend --logRotate rename --timeStampFormat ctime --fork --pidfilepath $PID_FILE --port $PORT --bind_ip $HOSTNAME --dbpath $DB_PATH --journalCommitInterval 100 --storageEngine wiredTiger --wiredTigerCacheSizeGB 4 --replSet $SHARD --enableMajorityReadConcern --shardsvr --oplogSize 10240 $3  
17  

cfgsvr.start

1  
2#!/bin/bash  
3  
4numactl --interleave=all /home/hadmin/mongodb/bin/mongod -f /home/hadmin/mongodb/conf/configserver.conf  
5  

route

1  
2#!/bin/bash  
3  
4/home/hadmin/mongodb/bin/mongos -f /home/hadmin/mongodb/conf/router.conf  
5  

shard_curr.start

1  
2#!/bin/bash  
3  
4/home/hadmin/mongodb/sbin/common.sh 1 curr  
5  

shard_next.start

1  
2#!/bin/bash  
3  
4/home/hadmin/mongodb/sbin/common.sh 2 next  
5  

shard_prev.start

1  
2#!/bin/bash  
3  
4/home/hadmin/mongodb/sbin/common.sh 5 prev  
5  

initConfig.sh

• 初始化副本

 1  
 2#!/bin/bash  
 3/home/hadmin/mongodb/bin/mongo dscn022:27018 <<EOF  
 4config={  
 5_id:"ConfigServer",  
 6configsvr:true,  
 7members:[  
 8{_id:0,host:"dscn022:27018",priority:2},  
 9{_id:1,host:"dscn023:27018"},  
10{_id:2,host:"dscn024:27018"}],  
11settings:{getLastErrorDefaults:{w:"majority",wtimeout:5000}}  
12};  
13rs.initiate(config);  
14exit;  
15EOF  
16  

initShard.sh

• 初始化Shard

 1  
 2#!/bin/bash  
 3/home/hadmin/mongodb/bin/mongo dscn022:27017 <<EOF  
 4sh.addShard("TOD-Shard-1/dscn022:27019,dscn023:27019,dscn026:27019");  
 5sh.addShard("TOD-Shard-2/dscn023:27020,dscn024:27020,dscn022:27020");  
 6sh.addShard("TOD-Shard-3/dscn024:27021,dscn025:27021,dscn023:27021");  
 7sh.addShard("TOD-Shard-4/dscn025:27022,dscn026:27022,dscn024:27022");  
 8sh.addShard("TOD-Shard-5/dscn025:27023,dscn026:27023,dscn022:27023");  
 9sh.setBalancerState(false);  
10exit;  
11EOF  
12  

initRS.sh

 1  
 2#!/bin/bash  
 3############  
 4# init rs1 #  
 5############  
 6/home/hadmin/mongodb/bin/mongo dscn022:27019 <<EOF  
 7config={_id:"TOD-Shard-1",  
 8members:[  
 9{_id:0,host:"dscn022:27019",priority:2},  
10{_id:1,host:"dscn026:27019"},  
11{_id:2,host:"dscn023:27019"}],  
12settings:{getLastErrorDefaults:{w:"majority",wtimeout:5000}}};  
13rs.initiate(config);  
14exit;  
15EOF  
16  
17############  
18# init rs2 #  
19############  
20/home/hadmin/mongodb/bin/mongo dscn022:27020 <<EOF  
21config={_id:"TOD-Shard-2",  
22members:[  
23{_id:0,host:"dscn023:27020",priority:2},  
24{_id:1,host:"dscn022:27020"},  
25{_id:2,host:"dscn024:27020"}],  
26settings:{getLastErrorDefaults:{w:"majority",wtimeout:5000}}};  
27rs.initiate(config);  
28exit;  
29EOF  
30  
31############  
32# init rs3 #  
33############  
34/home/hadmin/mongodb/bin/mongo dscn024:27021 <<EOF  
35config={_id:"TOD-Shard-3",  
36members:[  
37{_id:0,host:"dscn024:27021",priority:2},  
38{_id:1,host:"dscn023:27021"},  
39{_id:2,host:"dscn025:27021"}],  
40settings:{getLastErrorDefaults:{w:"majority",wtimeout:5000}}};  
41rs.initiate(config);  
42exit;  
43EOF  
44  
45############  
46# init rs4 #  
47############  
48/home/hadmin/mongodb/bin/mongo dscn025:27022 <<EOF  
49config={_id:"TOD-Shard-4",  
50members:[  
51{_id:0,host:"dscn025:27022",priority:2},  
52{_id:1,host:"dscn024:27022"},  
53{_id:2,host:"dscn026:27022"}],  
54settings:{getLastErrorDefaults:{w:"majority",wtimeout:5000}}};  
55rs.initiate(config);  
56exit;  
57EOF  
58  
59############  
60# init rs5 #  
61############  
62/home/hadmin/mongodb/bin/mongo dscn026:27023 <<EOF  
63config={_id:"TOD-Shard-5",  
64members:[  
65{_id:0,host:"dscn026:27023",priority:2},  
66{_id:1,host:"dscn025:27023"},  
67{_id:2,host:"dscn022:27023"}],  
68settings:{getLastErrorDefaults:{w:"majority",wtimeout:5000}}};  
69rs.initiate(config);  
70exit;  
71EOF  
72  

总结（总体流程）

• 第一步挂盘（mount）
• 复制Mongodb到5个节点
• 配置文件只需要配置（ConfigServer，Route）！Shard 不需要配置。
• 将脚本放倒Sbin下（与mongodb，Bin 同级别目录）
• 启动ConfigServer(3个节点)
• 启动Route 3个节点
• sbin包含文件
• stop 文件请看common.sh

脚本执行流程

• cfgsvr.start（三个节点）
• route.start（三个节点）
• shard_curr.start (每个节点)
• shard_next.start (每个节点)
• shard_prev.start (每个节点)
• initConfig.sh (任意节点)
• initRS.sh (任意节点)
• initShard.sh (任意节点)

提醒

• 注意 ConfigServer 名称
• 注意 Route指定的 ConfigServer地址
• 节点互信 以及hosts