MongoDB 集群终极方案
查看硬盘分配情况
1[root@dscn022 mongodb]# lsblk
2NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
3sda 8:0 0 278.5G 0 disk
4├─sda1 8:1 0 500M 0 part /boot
5└─sda2 8:2 0 224.9G 0 part
6├─rhel-root 253:0 0 50G 0 lvm /
7├─rhel-swap 253:1 0 4G 0 lvm [SWAP]
8├─rhel-home 253:2 0 100.9G 0 lvm /home
9├─rhel-var 253:3 0 20G 0 lvm /var
10└─rhel-var_log 253:4 0 50G 0 lvm /var/log
11sdb 8:16 0 3.7T 0 disk
12└─sdb1 8:17 0 3.7T 0 part /home/hadmin/logs
13sdc 8:32 0 3.7T 0 disk
14└─sdc1 8:33 0 3.7T 0 part
15sdd 8:48 0 3.7T 0 disk
16└─sdd1 8:49 0 3.7T 0 part
17sde 8:64 0 3.7T 0 disk
18└─sde1 8:65 0 3.7T 0 part
19sdf 8:80 0 3.7T 0 disk
20└─sdf1 8:81 0 3.7T 0 part /home/hadmin/data/mongodb/prev
21sdg 8:96 0 3.7T 0 disk
22└─sdg1 8:97 0 3.7T 0 part /home/hadmin/data/mongodb/curr
23sdh 8:112 0 3.7T 0 disk
24└─sdh1 8:113 0 3.7T 0 part /home/hadmin/data/mongodb/next
25sdi 8:128 0 3.7T 0 disk
26└─sdi1 8:129 0 3.7T 0 part
27sdj 8:144 0 3.7T 0 disk
28└─sdj1 8:145 0 3.7T 0 part
29sdk 8:160 0 3.7T 0 disk
30└─sdk1 8:161 0 3.7T 0 part
31sdl 8:176 0 3.7T 0 disk
32└─sdl1 8:177 0 3.7T 0 part
33sdm 8:192 0 3.7T 0 disk
34└─sdm1 8:193 0 3.7T 0 part
35loop0 7:0 0 3.8G 0 loop /mnt
进行配置
| ——- |
—— |
——- |
| curr |
1 |
2 |
| prev |
5 |
1 |
| next |
2 |
3 |
分配mongoDB 目录
1mkdir -p /home/hadmin/data/mongodb/prev
2mkdir -p /home/hadmin/data/mongodb/curr
3mkdir -p /home/hadmin/data/mongodb/next
进行mount
1mount /dev/sdf1 /home/hadmin/data/mongodb/prev
2mount /dev/sdg1 /home/hadmin/data/mongodb/curr
3mount /dev/sdh1 /home/hadmin/data/mongodb/next
4
查看硬盘大小
1
2[root@dscn022 mongodb]# df -h
3Filesystem Size Used Avail Use% Mounted on
4/dev/mapper/rhel-root 50G 5.1G 45G 11% /
5devtmpfs 126G 0 126G 0% /dev
6tmpfs 126G 0 126G 0% /dev/shm
7tmpfs 126G 50M 126G 1% /run
8tmpfs 126G 0 126G 0% /sys/fs/cgroup
9/dev/sda1 494M 127M 368M 26% /boot
10/dev/mapper/rhel-var 20G 101M 20G 1% /var
11/dev/mapper/rhel-home 101G 1.5G 100G 2% /home
12/dev/mapper/rhel-var_log 50G 58M 50G 1% /var/log
13/dev/loop0 3.8G 3.8G 0 100% /mnt
14tmpfs 126G 64K 126G 1% /tmp
15tmpfs 26G 0 26G 0% /run/user/0
16/dev/sdb1 3.6T 89M 3.4T 1% /home/hadmin/logs
17/dev/sdf1 3.6T 390M 3.4T 1% /home/hadmin/data/mongodb/prev
18/dev/sdg1 3.6T 390M 3.4T 1% /home/hadmin/data/mongodb/curr
19/dev/sdh1 3.6T 390M 3.4T 1% /home/hadmin/data/mongodb/next
20
脚本 (傻瓜模式)
- 重要
numactl
- yum install -y numactl
- 配置Mongodb用到的文件夹都要
mkdir
- common.sh
1#!/bin/bash
2
3# $1就是shard几
4# $2目录,重新规划磁盘后可以修改此处
5# $3是关闭位,启动不传,关闭传 --shutdown
6
7INDEX=$1
8
9PORT=$(( 27018 + $INDEX ))
10HOSTNAME=`cat /etc/hostname`
11SHARD="TOD-Shard-$INDEX"
12DB_PATH=" /home/hadmin/data/mongodb/"$2
13PID_FILE="/home/hadmin/data/pids/mongodb/$SHARD.pid"
14LOG_FILE="/home/hadmin/data/logs/mongodb/$SHARD.log"
15
16numactl --interleave=all /home/hadmin/mongodb/bin/mongod --logpath $LOG_FILE --logappend --logRotate rename --timeStampFormat ctime --fork --pidfilepath $PID_FILE --port $PORT --bind_ip $HOSTNAME --dbpath $DB_PATH --journalCommitInterval 100 --storageEngine wiredTiger --wiredTigerCacheSizeGB 4 --replSet $SHARD --enableMajorityReadConcern --shardsvr --oplogSize 10240 $3
17
cfgsvr.start
1
2#!/bin/bash
3
4numactl --interleave=all /home/hadmin/mongodb/bin/mongod -f /home/hadmin/mongodb/conf/configserver.conf
5
route
1
2#!/bin/bash
3
4/home/hadmin/mongodb/bin/mongos -f /home/hadmin/mongodb/conf/router.conf
5
shard_curr.start
1
2#!/bin/bash
3
4/home/hadmin/mongodb/sbin/common.sh 1 curr
5
shard_next.start
1
2#!/bin/bash
3
4/home/hadmin/mongodb/sbin/common.sh 2 next
5
shard_prev.start
1
2#!/bin/bash
3
4/home/hadmin/mongodb/sbin/common.sh 5 prev
5
initConfig.sh
1
2#!/bin/bash
3/home/hadmin/mongodb/bin/mongo dscn022:27018 <<EOF
4config={
5_id:"ConfigServer",
6configsvr:true,
7members:[
8{_id:0,host:"dscn022:27018",priority:2},
9{_id:1,host:"dscn023:27018"},
10{_id:2,host:"dscn024:27018"}],
11settings:{getLastErrorDefaults:{w:"majority",wtimeout:5000}}
12};
13rs.initiate(config);
14exit;
15EOF
16
initShard.sh
1
2#!/bin/bash
3/home/hadmin/mongodb/bin/mongo dscn022:27017 <<EOF
4sh.addShard("TOD-Shard-1/dscn022:27019,dscn023:27019,dscn026:27019");
5sh.addShard("TOD-Shard-2/dscn023:27020,dscn024:27020,dscn022:27020");
6sh.addShard("TOD-Shard-3/dscn024:27021,dscn025:27021,dscn023:27021");
7sh.addShard("TOD-Shard-4/dscn025:27022,dscn026:27022,dscn024:27022");
8sh.addShard("TOD-Shard-5/dscn025:27023,dscn026:27023,dscn022:27023");
9sh.setBalancerState(false);
10exit;
11EOF
12
1
2#!/bin/bash
3############
4# init rs1 #
5############
6/home/hadmin/mongodb/bin/mongo dscn022:27019 <<EOF
7config={_id:"TOD-Shard-1",
8members:[
9{_id:0,host:"dscn022:27019",priority:2},
10{_id:1,host:"dscn026:27019"},
11{_id:2,host:"dscn023:27019"}],
12settings:{getLastErrorDefaults:{w:"majority",wtimeout:5000}}};
13rs.initiate(config);
14exit;
15EOF
16
17############
18# init rs2 #
19############
20/home/hadmin/mongodb/bin/mongo dscn022:27020 <<EOF
21config={_id:"TOD-Shard-2",
22members:[
23{_id:0,host:"dscn023:27020",priority:2},
24{_id:1,host:"dscn022:27020"},
25{_id:2,host:"dscn024:27020"}],
26settings:{getLastErrorDefaults:{w:"majority",wtimeout:5000}}};
27rs.initiate(config);
28exit;
29EOF
30
31############
32# init rs3 #
33############
34/home/hadmin/mongodb/bin/mongo dscn024:27021 <<EOF
35config={_id:"TOD-Shard-3",
36members:[
37{_id:0,host:"dscn024:27021",priority:2},
38{_id:1,host:"dscn023:27021"},
39{_id:2,host:"dscn025:27021"}],
40settings:{getLastErrorDefaults:{w:"majority",wtimeout:5000}}};
41rs.initiate(config);
42exit;
43EOF
44
45############
46# init rs4 #
47############
48/home/hadmin/mongodb/bin/mongo dscn025:27022 <<EOF
49config={_id:"TOD-Shard-4",
50members:[
51{_id:0,host:"dscn025:27022",priority:2},
52{_id:1,host:"dscn024:27022"},
53{_id:2,host:"dscn026:27022"}],
54settings:{getLastErrorDefaults:{w:"majority",wtimeout:5000}}};
55rs.initiate(config);
56exit;
57EOF
58
59############
60# init rs5 #
61############
62/home/hadmin/mongodb/bin/mongo dscn026:27023 <<EOF
63config={_id:"TOD-Shard-5",
64members:[
65{_id:0,host:"dscn026:27023",priority:2},
66{_id:1,host:"dscn025:27023"},
67{_id:2,host:"dscn022:27023"}],
68settings:{getLastErrorDefaults:{w:"majority",wtimeout:5000}}};
69rs.initiate(config);
70exit;
71EOF
72
总结(总体流程)
- 第一步挂盘(mount)
- 复制Mongodb到5个节点
- 配置文件只需要配置(ConfigServer,Route)!Shard 不需要配置。
- 将脚本放倒Sbin下(与mongodb,Bin 同级别目录)
- 启动ConfigServer(3个节点)
- 启动Route 3个节点
- sbin包含文件
- stop 文件请看
common.sh

脚本执行流程
- cfgsvr.start(三个节点)
- route.start(三个节点)
- shard_curr.start (每个节点)
- shard_next.start (每个节点)
- shard_prev.start (每个节点)
- initConfig.sh (任意节点)
- initRS.sh (任意节点)
- initShard.sh (任意节点)
提醒
- 注意
ConfigServer 名称
- 注意
Route指定的 ConfigServer地址
- 节点互信 以及
hosts