샤딩 셋팅 방법

0. 준비

설치 파일은 다음을 기준으로 한다.

mongodb-linux-x86_64-ubuntu1204-3.2.9.tgz

(엔터프라이즈 버전의 경우 에러가 발생함.)

설치는 총 7대를 기준으로 산정.

MongoDB01  mongos,Config_Server1

MongoDB02  shard01_primary

MongoDB03  shard01_secondary,Config_Server2

MongoDB04  shard01_arbiter

MongoDB05  shard02_primary

MongoDB06  shard02_secondary,Config_Server3 

MongoDB07  shard02_arbiter

2개의 샤드, 3대의 config, 1대의 mongos로 우선 셋팅한다.

* 옵션

  mongos MongoDB02, MongoDB05

  추가 샤드

  MongoDB04 - shard03_primary

  MongoDB07 - shard03_secondary

  MongoDB01 - shard03_arbiter

경로는 다음과 같이 설정한다.

/usr/local/mongodb/bin : 바이너리 실행파일 위치

/usr/local/mongodb/config : 설정파일 위치

/usr/local/mongodb/database : 데이터 디렉토리

/usr/local/mongodb/data : 설정용 데이터 디렉토리

/usr/local/mongodb/key : 인증을 위한 key 파일 디렉토리

/usr/local/mongodb/logs : 로그파일 디렉토리

key 파일

인증을 위한 key파일 생성하기.

http://docs.mongodb.org/manual/tutorial/generate-key-file/ 참조

// key 파일 생성

[root@localhost]# openssl rand -base64 741 > /usr/local/mongodb/key/db.key

// 퍼미션 설정

[root@localhost]# chmod 0400 /usr/local/mongodb/key/db.key

// 전체 클러스터에 동일한 키 파일을 동일한 위치에 복사

1. ReplicaSet 구성

arbiter + primary + secondary 로 구성된 ReplicaSet x 2

1번째 Replica 셋팅

설정파일

shard01_arbiter.conf

위치: MongoDB04 /usr/local/mongodb/config/shard01_arbiter.conf

systemLog:

    destination: file

    path: /usr/local/mongodb/logs/shard01_arbiter.log

    logAppend: true

processManagement:

    pidFilePath: /usr/local/mongodb/logs/shard01_arbiter.pid

    fork: true

net:

    port: 27021

security:

    keyFile: /usr/local/mongodb/key/db.key

storage:

    dbPath: /usr/local/mongodb/database

    journal:

        enabled: false

    smallFiles: true

replication:

    replSetName: shard01

sharding: 

    clusterRole: shardsvr

shard01_primary.conf 

위치: MongoDB02 /usr/local/mongodb/config/shard01_primary.conf

systemLog:

    destination: file

    path: /usr/local/mongodb/logs/shard01_primary.log

    logAppend: true

processManagement:

    pidFilePath: /usr/local/mongodb/logs/shard01_primary.pid

    fork: true

net:

    port: 27021

    bindIp: 0.0.0.0

security:

    keyFile: /usr/local/mongodb/key/db.key

storage:

    dbPath: /usr/local/mongodb/database

replication:

    replSetName: shard01

sharding:

    clusterRole: shardsvr

shard01_secondary.conf

위치: MongoDB03 /usr/local/mongodb/config/shard01_secondary.conf

systemLog:

    path: /usr/local/mongodb/logs/shard01_secondary.log

    logAppend: true

    destination: file

processManagement:

    pidFilePath: /usr/local/mongodb/logs/shard01_secondary.pid

    fork: true

net:

    port: 27021

security:

    keyFile: /usr/local/mongodb/key/db.key

storage:

    dbPath: /usr/local/mongodb/database

replication:

    replSetName: shard01

sharding:

    clusterRole: shardsvr

프로세스 실행

각 서버에서 아래와 같이 arbiter, primary, secondary mongod를 실행해야 함.

// 각 프로세스가 실행되어야 하는 서버에서 각각 실행되어야 함.

// shard01_arbiter

[root@MongoDB04]# /usr/local/mongodb/bin/mongod -f /usr/local/mongodb/config/shard01_arbiter.conf

// shard01_primary

[root@MongoDB02]# /usr/local/mongodb/bin/mongod -f /usr/local/mongodb/config/shard01_primary.conf

// shard01_secondary

[root@MongoDB03]# /usr/local/mongodb/bin/mongod -f /usr/local/mongodb/config/shard01_secondary.conf

실행 후 로그파일을 보면서 preallocation이 끝날 때 까지 기다린다.

ReplicaSet 초기화

MongoDB02 서버에서 shard01_primary용 mongod에 접속하여 ReplicaSet 구성

[root@MongoDB02]# /usr/local/mongodb/bin/mongo localhost:27021

// ReplicaSet 초기화

> rs.initiate()

{

        "info2" : "no configuration explicitly specified -- making one",

        "me" : "MongoDB02:27021",

        "info" : "Config now saved locally.  Should come online in about a minute.",

        "ok" : 1

}

// root 추가

// ReplicaSet의 사용자는 추후 Sharding 구성 후 mongos에서 추가된 사용자와는 별개임에 주의할 것.

shard01:PRIMARY> use admin

switched to db admin

shard01:PRIMARY> db.createUser({user: "root", pwd: "비밀번호", roles: ["root"]})

Successfully added user: { "user" : "root", "roles" : [ "root" ] }

// 추가된 사용자로 로그인

shard01:PRIMARY> db.auth("root", "비밀번호")

1

// arbiter 추가

shard01:PRIMARY> rs.addArb("MongoDB04:27021")

{ "ok" : 1 }

// secondary 추가

shard01:PRIMARY> rs.add("MongoDB03:27021")

{ "ok" : 1 }

// 좀 기다렸다가 rs.status()로 제대로 추가되었는지 확인.

shard01:PRIMARY> rs.status()

{

        "set" : "shard001",

        "date" : ISODate("2014-12-29T01:32:15Z"),

        "myState" : 1,

        "members" : [

                {

                        "_id" : 0,

                        "name" : "MongoDB02:27021",

                        "health" : 1,

                        "state" : 1,

                        "stateStr" : "PRIMARY",

                        "uptime" : 184,

                        "optime" : Timestamp(1419816708, 1),

                        "optimeDate" : ISODate("2014-12-29T01:31:48Z"),

                        "electionTime" : Timestamp(1419816631, 2),

                        "electionDate" : ISODate("2014-12-29T01:30:31Z"),

                        "self" : true

                },

                {

                        "_id" : 1,

                        "name" : "MongoDB02:27021",

                        "health" : 1,

                        "state" : 7,

                        "stateStr" : "ARBITER",

                        "uptime" : 34,

                        "lastHeartbeat" : ISODate("2014-12-29T01:32:13Z"),

                        "lastHeartbeatRecv" : ISODate("2014-12-29T01:32:13Z"),

                        "pingMs" : 1

                },

                {

                        "_id" : 2,

                        "name" : "MongoDB02:27021",

                        "health" : 1,

                        "state" : 2,

                        "stateStr" : "SECONDARY",

                        "uptime" : 27,

                        "optime" : Timestamp(1419816708, 1),

                        "optimeDate" : ISODate("2014-12-29T01:31:48Z"),

                        "lastHeartbeat" : ISODate("2014-12-29T01:32:14Z"),

                        "lastHeartbeatRecv" : ISODate("2014-12-29T01:32:13Z"),

                        "pingMs" : 1,

                        "syncingTo" : "MongoDB02:27021"

                }

        ],

        "ok" : 1

}

2번째 ReplicaSet 구성 (shard02)

설정파일

shard02_arbiter.conf

위치: MongoDB07 /usr/local/mongodb/config/shard02_arbiter.conf

systemLog:

    destination: file

    path: /usr/local/mongodb/logs/shard02_arbiter.log

    logAppend: true

processManagement:

    pidFilePath: /usr/local/mongodb/logs/shard02_arbiter.pid

    fork: true

net:

    port: 27022

security:

    keyFile: /usr/local/mongodb/key/db.key

storage:

    dbPath: /usr/local/mongodb/database

    journal:

        enabled: false

    smallFiles: true

replication:

    replSetName: shard02

sharding: 

    clusterRole: shardsvr

shard02_primary.conf 

위치: MongoDB05 /usr/local/mongodb/config/shard02_primary.conf

systemLog:

    destination: file

    path: /usr/local/mongodb/logs/shard02_primary.log

    logAppend: true

processManagement:

    pidFilePath: /usr/local/mongodb/logs/shard02_primary.pid

    fork: true

net:

    port: 27022

security:

    keyFile: /usr/local/mongodb/key/db.key

storage:

    dbPath: /usr/local/mongodb/database

replication:

    replSetName: shard02

sharding:

    clusterRole: shardsvr

shard02_secondary.conf

위치: MongoDB06 /usr/local/mongodb/config/shard02_secondary.conf

systemLog:

    path: /usr/local/mongodb/logs/shard02_secondary.log

    logAppend: true

    destination: file

processManagement:

    pidFilePath: /usr/local/mongodb/logs/shard02_secondary.pid

    fork: true

net:

    port: 27022

security:

    keyFile: /usr/local/mongodb/key/db.key

storage:

    dbPath: /usr/local/mongodb/database

replication:

    replSetName: shard02

sharding:

    clusterRole: shardsvr

프로세스 실행

각 서버에서 아래와 같이 arbiter, primary, secondary mongod를 실행해야 함.

// 각 프로세스가 실행되어야 하는 서버에서 각각 실행되어야 함.

// shard02_arbiter

[root@MongoDB07]# /usr/local/mongodb/bin/mongod -f /usr/local/mongodb/config/shard02_arbiter.conf

// shard02_primary

[root@MongoDB05]# /usr/local/mongodb/bin/mongod -f /usr/local/mongodb/config/shard02_primary.conf

// shard02_secondary

[root@MongoDB06]# /usr/local/mongodb/bin/mongod -f /usr/local/mongodb/config/shard02_secondary.conf

실행 후 로그파일을 보면서 preallocation이 끝날 때 까지 기다려야 함.

ReplicaSet 초기화

MongoDB05 서버에서 shard02_primary용 mongod에 접속하여 ReplicaSet 구성

[root@MongoDB05]# /usr/local/mongodb/bin/mongo localhost:27022

// ReplicaSet 초기화

> rs.initiate()

{

        "info2" : "no configuration explicitly specified -- making one",

        "me" : "MongoDB05:27022",

        "info" : "Config now saved locally.  Should come online in about a minute.",

        "ok" : 1

}

// root 추가

// ReplicaSet의 사용자는 추후 Sharding 구성 후 mongos에서 추가된 사용자와는 별개임에 주의할 것.

shard02:PRIMARY> use admin

switched to db admin

shard02:PRIMARY> db.createUser({user: "root", pwd: "비밀번호", roles: ["root"]})

Successfully added user: { "user" : "root", "roles" : [ "root" ] }

// 추가된 사용자로 로그인

shard02:PRIMARY> db.auth("root", "비밀번호")

1

// arbiter 추가

shard02:PRIMARY> rs.addArb("MongoDB07:27022")

{ "ok" : 1 }

// secondary 추가

shard02:PRIMARY> rs.add("MongoDB06:27022")

{ "ok" : 1 }

// 좀 기다렸다가 rs.status()로 제대로 추가되었는지 확인.

shard02:PRIMARY> rs.status()

{

        "set" : "shard002",

        "date" : ISODate("2014-12-29T01:32:15Z"),

        "myState" : 1,

        "members" : [

                {

                        "_id" : 0,

                        "name" : "MongoDB05:27022",

                        "health" : 1,

                        "state" : 1,

                        "stateStr" : "PRIMARY",

                        "uptime" : 184,

                        "optime" : Timestamp(1419816708, 1),

                        "optimeDate" : ISODate("2014-12-29T01:31:48Z"),

                        "electionTime" : Timestamp(1419816631, 2),

                        "electionDate" : ISODate("2014-12-29T01:30:31Z"),

                        "self" : true

                },

                {

                        "_id" : 1,

                        "name" : "MongoDB07:27022",

                        "health" : 1,

                        "state" : 7,

                        "stateStr" : "ARBITER",

                        "uptime" : 34,

                        "lastHeartbeat" : ISODate("2014-12-29T01:32:13Z"),

                        "lastHeartbeatRecv" : ISODate("2014-12-29T01:32:13Z"),

                        "pingMs" : 1

                },

                {

                        "_id" : 2,

                        "name" : "MongoDB06:27022",

                        "health" : 1,

                        "state" : 2,

                        "stateStr" : "SECONDARY",

                        "uptime" : 27,

                        "optime" : Timestamp(1419816708, 1),

                        "optimeDate" : ISODate("2014-12-29T01:31:48Z"),

                        "lastHeartbeat" : ISODate("2014-12-29T01:32:14Z"),

                        "lastHeartbeatRecv" : ISODate("2014-12-29T01:32:13Z"),

                        "pingMs" : 1,

                        "syncingTo" : "MongoDB05:27022"

                }

        ],

        "ok" : 1

}

ReplicaSet 2개 준비 완료.

2. config

sharding cluster의 meta data를 저장할 config 서버 구성

설정파일

config01.conf

위치: MongoDB01 /usr/local/mongodb/config/config01.conf

systemLog:

    destination: file

    path: /usr/local/mongodb/logs/config01.log

    logAppend: true

processManagement:

    pidFilePath: /usr/local/mongodb/logs/config01.pid

    fork: true

net:

    port: 27019

security:

    keyFile: /usr/local/mongodb/key/db.key

storage:

    dbPath: /usr/local/mongodb/data

sharding:

    clusterRole: configsvr  

config02.conf

위치: MongoDB03 /usr/local/mongodb/config/config02.conf

systemLog:

    destination: file

    path: /usr/local/mongodb/logs/config02.log

    logAppend: true

processManagement:

    pidFilePath: /usr/local/mongodb/logs/config02.pid

    fork: true

net:

    port: 27019

security:

    keyFile: /usr/local/mongodb/key/db.key

storage:

    dbPath: /usr/local/mongodb/data

sharding:

    clusterRole: configsvr 

config03.conf

위치: MongoDB06 /usr/local/mongodb/config/config03.conf

systemLog:

    destination: file

    path: /usr/local/mongodb/logs/config03.log

    logAppend: true

processManagement:

    pidFilePath: /usr/local/mongodb/logs/config03.pid

    fork: true

net:

    port: 27019

security:

    keyFile: /usr/local/mongodb/key/db.key

storage:

    dbPath: /usr/local/mongodb/data

sharding:

    clusterRole: configsvr 

프로세스 실행

각 서버에서 아래와 같이 config용 mongod를 실행해야 함.

// config01

[root@MongoDB01]# /usr/local/mongodb/bin/mongod -f /usr/local/mongodb/config/config01.conf

// config02

[root@MongoDB03]# /usr/local/mongodb/bin/mongod -f /usr/local/mongodb/config/config02.conf

// config03

[root@MongoDB06]# /usr/local/mongodb/bin/mongod -f /usr/local/mongodb/config/config03.conf

3. mongos

MongoDB cluster와 외부를 이어주는 router인 mongos 구성.

설정파일

mongos01.conf

위치: MongoDB01 /usr/local/mongodb/config/mongos01.conf

systemLog:

    destination: file

    path: /usr/local/mongodb/logs/mongos01.log

    logAppend: true

processManagement:

    pidFilePath: /usr/local/mongodb/logs/mongos01.pid

    fork: true

net:

    port: 27017

security:

    keyFile: /usr/local/mongodb/key/db.key

sharding:

    configDB: mongodb01:27019,mongodb03:27019,mongodb06:27019 

프로세스 실행

서버에서 아래와 같이 mongos 프로세스를 실행해야 함.

// mongos01

[root@MongoDB01]# /usr/local/mongodb/bin/mongos -f /usr/local/mongodb/config/mongos01.conf

4. ReplicaSet을 Shard로 추가

// mongos 접속

[root@MongoDB01]# /usr/local/mongodb/bin/mongo localhost:27017

// shard 추가

mongos> sh.addShard("shard01/mongodb02:27021,mongodb03:27021,mongodb04:27021")

{ "shardAdded" : "shard01", "ok" : 1 }

mongos> sh.addShard("shard02/mongodb05:27022,mongodb06:27022,mongodb07:27022")

{ "shardAdded" : "shard02", "ok" : 1 }

// 확인

mongos> use config

switched to db config

mongos> db.shards.find()

{ "_id" : "shard01", "host" : "shard01/mongodb02:27021,mongodb03:27021" }

{ "_id" : "shard02", "host" : "shard02/mongodb05:27022,mongodb06:27022" }

5. root 추가

mongos> use admin

switched to db admin

mongos> db.createUser({user: "root", pwd: "비밀번호", roles: ["root"]})

Successfully added user: { "user" : "root", "roles" : [ "root" ] }

// 확인

mongos> db.auth("root", "비밀번호")

1

6. Sharding

// sharding할 DB 선택

mongos> use mydb

switched to db mydb

// DB sharding

mongos> sh.enableSharding("mydb")

{ "ok" : 1 }

// shard key로 사용할 인덱스 생성

mongos> db.mycol.ensureIndex({idx1: 1, idx2: 1})

{

        "raw" : {

                "shard001/m02.example.com:27020,m003.example.com:27020" : {

                        "createdCollectionAutomatically" : true,

                        "numIndexesBefore" : 1,

                        "numIndexesAfter" : 2,

                        "ok" : 1

                }

        },

        "ok" : 1

}

// 생성한 인덱스를 이용하여 collection sharding

mongos> sh.shardCollection("mydb.mycol", {idx1: 1, idx2: 1})

{ "collectionsharded" : "mydb.mycol", "ok" : 1 }