前期准备

磁盘 (所有节点)

所有磁盘要求至少两块磁盘，其中一块用于安装操作系统，另一块用于数据存储。

以 sdb 为例，必须是没有经过分区或格式化的原始块设备

NTP (所有节点)

yum install ntpdate
ntpdate ntp.aliyun.com

DNS (所有节点)

所有节点要求配置好可用的 DNS

hosts (所有节点)

echo "
# GlusterFS Start
192.168.1.10 node1
192.168.1.11 node2
192.168.1.12 node3
# GlusterFS End
" >> /etc/hosts

安装

安装 GlusterFS (所有节点)

yum install centos-release-gluster
yum install glusterfs-server
glusterfs -V
systemctl enable glusterd --now

创建集群 (第一台节点)

gluster peer probe node2
gluster peer probe node3
gluster peer status

安装 Heketi (第一台节点)

yum install heketi-client heketi -y

配置免密登录 (第一台节点)

ssh-keygen -t rsa -q -f /etc/heketi/private_key -N ""
ssh-copy-id -i /etc/heketi/private_key.pub node1
ssh-copy-id -i /etc/heketi/private_key.pub node2
ssh-copy-id -i /etc/heketi/private_key.pub node3
chown heketi:heketi /etc/heketi/ -R
chown heketi:heketi /var/lib/heketi -R
ssh node1
ssh node2
ssh node3

编辑配置文件

/etc/heketi/heketi.json

{
  "_port_comment": "Heketi Server Port Number",
  "port": "48080",

  "_use_auth": "Enable JWT authorization. Please enable for deployment",
  "use_auth": true,

  "_jwt": "Private keys for access",
  "jwt": {
    "_admin": "Admin has access to all APIs",
    "admin": {
      "key": "123456"
    },
    "_user": "User only has access to /volumes endpoint",
    "user": {
      "key": "123456"
    }
  },

  "_glusterfs_comment": "GlusterFS Configuration",
  "glusterfs": {
    "_executor_comment": [
      "Execute plugin. Possible choices: mock, ssh",
      "mock: This setting is used for testing and development.",
      "      It will not send commands to any node.",
      "ssh:  This setting will notify Heketi to ssh to the nodes.",
      "      It will need the values in sshexec to be configured.",
      "kubernetes: Communicate with GlusterFS containers over",
      "            Kubernetes exec api."
    ],
    "executor": "ssh",

    "_sshexec_comment": "SSH username and private key file information",
    "sshexec": {
      "keyfile": "/etc/heketi/private_key",
      "user": "root",
      "port": "22",
      "fstab": "/etc/fstab"
    },

    "_kubeexec_comment": "Kubernetes configuration",
    "kubeexec": {
      "host" :"https://kubernetes.host:8443",
      "cert" : "/path/to/crt.file",
      "insecure": false,
      "user": "kubernetes username",
      "password": "password for kubernetes user",
      "namespace": "OpenShift project or Kubernetes namespace",
      "fstab": "Optional: Specify fstab file on node.  Default is /etc/fstab"
    },

    "_db_comment": "Database file name",
    "db": "/var/lib/heketi/heketi.db",

    "_loglevel_comment": [
      "Set log level. Choices are:",
      "  none, critical, error, warning, info, debug",
      "Default is warning"
    ],
    "loglevel" : "debug"
  }
}

/etc/heketi/topology.json

{
    "clusters": [
        {
            "nodes": [
                {
                    "node": {
                        "hostnames": {
                            "manage": [
                                "192.168.1.10"
                            ],
                            "storage": [
                                "192.168.1.10"
                            ]
                        },
                        "zone": 1
                    },
                    "devices": [
                        "/dev/sdb"
                    ]
                },
                {
                    "node": {
                        "hostnames": {
                            "manage": [
                                "192.168.1.11"
                            ],
                            "storage": [
                                "192.168.1.11"
                            ]
                        },
                        "zone": 1
                    },
                    "devices": [
                        "/dev/sdb"
                    ]
                },
                {
                    "node": {
                        "hostnames": {
                            "manage": [
                                "192.168.1.12"
                            ],
                            "storage": [
                                "192.168.1.12"
                            ]
                        },
                        "zone": 1
                    },
                    "devices": [
                        "/dev/sdb"
                    ]
                }
            ]
        }
    ]
}

启动 Heketi

systemctl enable heketi --now
systemctl status heketi
echo "export HEKETI_CLI_SERVER=http://192.168.1.10:48080" >> /etc/profile
echo "alias heketi-cli='heketi-cli --server '$HEKETI_CLI_SERVER' --user admin --secret 123456'" >> /etc/profile
source /etc/profile
heketi-cli topology load --json=/etc/heketi/topology.json

预计输出如下信息: 

Creating cluster ... ID: d9ffcf9d18524d42caf87405e344f159
        Allowing file volumes on cluster.
        Allowing block volumes on cluster.
        Creating node 192.168.1.10 ... ID: 142dcaa68407a090a97578668caabf22
                Adding device /dev/sdb ... OK
        Creating node 192.168.1.11 ... ID: a5629ae957f6de44757fe56efc86ea1e
                Adding device /dev/sdb ... OK
        Creating node 192.168.1.12 ... ID: 0671491de22f9c4f84d46f2a83979d23
                Adding device /dev/sdb ... OK

查看集群信息

heketi-cli cluster info d9ffcf9d18524d42caf87405e344f159

预计输出如下: 

Cluster id: d9ffcf9d18524d42caf87405e344f159
Nodes:
0671491de22f9c4f84d46f2a83979d23
142dcaa68407a090a97578668caabf22
a5629ae957f6de44757fe56efc86ea1e
Volumes:

Block: true

File: true

测试

heketi-cli volume create --size=2 --replica=3
heketi-cli volume list
pvdisplay
vgdisplay
lvdisplay
heketi-cli volume delete 25df596b75202e23b27c4ab3f4b3f988

设置 Kubernetes Storage Class

/root/glusterfs.yaml

apiVersion: v1
kind: Secret
metadata:
  name: heketi-secret
  namespace: kube-system
type: kubernetes.io/glusterfs
data:
  key: MTIzNDU2
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  annotations:
    storageclass.beta.kubernetes.io/is-default-class: "true"
  name: glusterfs
parameters:
  clusterid: "d9ffcf9d18524d42caf87405e344f159"
  restauthenabled: "true"
  resturl: "http://192.168.1.10:48080"
  restuser: "admin"
  secretName: "heketi-secret"
  secretNamespace: "kube-system"
  volumetype: "replicate:3"
provisioner: kubernetes.io/glusterfs
reclaimPolicy: Delete