Hadoop框架高可用配置---HA

1.高可用集群搭建

2.准备安装环境

  tar -zxvf hadoop-3.1.2.tar.gz
  mv hadoop-3.1.2 /opt/
  cd /opt/hadoop-3.1.2/etc/hadoop/

3.修改集群环境

  vim hadoop-env.sh
  # 直接在文件的最后添加
  export JAVA_HOME=/usr/local/java/jdk1.8.0_261
  export HDFS_NAMENODE_USER=root
  export HDFS_DATANODE_USER=root
  export HDFS_ZKFC_USER=root
  export HDFS_JOURNALNODE_USER=root
  export YARN_RESOURCEMANAGER_USER=root
  export YARN_NODEMANAGER_USER=root

4.修改配置文件

  vim core-site.xml
  
  fs.defaultFS
  hdfs://bdp # 集群的名字
  
  
  hadoop.tmp.dir
  /var/bdp/hadoop/ha # 存储到linux本地的临时目录
  
  
  hadoop.http.staticuser.user # 登录的用户
  root
  
  
  ha.zookeeper.quorum # ha用到的zookeeper的集群地址
  node001:2181,node002:2181,node003:2181
  

  vim hdfs-site.xml
  # 关于集群
  
  dfs.nameservices
  bdp
  

  
  dfs.ha.namenodes.bdp
  nn1,nn2
  

  
  dfs.namenode.rpc-address.bdp.nn1
  node001:8020 # http连接时的端口和ip地址
  

  
  dfs.namenode.rpc-address.bdp.nn2
  node002:8020
  

  
  dfs.namenode.http-address.bdp.nn1
  node001:9870
  

  
  dfs.namenode.http-address.bdp.nn2
  node002:9870
  

  # 关于journalnode
  
  dfs.namenode.shared.edits.dir
  # journalnode中namenode存储的路径
  qjournal://node001:8485;node002:8485;node003:8485/bdp
  

  
  dfs.journalnode.edits.dir
  /var/bdp/hadoop/ha/qjm
  

  # 故障转移
  
  dfs.client.failover.proxy.provider.bdp
  org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
  

  
  dfs.ha.fencing.methods # 启动故障转移的类
  sshfence
  shell(true)
  

  
  dfs.ha.fencing.ssh.private-key-files # 启动故障转移的参数
  /root/.ssh/id_rsa
  

  
  dfs.ha.automatic-failover.enabled
  true
  
  # 启动2个进程
  
  dfs.replication
  2
  

  [root@node001 hadoop] vim workers
  node001
  node002
  node003

5.拷贝分发软件

  # 将配置好的软件分发到其他主机
  scp -r /opt/ root@node002:/opt/
  scp -r /opt/ root@node003:/opt/

6.修改环境变量

  vim /etc/profile
  export HADOOP_HOME=/opt/hadoop-3.1.2
  export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
  # 将环境变量拷贝到其他主机
  scp /etc/profile root@node002:/etc/profile
  scp /etc/profile root@node003:/etc/profile
  # 重新加载三台服务器的环境变量
  source /etc/profile

7.首先启动Zookeeper

  zkServer.sh start
  zkServer.sh status

启动JournalNode

  [123]hdfs --daemon start journalnode

格式化NameNode

  [root@node001]hdfs namenode -format
  # 开启namenode的使用
  [root@node001]hdfs --daemon start namenode
  [root@node002]hdfs namenode -bootstrapStandby
  [root@node001]hdfs zkfc -formatZK
  [root@node001]start-dfs.sh

8.测试集群

http://node001:9870
http://node002:9870

  [root@node001] hdfs dfs -mkdir -p /lzj
  [root@node001] hdfs dfs -put zookeeper-3.4.5.tar.gz /zj/
  # 设置块的大小
  [root@node001] hdfs dfs -D dfs.blocksize=1048576 -put zookeeper-3.4.5.tar.gz /lzj/

9.关闭集群

[root@node001]# stop-dfs.sh

10.关机拍摄快照

shutdown -h now