大数据平台配置文件汇总

1.Hadoop相关配置文件

1.1core-site.xml

<?xml version="1.0" encoding="utf-8"?>

<configuration> 
    
  <property>
    <name>fs.defaultFSname>
    <value>hdfs://bdphavalue>
  property>  
    
  <property>
    <name>hadoop.tmp.dirname>
    <value>/bdp/tmpvalue>
  property>  
    
  <property>
    <name>ha.zookeeper.quorumname>
    <value>aaa-node1:2181,aaa-node7:2181,aaa-node5:2181value>
  property>  
  <property>
    <name>hadoop.proxyuser.hadoop.hostsname>
    <value>*value>
  property>  
  <property>
    <name>hadoop.proxyuser.hadoop.groupsname>
    <value>*value>
  property>  
  <property>
    <name>hadoop.proxyuser.hue.hostsname>
    <value>*value>
  property>  
  <property>
    <name>hadoop.proxyuser.hue.groupsname>
    <value>*value>
  property>  
    
  <property>
    <name>fs.trash.intervalname>
    <value>1440value>
  property>  
    
  <property>
    <name>io.compression.codecsname>
    <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodecvalue>
  property> 
configuration>

1.2hadoop-env.sh

export JAVA_HOME=/opt/java/jre/
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"}
export HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true"
export HADOOP_NAMENODE_OPTS="-XX:MaxPermSize=2048m -Xmx16000m -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS"
export HADOOP_DATANODE_OPTS="-XX:MaxPermSize=1024m -Xmx2048m -Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS"
export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS"
export HADOOP_NFS3_OPTS="$HADOOP_NFS3_OPTS"
export HADOOP_PORTMAP_OPTS="-Xmx512m $HADOOP_PORTMAP_OPTS"
export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS"
export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER}
export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
export HADOOP_PID_DIR=/opt/hadoop/PID
export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR}
export HADOOP_IDENT_STRING=$USER
export LD_LIBRARY_PATH=/usr/local/hadoop/lzo/lib:$LD_LIBRARY_PATH

1.3hdfs-site.xml

<?xml version="1.0" encoding="utf-8"?>

<configuration> 
    
  <property>
    <name>dfs.nameservicesname>
    <value>bdphavalue>
  property>  
    
  <property>
    <name>dfs.ha.namenodes.bdphaname>
    <value>nn1,nn2value>
  property>  
    
  <property>
    <name>dfs.namenode.rpc-address.bdpha.nn1name>
    <value>aaa-master2:9000value>
  property>  
    
  <property>
    <name>dfs.namenode.rpc-address.bdpha.nn2name>
    <value>aaa-master1:9000value>
  property>  
    
  <property>
    <name>dfs.namenode.http-address.bdpha.nn1name>
    <value>aaa-master2:50070value>
  property>  
    
  <property>
    <name>dfs.namenode.http-address.bdpha.nn2name>
    <value>aaa-master1:50070value>
  property>  
    
  <property>
    <name>dfs.journalnode.edits.dirname>
    <value>/data/bdp/journalnodevalue>
  property>  
    
  <property>
    <name>dfs.namenode.shared.edits.dirname>
    <value>qjournal://aaa-node2:8485;aaa-master2:8485;aaa-master1:8485/bdphavalue>
  property>  
    
  <property>
    <name>dfs.ha.fencing.methodsname>
    <value>shell(/bin/true)value>
  property>  
    
  <property>
    <name>dfs.ha.fencing.ssh.private-key-filesname>
    <value>/home/hadoop/.ssh/id_rsavalue>
  property>  
    
  <property>
    <name>dfs.ha.fencing.ssh.connect-timeoutname>
    <value>30000value>
  property>  
    
  <property>
    <name>dfs.permissionsname>
    <value>falsevalue>
  property>  
    
  <property>
    <name>dfs.ha.automatic-failover.enabledname>
    <value>truevalue>
  property>  
    
  <property>
    <name>dfs.client.failover.proxy.provider.bdphaname>
    <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvidervalue>
  property>  
    
  <property>
    <name>dfs.replicationname>
    <value>3value>
  property>  
    
  <property>
    <name>dfs.name.dirname>
    <value>/data/bdp/metadatavalue>
  property>  
    
  <property>
    <name>dfs.data.dirname>
    <value>/data1,/data2,/data3,/data4,/data5,/data6,/data7,/data8,/data9,/data10value>
  property>  
    
  <property>
    <name>dfs.datanode.failed.volumes.toleratedname>
    <value>3value>
  property> 
configuration>

1.4mapred-site.xml

<?xml version="1.0" encoding="utf-8"?>

<configuration> 
    
  <property>
    <name>mapreduce.job.reduce.slowstart.completedmapsname>
    <value>0.8value>
  property>  
    
  <property>
    <name>mapreduce.framework.namename>
    <value>yarnvalue>
  property>  
  <property>
    <name>mapreduce.map.envname>
    <value>LD_LIBRARY_PATH=/usr/local/hadoop/lzo/lib:$LD_LIBRARY_PATHvalue>
  property>  
  <property>
    <name>mapreduce.reduce.envname>
    <value>LD_LIBRARY_PATH=/usr/local/hadoop/lzo/lib:$LD_LIBRARY_PATHvalue>
  property> 
configuration>

1.5yarn-env.sh

export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
export YARN_PID_DIR=./pids
export YARN_RESOURCEMANAGER_OPTS="-XX:PermSize=256m -XX:MaxPermSize=512m -Xms4096m -Xmx8192m"
export YARN_NODEMANAGER_OPTS="-XX:PermSize=256m -XX:MaxPermSize=512m -Xms4096m -Xmx4096m"
export LD_LIBRARY_PATH=/usr/local/hadoop/lzo/lib:$LD_LIBRARY_PATH

1.6yarn-site.xml

<?xml version="1.0" encoding="utf-8"?>

<configuration> 
    
  <property>
    <name>yarn.resourcemanager.ha.enabledname>
    <value>truevalue>
  property>  
    
  <property>
    <name>yarn.resourcemanager.cluster-idname>
    <value>yrcvalue>
  property>  
    
  <property>
    <name>yarn.resourcemanager.ha.rm-idsname>
    <value>rm1,rm2value>
  property>  
    
  <property>
    <name>yarn.resourcemanager.hostname.rm1name>
    <value>aaa-master2value>
  property>  
    
  <property>
    <name>yarn.resourcemanager.hostname.rm2name>
    <value>aaa-master1value>
  property>  
    
  <property>
    <name>yarn.resourcemanager.zk-addressname>
    <value>aaa-node1:2181,aaa-node7:2181,aaa-node5:2181value>
  property>  
    
  <property>
    <name>yarn.resourcemanager.recovery.enabledname>
    <value>truevalue>
  property>  
    
  <property>
    <name>yarn.nodemanager.local-dirsname>
    <value>/data1/local/tmp,/data2/local/tmp,/data3/local/tmp,/data4/local/tmp,/data5/local/tmp,/data6/local/tmp,/data7/local/tmp,/data8/local/tmpvalue>
  property>  
    
  <property>
    <name>yarn.nodemanager.log-dirsname>
    <value>/data1/log/tmp,/data2/log/tmp,/data3/log/tmp,/data4/log/tmp,/data5/log/tmp,/data6/log/tmp,/data7/log/tmp,/data8/log/tmpvalue>
  property>  
    
  <property>
    <name>yarn.resourcemanager.store.classname>
    <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStorevalue>
  property>  
    
  <property>
    <name>yarn.nodemanager.aux-servicesname>
    <value>mapreduce_shufflevalue>
  property>  
    
  <property>
    <name>yarn.log-aggregation-enablename>
    <value>truevalue>
  property>  
    
  <property>
    <name>yarn.nodemanager.resource.memory-mbname>
    <value>24576value>
  property>  
    
  <property>
    <name>yarn.scheduler.minimum-allocation-mbname>
    <value>2048value>
  property>  
    
  <property>
    <name>yarn.scheduler.maximum-allocation-mbname>
    <value>24576value>
  property>  
    
  <property>
    <name>yarn.nodemanager.pmem-check-enablename>
    <value>falsevalue>
  property>  
    
  <property>
    <name>yarn.nodemanager.vmem-check-enablename>
    <value>falsevalue>
  property>  
    
  <property>
    <name>yarn.nodemanager.resource.cpu-vcoresname>
    <value>32value>
  property>  
    
  <property>
    <name>yarn.scheduler.minimum-allocation-vcoresname>
    <value>1value>
  property>  
    
  <property>
    <name>yarn.scheduler.maximum-allocation-vcoresname>
    <value>32value>
  property> 
configuration>

1.7log4j.properties

hadoop.root.logger=warn,console

2.Hive相关配置

2.1hive-site.xml

<?xml version="1.0" encoding="utf-8"?>

<configuration> 
  <property>
    <name>hive.metastore.urisname>
    <value>thrift://aaa-master1:9083value>
  property>  
  <property>
    <name>javax.jdo.option.ConnectionUserNamename>
    <value>hadoopvalue>
  property>  
  <property>
    <name>javax.jdo.option.ConnectionDriverNamename>
    <value>com.mysql.jdbc.Drivervalue>
  property>  
  <property>
    <name>javax.jdo.option.ConnectionURLname>
    <value>jdbc:mysql://AAA-MASTER1:3306/hive?createDatabaseIfNotExist=true&useUnicode=true&characterEncoding=latin1value>
  property>  
  <property>
    <name>hive.server2.long.polling.timeoutname>
    <value>50000value>
  property>  
  <property>
    <name>hive.server2.thrift.portname>
    <value>10000value>
  property>  
  <property>
    <name>dfs.permissionsname>
    <value>falsevalue>
  property>  
  <property>
    <name>hive.security.authorization.createtable.owner.grantsname>
    <value>ALLvalue>
  property>  
  <property>
    <name>hive.security.authorization.task.factoryname>
    <value>org.apache.hadoop.hive.ql.parse.authorization.HiveAuthorizationTaskFactoryImplvalue>
  property>  
  <property>
    <name>hive.security.authorization.enabledname>
    <value>falsevalue>
  property>  
  <property>
    <name>javax.jdo.option.ConnectionPasswordname>
    <value>hadoop0928value>
  property>  
  <property>
    <name>hive.server2.enable.doAsname>
    <value>falsevalue>
  property>  
    
  <property>
    <name>datanucleus.autoCreateSchemaname>
    <value>truevalue>
  property>  
  <property>
    <name>datanucleus.autoCreateColumnsname>
    <value>truevalue>
  property>  
  <property>
    <name>datanucleus.autoCreateTablesname>
    <value>truevalue>
  property> 
configuration>

2.2hive-env.sh

export HADOOP_HOME=/opt/hadoop
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export HIVE_CONF_DIR=/opt/hive/conf
export HIVE_AUX_JARS_PATH=/opt/hive/lib
export SPARK_HOME=/opt/spark

2.3parquet-logging.properties

.level=WARN

3.Hbase相关配置

3.1hbase-site.xml

<?xml version="1.0" encoding="utf-8"?>

<configuration> 
  <property>
    <name>hbase.zookeeper.quorumname>
    <value>aaa-node1:2181,aaa-node7:2181,aaa-node5:2181value>
  property>  
  <property>
    <name>hbase.coprocessor.region.classesname>
    <value>org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.AccessControllervalue>
  property>  
  <property>
    <name>hbase.replicationname>
    <value>truevalue>
  property>  
  <property>
    <name>hbase.hstore.compaction.maxname>
    <value>20value>
  property>  
  <property>
    <name>replication.source.nb.capacityname>
    <value>1000value>
  property>  
  <property>
    <name>replication.source.rationame>
    <value>1.0value>
  property>  
  <property>
    <name>hbase.superusername>
    <value>hadoopvalue>
  property>  
  <property>
    <name>hbase.cluster.distributedname>
    <value>truevalue>
  property>  
  <property>
    <name>hbase.rpc.enginename>
    <value>org.apache.hadoop.hbase.ipc.SecureRpcEnginevalue>
  property>  
  <property>
    <name>zookeeper.session.timeoutname>
    <value>120000value>
  property>  
  <property>
    <name>replication.replicationsource.implementationname>
    <value>com.ngdata.sep.impl.SepReplicationSourcevalue>
  property>  
  <property>
    <name>hbase.coprocessor.regionserver.classesname>
    <value>org.apache.hadoop.hbase.security.access.AccessControllervalue>
  property>  
  <property>
    <name>hbase.security.authorizationname>
    <value>truevalue>
  property>  
  <property>
    <name>hbase.hregion.memstore.mslab.enabledname>
    <value>truevalue>
  property>  
  <property>
    <name>hbase.client.scanner.timeout.periodname>
    <value>120000value>
  property>  
  <property>
    <name>hbase.hregion.majorcompactionname>
    <value>0value>
  property>  
  <property>
    <name>hbase.rootdirname>
    <value>hdfs://bdpha/hbasevalue>
  property>  
  <property>
    <name>hbase.hstore.blockingStoreFilesname>
    <value>2100000000value>
  property>  
  <property>
    <name>hbase.security.authenticationname>
    <value>simplevalue>
  property>  
  <property>
    <name>hbase.hregion.max.filesizename>
    <value>21474836480value>
  property>  
  <property>
    <name>hbase.coprocessor.master.classesname>
    <value>org.apache.hadoop.hbase.security.access.AccessControllervalue>
  property>  
  <property>
    <name>hbase.master.maxclockskewname>
    <value>30000value>
  property>  
  <property>
    <name>hbase.regionserver.handler.countname>
    <value>20value>
  property> 
configuration>

3.2hbase-env.sh

export JAVA_HOME=/opt/java/jre/ 
export HBASE_MANAGES_ZK=false 
export HBASE_CLASSPATH=/opt/hadoop/etc/hadoop
export LD_LIBRARY_PATH=/usr/local/hadoop/lzo/lib:$LD_LIBRARY_PATH
export HBASE_PID_DIR=/opt/hbase/PID
export HBASE_MASTER_OPTS="-XX:PermSize=256m -XX:MaxPermSize=512m -Xms4096m -Xmx8192m"
export HBASE_REGIONSERVER_OPTS="-XX:PermSize=256m -XX:MaxPermSize=512m -Xms4096m -Xmx8192m"

3.3log4j-properties

hbase.root.logger=WARN,console
hbase.security.logger= WARN,console

4.Spark相关配置

5.Flink相关配置

6.Solr相关配置

7.Hue相关配置

8.Presto-sever相关配置

9.MySQL相关配置

Hadoop

相关

04 Hadoop思想与原理，Hbase原理

zz手把手教你在腾讯云上搭建hadoop3.x伪集群的方法（参考该文建立伪集群）

Hadoop与大数据平台

hadoop安装教程

hadoop系列整理---PySpark

hadoop3.2.1下MapReduce操作出现错误: 找不到或无法加载主类org.apache.hadoop.mapred

为hadoop集群设置静态IP

hadoop-eclipse-plugin-2.x.x 插件编译

Apache Hadoop 完全分布式集群搭建

大数据学习之旅2——从零开始搭hadoop完全分布式集群

七、Hadoop3.3.1 HA 高可用集群QJM （基于Zookeeper，NameNode高可用+Yarn高可用）

5-Hadoop学习笔记2

标签