常州市网站优化免备案域名

---

前言

hadoop学习——伪分布式环境——普通用户搭建

---

一、准备

1. 下载Hadoop

2. 配置环境变量

vi ~/.bash_profile# 增加如下配置
export HADOOP_HOME=/home/install/hadoop-3.3.4
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH# 使配置生效
source ~/.bash_profile

3. 配置免密

二、Hadoop配置

配置的目录是$HADOOP_HOME/etc/hadop

1. hadoop-env.sh

修改JAVA_HOME，设置为JDK所在的位置

# Linux命令行输出 JDK 位置
echo $JAVA_HOME

2. hdfs-site.xml

<!--指定数据冗余份数-->
<property><name>dfs.replication</name><value>1</value>
</property>
<property><name>dfs.http.address</name><value>hadoop01:50070</value>
</property>

3. core-site.xml

 <!--hadoop01是机器名  hostname-->
<property><name>fs.defaultFS</name><value>hdfs://hadoop01:9000</value>
</property>
<!-- hadoop临时目录 -->
<property><name>hadoop.tmp.dir</name><value>/home/install/data/hadoop/HADOOP_TMP_DIR</value><description>Abase for other temporary directories.</description>
</property>
<!--所有用户可访问-->
<property><name>hadoop.proxyuser.hduser.groups</name><value>*</value>
</property>
<!--设置buffer size-->
<property><name>io.file.buffer.size</name><value>131072</value>
</property>
<!--任何IP可访问-->
<property><name>hadoop.proxyuser.hduser.hosts</name><value>*</value>
</property>

4. mapred-site.xml

<property><name>mapreduce.framework.name</name><value>yarn</value>
</property>
<property><name>mapreduce.job.counters.max</name><value>1200000</value>
</property>
<property><name>mapreduce.map.memory.mb</name><value>4096</value>
</property>
<property><name>mapreduce.reduce.memory.mb</name><value>8192</value>
</property>
<property><name>mapreduce.map.java.opts</name><value>-Xmx3072m</value>
</property>
<property><name>mapreduce.reduce.java.opts</name><value>-Xmx6144m</value>
</property>

5. yarn-site.xml

<property><name>yarn.nodemanager.aux-services</name><value>mapreduce_shuffle</value>
</property>
<property><name>yarn.resourcemanager.hostname</name><value>hadoop01</value>
</property>
<property><name>yarn.nodemanager.vmem-pmem-ratio</name><value>4</value>
</property>
<property><name>yarn.nodemanager.vmem-check-enabled</name><value>false</value>
</property>
<property><name>yarn.nodemanager.resource.memory-mb</name><value>20480</value>
</property>
<property><name>yarn.scheduler.minimum-allocation-mb</name><value>1024</value>
</property>
<property><name>yarn.scheduler.maximum-allocation-mb</name><value>20480</value>
</property>

三、格式化

hadoop namenode -format

四、启动

# 懒一点的方式
start-all.sh

---